fmt : handle invalid UTF-8 input by replacing malformed sequences

src/uu/fmt/src/parasplit.rs

@@ -5,7 +5,7 @@
 
 // spell-checker:ignore (ToDO) INFTY MULT PSKIP accum aftertab beforetab breakwords fmt's formatline linebreak linebreaking linebreaks linelen maxlength minlength nchars noformat noformatline ostream overlen parasplit plass pmatch poffset posn powf prefixindent punct signum slen sstart tabwidth tlen underlen winfo wlen wordlen wordsplits xanti xprefix
 
-use std::io::{BufRead, Lines};
+use std::io::BufRead;
 use std::iter::Peekable;
 use std::slice::Iter;
 use unicode_width::UnicodeWidthChar;
@@ -78,12 +78,12 @@ pub struct FileLine {
 /// Iterator that produces a stream of Lines from a file
 pub struct FileLines<'a> {
     opts: &'a FmtOptions,
-    lines: Lines<&'a mut FileOrStdReader>,
+    reader: &'a mut FileOrStdReader,
 }
 
 impl FileLines<'_> {
-    fn new<'b>(opts: &'b FmtOptions, lines: Lines<&'b mut FileOrStdReader>) -> FileLines<'b> {
-        FileLines { opts, lines }
+    fn new<'b>(opts: &'b FmtOptions, reader: &'b mut FileOrStdReader) -> FileLines<'b> {
+        FileLines { opts, reader }
     }
 
     /// returns true if this line should be formatted
@@ -156,7 +156,19 @@ impl Iterator for FileLines<'_> {
     type Item = Line;
 
     fn next(&mut self) -> Option<Line> {
-        let n = self.lines.next()?.ok()?;
+        let mut buf = Vec::new();
+        match self.reader.read_until(b'\n', &mut buf) {
+            Ok(0) => return None,
+            Ok(_) => {}
+            Err(_) => return None,
+        }
+        if buf.ends_with(b"\n") {
+            buf.pop();
+            if buf.ends_with(b"\r") {
+                buf.pop();
+            }
+        }
+        let n = String::from_utf8_lossy(&buf).into_owned();
 
         // if this line is entirely whitespace,
         // emit a blank line
@@ -242,7 +254,7 @@ pub struct ParagraphStream<'a> {
 
 impl ParagraphStream<'_> {
     pub fn new<'b>(opts: &'b FmtOptions, reader: &'b mut FileOrStdReader) -> ParagraphStream<'b> {
-        let lines = FileLines::new(opts, reader.lines()).peekable();
+        let lines = FileLines::new(opts, reader).peekable();
         // at the beginning of the file, we might find mail headers
         ParagraphStream {
             lines,

tests/by-util/test_fmt.rs

@@ -3,7 +3,7 @@
 // For the full copyright and license information, please view the LICENSE
 // file that was distributed with this source code.
 
-// spell-checker:ignore plass samp
+// spell-checker:ignore plass samp FFFD
 #[cfg(target_os = "linux")]
 use std::os::unix::ffi::OsStringExt;
 use uutests::new_ucmd;
@@ -323,6 +323,8 @@ fn test_fmt_unicode_whitespace_handling() {
         ("non-breaking space", non_breaking_space),
         ("figure space", figure_space),
         ("narrow no-break space", narrow_no_break_space),
+        ("word joiner", "\u{2060}"),
+        ("cyrillic kha", "\u{0445}"),
     ] {
         let input = format!("={char}=");
         let result = new_ucmd!()
@@ -397,3 +399,17 @@ fn fmt_reflow_unicode() {
         .succeeds()
         .stdout_is("漢字漢字\n💐\n日本語の文字\n");
 }
+
+#[test]
+fn test_fmt_invalid_utf8() {
+    // Regression test for handling invalid UTF-8 input (e.g. ISO-8859-1)
+    // fmt should not drop lines with invalid UTF-8.
+    // \xA0 is non-breaking space in ISO-8859-1, but invalid in UTF-8.
+    // We expect it to be replaced by replacement character and treated as non-space.
+    let input = b"=\xA0=";
+    new_ucmd!()
+        .args(&["-s", "-w1"])
+        .pipe_in(input)
+        .succeeds()
+        .stdout_is("=\u{FFFD}=\n");
+}