comm: sorted input rejected as "not in sorted order" when a line is a prefix of the next followed by a tab

From 8b550961ea7fd2e71f803ae8013dd3c614268575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alperen=20Kele=C5=9F?= <[email protected]>
Date: Fri, 16 Oct 2026 21:30:39 +0000
Subject: [PATCH] comm: do not compare the line terminator

LineReader::read_line keeps the line terminator in the buffer (and adds
one to an unterminated last line), and line_cmp compared those buffers
as they are, both for --check-order and for merging the two files. When
one line is a prefix of the next and the next continues with a byte
below the terminator, the comparison reached the terminator: "" sorts
before "\t", yet "\n" is greater than "\t\n". Sorted input such as the
output of LC_ALL=C sort was then reported as not in sorted order, and
lines from the two files were printed in the wrong order.

Strip the terminator before comparing. OrderChecker keeps the line
ending so it can do the same.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
 src/uu/comm/src/comm.rs    | 25 +++++++++++++++++++------
 tests/by-util/test_comm.rs | 24 ++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/src/uu/comm/src/comm.rs b/src/uu/comm/src/comm.rs
index bfe5ffd50..cbd81e363 100644
--- a/src/uu/comm/src/comm.rs
+++ b/src/uu/comm/src/comm.rs
@@ -62,6 +62,7 @@ struct OrderChecker {
     mode: CheckOrder,
     has_error: bool,
     use_locale: bool,
+    line_ending: u8,
 }
 
 /// Compare two lines the way the input was ordered.
@@ -69,7 +70,12 @@ struct OrderChecker {
 /// `sort` orders with the locale collation, and so do `join` and `ls`. Reading
 /// that order back with a byte comparison rejects input that is in order and
 /// puts lines in the wrong column, so `comm` has to measure it the same way.
-fn line_cmp(a: &[u8], b: &[u8], use_locale: bool) -> Ordering {
+///
+/// The line terminator is not part of the line and must not take part in the
+/// comparison: `""` sorts before `"\t"`, yet `"\n"` is greater than `"\t\n"`.
+fn line_cmp(a: &[u8], b: &[u8], line_ending: u8, use_locale: bool) -> Ordering {
+    let a = a.strip_suffix(&[line_ending]).unwrap_or(a);
+    let b = b.strip_suffix(&[line_ending]).unwrap_or(b);
     if use_locale {
         locale_cmp(a, b)
     } else {
@@ -121,13 +127,14 @@ impl LineReader {
 }
 
 impl OrderChecker {
-    fn new(file_num: FileNumber, mode: CheckOrder, use_locale: bool) -> Self {
+    fn new(file_num: FileNumber, mode: CheckOrder, use_locale: bool, line_ending: u8) -> Self {
         Self {
             last_line: Vec::new(),
             file_num,
             mode,
             has_error: false,
             use_locale,
+            line_ending,
         }
     }
 
@@ -143,7 +150,12 @@ impl OrderChecker {
             return true;
         }
 
-        let is_ordered = line_cmp(current_line, &self.last_line, self.use_locale) != Ordering::Less;
+        let is_ordered = line_cmp(
+            current_line,
+            &self.last_line,
+            self.line_ending,
+            self.use_locale,
+        ) != Ordering::Less;
         if !is_ordered && !self.has_error {
             let _ = writeln!(
                 stderr(),
@@ -211,8 +223,9 @@ fn comm(
     };
 
     let use_locale = should_use_locale_collation();
-    let mut checker1 = OrderChecker::new(FileNumber::One, mode, use_locale);
-    let mut checker2 = OrderChecker::new(FileNumber::Two, mode, use_locale);
+    let line_ending: u8 = a.line_ending.into();
+    let mut checker1 = OrderChecker::new(FileNumber::One, mode, use_locale, line_ending);
+    let mut checker2 = OrderChecker::new(FileNumber::Two, mode, use_locale, line_ending);
     let mut files_differ = false;
     let mut input_error = false;
 
@@ -220,7 +233,7 @@ fn comm(
         let ord = match (na, nb) {
             (0, _) => Ordering::Greater,
             (_, 0) => Ordering::Less,
-            (_, _) => line_cmp(ra, rb, use_locale),
+            (_, _) => line_cmp(ra, rb, line_ending, use_locale),
         };
 
         if ord != Ordering::Equal {
diff --git a/tests/by-util/test_comm.rs b/tests/by-util/test_comm.rs
index 88a01f211..95deaebd8 100644
--- a/tests/by-util/test_comm.rs
+++ b/tests/by-util/test_comm.rs
@@ -859,3 +859,27 @@ fn test_both_inputs_stdin() {
         .stdout_is("\ta\nb\n\t\tc\n")
         .no_stderr();
 }
+
+#[test]
+fn test_line_terminator_not_compared() {
+    // "" sorts before "\t" and "a" before "a\tb"; the newline that ends each
+    // line must not take part in the comparison.
+    let scene = TestScenario::new(util_name!());
+    let at = &scene.fixtures;
+    at.write("empty", "");
+    at.write("sorted", "\n\t\n");
+    at.write("f1", "a\n");
+    at.write("f2", "a\tb\n");
+    scene
+        .ucmd()
+        .env("LC_ALL", "C")
+        .args(&["--check-order", "empty", "sorted"])
+        .succeeds()
+        .stdout_only("\t\n\t\t\n");
+    scene
+        .ucmd()
+        .env("LC_ALL", "C")
+        .args(&["f1", "f2"])
+        .succeeds()
+        .stdout_only("a\n\ta\tb\n");
+}
-- 
2.55.0