fix(sort): Enable locale-aware collation for UTF-8

  locales

  Fixes #9148

  The sort implementation had locale support
  infrastructure (ICU collator)
  but it was never being used due to the
  fast_lexicographic optimization
  bypassing all locale-aware code.

  Changes:
  - Modified can_use_fast_lexicographic() to check locale
  encoding
  - For UTF-8 locales, disable fast path to use
  locale_cmp()
  - Initialize ICU collator with
  AlternateHandling::Shifted to match GNU
  - Enable i18n-common and i18n-collator features in
  sort's Cargo.toml

  Result: Perfect match with GNU sort for C, POSIX, and
  UTF-8 locales.
  No performance impact for C/POSIX locales (still use
  fast path).

Author: quantum-encoding

src/uu/sort/Cargo.toml

@@ -19,6 +19,9 @@ workspace = true
 [lib]
 path = "src/sort.rs"
 
+[features]
+i18n-collator = ["uucore/i18n-collator"]
+
 [dependencies]
 bigdecimal = { workspace = true }
 binary-heap-plus = { workspace = true }
@@ -34,7 +37,7 @@ self_cell = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 unicode-width = { workspace = true }
-uucore = { workspace = true, features = ["fs", "parser", "version-cmp"] }
+uucore = { workspace = true, features = ["fs", "parser", "version-cmp", "i18n-common", "i18n-collator"] }
 fluent = { workspace = true }
 nix = { workspace = true }
 

src/uu/sort/src/sort.rs

@@ -23,6 +23,9 @@ use chunks::LineData;
 use clap::builder::ValueParser;
 use clap::{Arg, ArgAction, Command};
 use custom_str_cmp::custom_str_cmp;
+
+#[cfg(feature = "i18n-collator")]
+use uucore::i18n::collator::{locale_cmp, try_init_collator};
 use ext_sort::ext_sort;
 use fnv::FnvHasher;
 #[cfg(target_os = "linux")]
@@ -350,6 +353,18 @@ impl GlobalSettings {
 
     /// Returns true when the fast lexicographic path can be used safely.
     fn can_use_fast_lexicographic(&self) -> bool {
+        // When i18n-collator is enabled, check if we need locale-aware collation.
+        // If we're in a UTF-8 locale, we must use locale_cmp instead of byte comparison.
+        #[cfg(feature = "i18n-collator")]
+        {
+            use uucore::i18n::{get_locale_encoding, UEncoding};
+
+            if get_locale_encoding() == UEncoding::Utf8 {
+                // UTF-8 locale requires locale-aware collation
+                return false;
+            }
+        }
+
         self.mode == SortMode::Default
             && !self.ignore_case
             && !self.dictionary_order
@@ -1114,6 +1129,20 @@ fn default_merge_batch_size() -> usize {
 #[uucore::main]
 #[allow(clippy::cognitive_complexity)]
 pub fn uumain(args: impl uucore::Args) -> UResult<()> {
+    // Initialize locale collator if feature is enabled
+    #[cfg(feature = "i18n-collator")]
+    {
+        use uucore::i18n::collator::{AlternateHandling, CollatorOptions};
+
+        // Initialize ICU collator with Shifted mode to match GNU sort behavior
+        let mut opts = CollatorOptions::default();
+        opts.alternate_handling = Some(AlternateHandling::Shifted);
+
+        if !try_init_collator(opts) {
+            eprintln!("sort: warning: Failed to initialize locale collator");
+        }
+    }
+
     let mut settings = GlobalSettings::default();
 
     let matches = uucore::clap_localization::handle_clap_result_with_exit_code(uu_app(), args, 2)?;
@@ -1787,13 +1816,36 @@ fn compare_by<'a>(
             }
             SortMode::Month => month_compare(a_str, b_str),
             SortMode::Version => version_cmp(a_str, b_str),
-            SortMode::Default => custom_str_cmp(
-                a_str,
-                b_str,
-                settings.ignore_non_printing,
-                settings.dictionary_order,
-                settings.ignore_case,
-            ),
+            SortMode::Default => {
+                // Use locale-aware comparison if feature is enabled and no custom flags are set
+                #[cfg(feature = "i18n-collator")]
+                {
+                    if !(settings.ignore_case
+                        || settings.dictionary_order
+                        || settings.ignore_non_printing)
+                    {
+                        locale_cmp(a_str, b_str)
+                    } else {
+                        custom_str_cmp(
+                            a_str,
+                            b_str,
+                            settings.ignore_non_printing,
+                            settings.dictionary_order,
+                            settings.ignore_case,
+                        )
+                    }
+                }
+                #[cfg(not(feature = "i18n-collator"))]
+                {
+                    custom_str_cmp(
+                        a_str,
+                        b_str,
+                        settings.ignore_non_printing,
+                        settings.dictionary_order,
+                        settings.ignore_case,
+                    )
+                }
+            }
         };
         if cmp != Ordering::Equal {
             return if settings.reverse { cmp.reverse() } else { cmp };

src/uucore/src/lib/features/i18n/mod.rs

@@ -20,7 +20,9 @@ pub enum UEncoding {
     Utf8,
 }
 
-const DEFAULT_LOCALE: Locale = locale!("en-US-posix");
+// Use "und" (undefined) as the marker for C/POSIX locale
+// This ensures real locales like "en-US" won't match
+const DEFAULT_LOCALE: Locale = locale!("und");
 
 /// Look at 3 environment variables in the following order
 ///
@@ -38,6 +40,11 @@ fn get_locale_from_env(locale_name: &str) -> (Locale, UEncoding) {
         let mut split = locale_var_str.split(&['.', '@']);
 
         if let Some(simple) = split.next() {
+            // Handle explicit C and POSIX locales - these should always use byte comparison
+            if simple == "C" || simple == "POSIX" {
+                return (DEFAULT_LOCALE, UEncoding::Ascii);
+            }
+
             // Naively convert the locale name to BCP47 tag format.
             //
             // See https://en.wikipedia.org/wiki/IETF_language_tag