DCS: Add more formal BOM-detection to Json encoding stream wrappers.

src/libraries/System.Private.DataContractSerialization/src/System/Runtime/Serialization/Json/JsonEncodingStreamWrapper.cs

@@ -122,15 +122,12 @@ public static ArraySegment<byte> ProcessBuffer(byte[] buffer, int offset, int co
             try
             {
                 SupportedEncoding expectedEnc = GetSupportedEncoding(encoding);
-                SupportedEncoding dataEnc;
-                if (count < 2)
-                {
-                    dataEnc = SupportedEncoding.UTF8;
-                }
-                else
-                {
-                    dataEnc = ReadEncoding(buffer[offset], buffer[offset + 1]);
-                }
+                SupportedEncoding dataEnc = DetectEncoding(buffer.AsSpan(offset, count), out int bomLength);
+
+                // Skip past any byte order mark; it is not part of the document.
+                offset += bomLength;
+                count -= bomLength;
+
                 if ((expectedEnc != SupportedEncoding.None) && (expectedEnc != dataEnc))
                 {
                     ThrowExpectedEncodingMismatch(expectedEnc, dataEnc);
@@ -320,26 +317,53 @@ private static SupportedEncoding GetSupportedEncoding(Encoding? encoding)
             }
         }
 
-        private static SupportedEncoding ReadEncoding(byte b1, byte b2)
+        // Determines the encoding of a JSON document from its leading bytes. A leading byte order
+        // mark, when present, authoritatively selects the encoding and its length is reported via
+        // bomLength so callers can skip past it. When no BOM is present, the encoding is inferred
+        // from the position of the zero byte in the leading (always ASCII) JSON character. Both the
+        // stream and the buffer code paths funnel through this single method so the detection logic
+        // lives in one place.
+        private static SupportedEncoding DetectEncoding(ReadOnlySpan<byte> data, out int bomLength)
         {
-            if (b1 == 0x00 && b2 != 0x00)
-            {
-                return SupportedEncoding.UTF16BE;
-            }
-            else if (b1 != 0x00 && b2 == 0x00)
+            bomLength = 0;
+
+            // Not enough characters for a BOM
+            if (data.Length < 2)
             {
-                // 857 It's possible to misdetect UTF-32LE as UTF-16LE, but that's OK.
-                return SupportedEncoding.UTF16LE;
+                // A single-byte (or empty) JSON document is necessarily UTF-8.
+                return SupportedEncoding.UTF8;
             }
-            else if (b1 == 0x00 && b2 == 0x00)
+
+            switch ((data[0], data[1]))
             {
+                // Detect known BOMs
+                case (0xFF, 0xFE):
+                    bomLength = 2;
+                    return SupportedEncoding.UTF16LE;
+                case (0xFE, 0xFF):
+                    bomLength = 2;
+                    return SupportedEncoding.UTF16BE;
+                case (0xEF, 0xBB):
+                    if (data.Length >= 3 && data[2] == 0xBF)
+                    {
+                        bomLength = 3;
+                        return SupportedEncoding.UTF8;
+                    }
+                    break;
+
+                // No byte order mark or inference from the leading ASCII character.
+                case (0x00, not 0x00):
+                    return SupportedEncoding.UTF16BE;
+                case (not 0x00, 0x00):
+                    return SupportedEncoding.UTF16LE;
+
                 // UTF-32BE not supported
-                throw new XmlException(SR.JsonInvalidBytes);
-            }
-            else
-            {
-                return SupportedEncoding.UTF8;
+                case (0x00, 0x00):
+                    throw new XmlException(SR.JsonInvalidBytes);
             }
+
+            // No BOM detected or inferred. Assume UTF8
+            return SupportedEncoding.UTF8;
         }
 
         private static void ThrowExpectedEncodingMismatch(SupportedEncoding expEnc, SupportedEncoding actualEnc)
@@ -473,31 +497,23 @@ private void InitForWriting(Stream outputStream, Encoding writeEncoding)
 
         private SupportedEncoding ReadEncoding()
         {
-            int b1 = _stream.ReadByte();
-            int b2 = _stream.ReadByte();
-
             EnsureByteBuffer();
 
-            SupportedEncoding e;
-
-            if (b1 == -1)
-            {
-                e = SupportedEncoding.UTF8;
-                _byteCount = 0;
-            }
-            else if (b2 == -1)
-            {
-                e = SupportedEncoding.UTF8;
-                _bytes[0] = (byte)b1;
-                _byteCount = 1;
-            }
-            else
-            {
-                e = ReadEncoding((byte)b1, (byte)b2);
-                _bytes[0] = (byte)b1;
-                _bytes[1] = (byte)b2;
-                _byteCount = 2;
-            }
+            // Read whatever bytes are immediately available, up to the three occupied by the longest
+            // byte order mark. A single Read is used deliberately instead of ReadAtLeast: Read performs
+            // one underlying read and returns however many bytes were available. If it's enough for
+            // BOM detection, we will try to determine encoding. If not, we continue BOM-less.
+            // `_stream` here is buffered, so `Read()` should be able to return a full BOM if it's there.
+            // We need 3 bytes for full ASCII/UTF-8/16 detection.
+            Span<byte> leading = stackalloc byte[3];
+            int read = _stream.Read(leading);
+
+            SupportedEncoding e = DetectEncoding(leading.Slice(0, read), out int bomLength);
+
+            // Preserve any bytes that follow the byte order mark; they belong to the document.
+            int preserve = read - bomLength;
+            leading.Slice(bomLength, preserve).CopyTo(_bytes);
+            _byteCount = preserve;
 
             return e;
         }

src/libraries/System.Runtime.Serialization.Json/tests/DataContractJsonSerializer.cs

@@ -2319,6 +2319,71 @@ public static void DCJS_CreateJsonWriterTest()
         }
     }
 
+    public static IEnumerable<object[]> ByteOrderMarkEncodings()
+    {
+        yield return new object[] { new UnicodeEncoding(bigEndian: false, byteOrderMark: true) }; // UTF-16LE with BOM
+        yield return new object[] { new UnicodeEncoding(bigEndian: true, byteOrderMark: true) };  // UTF-16BE with BOM
+        yield return new object[] { new UTF8Encoding(encoderShouldEmitUTF8Identifier: true) };    // UTF-8 with BOM
+    }
+
+    [Theory]
+    [MemberData(nameof(ByteOrderMarkEncodings))]
+    public static void DCJS_DeserializeStreamWithByteOrderMark(Encoding encoding)
+    {
+        var serializer = new DataContractJsonSerializer(typeof(Person1));
+        var value = new Person1 { Name = "John", Age = 42 };
+        byte[] bytes = GetJsonBytesWithByteOrderMark(serializer, value, encoding);
+
+        // Auto-detected encoding (no encoding specified). This is the scenario from the bug report.
+        using (var stream = new MemoryStream(bytes))
+        {
+            var result = (Person1)serializer.ReadObject(stream);
+            Assert.Equal(value.Name, result.Name);
+            Assert.Equal(value.Age, result.Age);
+        }
+
+        // Encoding explicitly specified and matching the byte order mark.
+        using (var stream = new MemoryStream(bytes))
+        {
+            XmlDictionaryReader reader = JsonReaderWriterFactory.CreateJsonReader(stream, encoding, XmlDictionaryReaderQuotas.Max, null);
+            var result = (Person1)serializer.ReadObject(reader);
+            Assert.Equal(value.Name, result.Name);
+            Assert.Equal(value.Age, result.Age);
+        }
+    }
+
+    [Theory]
+    [MemberData(nameof(ByteOrderMarkEncodings))]
+    public static void DCJS_DeserializeBufferWithByteOrderMark(Encoding encoding)
+    {
+        var serializer = new DataContractJsonSerializer(typeof(Person1));
+        var value = new Person1 { Name = "John", Age = 42 };
+        byte[] bytes = GetJsonBytesWithByteOrderMark(serializer, value, encoding);
+
+        XmlDictionaryReader reader = JsonReaderWriterFactory.CreateJsonReader(bytes, 0, bytes.Length, XmlDictionaryReaderQuotas.Max);
+        var result = (Person1)serializer.ReadObject(reader);
+        Assert.Equal(value.Name, result.Name);
+        Assert.Equal(value.Age, result.Age);
+    }
+
+    private static byte[] GetJsonBytesWithByteOrderMark(DataContractJsonSerializer serializer, object value, Encoding encoding)
+    {
+        string json;
+        using (var utf8Stream = new MemoryStream())
+        {
+            serializer.WriteObject(utf8Stream, value);
+            json = Encoding.UTF8.GetString(utf8Stream.ToArray());
+        }
+
+        byte[] preamble = encoding.GetPreamble();
+        Assert.NotEmpty(preamble);
+        byte[] body = encoding.GetBytes(json);
+        byte[] bytes = new byte[preamble.Length + body.Length];
+        preamble.CopyTo(bytes, 0);
+        body.CopyTo(bytes, preamble.Length);
+        return bytes;
+    }
+
     [Fact]
     public static void DCJS_MyISerializableType()
     {