diff --git a/src/libraries/System.Private.DataContractSerialization/src/System/Runtime/Serialization/Json/JsonEncodingStreamWrapper.cs b/src/libraries/System.Private.DataContractSerialization/src/System/Runtime/Serialization/Json/JsonEncodingStreamWrapper.cs index 2e07b82314dfdf..a96f756e0a1ae8 100644 --- a/src/libraries/System.Private.DataContractSerialization/src/System/Runtime/Serialization/Json/JsonEncodingStreamWrapper.cs +++ b/src/libraries/System.Private.DataContractSerialization/src/System/Runtime/Serialization/Json/JsonEncodingStreamWrapper.cs @@ -122,15 +122,12 @@ public static ArraySegment ProcessBuffer(byte[] buffer, int offset, int co try { SupportedEncoding expectedEnc = GetSupportedEncoding(encoding); - SupportedEncoding dataEnc; - if (count < 2) - { - dataEnc = SupportedEncoding.UTF8; - } - else - { - dataEnc = ReadEncoding(buffer[offset], buffer[offset + 1]); - } + SupportedEncoding dataEnc = DetectEncoding(buffer.AsSpan(offset, count), out int bomLength); + + // Skip past any byte order mark; it is not part of the document. + offset += bomLength; + count -= bomLength; + if ((expectedEnc != SupportedEncoding.None) && (expectedEnc != dataEnc)) { ThrowExpectedEncodingMismatch(expectedEnc, dataEnc); @@ -320,26 +317,53 @@ private static SupportedEncoding GetSupportedEncoding(Encoding? encoding) } } - private static SupportedEncoding ReadEncoding(byte b1, byte b2) + // Determines the encoding of a JSON document from its leading bytes. A leading byte order + // mark, when present, authoritatively selects the encoding and its length is reported via + // bomLength so callers can skip past it. When no BOM is present, the encoding is inferred + // from the position of the zero byte in the leading (always ASCII) JSON character. Both the + // stream and the buffer code paths funnel through this single method so the detection logic + // lives in one place. + private static SupportedEncoding DetectEncoding(ReadOnlySpan data, out int bomLength) { - if (b1 == 0x00 && b2 != 0x00) - { - return SupportedEncoding.UTF16BE; - } - else if (b1 != 0x00 && b2 == 0x00) + bomLength = 0; + + // Not enough characters for a BOM + if (data.Length < 2) { - // 857 It's possible to misdetect UTF-32LE as UTF-16LE, but that's OK. - return SupportedEncoding.UTF16LE; + // A single-byte (or empty) JSON document is necessarily UTF-8. + return SupportedEncoding.UTF8; } - else if (b1 == 0x00 && b2 == 0x00) + + switch ((data[0], data[1])) { + // Detect known BOMs + case (0xFF, 0xFE): + bomLength = 2; + return SupportedEncoding.UTF16LE; + case (0xFE, 0xFF): + bomLength = 2; + return SupportedEncoding.UTF16BE; + case (0xEF, 0xBB): + if (data.Length >= 3 && data[2] == 0xBF) + { + bomLength = 3; + return SupportedEncoding.UTF8; + } + break; + + // No byte order mark or inference from the leading ASCII character. + case (0x00, not 0x00): + return SupportedEncoding.UTF16BE; + case (not 0x00, 0x00): + return SupportedEncoding.UTF16LE; + // UTF-32BE not supported - throw new XmlException(SR.JsonInvalidBytes); - } - else - { - return SupportedEncoding.UTF8; + case (0x00, 0x00): + throw new XmlException(SR.JsonInvalidBytes); } + + // No BOM detected or inferred. Assume UTF8 + return SupportedEncoding.UTF8; } private static void ThrowExpectedEncodingMismatch(SupportedEncoding expEnc, SupportedEncoding actualEnc) @@ -473,31 +497,23 @@ private void InitForWriting(Stream outputStream, Encoding writeEncoding) private SupportedEncoding ReadEncoding() { - int b1 = _stream.ReadByte(); - int b2 = _stream.ReadByte(); - EnsureByteBuffer(); - SupportedEncoding e; - - if (b1 == -1) - { - e = SupportedEncoding.UTF8; - _byteCount = 0; - } - else if (b2 == -1) - { - e = SupportedEncoding.UTF8; - _bytes[0] = (byte)b1; - _byteCount = 1; - } - else - { - e = ReadEncoding((byte)b1, (byte)b2); - _bytes[0] = (byte)b1; - _bytes[1] = (byte)b2; - _byteCount = 2; - } + // Read whatever bytes are immediately available, up to the three occupied by the longest + // byte order mark. A single Read is used deliberately instead of ReadAtLeast: Read performs + // one underlying read and returns however many bytes were available. If it's enough for + // BOM detection, we will try to determine encoding. If not, we continue BOM-less. + // `_stream` here is buffered, so `Read()` should be able to return a full BOM if it's there. + // We need 3 bytes for full ASCII/UTF-8/16 detection. + Span leading = stackalloc byte[3]; + int read = _stream.Read(leading); + + SupportedEncoding e = DetectEncoding(leading.Slice(0, read), out int bomLength); + + // Preserve any bytes that follow the byte order mark; they belong to the document. + int preserve = read - bomLength; + leading.Slice(bomLength, preserve).CopyTo(_bytes); + _byteCount = preserve; return e; } diff --git a/src/libraries/System.Runtime.Serialization.Json/tests/DataContractJsonSerializer.cs b/src/libraries/System.Runtime.Serialization.Json/tests/DataContractJsonSerializer.cs index 0df7cb5f484b63..91060d57221ae4 100644 --- a/src/libraries/System.Runtime.Serialization.Json/tests/DataContractJsonSerializer.cs +++ b/src/libraries/System.Runtime.Serialization.Json/tests/DataContractJsonSerializer.cs @@ -2319,6 +2319,71 @@ public static void DCJS_CreateJsonWriterTest() } } + public static IEnumerable ByteOrderMarkEncodings() + { + yield return new object[] { new UnicodeEncoding(bigEndian: false, byteOrderMark: true) }; // UTF-16LE with BOM + yield return new object[] { new UnicodeEncoding(bigEndian: true, byteOrderMark: true) }; // UTF-16BE with BOM + yield return new object[] { new UTF8Encoding(encoderShouldEmitUTF8Identifier: true) }; // UTF-8 with BOM + } + + [Theory] + [MemberData(nameof(ByteOrderMarkEncodings))] + public static void DCJS_DeserializeStreamWithByteOrderMark(Encoding encoding) + { + var serializer = new DataContractJsonSerializer(typeof(Person1)); + var value = new Person1 { Name = "John", Age = 42 }; + byte[] bytes = GetJsonBytesWithByteOrderMark(serializer, value, encoding); + + // Auto-detected encoding (no encoding specified). This is the scenario from the bug report. + using (var stream = new MemoryStream(bytes)) + { + var result = (Person1)serializer.ReadObject(stream); + Assert.Equal(value.Name, result.Name); + Assert.Equal(value.Age, result.Age); + } + + // Encoding explicitly specified and matching the byte order mark. + using (var stream = new MemoryStream(bytes)) + { + XmlDictionaryReader reader = JsonReaderWriterFactory.CreateJsonReader(stream, encoding, XmlDictionaryReaderQuotas.Max, null); + var result = (Person1)serializer.ReadObject(reader); + Assert.Equal(value.Name, result.Name); + Assert.Equal(value.Age, result.Age); + } + } + + [Theory] + [MemberData(nameof(ByteOrderMarkEncodings))] + public static void DCJS_DeserializeBufferWithByteOrderMark(Encoding encoding) + { + var serializer = new DataContractJsonSerializer(typeof(Person1)); + var value = new Person1 { Name = "John", Age = 42 }; + byte[] bytes = GetJsonBytesWithByteOrderMark(serializer, value, encoding); + + XmlDictionaryReader reader = JsonReaderWriterFactory.CreateJsonReader(bytes, 0, bytes.Length, XmlDictionaryReaderQuotas.Max); + var result = (Person1)serializer.ReadObject(reader); + Assert.Equal(value.Name, result.Name); + Assert.Equal(value.Age, result.Age); + } + + private static byte[] GetJsonBytesWithByteOrderMark(DataContractJsonSerializer serializer, object value, Encoding encoding) + { + string json; + using (var utf8Stream = new MemoryStream()) + { + serializer.WriteObject(utf8Stream, value); + json = Encoding.UTF8.GetString(utf8Stream.ToArray()); + } + + byte[] preamble = encoding.GetPreamble(); + Assert.NotEmpty(preamble); + byte[] body = encoding.GetBytes(json); + byte[] bytes = new byte[preamble.Length + body.Length]; + preamble.CopyTo(bytes, 0); + body.CopyTo(bytes, preamble.Length); + return bytes; + } + [Fact] public static void DCJS_MyISerializableType() {