From 3e9c0fc3341ade0ead898cb41aa1b73ddbbb9cbe Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 17 Mar 2021 20:49:15 +0000 Subject: [PATCH 01/27] implement IAsyncEnumerable JsonConverter --- .../src/Resources/Strings.resx | 5 +- .../src/System.Text.Json.csproj | 8 +- .../Text/Json/Serialization/ClassType.cs | 2 +- .../IAsyncEnumerableConverterFactory.cs | 55 ++++ .../IAsyncEnumerableOfTConverter.cs | 144 +++++++++ .../Collection/IEnumerableDefaultConverter.cs | 2 +- .../Converters/Object/JsonObjectConverter.cs | 13 +- .../Text/Json/Serialization/JsonClassInfo.cs | 12 +- .../JsonResumableConverterOfT.cs | 10 +- .../JsonSerializer.Write.Helpers.cs | 12 +- .../JsonSerializer.Write.Stream.cs | 43 ++- .../JsonSerializerOptions.Converters.cs | 2 + .../Text/Json/Serialization/WriteStack.cs | 178 +++++++++++ .../Json/Serialization/WriteStackFrame.cs | 13 + .../Text/Json/ThrowHelper.Serialization.cs | 7 + .../CollectionTests.AsyncEnumerable.cs | 276 ++++++++++++++++++ .../tests/System.Text.Json.Tests.csproj | 1 + 17 files changed, 760 insertions(+), 23 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs create mode 100644 src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs diff --git a/src/libraries/System.Text.Json/src/Resources/Strings.resx b/src/libraries/System.Text.Json/src/Resources/Strings.resx index c6039fdeba147f..47c651107dfa33 100644 --- a/src/libraries/System.Text.Json/src/Resources/Strings.resx +++ b/src/libraries/System.Text.Json/src/Resources/Strings.resx @@ -357,6 +357,9 @@ The type '{0}' is not supported. + + The type '{0}' can only be serialized using async serialization methods. + '{0}' is invalid after '/' at the beginning of the comment. Expected either '/' or '*'. @@ -557,4 +560,4 @@ The converter '{0}' cannot return an instance of JsonConverterFactory. - \ No newline at end of file + diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index cb91cbdfdb4efb..8a6d6410164873 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -65,10 +65,12 @@ + + @@ -233,11 +235,9 @@ - + - + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ClassType.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ClassType.cs index 7dd4fe798d1315..aedc4343192f53 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ClassType.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ClassType.cs @@ -20,7 +20,7 @@ internal enum ClassType : byte Value = 0x2, // JsonValueConverter<> - simple values that need to re-enter the serializer such as KeyValuePair. NewValue = 0x4, - // JsonIEnumerbleConverter<> - all enumerable collections except dictionaries. + // JsonIEnumerableConverter<> - all enumerable collections except dictionaries. Enumerable = 0x8, // JsonDictionaryConverter<,> - dictionary types. Dictionary = 0x10, diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs new file mode 100644 index 00000000000000..a181a2aaa1b1b2 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization.Converters; + +namespace System.Text.Json.Serialization +{ + /// + /// Converter for streaming values. + /// + internal class IAsyncEnumerableConverterFactory : JsonConverterFactory + { + public override bool CanConvert(Type typeToConvert) => TryGetAsyncEnumerableInterface(typeToConvert, out _); + + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + if (!TryGetAsyncEnumerableInterface(typeToConvert, out Type? asyncEnumerableInterface)) + { + Debug.Fail("type not supported by the converter."); + throw new Exception(); + } + + Type elementType = asyncEnumerableInterface.GetGenericArguments()[0]; + Type converterType = typeof(IAsyncEnumerableOfTConverter<,>).MakeGenericType(typeToConvert, elementType); + return (JsonConverter)Activator.CreateInstance(converterType)!; + } + + internal static bool TryGetAsyncEnumerableInterface(Type type, [NotNullWhen(true)] out Type? asyncEnumerableInterface) + { + if (type.IsInterface && IsAsyncEnumerableInterface(type)) + { + asyncEnumerableInterface = type; + return true; + } + + foreach (Type interfaceTy in type.GetInterfaces()) + { + if (IsAsyncEnumerableInterface(interfaceTy)) + { + asyncEnumerableInterface = interfaceTy; + return true; + } + } + + asyncEnumerableInterface = null; + return false; + + static bool IsAsyncEnumerableInterface(Type interfaceTy) + => interfaceTy.IsGenericType && interfaceTy.GetGenericTypeDefinition() == typeof(IAsyncEnumerable<>); + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs new file mode 100644 index 00000000000000..a876fa3c4784fa --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs @@ -0,0 +1,144 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace System.Text.Json.Serialization.Converters +{ + internal sealed class IAsyncEnumerableOfTConverter + : IEnumerableDefaultConverter + where TAsyncEnumerable : IAsyncEnumerable + { + internal override bool OnTryWrite(Utf8JsonWriter writer, TAsyncEnumerable value, JsonSerializerOptions options, ref WriteStack state) + { + if (!state.SupportContinuation) + { + ThrowHelper.ThrowNotSupportedException_TypeRequiresAsyncSerialization(TypeToConvert); + } + + return base.OnTryWrite(writer, value, options, ref state); + } + + [Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2012:Use ValueTasks correctly", Justification = "Converter needs to consume ValueTask's in a non-async context")] + protected override bool OnWriteResume(Utf8JsonWriter writer, TAsyncEnumerable value, JsonSerializerOptions options, ref WriteStack state) + { + IAsyncEnumerator enumerator; + ValueTask moveNextTask; + + if (state.Current.AsyncEnumerator is null) + { + enumerator = value.GetAsyncEnumerator(state.CancellationToken); + moveNextTask = enumerator.MoveNextAsync(); + // we always need to attach the enumerator to the stack + // since it will need to be disposed asynchronously. + state.Current.AsyncEnumerator = enumerator; + } + else + { + Debug.Assert(state.Current.AsyncEnumerator is IAsyncEnumerator); + enumerator = (IAsyncEnumerator)state.Current.AsyncEnumerator; + + if (state.Current.AsyncEnumeratorIsPendingCompletion) + { + // converter was previously suspended due to a pending MoveNextAsync() task + Debug.Assert(state.PendingTask is Task && state.PendingTask.IsCompleted); + moveNextTask = new ValueTask((Task)state.PendingTask); + state.Current.AsyncEnumeratorIsPendingCompletion = false; + state.PendingTask = null; + } + else + { + // converter was suspended for a different reason; + // the last MoveNextAsync() call can only have completed with 'true'. + moveNextTask = new ValueTask(true); + } + } + + JsonConverter converter = GetElementConverter(ref state); + + // iterate through the enumerator while elements are being returned synchronously + for (; moveNextTask.IsCompleted; moveNextTask = enumerator.MoveNextAsync()) + { + if (!moveNextTask.Result) + { + return true; + } + + if (ShouldFlush(writer, ref state)) + { + return false; + } + + TElement element = enumerator.Current; + if (!converter.TryWrite(writer, element, options, ref state)) + { + return false; + } + } + + // we have a pending MoveNextAsync() call; + // wrap inside a regular task so that it can be awaited multiple times; + // mark the current stackframe as pending completion. + Debug.Assert(state.PendingTask is null); + state.PendingTask = moveNextTask.AsTask(); + state.Current.AsyncEnumeratorIsPendingCompletion = true; + return false; + } + + internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, ref ReadStack state, out TAsyncEnumerable value) + { + if (!typeToConvert.IsAssignableFrom(typeof(IAsyncEnumerable))) + { + ThrowHelper.ThrowNotSupportedException_CannotPopulateCollection(TypeToConvert, ref reader, ref state); + } + + return base.OnTryRead(ref reader, typeToConvert, options, ref state, out value!); + } + + protected override void Add(in TElement value, ref ReadStack state) + { + ((BufferedAsyncEnumerable)state.Current.ReturnValue!)._buffer.Add(value); + } + + protected override void CreateCollection(ref Utf8JsonReader reader, ref ReadStack state, JsonSerializerOptions options) + { + state.Current.ReturnValue = new BufferedAsyncEnumerable(); + } + + private class BufferedAsyncEnumerable : IAsyncEnumerable + { + public readonly List _buffer = new(); + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken _) => + new BufferedAsyncEnumerator(_buffer); + + private class BufferedAsyncEnumerator : IAsyncEnumerator + { + private readonly List _buffer; + private int _index; + + public BufferedAsyncEnumerator(List buffer) + { + _buffer = buffer; + _index = -1; + } + + public TElement Current => _index < 0 ? default! : _buffer[_index]; + public ValueTask DisposeAsync() => default; + public ValueTask MoveNextAsync() + { + if (_index == _buffer.Count - 1) + { + return new ValueTask(false); + } + + _index++; + return new ValueTask(true); + } + } + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IEnumerableDefaultConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IEnumerableDefaultConverter.cs index 67613d90973a30..0400b617121731 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IEnumerableDefaultConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IEnumerableDefaultConverter.cs @@ -218,7 +218,7 @@ internal override bool OnTryRead( return true; } - internal sealed override bool OnTryWrite( + internal override bool OnTryWrite( Utf8JsonWriter writer, TCollection value, JsonSerializerOptions options, diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/JsonObjectConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/JsonObjectConverter.cs index 02752db31d549d..9b5aaea790b07f 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/JsonObjectConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/JsonObjectConverter.cs @@ -9,7 +9,18 @@ namespace System.Text.Json.Serialization /// internal abstract class JsonObjectConverter : JsonResumableConverter { + internal JsonObjectConverter() + { + // Populate ElementType if the runtime type implements IAsyncEnumerable. + // Used to feed the (converter-agnostic) JsonClassInfo.ElementType instace + // which is subsequently consulted by custom enumerable converters fed via JsonConverterAttribute. + if (IAsyncEnumerableConverterFactory.TryGetAsyncEnumerableInterface(typeof(T), out Type? asyncEnumerableInterface)) + { + ElementType = asyncEnumerableInterface.GetGenericArguments()[0]; + } + } + internal sealed override ClassType ClassType => ClassType.Object; - internal sealed override Type? ElementType => null; + internal sealed override Type? ElementType { get; } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonClassInfo.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonClassInfo.cs index 16374a3e180965..b1d3460ac09d50 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonClassInfo.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonClassInfo.cs @@ -46,8 +46,11 @@ public JsonClassInfo? ElementClassInfo { if (_elementClassInfo == null && ElementType != null) { - Debug.Assert(ClassType == ClassType.Enumerable || - ClassType == ClassType.Dictionary); + Debug.Assert( + ClassType == ClassType.Enumerable || + ClassType == ClassType.Dictionary || + (ClassType == ClassType.Object && + IAsyncEnumerableConverterFactory.TryGetAsyncEnumerableInterface(Type, out _))); _elementClassInfo = Options.GetOrAddClass(ElementType); } @@ -246,6 +249,11 @@ public JsonClassInfo(Type type, JsonSerializerOptions options) { InitializeConstructorParameters(converter.ConstructorInfo!); } + + // Types like IAsyncEnumerable can default to the Object class type + // but still require the ElementType for other converters specified + // via JsonConverterAttribute. + ElementType = converter.ElementType; } break; case ClassType.Enumerable: diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonResumableConverterOfT.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonResumableConverterOfT.cs index 05efb460b64f1b..e1d7eb14c6b0f8 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonResumableConverterOfT.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonResumableConverterOfT.cs @@ -36,7 +36,15 @@ public sealed override void Write(Utf8JsonWriter writer, T value, JsonSerializer WriteStack state = default; state.Initialize(typeof(T), options, supportContinuation: false); - TryWrite(writer, value, options, ref state); + try + { + TryWrite(writer, value, options, ref state); + } + catch + { + state.DisposePendingDisposablesOnException(); + throw; + } } public sealed override bool HandleNull => false; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Helpers.cs index 6f9dea9343c6e1..75aa2de65f0848 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Helpers.cs @@ -29,8 +29,16 @@ private static void WriteCore( WriteStack state = default; JsonConverter jsonConverter = state.Initialize(inputType, options, supportContinuation: false); - bool success = WriteCore(jsonConverter, writer, value, options, ref state); - Debug.Assert(success); + try + { + bool success = WriteCore(jsonConverter, writer, value, options, ref state); + Debug.Assert(success); + } + catch + { + state.DisposePendingDisposablesOnException(); + throw; + } } private static bool WriteCore( diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs index 6e4a3ef256442b..a44d66cc990514 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs @@ -113,21 +113,44 @@ private static async Task WriteAsyncCore( inputType = value!.GetType(); } - WriteStack state = default; + WriteStack state = new WriteStack { CancellationToken = cancellationToken }; JsonConverter converterBase = state.Initialize(inputType, options, supportContinuation: true); bool isFinalBlock; - do + try { - state.FlushThreshold = (int)(bufferWriter.Capacity * FlushThreshold); - - isFinalBlock = WriteCore(converterBase, writer, value, options, ref state); - - await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); - - bufferWriter.Clear(); - } while (!isFinalBlock); + do + { + state.FlushThreshold = (int)(bufferWriter.Capacity * FlushThreshold); + + try + { + isFinalBlock = WriteCore(converterBase, writer, value, options, ref state); + } + finally + { + if (state.PendingAsyncDisposables?.Count > 0) + { + await state.DisposePendingAsyncDisposables().ConfigureAwait(false); + } + } + + await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); + bufferWriter.Clear(); + + if (state.PendingTask is not null) + { + await state.AwaitPendingTask().ConfigureAwait(false); + } + + } while (!isFinalBlock); + } + catch + { + await state.DisposePendingDisposablesOnExceptionAsync().ConfigureAwait(false); + throw; + } } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptions.Converters.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptions.Converters.cs index 2f17deede6d5b1..226f86f171a560 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptions.Converters.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptions.Converters.cs @@ -26,6 +26,8 @@ public sealed partial class JsonSerializerOptions // Nullable converter should always be next since it forwards to any nullable type. new NullableConverterFactory(), new EnumConverterFactory(), + // IAsyncEnumerable takes precedence over IEnumerable. + new IAsyncEnumerableConverterFactory(), // IEnumerable should always be second to last since they can convert any IEnumerable. new IEnumerableConverterFactory(), // Object should always be last since it converts any type. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs index 7f231046fc036d..82a1a4814afe67 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs @@ -1,9 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.Runtime.ExceptionServices; using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; namespace System.Text.Json { @@ -20,6 +24,22 @@ internal struct WriteStack /// private int _count; + /// + /// Cancellation token used by converters performing async serialization (e.g. IAsyncEnumerable) + /// + public CancellationToken CancellationToken; + + /// + /// Stores a pending task that a resumable converter depends on to continue work. + /// It must be awaited by the root context before serialization is resumed. + /// + public Task? PendingTask; + + /// + /// List of IAsyncDisposables that have been scheduled for disposal by converters. + /// + public List? PendingAsyncDisposables; + private List _previous; // A field is used instead of a property to avoid value semantics. @@ -172,6 +192,14 @@ public void Pop(bool success) else { Debug.Assert(_continuationCount == 0); + + if (Current.AsyncEnumerator is not null) + { + // we have completed serialization of an AsyncEnumerator, + // pop from the stack and schedule for async disposal. + PendingAsyncDisposables ??= new(); + PendingAsyncDisposables.Add(Current.AsyncEnumerator); + } } if (_count > 1) @@ -180,6 +208,156 @@ public void Pop(bool success) } } + // asynchronously await any pending stacks that resumable converters depend on. + public async ValueTask AwaitPendingTask() + { + Debug.Assert(PendingTask != null); + + if (!PendingTask.IsCompleted) + { + // wrap with Task.WhenAny to avoid surfacing any exceptions here + await Task.WhenAny(PendingTask).ConfigureAwait(false); + } + + // Do not clear the `PendingTask` field here since the result + // will need to be consumed by a resumable converter. + } + + // Asynchronously dispose of any AsyncDisposables that have been scheduled for disposal + public async ValueTask DisposePendingAsyncDisposables() + { + Debug.Assert(PendingAsyncDisposables?.Count > 0); + List? exceptions = null; + + foreach (IAsyncDisposable asyncDisposable in PendingAsyncDisposables) + { + try + { + await asyncDisposable.DisposeAsync().ConfigureAwait(false); + } + catch (Exception exn) + { + exceptions ??= new(); + exceptions.Add(exn); + } + } + + if (exceptions is not null) + { + if (exceptions.Count == 1) + { + ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); + } + + throw new AggregateException(exceptions); + } + + PendingAsyncDisposables.Clear(); + } + + /// + /// Walks the stack cleaning up any leftover IDisposables + /// in the event of an exception on serialization + /// + public void DisposePendingDisposablesOnException() + { + List? exceptions = null; + + Debug.Assert(Current.AsyncEnumerator is null); + DisposeFrame(Current.CollectionEnumerator); + + int stackSize = Math.Max(_count, _continuationCount); + if (stackSize > 1) + { + for (int i = 0; i < stackSize - 1; i++) + { + Debug.Assert(_previous[i].AsyncEnumerator is null); + DisposeFrame(_previous[i].CollectionEnumerator); + } + } + + if (exceptions is not null) + { + if (exceptions.Count == 1) + { + ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); + } + + throw new AggregateException(exceptions); + } + + void DisposeFrame(IEnumerator? collectionEnumerator) + { + try + { + if (collectionEnumerator is IDisposable disposable) + { + disposable.Dispose(); + return; + } + } + catch (Exception e) + { + exceptions ??= new(); + exceptions.Add(e); + } + } + } + + /// + /// Walks the stack cleaning up any leftover I(Async)Disposables + /// in the event of an exception on async serialization + /// + public async ValueTask DisposePendingDisposablesOnExceptionAsync() + { + List? exceptions = null; + + await DisposeFrame(Current.CollectionEnumerator, Current.AsyncEnumerator).ConfigureAwait(false); + + int stackSize = Math.Max(_count, _continuationCount); + if (stackSize > 1) + { + for (int i = 0; i < stackSize - 1; i++) + { + await DisposeFrame(_previous[i].CollectionEnumerator, _previous[i].AsyncEnumerator).ConfigureAwait(false); + } + } + + if (exceptions is not null) + { + if (exceptions.Count == 1) + { + ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); + } + + throw new AggregateException(exceptions); + } + + async ValueTask DisposeFrame(IEnumerator? collectionEnumerator, IAsyncDisposable? asyncDisposable) + { + Debug.Assert(!(collectionEnumerator is not null && asyncDisposable is not null)); + + try + { + if (collectionEnumerator is IDisposable disposable) + { + disposable.Dispose(); + return; + } + + if (asyncDisposable is not null) + { + await asyncDisposable.DisposeAsync().ConfigureAwait(false); + } + } + catch (Exception e) + { + exceptions ??= new(); + exceptions.Add(e); + } + } + } + // Return a property path as a simple JSONPath using dot-notation when possible. When special characters are present, bracket-notation is used: // $.x.y.z // $['PropertyName.With.Special.Chars'] diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStackFrame.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStackFrame.cs index 59533e5c542310..bdb299c3b24372 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStackFrame.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStackFrame.cs @@ -3,6 +3,7 @@ using System.Collections; using System.Diagnostics; +using System.Threading.Tasks; using System.Text.Json.Serialization; namespace System.Text.Json @@ -15,6 +16,17 @@ internal struct WriteStackFrame /// public IEnumerator? CollectionEnumerator; + /// + /// The enumerator for resumable async disposables. + /// + public IAsyncDisposable? AsyncEnumerator; + + /// + /// The current stackframe has suspended serialization due to a pending task, + /// stored in the property. + /// + public bool AsyncEnumeratorIsPendingCompletion; + /// /// The original JsonPropertyInfo that is not changed. It contains all properties. /// @@ -112,6 +124,7 @@ public JsonConverter InitializeReEntry(Type type, JsonSerializerOptions options) public void Reset() { CollectionEnumerator = null; + AsyncEnumerator = null; EnumeratorIndex = 0; IgnoreDictionaryKeyPolicy = false; JsonClassInfo = null!; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs index 34868bbf4edef1..355cbd99ced919 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs @@ -26,6 +26,13 @@ public static void ThrowNotSupportedException_SerializationNotSupported(Type pro throw new NotSupportedException(SR.Format(SR.SerializationNotSupportedType, propertyType)); } + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ThrowNotSupportedException_TypeRequiresAsyncSerialization(Type propertyType) + { + throw new NotSupportedException(SR.Format(SR.TypeRequiresAsyncSerialization, propertyType)); + } + [DoesNotReturn] [MethodImpl(MethodImplOptions.NoInlining)] public static void ThrowNotSupportedException_ConstructorMaxOf64Parameters(ConstructorInfo constructorInfo, Type type) diff --git a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs new file mode 100644 index 00000000000000..5616cc7a04f2fa --- /dev/null +++ b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs @@ -0,0 +1,276 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Text.Json.Serialization; +using Xunit; + +namespace System.Text.Json.Tests.Serialization +{ + public static partial class CollectionTests + { + [Theory] + [MemberData(nameof(GetAsyncEnumerableSources))] + public static async Task WriteRootLevelAsyncEnumerable(IEnumerable source, int delayInterval) + { + string expectedJson = JsonSerializer.Serialize(source); + + using var stream = new Utf8MemoryStream(); + var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); + await JsonSerializer.SerializeAsync(stream, asyncEnumerable); + + Assert.Equal(expectedJson, stream.ToString()); + Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); + Assert.Equal(1, asyncEnumerable.TotalDisposedEnumerators); + } + + [Theory] + [MemberData(nameof(GetAsyncEnumerableSources))] + public static async Task WriteNestedAsyncEnumerable(IEnumerable source, int delayInterval) + { + string expectedJson = JsonSerializer.Serialize(new { Data = source }); + + using var stream = new Utf8MemoryStream(); + var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); + await JsonSerializer.SerializeAsync(stream, new { Data = asyncEnumerable }); + + Assert.Equal(expectedJson, stream.ToString()); + Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); + Assert.Equal(1, asyncEnumerable.TotalDisposedEnumerators); + } + + [Theory] + [MemberData(nameof(GetAsyncEnumerableSources))] + public static async Task WriteNestedAsyncEnumerable_DTO(IEnumerable source, int delayInterval) + { + string expectedJson = JsonSerializer.Serialize(new { Data = source }); + + using var stream = new Utf8MemoryStream(); + var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); + await JsonSerializer.SerializeAsync(stream, new AsyncEnumerableDto { Data = asyncEnumerable }); + + Assert.Equal(expectedJson, stream.ToString()); + Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); + Assert.Equal(1, asyncEnumerable.TotalDisposedEnumerators); + } + + [Fact] + public static async Task WriteAsyncEnumerable_LongRunningEnumeration_Cancellation() + { + var longRunningEnumerable = new MockedAsyncEnumerable( + source: Enumerable.Range(1, 100), + delayInterval: 1, + delay: TimeSpan.FromMinutes(1)); + + using var utf8Stream = new Utf8MemoryStream(); + using var cts = new CancellationTokenSource(delay: TimeSpan.FromSeconds(5)); + await Assert.ThrowsAsync(async () => + await JsonSerializer.SerializeAsync(utf8Stream, longRunningEnumerable, cancellationToken: cts.Token)); + + Assert.Equal(1, longRunningEnumerable.TotalCreatedEnumerators); + Assert.Equal(1, longRunningEnumerable.TotalDisposedEnumerators); + } + + public class AsyncEnumerableDto + { + public IAsyncEnumerable Data { get; set; } + } + + [Theory] + [MemberData(nameof(GetAsyncEnumerableSources))] + public static async Task WriteSequentialNestedAsyncEnumerables(IEnumerable source, int delayInterval) + { + string expectedJson = JsonSerializer.Serialize(new { Data1 = source, Data2 = source }); + + using var stream = new Utf8MemoryStream(); + var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); + await JsonSerializer.SerializeAsync(stream, new { Data1 = asyncEnumerable, Data2 = asyncEnumerable }); + + Assert.Equal(expectedJson, stream.ToString()); + Assert.Equal(2, asyncEnumerable.TotalCreatedEnumerators); + Assert.Equal(2, asyncEnumerable.TotalDisposedEnumerators); + } + + [Theory] + [MemberData(nameof(GetAsyncEnumerableSources))] + public static async Task WriteAsyncEnumerableOfAsyncEnumerables(IEnumerable source, int delayInterval) + { + const int OuterEnumerableCount = 5; + string expectedJson = JsonSerializer.Serialize(Enumerable.Repeat(source, OuterEnumerableCount)); + + var innerAsyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); + var outerAsyncEnumerable = + new MockedAsyncEnumerable>( + Enumerable.Repeat(innerAsyncEnumerable, OuterEnumerableCount), delayInterval); + + using var stream = new Utf8MemoryStream(); + await JsonSerializer.SerializeAsync(stream, outerAsyncEnumerable); + + Assert.Equal(expectedJson, stream.ToString()); + Assert.Equal(1, outerAsyncEnumerable.TotalCreatedEnumerators); + Assert.Equal(1, outerAsyncEnumerable.TotalDisposedEnumerators); + Assert.Equal(OuterEnumerableCount, innerAsyncEnumerable.TotalCreatedEnumerators); + Assert.Equal(OuterEnumerableCount, innerAsyncEnumerable.TotalDisposedEnumerators); + } + + [Fact] + public static void WriteRootLevelAsyncEnumerableSync_ThrowsNotSupportedException() + { + IAsyncEnumerable asyncEnumerable = new MockedAsyncEnumerable(Enumerable.Range(1, 10)); + Assert.Throws(() => JsonSerializer.Serialize(asyncEnumerable)); + } + + [Fact] + public static void WriteNestedAsyncEnumerableSync_ThrowsNotSupportedException() + { + IAsyncEnumerable asyncEnumerable = new MockedAsyncEnumerable(Enumerable.Range(1, 10)); + Assert.Throws(() => JsonSerializer.Serialize(new { Data = asyncEnumerable })); + } + + [Fact] + public static async Task ReadRootLevelAsyncEnumerable() + { + var utf8Stream = new Utf8MemoryStream("[0,1,2,3,4]"); + + IAsyncEnumerable result = await JsonSerializer.DeserializeAsync>(utf8Stream); + Assert.Equal(new int[] { 0, 1, 2, 3, 4 }, await result.ToListAsync()); + } + + [Fact] + public static async Task ReadNestedAsyncEnumerable() + { + var utf8Stream = new Utf8MemoryStream(@"{ ""Data"" : [0,1,2,3,4] }"); + + var result = await JsonSerializer.DeserializeAsync>(utf8Stream); + Assert.Equal(new int[] { 0, 1, 2, 3, 4 }, await result.Data.ToListAsync()); + } + + [Fact] + public static async Task ReadAsyncEnumerableOfAsyncEnumerables() + { + var utf8Stream = new Utf8MemoryStream("[[0,1,2,3,4], []]"); + + var result = await JsonSerializer.DeserializeAsync>>(utf8Stream); + var resultArray = await result.ToListAsync(); + + Assert.Equal(2, resultArray.Count); + Assert.Equal(new int[] { 0, 1, 2, 3, 4 }, await resultArray[0].ToListAsync()); + Assert.Equal(Array.Empty(), await resultArray[1].ToListAsync()); + } + + [Fact] + public static async Task ReadRootLevelAsyncEnumerableDerivative_ThrowsNotSupportedException() + { + var utf8Stream = new Utf8MemoryStream("[0,1,2,3,4]"); + await Assert.ThrowsAsync(async () => await JsonSerializer.DeserializeAsync>(utf8Stream)); + } + + public static IEnumerable GetAsyncEnumerableSources() + { + yield return WrapArgs(Enumerable.Empty(), 0); + yield return WrapArgs(Enumerable.Range(0, 20), 0); + yield return WrapArgs(Enumerable.Range(0, 100), 20); + + static object[] WrapArgs(IEnumerable source, int delayInterval) => new object[]{ source, delayInterval }; + } + + private static async Task> ToListAsync(this IAsyncEnumerable source) + { + var list = new List(); + await foreach (T item in source) + { + list.Add(item); + } + + return list; + } + + private class MockedAsyncEnumerable : IAsyncEnumerable, IEnumerable + { + private readonly IEnumerable _source; + private readonly TimeSpan _delay; + private readonly int _delayInterval; + + internal int TotalCreatedEnumerators { get; private set; } + internal int TotalDisposedEnumerators { get; private set; } + internal int TotalEnumeratedElements { get; private set; } + + public MockedAsyncEnumerable(IEnumerable source, int delayInterval = 0, TimeSpan? delay = null) + { + _source = source; + _delay = delay ?? TimeSpan.FromMilliseconds(20); + _delayInterval = delayInterval; + } + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + return new MockedAsyncEnumerator(this, cancellationToken); + } + + // Enumerator class required to instrument IAsyncDisposable calls + private class MockedAsyncEnumerator : IAsyncEnumerator + { + private readonly MockedAsyncEnumerable _enumerable; + private IAsyncEnumerator _innerEnumerator; + + public MockedAsyncEnumerator(MockedAsyncEnumerable enumerable, CancellationToken token) + { + _enumerable = enumerable; + _innerEnumerator = enumerable.GetAsyncEnumeratorInner(token); + } + + public TElement Current => _innerEnumerator.Current; + public ValueTask DisposeAsync() + { + _enumerable.TotalDisposedEnumerators++; + return _innerEnumerator.DisposeAsync(); + } + + public ValueTask MoveNextAsync() => _innerEnumerator.MoveNextAsync(); + } + + private async IAsyncEnumerator GetAsyncEnumeratorInner(CancellationToken cancellationToken = default) + { + TotalCreatedEnumerators++; + int i = 0; + foreach (TElement element in _source) + { + if (i > 0 && _delayInterval > 0 && i % _delayInterval == 0) + { + await Task.Delay(_delay, cancellationToken); + } + + if (cancellationToken.IsCancellationRequested) + { + yield break; + } + + TotalEnumeratedElements++; + yield return element; + i++; + } + } + + public IEnumerator GetEnumerator() => throw new InvalidOperationException("Collection should not be enumerated synchronously."); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + private class Utf8MemoryStream : MemoryStream + { + public Utf8MemoryStream() : base() + { + } + + public Utf8MemoryStream(string text) : base(Encoding.UTF8.GetBytes(text)) + { + } + + public override string ToString() => Encoding.UTF8.GetString(ToArray()); + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj index dbebf9b1814f3a..c5f73c89c2f85f 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj @@ -40,6 +40,7 @@ + From b758985ae2f882f8869cad2461d386392adf0842 Mon Sep 17 00:00:00 2001 From: Steve Harter Date: Tue, 8 Dec 2020 13:58:05 -0600 Subject: [PATCH 02/27] Prototype of IAsyncEnumerable deserialize with Stream --- .../System.Text.Json/ref/System.Text.Json.cs | 1 + .../src/System.Text.Json.csproj | 3 + .../JsonSerializer.Read.Stream.cs | 233 ++++++++++-------- .../Text/Json/Serialization/ReadAsyncState.cs | 50 ++++ .../SerializerReadAsyncEnumerable.cs | 38 +++ .../SerializerReadAsyncEnumerator.cs | 110 +++++++++ .../Serialization/Stream.IAsyncEnumerable.cs | 52 ++++ .../tests/System.Text.Json.Tests.csproj | 1 + 8 files changed, 388 insertions(+), 100 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerable.cs create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs create mode 100644 src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index cf0f709ad2d2c5..48a4c8c5f9e519 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -192,6 +192,7 @@ public static partial class JsonSerializer public static object? Deserialize(ref System.Text.Json.Utf8JsonReader reader, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type returnType, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static System.Threading.Tasks.ValueTask DeserializeAsync(System.IO.Stream utf8Json, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type returnType, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.ValueTask DeserializeAsync<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Collections.Generic.IAsyncEnumerable DeserializeAsyncEnumerable<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.ReadOnlySpan utf8Json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(string json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.ReadOnlySpan json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 8a6d6410164873..a45636c3015f21 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -175,6 +175,7 @@ + @@ -183,6 +184,8 @@ + + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs index 5c695a62c43f27..a03e0100a15032 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; @@ -45,7 +46,12 @@ public static partial class JsonSerializer throw new ArgumentNullException(nameof(utf8Json)); } - return ReadAsync(utf8Json, typeof(TValue), options, cancellationToken); + if (utf8Json == null) + throw new ArgumentNullException(nameof(utf8Json)); + + ReadAsyncState asyncState = new ReadAsyncState(typeof(TValue), cancellationToken, options); + + return ReadAllAsync(utf8Json, asyncState); } /// @@ -83,137 +89,153 @@ public static partial class JsonSerializer if (returnType == null) throw new ArgumentNullException(nameof(returnType)); - return ReadAsync(utf8Json, returnType, options, cancellationToken); + ReadAsyncState asyncState = new ReadAsyncState(returnType, cancellationToken, options); + + return ReadAllAsync(utf8Json, asyncState); } - private static async ValueTask ReadAsync( + /// + /// todo + /// + /// + /// + /// + /// + /// + public static IAsyncEnumerable DeserializeAsyncEnumerable<[DynamicallyAccessedMembers(MembersAccessedOnRead)] TValue>( Stream utf8Json, - Type returnType, - JsonSerializerOptions? options, - CancellationToken cancellationToken) + JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) { - if (options == null) + if (utf8Json == null) { - options = JsonSerializerOptions.s_defaultOptions; + throw new ArgumentNullException(nameof(utf8Json)); } - ReadStack state = default; - state.Initialize(returnType, options, supportContinuation: true); - - JsonConverter converter = state.Current.JsonPropertyInfo!.ConverterBase; - - var readerState = new JsonReaderState(options.GetReaderOptions()); - - // todo: https://github.com/dotnet/runtime/issues/32355 - int utf8BomLength = JsonConstants.Utf8Bom.Length; - byte[] buffer = ArrayPool.Shared.Rent(Math.Max(options.DefaultBufferSize, utf8BomLength)); - int bytesInBuffer = 0; - long totalBytesRead = 0; - int clearMax = 0; - bool isFirstIteration = true; + return new SerializerReadAsyncEnumerable(utf8Json, options); + } + internal static async ValueTask ReadAllAsync(Stream utf8Json, ReadAsyncState asyncState) + { try { while (true) { - // Read from the stream until either our buffer is filled or we hit EOF. - // Calling ReadCore is relatively expensive, so we minimize the number of times - // we need to call it. - bool isFinalBlock = false; - while (true) + bool isFinalBlock = await ReadFromStream(utf8Json, asyncState).ConfigureAwait(false); + + TValue value = ContinueDeserialize(asyncState, isFinalBlock); + if (isFinalBlock) { - int bytesRead = await utf8Json.ReadAsync( + return value!; + } + } + } + finally + { + asyncState.Dispose(); + } + } + + /// + /// Read from the stream until either our buffer is filled or we hit EOF. + /// Calling ReadCore is relatively expensive, so we minimize the number of times + /// we need to call it. + /// + internal static async ValueTask ReadFromStream(Stream utf8Json, ReadAsyncState asyncState) + { + bool isFinalBlock = false; + while (!isFinalBlock) + { + int bytesRead = await utf8Json.ReadAsync( #if BUILDING_INBOX_LIBRARY - buffer.AsMemory(bytesInBuffer), + asyncState.Buffer.AsMemory(asyncState.BytesInBuffer), #else - buffer, bytesInBuffer, buffer.Length - bytesInBuffer, + asyncState.Buffer, asyncState.BytesInBuffer, asyncState.Buffer.Length - asyncState.BytesInBuffer, #endif - cancellationToken).ConfigureAwait(false); + asyncState.CancellationToken).ConfigureAwait(false); - if (bytesRead == 0) - { - isFinalBlock = true; - break; - } + if (bytesRead == 0) + { + isFinalBlock = true; + break; + } - totalBytesRead += bytesRead; - bytesInBuffer += bytesRead; + asyncState.TotalBytesRead += bytesRead; + asyncState.BytesInBuffer += bytesRead; - if (bytesInBuffer == buffer.Length) - { - break; - } - } + if (asyncState.BytesInBuffer == asyncState.Buffer.Length) + { + break; + } + } - if (bytesInBuffer > clearMax) - { - clearMax = bytesInBuffer; - } + return isFinalBlock; + } - int start = 0; - if (isFirstIteration) - { - isFirstIteration = false; - - // Handle the UTF-8 BOM if present - Debug.Assert(buffer.Length >= JsonConstants.Utf8Bom.Length); - if (buffer.AsSpan().StartsWith(JsonConstants.Utf8Bom)) - { - start += utf8BomLength; - bytesInBuffer -= utf8BomLength; - } - } + internal static TValue ContinueDeserialize(ReadAsyncState asyncState, bool isFinalBlock) + { + if (asyncState.BytesInBuffer > asyncState.ClearMax) + { + asyncState.ClearMax = asyncState.BytesInBuffer; + } - // Process the data available - TValue value = ReadCore( - ref readerState, - isFinalBlock, - new ReadOnlySpan(buffer, start, bytesInBuffer), - options, - ref state, - converter); + int start = 0; + if (asyncState.IsFirstIteration) + { + asyncState.IsFirstIteration = false; - Debug.Assert(state.BytesConsumed <= bytesInBuffer); - int bytesConsumed = checked((int)state.BytesConsumed); + // Handle the UTF-8 BOM if present + Debug.Assert(asyncState.Buffer.Length >= JsonConstants.Utf8Bom.Length); + if (asyncState.Buffer.AsSpan().StartsWith(JsonConstants.Utf8Bom)) + { + start += JsonConstants.Utf8Bom.Length; + asyncState.BytesInBuffer -= JsonConstants.Utf8Bom.Length; + } + } - bytesInBuffer -= bytesConsumed; + // Process the data available + TValue value = ReadCore( + ref asyncState.ReaderState, + isFinalBlock, + new ReadOnlySpan(asyncState.Buffer, start, asyncState.BytesInBuffer), + asyncState.Options, + ref asyncState.ReadStack, + asyncState.Converter); - if (isFinalBlock) - { - // The reader should have thrown if we have remaining bytes. - Debug.Assert(bytesInBuffer == 0); + Debug.Assert(asyncState.ReadStack.BytesConsumed <= asyncState.BytesInBuffer); + int bytesConsumed = checked((int)asyncState.ReadStack.BytesConsumed); - return value; - } + asyncState.BytesInBuffer -= bytesConsumed; - // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. - if ((uint)bytesInBuffer > ((uint)buffer.Length / 2)) - { - // We have less than half the buffer available, double the buffer size. - byte[] dest = ArrayPool.Shared.Rent((buffer.Length < (int.MaxValue / 2)) ? buffer.Length * 2 : int.MaxValue); + if (isFinalBlock) + { + // The reader should have thrown if we have remaining bytes. + Debug.Assert(asyncState.BytesInBuffer == 0); + return value; + } - // Copy the unprocessed data to the new buffer while shifting the processed bytes. - Buffer.BlockCopy(buffer, bytesConsumed + start, dest, 0, bytesInBuffer); + // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. + if ((uint)asyncState.BytesInBuffer > ((uint)asyncState.Buffer.Length / 2)) + { + // We have less than half the buffer available, double the buffer size. + byte[] dest = ArrayPool.Shared.Rent((asyncState.Buffer.Length < (int.MaxValue / 2)) ? asyncState.Buffer.Length * 2 : int.MaxValue); - new Span(buffer, 0, clearMax).Clear(); - ArrayPool.Shared.Return(buffer); + // Copy the unprocessed data to the new buffer while shifting the processed bytes. + Buffer.BlockCopy(asyncState.Buffer, bytesConsumed + start, dest, 0, asyncState.BytesInBuffer); - clearMax = bytesInBuffer; - buffer = dest; - } - else if (bytesInBuffer != 0) - { - // Shift the processed bytes to the beginning of buffer to make more room. - Buffer.BlockCopy(buffer, bytesConsumed + start, buffer, 0, bytesInBuffer); - } - } + new Span(asyncState.Buffer, 0, asyncState.ClearMax).Clear(); + ArrayPool.Shared.Return(asyncState.Buffer); + + asyncState.ClearMax = asyncState.BytesInBuffer; + asyncState.Buffer = dest; } - finally + else if (asyncState.BytesInBuffer != 0) { - // Clear only what we used and return the buffer to the pool - new Span(buffer, 0, clearMax).Clear(); - ArrayPool.Shared.Return(buffer); + // Shift the processed bytes to the beginning of buffer to make more room. + Buffer.BlockCopy(asyncState.Buffer, bytesConsumed + start, asyncState.Buffer, 0, asyncState.BytesInBuffer); } + + return value!; // Return the partial value. } private static TValue ReadCore( @@ -233,7 +255,18 @@ private static TValue ReadCore( state.ReadAhead = !isFinalBlock; state.BytesConsumed = 0; - TValue? value = ReadCore(converterBase, ref reader, options, ref state); + TValue? value; + if (isFinalBlock) + { + value = ReadCore(converterBase, ref reader, options, ref state); + } + else + { + ReadCore(converterBase, ref reader, options, ref state); + + // Obtain the partial value. + value = (TValue)state.Current.ReturnValue!; + } readerState = reader.CurrentState; return value!; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs new file mode 100644 index 00000000000000..dc789926dd2cb3 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Threading; + +namespace System.Text.Json.Serialization +{ + internal class ReadAsyncState : IDisposable + { + public CancellationToken CancellationToken; + public byte[] Buffer; + public int BytesInBuffer; + public int ClearMax; + public JsonConverter Converter; + public bool IsFirstIteration; + public JsonReaderState ReaderState; + public ReadStack ReadStack; + public JsonSerializerOptions Options; + public long TotalBytesRead; + + public ReadAsyncState(Type returnType, CancellationToken cancellationToken = default, JsonSerializerOptions? options = null) + { + Options = options ??= JsonSerializerOptions.s_defaultOptions; + Buffer = ArrayPool.Shared.Rent(Math.Max(Options.DefaultBufferSize, JsonConstants.Utf8Bom.Length)); + ReadStack.Initialize(returnType, Options, supportContinuation: true); + Converter = ReadStack.Current.JsonPropertyInfo!.ConverterBase; + ReaderState = new JsonReaderState(Options.GetReaderOptions()); + CancellationToken = cancellationToken; + IsFirstIteration = true; + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + // Clear only what we used and return the buffer to the pool + new Span(Buffer, 0, ClearMax).Clear(); + ArrayPool.Shared.Return(Buffer); + Buffer = null!; + } + } + + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerable.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerable.cs new file mode 100644 index 00000000000000..cc0e02e593d180 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerable.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Threading; + +namespace System.Text.Json.Serialization +{ + internal struct SerializerReadAsyncEnumerable : IAsyncEnumerable + { + public SerializerReadAsyncEnumerable(Stream stream, JsonSerializerOptions? options) + { + Stream = stream; + Options = options; + } + + /// + /// todo + /// + public Stream Stream { get; set; } + + /// + /// todo + /// + public JsonSerializerOptions? Options { get; set; } + + public SerializerReadAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken) + { + return new SerializerReadAsyncEnumerator(Stream, Options); + } + + IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) + { + return GetAsyncEnumerator(cancellationToken); + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs new file mode 100644 index 00000000000000..31f42febaadbe4 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; + +namespace System.Text.Json.Serialization +{ + /// + /// todo + /// + /// + // todo: change to struct? + internal sealed class SerializerReadAsyncEnumerator : IAsyncEnumerator + { + private Stream _utf8Json; + private ReadAsyncState _asyncState; + private List? _list; + private TValue? _current; + private bool _isFinalBlock; + private bool _doneProcessing; + + public SerializerReadAsyncEnumerator(Stream utf8Json, JsonSerializerOptions? options) + { + _utf8Json = utf8Json; + _asyncState = new ReadAsyncState(typeof(List), cancellationToken: default, options); + } + + public TValue Current + { + get + { + return _current!; + } + } + + /// + /// todo + /// + /// + public ValueTask DisposeAsync() + { + return default; + } + + /// + /// todo + /// + /// + public async ValueTask MoveNextAsync() + { + if (_doneProcessing) + { + return false; + } + + if (HasLocalDataToReturn()) + { + return true; + } + + await ContinueRead().ConfigureAwait(false); + HasLocalDataToReturn(); + return true; + + bool HasLocalDataToReturn() + { + if (_list?.Count > 1) + { + _current = _list[0]; + _list.RemoveAt(0); + return true; + } + + if (_isFinalBlock) + { + if (_list?.Count >= 1) + { + _current = _list[0]; + _list.RemoveAt(0); + } + + _doneProcessing = true; + _asyncState.Dispose(); + _asyncState = null!; + + return true; + } + + return false; + } + } + + private async ValueTask ContinueRead() + { + while (!HaveDataToReturn()) + { + _isFinalBlock = await JsonSerializer.ReadFromStream(_utf8Json, _asyncState).ConfigureAwait(false); + _list = JsonSerializer.ContinueDeserialize>(_asyncState, _isFinalBlock); + } + + return true; + + // If .Count is 1, we may still be processing the item. + bool HaveDataToReturn() => _isFinalBlock || _list?.Count > 1; + } + } +} diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs new file mode 100644 index 00000000000000..8ac15094355294 --- /dev/null +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace System.Text.Json.Serialization.Tests +{ + public static partial class StreamTests_IAsyncEnumerable + { + [Theory] + [InlineData(1)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + public static async Task ReadSimpleObjectAsync(int count) + { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = 1 + }; + + // Produce the JSON + SimpleTestClass[] collection = new SimpleTestClass[count]; + for (int i = 0; i < collection.Length; i++) + { + var obj = new SimpleTestClass(); + obj.Initialize(); + collection[i] = obj; + } + + byte[] data = JsonSerializer.SerializeToUtf8Bytes(collection); + + // Use async await on the Stream. + using (MemoryStream stream = new MemoryStream(data)) + { + int callbackCount = 0; + + await foreach(SimpleTestClass item in + JsonSerializer.DeserializeAsyncEnumerable(stream, options)) + { + item.Verify(); + callbackCount++; + } + + Assert.Equal(count, callbackCount); + } + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj index c5f73c89c2f85f..238acba6712657 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj @@ -124,6 +124,7 @@ + From 2dab454751d8635a6e778de11fe5a964dee06e5a Mon Sep 17 00:00:00 2001 From: Steve Harter Date: Wed, 9 Dec 2020 08:52:10 -0600 Subject: [PATCH 03/27] Use a Queue + test buffersizes --- .../SerializerReadAsyncEnumerator.cs | 22 +++++++++---------- .../Serialization/Stream.IAsyncEnumerable.cs | 14 +++++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs index 31f42febaadbe4..45300e591f53fa 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers; using System.Collections.Generic; using System.IO; using System.Threading.Tasks; @@ -17,7 +16,7 @@ internal sealed class SerializerReadAsyncEnumerator : IAsyncEnumerator? _list; + private Queue? _valuesToReturn; private TValue? _current; private bool _isFinalBlock; private bool _doneProcessing; @@ -25,7 +24,7 @@ internal sealed class SerializerReadAsyncEnumerator : IAsyncEnumerator), cancellationToken: default, options); + _asyncState = new ReadAsyncState(typeof(Queue), cancellationToken: default, options); } public TValue Current @@ -67,19 +66,17 @@ public async ValueTask MoveNextAsync() bool HasLocalDataToReturn() { - if (_list?.Count > 1) + if (_valuesToReturn?.Count > 1) { - _current = _list[0]; - _list.RemoveAt(0); + _current = _valuesToReturn.Dequeue(); return true; } if (_isFinalBlock) { - if (_list?.Count >= 1) + if (_valuesToReturn?.Count >= 1) { - _current = _list[0]; - _list.RemoveAt(0); + _current = _valuesToReturn.Dequeue(); } _doneProcessing = true; @@ -98,13 +95,14 @@ private async ValueTask ContinueRead() while (!HaveDataToReturn()) { _isFinalBlock = await JsonSerializer.ReadFromStream(_utf8Json, _asyncState).ConfigureAwait(false); - _list = JsonSerializer.ContinueDeserialize>(_asyncState, _isFinalBlock); + _valuesToReturn = JsonSerializer.ContinueDeserialize>(_asyncState, _isFinalBlock); } return true; - // If .Count is 1, we may still be processing the item. - bool HaveDataToReturn() => _isFinalBlock || _list?.Count > 1; + // If .Count is 1, the item may be a partial object. + // todo: if feasible, have a better way to detect that the item is finished. + bool HaveDataToReturn() => _isFinalBlock || _valuesToReturn?.Count > 1; } } } diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs index 8ac15094355294..a8e2ad9f3ea3ee 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs @@ -11,15 +11,17 @@ namespace System.Text.Json.Serialization.Tests public static partial class StreamTests_IAsyncEnumerable { [Theory] - [InlineData(1)] - [InlineData(10)] - [InlineData(100)] - [InlineData(1000)] - public static async Task ReadSimpleObjectAsync(int count) + [InlineData(1, 1)] + [InlineData(10, 1)] + [InlineData(100, 1)] + [InlineData(1000, 1)] + [InlineData(1000, 1000)] + [InlineData(1000, 32000)] + public static async Task ReadSimpleObjectAsync(int count, int bufferSize) { JsonSerializerOptions options = new JsonSerializerOptions { - DefaultBufferSize = 1 + DefaultBufferSize = bufferSize }; // Produce the JSON From d00415d2d90c0ac889043dfa12ff46c3ce85d329 Mon Sep 17 00:00:00 2001 From: Steve Harter Date: Wed, 9 Dec 2020 13:25:48 -0600 Subject: [PATCH 04/27] Avoid 1 item lag --- .../SerializerReadAsyncEnumerator.cs | 27 ++++++++++--------- .../Serialization/Stream.IAsyncEnumerable.cs | 6 ++++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs index 45300e591f53fa..d1ad72afef8790 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Threading.Tasks; @@ -53,6 +54,7 @@ public async ValueTask MoveNextAsync() if (_doneProcessing) { return false; + // todo: throw InvalidOperationException if MoveNext is called again? } if (HasLocalDataToReturn()) @@ -60,29 +62,32 @@ public async ValueTask MoveNextAsync() return true; } + // Read additional data. await ContinueRead().ConfigureAwait(false); - HasLocalDataToReturn(); - return true; + return ApplyReturnValue(); bool HasLocalDataToReturn() { - if (_valuesToReturn?.Count > 1) + if (ApplyReturnValue()) { - _current = _valuesToReturn.Dequeue(); return true; } if (_isFinalBlock) { - if (_valuesToReturn?.Count >= 1) - { - _current = _valuesToReturn.Dequeue(); - } - _doneProcessing = true; _asyncState.Dispose(); _asyncState = null!; + } + + return false; + } + bool ApplyReturnValue() + { + if (_valuesToReturn?.Count > 0) + { + _current = _valuesToReturn.Dequeue(); return true; } @@ -100,9 +105,7 @@ private async ValueTask ContinueRead() return true; - // If .Count is 1, the item may be a partial object. - // todo: if feasible, have a better way to detect that the item is finished. - bool HaveDataToReturn() => _isFinalBlock || _valuesToReturn?.Count > 1; + bool HaveDataToReturn() => _isFinalBlock || _valuesToReturn?.Count > 0; } } } diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs index a8e2ad9f3ea3ee..799becb4b5d2f6 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using Xunit; @@ -30,6 +29,7 @@ public static async Task ReadSimpleObjectAsync(int count, int bufferSize) { var obj = new SimpleTestClass(); obj.Initialize(); + obj.MyInt32 = i; // verify order correctness collection[i] = obj; } @@ -43,7 +43,11 @@ public static async Task ReadSimpleObjectAsync(int count, int bufferSize) await foreach(SimpleTestClass item in JsonSerializer.DeserializeAsyncEnumerable(stream, options)) { + Assert.Equal(callbackCount, item.MyInt32); + + item.MyInt32 = 2; // Put correct value back for Verify() item.Verify(); + callbackCount++; } From e497c82cc711aa1a59985beec63c80e040d55438 Mon Sep 17 00:00:00 2001 From: Steve Harter Date: Fri, 11 Dec 2020 12:18:31 -0600 Subject: [PATCH 05/27] Add support for Serialize --- .../System.Text.Json/ref/System.Text.Json.cs | 1 + .../JsonSerializer.Read.Stream.cs | 82 +++++++--------- .../JsonSerializer.Write.Stream.cs | 93 ++++++++++++++++--- .../SerializerReadAsyncEnumerator.cs | 5 +- .../Serialization/Stream.IAsyncEnumerable.cs | 71 ++++++++++++++ 5 files changed, 185 insertions(+), 67 deletions(-) diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index 48a4c8c5f9e519..043ca627336408 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -201,6 +201,7 @@ public static partial class JsonSerializer public static void Serialize(System.Text.Json.Utf8JsonWriter writer, object? value, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type inputType, System.Text.Json.JsonSerializerOptions? options = null) { } public static System.Threading.Tasks.Task SerializeAsync(System.IO.Stream utf8Json, object? value, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type inputType, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.Task SerializeAsync<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields)]TValue>(System.IO.Stream utf8Json, TValue value, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Threading.Tasks.Task SerializeAsyncEnumerable<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields)]TValue>(System.IO.Stream utf8Json, System.Collections.Generic.IAsyncEnumerable value, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default) { throw null; } public static byte[] SerializeToUtf8Bytes(object? value, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type inputType, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static byte[] SerializeToUtf8Bytes<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(TValue value, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static void Serialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.Text.Json.Utf8JsonWriter writer, TValue value, System.Text.Json.JsonSerializerOptions? options = null) { } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs index a03e0100a15032..37c0c17550ed64 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs @@ -46,12 +46,7 @@ public static partial class JsonSerializer throw new ArgumentNullException(nameof(utf8Json)); } - if (utf8Json == null) - throw new ArgumentNullException(nameof(utf8Json)); - - ReadAsyncState asyncState = new ReadAsyncState(typeof(TValue), cancellationToken, options); - - return ReadAllAsync(utf8Json, asyncState); + return ReadAllAsync(utf8Json, typeof(TValue), options, cancellationToken); } /// @@ -89,9 +84,7 @@ public static partial class JsonSerializer if (returnType == null) throw new ArgumentNullException(nameof(returnType)); - ReadAsyncState asyncState = new ReadAsyncState(returnType, cancellationToken, options); - - return ReadAllAsync(utf8Json, asyncState); + return ReadAllAsync(utf8Json, returnType, options, cancellationToken); } /// @@ -115,25 +108,25 @@ public static partial class JsonSerializer return new SerializerReadAsyncEnumerable(utf8Json, options); } - internal static async ValueTask ReadAllAsync(Stream utf8Json, ReadAsyncState asyncState) + internal static async ValueTask ReadAllAsync( + Stream utf8Json, + Type inputType, + JsonSerializerOptions? options, + CancellationToken cancellationToken) { - try + using (var asyncState = new ReadAsyncState(inputType, cancellationToken, options)) { while (true) { bool isFinalBlock = await ReadFromStream(utf8Json, asyncState).ConfigureAwait(false); - TValue value = ContinueDeserialize(asyncState, isFinalBlock); + if (isFinalBlock) { return value!; } } } - finally - { - asyncState.Dispose(); - } } /// @@ -207,35 +200,34 @@ internal static TValue ContinueDeserialize(ReadAsyncState asyncState, bo asyncState.BytesInBuffer -= bytesConsumed; - if (isFinalBlock) - { - // The reader should have thrown if we have remaining bytes. - Debug.Assert(asyncState.BytesInBuffer == 0); - return value; - } + // The reader should have thrown if we have remaining bytes. + Debug.Assert(!isFinalBlock || asyncState.BytesInBuffer == 0); - // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. - if ((uint)asyncState.BytesInBuffer > ((uint)asyncState.Buffer.Length / 2)) + if (!isFinalBlock) { - // We have less than half the buffer available, double the buffer size. - byte[] dest = ArrayPool.Shared.Rent((asyncState.Buffer.Length < (int.MaxValue / 2)) ? asyncState.Buffer.Length * 2 : int.MaxValue); + // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. + if ((uint)asyncState.BytesInBuffer > ((uint)asyncState.Buffer.Length / 2)) + { + // We have less than half the buffer available, double the buffer size. + byte[] dest = ArrayPool.Shared.Rent((asyncState.Buffer.Length < (int.MaxValue / 2)) ? asyncState.Buffer.Length * 2 : int.MaxValue); - // Copy the unprocessed data to the new buffer while shifting the processed bytes. - Buffer.BlockCopy(asyncState.Buffer, bytesConsumed + start, dest, 0, asyncState.BytesInBuffer); + // Copy the unprocessed data to the new buffer while shifting the processed bytes. + Buffer.BlockCopy(asyncState.Buffer, bytesConsumed + start, dest, 0, asyncState.BytesInBuffer); - new Span(asyncState.Buffer, 0, asyncState.ClearMax).Clear(); - ArrayPool.Shared.Return(asyncState.Buffer); + new Span(asyncState.Buffer, 0, asyncState.ClearMax).Clear(); + ArrayPool.Shared.Return(asyncState.Buffer); - asyncState.ClearMax = asyncState.BytesInBuffer; - asyncState.Buffer = dest; - } - else if (asyncState.BytesInBuffer != 0) - { - // Shift the processed bytes to the beginning of buffer to make more room. - Buffer.BlockCopy(asyncState.Buffer, bytesConsumed + start, asyncState.Buffer, 0, asyncState.BytesInBuffer); + asyncState.ClearMax = asyncState.BytesInBuffer; + asyncState.Buffer = dest; + } + else if (asyncState.BytesInBuffer != 0) + { + // Shift the processed bytes to the beginning of buffer to make more room. + Buffer.BlockCopy(asyncState.Buffer, bytesConsumed + start, asyncState.Buffer, 0, asyncState.BytesInBuffer); + } } - return value!; // Return the partial value. + return value; } private static TValue ReadCore( @@ -255,19 +247,7 @@ private static TValue ReadCore( state.ReadAhead = !isFinalBlock; state.BytesConsumed = 0; - TValue? value; - if (isFinalBlock) - { - value = ReadCore(converterBase, ref reader, options, ref state); - } - else - { - ReadCore(converterBase, ref reader, options, ref state); - - // Obtain the partial value. - value = (TValue)state.Current.ReturnValue!; - } - + TValue? value = ReadCore(converterBase, ref reader, options, ref state); readerState = reader.CurrentState; return value!; } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs index a44d66cc990514..b5e4e7d4fa5d25 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs @@ -1,9 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Converters; using System.Threading; using System.Threading.Tasks; @@ -11,6 +14,14 @@ namespace System.Text.Json { public static partial class JsonSerializer { + // We flush the Stream when the buffer is >=90% of capacity. + // This threshold is a compromise between buffer utilization and minimizing cases where the buffer + // needs to be expanded\doubled because it is not large enough to write the current property or element. + // We check for flush after each object property and array element is written to the buffer. + // Once the buffer is expanded to contain the largest single element\property, a 90% thresold + // means the buffer may be expanded a maximum of 4 times: 1-(1\(2^4))==.9375. + private const float FlushThreshold = .9f; + /// /// Convert the provided value to UTF-8 encoded JSON text and write it to the . /// @@ -82,6 +93,66 @@ public static Task SerializeAsync( return WriteAsyncCore(utf8Json, value!, inputType, options, cancellationToken); } + /// + /// todo + /// + /// + /// + /// + /// + /// + /// + public static async Task SerializeAsyncEnumerable<[DynamicallyAccessedMembers(MembersAccessedOnWrite)] TValue>( + Stream utf8Json, + IAsyncEnumerable value, + JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) + { + if (options == null) + { + options = JsonSerializerOptions.s_defaultOptions; + } + + Type inputType = typeof(TValue); + JsonWriterOptions writerOptions = options.GetWriterOptions(); + + WriteStack state = default; + + using (var bufferWriter = new PooledByteBufferWriter(options.DefaultBufferSize)) + using (var writer = new Utf8JsonWriter(bufferWriter, writerOptions)) + { + writer.WriteStartArray(); + + await foreach (TValue item in value) + { + bool isFinalBlock; + + // todo: make this faster by caching converter and avoiding dictionary lookup + state = default; + JsonConverter converterBase = state.Initialize(inputType, options, supportContinuation: true); + + while (true) + { + state.FlushThreshold = (int)(bufferWriter.Capacity * FlushThreshold); + isFinalBlock = WriteCore(converterBase, writer, item, options, ref state); + if (isFinalBlock) + { + // We finished successfully; no need to flush to Stream yet. + break; + } + + // We hit the flush threshold. + await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); + bufferWriter.Clear(); + }; + } + + writer.WriteEndArray(); + writer.Flush(); + await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); + } + } + private static async Task WriteAsyncCore( Stream utf8Json, TValue value, @@ -89,14 +160,6 @@ private static async Task WriteAsyncCore( JsonSerializerOptions? options, CancellationToken cancellationToken) { - // We flush the Stream when the buffer is >=90% of capacity. - // This threshold is a compromise between buffer utilization and minimizing cases where the buffer - // needs to be expanded\doubled because it is not large enough to write the current property or element. - // We check for flush after each object property and array element is written to the buffer. - // Once the buffer is expanded to contain the largest single element\property, a 90% thresold - // means the buffer may be expanded a maximum of 4 times: 1-(1\(2^4))==.9375. - const float FlushThreshold = .9f; - if (options == null) { options = JsonSerializerOptions.s_defaultOptions; @@ -104,18 +167,18 @@ private static async Task WriteAsyncCore( JsonWriterOptions writerOptions = options.GetWriterOptions(); - using (var bufferWriter = new PooledByteBufferWriter(options.DefaultBufferSize)) - using (var writer = new Utf8JsonWriter(bufferWriter, writerOptions)) + // We treat typeof(object) special and allow polymorphic behavior. + if (inputType == JsonClassInfo.ObjectType && value != null) { - // We treat typeof(object) special and allow polymorphic behavior. - if (inputType == JsonClassInfo.ObjectType && value != null) - { - inputType = value!.GetType(); - } + inputType = value!.GetType(); + } WriteStack state = new WriteStack { CancellationToken = cancellationToken }; JsonConverter converterBase = state.Initialize(inputType, options, supportContinuation: true); + using (var bufferWriter = new PooledByteBufferWriter(options.DefaultBufferSize)) + using (var writer = new Utf8JsonWriter(bufferWriter, writerOptions)) + { bool isFinalBlock; try diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs index d1ad72afef8790..2790c291594aac 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs @@ -100,7 +100,10 @@ private async ValueTask ContinueRead() while (!HaveDataToReturn()) { _isFinalBlock = await JsonSerializer.ReadFromStream(_utf8Json, _asyncState).ConfigureAwait(false); - _valuesToReturn = JsonSerializer.ContinueDeserialize>(_asyncState, _isFinalBlock); + JsonSerializer.ContinueDeserialize>(_asyncState, _isFinalBlock); + + // Obtain the partial collection. + _valuesToReturn = (Queue?)_asyncState.ReadStack.Current.ReturnValue; } return true; diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs index 799becb4b5d2f6..6b896fe00b8f9a 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs @@ -1,7 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.IO; +using System.Threading; using System.Threading.Tasks; using Xunit; @@ -54,5 +56,74 @@ public static async Task ReadSimpleObjectAsync(int count, int bufferSize) Assert.Equal(count, callbackCount); } } + + private class SimpleObjectProvider : IAsyncEnumerable + { + private int _count; + + public SimpleObjectProvider(int count) + { + _count = count; + } + + public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + for (int i = 0; i < _count; i++) + { + await Task.Delay(1); + var obj = new SimpleTestClass(); + obj.Initialize(); + yield return obj; + } + } + } + + [Theory] + [InlineData(1, 1)] + [InlineData(10, 1)] + [InlineData(100, 1)] + [InlineData(100, 1000)] + [InlineData(100, 32000)] + public static async Task WriteSimpleObjectAsync(int count, int bufferSize) + { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = bufferSize + }; + + long singleLength = 0; + using (MemoryStream singleObjectStream = new MemoryStream()) + { + var obj = new SimpleTestClass(); + obj.Initialize(); + + await JsonSerializer.SerializeAsync(singleObjectStream, obj, options); + singleLength = singleObjectStream.Length; + } + + long allLength = 0; + using (MemoryStream stream = new MemoryStream()) + { + await JsonSerializer.SerializeAsyncEnumerable( + stream, + new SimpleObjectProvider(count), + options); + + allLength = stream.Length; + allLength -= 1; // account for start array token. + allLength -= count; // account for commas; includes end array token since there is no trailing comma. + + Assert.Equal(singleLength * count, allLength); + + // Verify the contents. + stream.Position = 0; + SimpleTestClass[] result = await JsonSerializer.DeserializeAsync(stream); + Assert.Equal(count, result.Length); + for (int i = 0; i < count; i++) + { + result[i].Verify(); + } + } + } } } From 0ba71722195156fa0c2c68eaa3c5926e893d5602 Mon Sep 17 00:00:00 2001 From: Steve Harter Date: Fri, 11 Dec 2020 12:32:38 -0600 Subject: [PATCH 06/27] Misc cleanup on test --- .../tests/Serialization/Stream.IAsyncEnumerable.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs index 6b896fe00b8f9a..e4362683cfe75b 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs @@ -91,7 +91,8 @@ public static async Task WriteSimpleObjectAsync(int count, int bufferSize) DefaultBufferSize = bufferSize }; - long singleLength = 0; + // Calculate the byte length of a single item not in a JSON array. + long singleLength; using (MemoryStream singleObjectStream = new MemoryStream()) { var obj = new SimpleTestClass(); @@ -101,7 +102,6 @@ public static async Task WriteSimpleObjectAsync(int count, int bufferSize) singleLength = singleObjectStream.Length; } - long allLength = 0; using (MemoryStream stream = new MemoryStream()) { await JsonSerializer.SerializeAsyncEnumerable( @@ -109,7 +109,7 @@ await JsonSerializer.SerializeAsyncEnumerable( new SimpleObjectProvider(count), options); - allLength = stream.Length; + long allLength = stream.Length; allLength -= 1; // account for start array token. allLength -= count; // account for commas; includes end array token since there is no trailing comma. From 4d697550912f86673628b657cbf0b176c52559bc Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 12:32:18 +0100 Subject: [PATCH 07/27] extend DeserializeAsyncEnumerable test coverage also removes SerializeAsyncEnumerable components --- .../System.Text.Json/ref/System.Text.Json.cs | 3 +- .../src/System.Text.Json.csproj | 10 +- .../AsyncEnumerableStreamingDeserializer.cs | 113 +++++++++++++++ .../JsonSerializer.Read.Stream.cs | 37 ++--- .../JsonSerializer.Write.Stream.cs | 93 ++----------- .../Text/Json/Serialization/ReadAsyncState.cs | 8 +- .../SerializerReadAsyncEnumerable.cs | 38 ------ .../SerializerReadAsyncEnumerator.cs | 114 ---------------- .../CollectionTests.AsyncEnumerable.cs | 61 ++++++--- .../Stream.DeserializeAsyncEnumerable.cs | 126 +++++++++++++++++ .../Serialization/Stream.IAsyncEnumerable.cs | 129 ------------------ .../tests/System.Text.Json.Tests.csproj | 2 +- 12 files changed, 328 insertions(+), 406 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs delete mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerable.cs delete mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs create mode 100644 src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs delete mode 100644 src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index 043ca627336408..b24167f6f07fca 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -192,7 +192,7 @@ public static partial class JsonSerializer public static object? Deserialize(ref System.Text.Json.Utf8JsonReader reader, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type returnType, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static System.Threading.Tasks.ValueTask DeserializeAsync(System.IO.Stream utf8Json, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type returnType, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.ValueTask DeserializeAsync<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public static System.Collections.Generic.IAsyncEnumerable DeserializeAsyncEnumerable<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Collections.Generic.IAsyncEnumerable DeserializeAsyncEnumerable<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.ReadOnlySpan utf8Json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(string json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.ReadOnlySpan json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } @@ -201,7 +201,6 @@ public static partial class JsonSerializer public static void Serialize(System.Text.Json.Utf8JsonWriter writer, object? value, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type inputType, System.Text.Json.JsonSerializerOptions? options = null) { } public static System.Threading.Tasks.Task SerializeAsync(System.IO.Stream utf8Json, object? value, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type inputType, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.Task SerializeAsync<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields)]TValue>(System.IO.Stream utf8Json, TValue value, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public static System.Threading.Tasks.Task SerializeAsyncEnumerable<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields)]TValue>(System.IO.Stream utf8Json, System.Collections.Generic.IAsyncEnumerable value, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default) { throw null; } public static byte[] SerializeToUtf8Bytes(object? value, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type inputType, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static byte[] SerializeToUtf8Bytes<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(TValue value, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static void Serialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.Text.Json.Utf8JsonWriter writer, TValue value, System.Text.Json.JsonSerializerOptions? options = null) { } diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index a45636c3015f21..5ec3035a51e139 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -1,4 +1,4 @@ - + true $(NetCoreAppCurrent);netstandard2.0;netcoreapp3.0;net461 @@ -65,11 +65,11 @@ - + @@ -184,8 +184,7 @@ - - + @@ -240,7 +239,8 @@ - + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs new file mode 100644 index 00000000000000..b4afbdcfc24052 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs @@ -0,0 +1,113 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace System.Text.Json.Serialization +{ + internal sealed class AsyncEnumerableStreamingDeserializer : IAsyncEnumerable + { + private readonly Stream _utf8Json; + private readonly JsonSerializerOptions? _options; + + public AsyncEnumerableStreamingDeserializer(Stream utf8Json, JsonSerializerOptions? options) + { + _utf8Json = utf8Json; + _options = options; + } + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken) + { + return new Enumerator(_utf8Json, _options, cancellationToken); + } + + private sealed class Enumerator : IAsyncEnumerator + { + private readonly Stream _utf8Json; + private ReadAsyncState _asyncState; + private Queue? _valuesToReturn; + private TValue? _current; + private bool _isFinalBlock; + private bool _doneProcessing; + + public Enumerator(Stream utf8Json, JsonSerializerOptions? options, CancellationToken cancellationToken) + { + _utf8Json = utf8Json; + _asyncState = new ReadAsyncState(typeof(Queue), options, cancellationToken); + } + + public TValue Current => _current!; + + public ValueTask DisposeAsync() + { + _asyncState.Dispose(); + _asyncState = null!; + return default; + } + + public async ValueTask MoveNextAsync() + { + if (_doneProcessing) + { + return false; + } + + if (HasLocalDataToReturn()) + { + return true; + } + + // Read additional data. + await ContinueRead().ConfigureAwait(false); + return ApplyReturnValue(); + + bool HasLocalDataToReturn() + { + if (ApplyReturnValue()) + { + return true; + } + + if (_isFinalBlock) + { + _doneProcessing = true; + _asyncState.Dispose(); + _asyncState = null!; + } + + return false; + } + + bool ApplyReturnValue() + { + if (_valuesToReturn?.Count > 0) + { + _current = _valuesToReturn.Dequeue(); + return true; + } + + return false; + } + } + + private async ValueTask ContinueRead() + { + while (!HaveDataToReturn()) + { + _isFinalBlock = await JsonSerializer.ReadFromStream(_utf8Json, _asyncState).ConfigureAwait(false); + JsonSerializer.ContinueDeserialize>(_asyncState, _isFinalBlock); + + // Obtain the partial collection. + _valuesToReturn = (Queue?)_asyncState.ReadStack.Current.ReturnValue; + } + + return true; + + bool HaveDataToReturn() => _isFinalBlock || _valuesToReturn?.Count > 0; + } + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs index 37c0c17550ed64..42f9e6be1bbf13 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs @@ -88,24 +88,26 @@ public static partial class JsonSerializer } /// - /// todo + /// Wraps the UTF-8 encoded text into an + /// that can be used to deserialize root-level JSON arrays in a streaming manner. /// - /// - /// - /// - /// - /// + /// An representation of the provided JSON array. + /// JSON data to parse. + /// Options to control the behavior during reading. + /// An representation of the JSON value. + /// + /// is . + /// public static IAsyncEnumerable DeserializeAsyncEnumerable<[DynamicallyAccessedMembers(MembersAccessedOnRead)] TValue>( Stream utf8Json, - JsonSerializerOptions? options = null, - CancellationToken cancellationToken = default) + JsonSerializerOptions? options = null) { if (utf8Json == null) { throw new ArgumentNullException(nameof(utf8Json)); } - return new SerializerReadAsyncEnumerable(utf8Json, options); + return new AsyncEnumerableStreamingDeserializer(utf8Json, options); } internal static async ValueTask ReadAllAsync( @@ -114,17 +116,16 @@ public static partial class JsonSerializer JsonSerializerOptions? options, CancellationToken cancellationToken) { - using (var asyncState = new ReadAsyncState(inputType, cancellationToken, options)) + using var asyncState = new ReadAsyncState(inputType, options, cancellationToken); + + while (true) { - while (true) - { - bool isFinalBlock = await ReadFromStream(utf8Json, asyncState).ConfigureAwait(false); - TValue value = ContinueDeserialize(asyncState, isFinalBlock); + bool isFinalBlock = await ReadFromStream(utf8Json, asyncState).ConfigureAwait(false); + TValue value = ContinueDeserialize(asyncState, isFinalBlock); - if (isFinalBlock) - { - return value!; - } + if (isFinalBlock) + { + return value!; } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs index b5e4e7d4fa5d25..a44d66cc990514 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs @@ -1,12 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Text.Json.Serialization; -using System.Text.Json.Serialization.Converters; using System.Threading; using System.Threading.Tasks; @@ -14,14 +11,6 @@ namespace System.Text.Json { public static partial class JsonSerializer { - // We flush the Stream when the buffer is >=90% of capacity. - // This threshold is a compromise between buffer utilization and minimizing cases where the buffer - // needs to be expanded\doubled because it is not large enough to write the current property or element. - // We check for flush after each object property and array element is written to the buffer. - // Once the buffer is expanded to contain the largest single element\property, a 90% thresold - // means the buffer may be expanded a maximum of 4 times: 1-(1\(2^4))==.9375. - private const float FlushThreshold = .9f; - /// /// Convert the provided value to UTF-8 encoded JSON text and write it to the . /// @@ -93,66 +82,6 @@ public static Task SerializeAsync( return WriteAsyncCore(utf8Json, value!, inputType, options, cancellationToken); } - /// - /// todo - /// - /// - /// - /// - /// - /// - /// - public static async Task SerializeAsyncEnumerable<[DynamicallyAccessedMembers(MembersAccessedOnWrite)] TValue>( - Stream utf8Json, - IAsyncEnumerable value, - JsonSerializerOptions? options = null, - CancellationToken cancellationToken = default) - { - if (options == null) - { - options = JsonSerializerOptions.s_defaultOptions; - } - - Type inputType = typeof(TValue); - JsonWriterOptions writerOptions = options.GetWriterOptions(); - - WriteStack state = default; - - using (var bufferWriter = new PooledByteBufferWriter(options.DefaultBufferSize)) - using (var writer = new Utf8JsonWriter(bufferWriter, writerOptions)) - { - writer.WriteStartArray(); - - await foreach (TValue item in value) - { - bool isFinalBlock; - - // todo: make this faster by caching converter and avoiding dictionary lookup - state = default; - JsonConverter converterBase = state.Initialize(inputType, options, supportContinuation: true); - - while (true) - { - state.FlushThreshold = (int)(bufferWriter.Capacity * FlushThreshold); - isFinalBlock = WriteCore(converterBase, writer, item, options, ref state); - if (isFinalBlock) - { - // We finished successfully; no need to flush to Stream yet. - break; - } - - // We hit the flush threshold. - await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); - bufferWriter.Clear(); - }; - } - - writer.WriteEndArray(); - writer.Flush(); - await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); - } - } - private static async Task WriteAsyncCore( Stream utf8Json, TValue value, @@ -160,6 +89,14 @@ private static async Task WriteAsyncCore( JsonSerializerOptions? options, CancellationToken cancellationToken) { + // We flush the Stream when the buffer is >=90% of capacity. + // This threshold is a compromise between buffer utilization and minimizing cases where the buffer + // needs to be expanded\doubled because it is not large enough to write the current property or element. + // We check for flush after each object property and array element is written to the buffer. + // Once the buffer is expanded to contain the largest single element\property, a 90% thresold + // means the buffer may be expanded a maximum of 4 times: 1-(1\(2^4))==.9375. + const float FlushThreshold = .9f; + if (options == null) { options = JsonSerializerOptions.s_defaultOptions; @@ -167,18 +104,18 @@ private static async Task WriteAsyncCore( JsonWriterOptions writerOptions = options.GetWriterOptions(); - // We treat typeof(object) special and allow polymorphic behavior. - if (inputType == JsonClassInfo.ObjectType && value != null) + using (var bufferWriter = new PooledByteBufferWriter(options.DefaultBufferSize)) + using (var writer = new Utf8JsonWriter(bufferWriter, writerOptions)) { - inputType = value!.GetType(); - } + // We treat typeof(object) special and allow polymorphic behavior. + if (inputType == JsonClassInfo.ObjectType && value != null) + { + inputType = value!.GetType(); + } WriteStack state = new WriteStack { CancellationToken = cancellationToken }; JsonConverter converterBase = state.Initialize(inputType, options, supportContinuation: true); - using (var bufferWriter = new PooledByteBufferWriter(options.DefaultBufferSize)) - using (var writer = new Utf8JsonWriter(bufferWriter, writerOptions)) - { bool isFinalBlock; try diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs index dc789926dd2cb3..2574ff0538c7f7 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs @@ -6,9 +6,9 @@ namespace System.Text.Json.Serialization { - internal class ReadAsyncState : IDisposable + internal sealed class ReadAsyncState : IDisposable { - public CancellationToken CancellationToken; + public readonly CancellationToken CancellationToken; public byte[] Buffer; public int BytesInBuffer; public int ClearMax; @@ -19,7 +19,7 @@ internal class ReadAsyncState : IDisposable public JsonSerializerOptions Options; public long TotalBytesRead; - public ReadAsyncState(Type returnType, CancellationToken cancellationToken = default, JsonSerializerOptions? options = null) + public ReadAsyncState(Type returnType, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) { Options = options ??= JsonSerializerOptions.s_defaultOptions; Buffer = ArrayPool.Shared.Rent(Math.Max(Options.DefaultBufferSize, JsonConstants.Utf8Bom.Length)); @@ -30,7 +30,7 @@ public ReadAsyncState(Type returnType, CancellationToken cancellationToken = def IsFirstIteration = true; } - protected virtual void Dispose(bool disposing) + private void Dispose(bool disposing) { if (disposing) { diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerable.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerable.cs deleted file mode 100644 index cc0e02e593d180..00000000000000 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerable.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.IO; -using System.Threading; - -namespace System.Text.Json.Serialization -{ - internal struct SerializerReadAsyncEnumerable : IAsyncEnumerable - { - public SerializerReadAsyncEnumerable(Stream stream, JsonSerializerOptions? options) - { - Stream = stream; - Options = options; - } - - /// - /// todo - /// - public Stream Stream { get; set; } - - /// - /// todo - /// - public JsonSerializerOptions? Options { get; set; } - - public SerializerReadAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken) - { - return new SerializerReadAsyncEnumerator(Stream, Options); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs deleted file mode 100644 index 2790c291594aac..00000000000000 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/SerializerReadAsyncEnumerator.cs +++ /dev/null @@ -1,114 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Threading.Tasks; - -namespace System.Text.Json.Serialization -{ - /// - /// todo - /// - /// - // todo: change to struct? - internal sealed class SerializerReadAsyncEnumerator : IAsyncEnumerator - { - private Stream _utf8Json; - private ReadAsyncState _asyncState; - private Queue? _valuesToReturn; - private TValue? _current; - private bool _isFinalBlock; - private bool _doneProcessing; - - public SerializerReadAsyncEnumerator(Stream utf8Json, JsonSerializerOptions? options) - { - _utf8Json = utf8Json; - _asyncState = new ReadAsyncState(typeof(Queue), cancellationToken: default, options); - } - - public TValue Current - { - get - { - return _current!; - } - } - - /// - /// todo - /// - /// - public ValueTask DisposeAsync() - { - return default; - } - - /// - /// todo - /// - /// - public async ValueTask MoveNextAsync() - { - if (_doneProcessing) - { - return false; - // todo: throw InvalidOperationException if MoveNext is called again? - } - - if (HasLocalDataToReturn()) - { - return true; - } - - // Read additional data. - await ContinueRead().ConfigureAwait(false); - return ApplyReturnValue(); - - bool HasLocalDataToReturn() - { - if (ApplyReturnValue()) - { - return true; - } - - if (_isFinalBlock) - { - _doneProcessing = true; - _asyncState.Dispose(); - _asyncState = null!; - } - - return false; - } - - bool ApplyReturnValue() - { - if (_valuesToReturn?.Count > 0) - { - _current = _valuesToReturn.Dequeue(); - return true; - } - - return false; - } - } - - private async ValueTask ContinueRead() - { - while (!HaveDataToReturn()) - { - _isFinalBlock = await JsonSerializer.ReadFromStream(_utf8Json, _asyncState).ConfigureAwait(false); - JsonSerializer.ContinueDeserialize>(_asyncState, _isFinalBlock); - - // Obtain the partial collection. - _valuesToReturn = (Queue?)_asyncState.ReadStack.Current.ReturnValue; - } - - return true; - - bool HaveDataToReturn() => _isFinalBlock || _valuesToReturn?.Count > 0; - } - } -} diff --git a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs index 5616cc7a04f2fa..ed3b16b2cdbe59 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs @@ -7,7 +7,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using System.Text.Json.Serialization; using Xunit; namespace System.Text.Json.Tests.Serialization @@ -16,13 +15,18 @@ public static partial class CollectionTests { [Theory] [MemberData(nameof(GetAsyncEnumerableSources))] - public static async Task WriteRootLevelAsyncEnumerable(IEnumerable source, int delayInterval) + public static async Task WriteRootLevelAsyncEnumerable(IEnumerable source, int delayInterval, int bufferSize) { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = bufferSize + }; + string expectedJson = JsonSerializer.Serialize(source); using var stream = new Utf8MemoryStream(); var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); - await JsonSerializer.SerializeAsync(stream, asyncEnumerable); + await JsonSerializer.SerializeAsync(stream, asyncEnumerable, options); Assert.Equal(expectedJson, stream.ToString()); Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); @@ -31,13 +35,18 @@ public static async Task WriteRootLevelAsyncEnumerable(IEnumerable(IEnumerable source, int delayInterval) + public static async Task WriteNestedAsyncEnumerable(IEnumerable source, int delayInterval, int bufferSize) { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = bufferSize + }; + string expectedJson = JsonSerializer.Serialize(new { Data = source }); using var stream = new Utf8MemoryStream(); var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); - await JsonSerializer.SerializeAsync(stream, new { Data = asyncEnumerable }); + await JsonSerializer.SerializeAsync(stream, new { Data = asyncEnumerable }, options); Assert.Equal(expectedJson, stream.ToString()); Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); @@ -46,13 +55,18 @@ public static async Task WriteNestedAsyncEnumerable(IEnumerable(IEnumerable source, int delayInterval) + public static async Task WriteNestedAsyncEnumerable_DTO(IEnumerable source, int delayInterval, int bufferSize) { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = bufferSize + }; + string expectedJson = JsonSerializer.Serialize(new { Data = source }); using var stream = new Utf8MemoryStream(); var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); - await JsonSerializer.SerializeAsync(stream, new AsyncEnumerableDto { Data = asyncEnumerable }); + await JsonSerializer.SerializeAsync(stream, new AsyncEnumerableDto { Data = asyncEnumerable }, options); Assert.Equal(expectedJson, stream.ToString()); Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); @@ -83,13 +97,18 @@ public class AsyncEnumerableDto [Theory] [MemberData(nameof(GetAsyncEnumerableSources))] - public static async Task WriteSequentialNestedAsyncEnumerables(IEnumerable source, int delayInterval) + public static async Task WriteSequentialNestedAsyncEnumerables(IEnumerable source, int delayInterval, int bufferSize) { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = bufferSize + }; + string expectedJson = JsonSerializer.Serialize(new { Data1 = source, Data2 = source }); using var stream = new Utf8MemoryStream(); var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); - await JsonSerializer.SerializeAsync(stream, new { Data1 = asyncEnumerable, Data2 = asyncEnumerable }); + await JsonSerializer.SerializeAsync(stream, new { Data1 = asyncEnumerable, Data2 = asyncEnumerable }, options); Assert.Equal(expectedJson, stream.ToString()); Assert.Equal(2, asyncEnumerable.TotalCreatedEnumerators); @@ -98,8 +117,13 @@ public static async Task WriteSequentialNestedAsyncEnumerables(IEnumer [Theory] [MemberData(nameof(GetAsyncEnumerableSources))] - public static async Task WriteAsyncEnumerableOfAsyncEnumerables(IEnumerable source, int delayInterval) + public static async Task WriteAsyncEnumerableOfAsyncEnumerables(IEnumerable source, int delayInterval, int bufferSize) { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = bufferSize + }; + const int OuterEnumerableCount = 5; string expectedJson = JsonSerializer.Serialize(Enumerable.Repeat(source, OuterEnumerableCount)); @@ -109,7 +133,7 @@ public static async Task WriteAsyncEnumerableOfAsyncEnumerables(IEnume Enumerable.Repeat(innerAsyncEnumerable, OuterEnumerableCount), delayInterval); using var stream = new Utf8MemoryStream(); - await JsonSerializer.SerializeAsync(stream, outerAsyncEnumerable); + await JsonSerializer.SerializeAsync(stream, outerAsyncEnumerable, options); Assert.Equal(expectedJson, stream.ToString()); Assert.Equal(1, outerAsyncEnumerable.TotalCreatedEnumerators); @@ -172,11 +196,15 @@ public static async Task ReadRootLevelAsyncEnumerableDerivative_ThrowsNotSupport public static IEnumerable GetAsyncEnumerableSources() { - yield return WrapArgs(Enumerable.Empty(), 0); - yield return WrapArgs(Enumerable.Range(0, 20), 0); - yield return WrapArgs(Enumerable.Range(0, 100), 20); - - static object[] WrapArgs(IEnumerable source, int delayInterval) => new object[]{ source, delayInterval }; + yield return WrapArgs(Enumerable.Empty(), 0, 1); + yield return WrapArgs(Enumerable.Range(0, 20), 0, 1); + yield return WrapArgs(Enumerable.Range(0, 100), 20, 1); + yield return WrapArgs(Enumerable.Range(0, 1000), 20, 1000); + yield return WrapArgs(Enumerable.Range(0, 100).Select(i => $"lorem ipsum dolor: {i}"), 1, 100); + yield return WrapArgs(Enumerable.Range(0, 1000).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1, 100); + yield return WrapArgs(Enumerable.Range(0, 1000).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1, 1000); + + static object[] WrapArgs(IEnumerable source, int delayInterval, int bufferSize) => new object[]{ source, delayInterval, bufferSize }; } private static async Task> ToListAsync(this IAsyncEnumerable source) @@ -186,7 +214,6 @@ private static async Task> ToListAsync(this IAsyncEnumerable sourc { list.Add(item); } - return list; } diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs new file mode 100644 index 00000000000000..be647aa926650b --- /dev/null +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace System.Text.Json.Serialization.Tests +{ + public static partial class StreamTests_DeserializeAsyncEnumerable + { + [Theory] + [InlineData(0, 1)] + [InlineData(1, 1)] + [InlineData(10, 1)] + [InlineData(100, 1)] + [InlineData(1000, 1)] + [InlineData(1000, 1000)] + [InlineData(1000, 32000)] + public static async Task DeserializeAsyncEnumerable_ReadSimpleObjectAsync(int count, int bufferSize) + { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = bufferSize + }; + + using var stream = new MemoryStream(GenerateJsonArray(count)); + + int callbackCount = 0; + await foreach(SimpleTestClass item in JsonSerializer.DeserializeAsyncEnumerable(stream, options)) + { + Assert.Equal(callbackCount, item.MyInt32); + + item.MyInt32 = 2; // Put correct value back for Verify() + item.Verify(); + + callbackCount++; + } + + Assert.Equal(count, callbackCount); + + static byte[] GenerateJsonArray(int count) + { + SimpleTestClass[] collection = new SimpleTestClass[count]; + for (int i = 0; i < collection.Length; i++) + { + var obj = new SimpleTestClass(); + obj.Initialize(); + obj.MyInt32 = i; // verify order correctness + collection[i] = obj; + } + + return JsonSerializer.SerializeToUtf8Bytes(collection); + } + } + + [Theory] + [MemberData(nameof(GetAsyncEnumerableSources))] + public static async Task DeserializeAsyncEnumerable_ReadSourceAsync(IEnumerable source, int bufferSize) + { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = bufferSize + }; + + byte[] data = JsonSerializer.SerializeToUtf8Bytes(source); + + using var stream = new MemoryStream(data); + List results = await JsonSerializer.DeserializeAsyncEnumerable(stream, options).ToListAsync(); + Assert.Equal(source, results); + } + + [Fact] + public static void DeserializeAsyncEnumerable_NullStream_ThrowsArgumentNullException() + { + AssertExtensions.Throws("utf8Json", () => JsonSerializer.DeserializeAsyncEnumerable(utf8Json: null)); + } + + [Fact] + public static async Task DeserializeAsyncEnumerable_CancellationToken_ThrowsOnCancellation() + { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = 1 + }; + + byte[] data = JsonSerializer.SerializeToUtf8Bytes(Enumerable.Range(1, 100)); + + var token = new CancellationToken(canceled: true); + using var stream = new MemoryStream(data); + IAsyncEnumerable asyncEnumerable = JsonSerializer.DeserializeAsyncEnumerable(stream, options); + + await Assert.ThrowsAsync(async () => + { + await foreach (int element in asyncEnumerable.WithCancellation(token)) + { + } + }); + } + + public static IEnumerable GetAsyncEnumerableSources() + { + yield return WrapArgs(Enumerable.Empty(), 1); + yield return WrapArgs(Enumerable.Range(0, 20), 1); + yield return WrapArgs(Enumerable.Range(0, 100), 1); + yield return WrapArgs(Enumerable.Range(0, 100).Select(i => $"lorem ipsum dolor: {i}"), 50); + yield return WrapArgs(Enumerable.Range(0, 1000).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 100); + yield return WrapArgs(Enumerable.Range(0, 1000).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1000); + + static object[] WrapArgs(IEnumerable source, int bufferSize) => new object[] { source, bufferSize }; + } + + private static async Task> ToListAsync(this IAsyncEnumerable source) + { + var list = new List(); + await foreach (T item in source) + { + list.Add(item); + } + return list; + } + } +} diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs deleted file mode 100644 index e4362683cfe75b..00000000000000 --- a/src/libraries/System.Text.Json/tests/Serialization/Stream.IAsyncEnumerable.cs +++ /dev/null @@ -1,129 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Xunit; - -namespace System.Text.Json.Serialization.Tests -{ - public static partial class StreamTests_IAsyncEnumerable - { - [Theory] - [InlineData(1, 1)] - [InlineData(10, 1)] - [InlineData(100, 1)] - [InlineData(1000, 1)] - [InlineData(1000, 1000)] - [InlineData(1000, 32000)] - public static async Task ReadSimpleObjectAsync(int count, int bufferSize) - { - JsonSerializerOptions options = new JsonSerializerOptions - { - DefaultBufferSize = bufferSize - }; - - // Produce the JSON - SimpleTestClass[] collection = new SimpleTestClass[count]; - for (int i = 0; i < collection.Length; i++) - { - var obj = new SimpleTestClass(); - obj.Initialize(); - obj.MyInt32 = i; // verify order correctness - collection[i] = obj; - } - - byte[] data = JsonSerializer.SerializeToUtf8Bytes(collection); - - // Use async await on the Stream. - using (MemoryStream stream = new MemoryStream(data)) - { - int callbackCount = 0; - - await foreach(SimpleTestClass item in - JsonSerializer.DeserializeAsyncEnumerable(stream, options)) - { - Assert.Equal(callbackCount, item.MyInt32); - - item.MyInt32 = 2; // Put correct value back for Verify() - item.Verify(); - - callbackCount++; - } - - Assert.Equal(count, callbackCount); - } - } - - private class SimpleObjectProvider : IAsyncEnumerable - { - private int _count; - - public SimpleObjectProvider(int count) - { - _count = count; - } - - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) - { - for (int i = 0; i < _count; i++) - { - await Task.Delay(1); - var obj = new SimpleTestClass(); - obj.Initialize(); - yield return obj; - } - } - } - - [Theory] - [InlineData(1, 1)] - [InlineData(10, 1)] - [InlineData(100, 1)] - [InlineData(100, 1000)] - [InlineData(100, 32000)] - public static async Task WriteSimpleObjectAsync(int count, int bufferSize) - { - JsonSerializerOptions options = new JsonSerializerOptions - { - DefaultBufferSize = bufferSize - }; - - // Calculate the byte length of a single item not in a JSON array. - long singleLength; - using (MemoryStream singleObjectStream = new MemoryStream()) - { - var obj = new SimpleTestClass(); - obj.Initialize(); - - await JsonSerializer.SerializeAsync(singleObjectStream, obj, options); - singleLength = singleObjectStream.Length; - } - - using (MemoryStream stream = new MemoryStream()) - { - await JsonSerializer.SerializeAsyncEnumerable( - stream, - new SimpleObjectProvider(count), - options); - - long allLength = stream.Length; - allLength -= 1; // account for start array token. - allLength -= count; // account for commas; includes end array token since there is no trailing comma. - - Assert.Equal(singleLength * count, allLength); - - // Verify the contents. - stream.Position = 0; - SimpleTestClass[] result = await JsonSerializer.DeserializeAsync(stream); - Assert.Equal(count, result.Length); - for (int i = 0; i < count; i++) - { - result[i].Verify(); - } - } - } - } -} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj index 238acba6712657..26116948c43104 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj @@ -124,7 +124,7 @@ - + From 8ce7722047cace95ade050a966324bad84e3abef Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 16:17:24 +0100 Subject: [PATCH 08/27] Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs Co-authored-by: Stephen Toub --- .../Converters/Collection/IAsyncEnumerableConverterFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs index a181a2aaa1b1b2..e5125972fc8cd7 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs @@ -11,7 +11,7 @@ namespace System.Text.Json.Serialization /// /// Converter for streaming values. /// - internal class IAsyncEnumerableConverterFactory : JsonConverterFactory + internal sealed class IAsyncEnumerableConverterFactory : JsonConverterFactory { public override bool CanConvert(Type typeToConvert) => TryGetAsyncEnumerableInterface(typeToConvert, out _); From 4a88d649242212d9b6fc306b43f20ec234e095f4 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 17:22:40 +0100 Subject: [PATCH 09/27] address feedback --- .../AsyncEnumerableStreamingDeserializer.cs | 93 ++----------------- .../IAsyncEnumerableConverterFactory.cs | 7 +- 2 files changed, 12 insertions(+), 88 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs index b4afbdcfc24052..feb936fb9a7aa2 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Threading; -using System.Threading.Tasks; namespace System.Text.Json.Serialization { @@ -19,95 +18,23 @@ public AsyncEnumerableStreamingDeserializer(Stream utf8Json, JsonSerializerOptio _options = options; } - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken) + public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken) { - return new Enumerator(_utf8Json, _options, cancellationToken); - } - - private sealed class Enumerator : IAsyncEnumerator - { - private readonly Stream _utf8Json; - private ReadAsyncState _asyncState; - private Queue? _valuesToReturn; - private TValue? _current; - private bool _isFinalBlock; - private bool _doneProcessing; - - public Enumerator(Stream utf8Json, JsonSerializerOptions? options, CancellationToken cancellationToken) - { - _utf8Json = utf8Json; - _asyncState = new ReadAsyncState(typeof(Queue), options, cancellationToken); - } - - public TValue Current => _current!; - - public ValueTask DisposeAsync() - { - _asyncState.Dispose(); - _asyncState = null!; - return default; - } - - public async ValueTask MoveNextAsync() + using var asyncState = new ReadAsyncState(typeof(Queue), _options, cancellationToken); + bool isFinalBlock = false; + do { - if (_doneProcessing) - { - return false; - } - - if (HasLocalDataToReturn()) - { - return true; - } - - // Read additional data. - await ContinueRead().ConfigureAwait(false); - return ApplyReturnValue(); - - bool HasLocalDataToReturn() + isFinalBlock = await JsonSerializer.ReadFromStream(_utf8Json, asyncState).ConfigureAwait(false); + JsonSerializer.ContinueDeserialize>(asyncState, isFinalBlock); + if (asyncState.ReadStack.Current.ReturnValue is Queue queue) { - if (ApplyReturnValue()) + while (queue.Count > 0) { - return true; + yield return queue.Dequeue(); } - - if (_isFinalBlock) - { - _doneProcessing = true; - _asyncState.Dispose(); - _asyncState = null!; - } - - return false; - } - - bool ApplyReturnValue() - { - if (_valuesToReturn?.Count > 0) - { - _current = _valuesToReturn.Dequeue(); - return true; - } - - return false; } } - - private async ValueTask ContinueRead() - { - while (!HaveDataToReturn()) - { - _isFinalBlock = await JsonSerializer.ReadFromStream(_utf8Json, _asyncState).ConfigureAwait(false); - JsonSerializer.ContinueDeserialize>(_asyncState, _isFinalBlock); - - // Obtain the partial collection. - _valuesToReturn = (Queue?)_asyncState.ReadStack.Current.ReturnValue; - } - - return true; - - bool HaveDataToReturn() => _isFinalBlock || _valuesToReturn?.Count > 0; - } + while (!isFinalBlock); } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs index e5125972fc8cd7..ea4cae9662c9ef 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs @@ -17,11 +17,8 @@ internal sealed class IAsyncEnumerableConverterFactory : JsonConverterFactory public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) { - if (!TryGetAsyncEnumerableInterface(typeToConvert, out Type? asyncEnumerableInterface)) - { - Debug.Fail("type not supported by the converter."); - throw new Exception(); - } + TryGetAsyncEnumerableInterface(typeToConvert, out Type? asyncEnumerableInterface); + Debug.Assert(asyncEnumerableInterface is not null, $"{typeToConvert} not supported by converter."); Type elementType = asyncEnumerableInterface.GetGenericArguments()[0]; Type converterType = typeof(IAsyncEnumerableOfTConverter<,>).MakeGenericType(typeToConvert, elementType); From 3eec24c5a5b8ff96db598b8a88624bda86304000 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 17:23:27 +0100 Subject: [PATCH 10/27] tweak test buffer values --- .../CollectionTests/CollectionTests.AsyncEnumerable.cs | 8 ++++---- .../Serialization/Stream.DeserializeAsyncEnumerable.cs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs index ed3b16b2cdbe59..9453a59fde478c 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs @@ -198,11 +198,11 @@ public static IEnumerable GetAsyncEnumerableSources() { yield return WrapArgs(Enumerable.Empty(), 0, 1); yield return WrapArgs(Enumerable.Range(0, 20), 0, 1); - yield return WrapArgs(Enumerable.Range(0, 100), 20, 1); + yield return WrapArgs(Enumerable.Range(0, 100), 20, 20); yield return WrapArgs(Enumerable.Range(0, 1000), 20, 1000); - yield return WrapArgs(Enumerable.Range(0, 100).Select(i => $"lorem ipsum dolor: {i}"), 1, 100); - yield return WrapArgs(Enumerable.Range(0, 1000).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1, 100); - yield return WrapArgs(Enumerable.Range(0, 1000).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1, 1000); + yield return WrapArgs(Enumerable.Range(0, 100).Select(i => $"lorem ipsum dolor: {i}"), 1, 500); + yield return WrapArgs(Enumerable.Range(0, 10).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1, 100); + yield return WrapArgs(Enumerable.Range(0, 100).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1, 500); static object[] WrapArgs(IEnumerable source, int delayInterval, int bufferSize) => new object[]{ source, delayInterval, bufferSize }; } diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs index be647aa926650b..75f16aadd9b028 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs @@ -105,10 +105,10 @@ public static IEnumerable GetAsyncEnumerableSources() { yield return WrapArgs(Enumerable.Empty(), 1); yield return WrapArgs(Enumerable.Range(0, 20), 1); - yield return WrapArgs(Enumerable.Range(0, 100), 1); - yield return WrapArgs(Enumerable.Range(0, 100).Select(i => $"lorem ipsum dolor: {i}"), 50); - yield return WrapArgs(Enumerable.Range(0, 1000).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 100); - yield return WrapArgs(Enumerable.Range(0, 1000).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1000); + yield return WrapArgs(Enumerable.Range(0, 100), 20); + yield return WrapArgs(Enumerable.Range(0, 100).Select(i => $"lorem ipsum dolor: {i}"), 500); + yield return WrapArgs(Enumerable.Range(0, 10).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 100); + yield return WrapArgs(Enumerable.Range(0, 100).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 500); static object[] WrapArgs(IEnumerable source, int bufferSize) => new object[] { source, bufferSize }; } From 917d63027cd198eeabae9e245a94443e85bf0ac1 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 17:24:27 +0100 Subject: [PATCH 11/27] Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs Co-authored-by: Stephen Toub --- .../Converters/Collection/IAsyncEnumerableOfTConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs index a876fa3c4784fa..bbaea18c97c7dc 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs @@ -108,7 +108,7 @@ protected override void CreateCollection(ref Utf8JsonReader reader, ref ReadStac state.Current.ReturnValue = new BufferedAsyncEnumerable(); } - private class BufferedAsyncEnumerable : IAsyncEnumerable + private sealed class BufferedAsyncEnumerable : IAsyncEnumerable { public readonly List _buffer = new(); From ebdce43c6ef7796db72010404cebd011f4fa279a Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 17:24:32 +0100 Subject: [PATCH 12/27] Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs Co-authored-by: Stephen Toub --- .../Converters/Collection/IAsyncEnumerableOfTConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs index bbaea18c97c7dc..08612cae8253cb 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs @@ -115,7 +115,7 @@ private sealed class BufferedAsyncEnumerable : IAsyncEnumerable public IAsyncEnumerator GetAsyncEnumerator(CancellationToken _) => new BufferedAsyncEnumerator(_buffer); - private class BufferedAsyncEnumerator : IAsyncEnumerator + private sealed class BufferedAsyncEnumerator : IAsyncEnumerator { private readonly List _buffer; private int _index; From ae83e8cb88f127be897d662ac53d497b0fea6d28 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 17:42:28 +0100 Subject: [PATCH 13/27] address feedback --- .../Serialization/AsyncEnumerableStreamingDeserializer.cs | 2 +- .../Text/Json/Serialization/JsonSerializer.Read.Stream.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs index feb936fb9a7aa2..fbb549f8375674 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs @@ -24,7 +24,7 @@ public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cance bool isFinalBlock = false; do { - isFinalBlock = await JsonSerializer.ReadFromStream(_utf8Json, asyncState).ConfigureAwait(false); + isFinalBlock = await JsonSerializer.ReadFromStreamAsync(_utf8Json, asyncState).ConfigureAwait(false); JsonSerializer.ContinueDeserialize>(asyncState, isFinalBlock); if (asyncState.ReadStack.Current.ReturnValue is Queue queue) { diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs index 42f9e6be1bbf13..640d701e561e25 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs @@ -120,7 +120,7 @@ public static partial class JsonSerializer while (true) { - bool isFinalBlock = await ReadFromStream(utf8Json, asyncState).ConfigureAwait(false); + bool isFinalBlock = await ReadFromStreamAsync(utf8Json, asyncState).ConfigureAwait(false); TValue value = ContinueDeserialize(asyncState, isFinalBlock); if (isFinalBlock) @@ -135,7 +135,7 @@ public static partial class JsonSerializer /// Calling ReadCore is relatively expensive, so we minimize the number of times /// we need to call it. /// - internal static async ValueTask ReadFromStream(Stream utf8Json, ReadAsyncState asyncState) + internal static async ValueTask ReadFromStreamAsync(Stream utf8Json, ReadAsyncState asyncState) { bool isFinalBlock = false; while (!isFinalBlock) From 102ed443b0beae19134b2742618c04e087f6a0fd Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 18:32:12 +0100 Subject: [PATCH 14/27] increase delayInterval in serialization tests --- .../CollectionTests/CollectionTests.AsyncEnumerable.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs index 9453a59fde478c..4b09bdf60d6ec7 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs @@ -199,10 +199,10 @@ public static IEnumerable GetAsyncEnumerableSources() yield return WrapArgs(Enumerable.Empty(), 0, 1); yield return WrapArgs(Enumerable.Range(0, 20), 0, 1); yield return WrapArgs(Enumerable.Range(0, 100), 20, 20); - yield return WrapArgs(Enumerable.Range(0, 1000), 20, 1000); - yield return WrapArgs(Enumerable.Range(0, 100).Select(i => $"lorem ipsum dolor: {i}"), 1, 500); - yield return WrapArgs(Enumerable.Range(0, 10).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1, 100); - yield return WrapArgs(Enumerable.Range(0, 100).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 1, 500); + yield return WrapArgs(Enumerable.Range(0, 1000), 20, 20); + yield return WrapArgs(Enumerable.Range(0, 100).Select(i => $"lorem ipsum dolor: {i}"), 20, 100); + yield return WrapArgs(Enumerable.Range(0, 10).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 3, 100); + yield return WrapArgs(Enumerable.Range(0, 100).Select(i => new { Field1 = i, Field2 = $"lorem ipsum dolor: {i}", Field3 = i % 2 == 0 }), 20, 100); static object[] WrapArgs(IEnumerable source, int delayInterval, int bufferSize) => new object[]{ source, delayInterval, bufferSize }; } From 197fdcece66cb99cdcc301b9692b9188baaa93db Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Tue, 6 Apr 2021 19:50:39 +0100 Subject: [PATCH 15/27] address feedback --- .../Serialization/JsonSerializer.Read.Stream.cs | 16 +++++++++------- .../Text/Json/Serialization/ReadAsyncState.cs | 17 ++++------------- .../Text/Json/Serialization/WriteStack.cs | 11 ++++------- 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs index 640d701e561e25..97bd10ad8ccfe7 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs @@ -210,16 +210,18 @@ internal static TValue ContinueDeserialize(ReadAsyncState asyncState, bo if ((uint)asyncState.BytesInBuffer > ((uint)asyncState.Buffer.Length / 2)) { // We have less than half the buffer available, double the buffer size. - byte[] dest = ArrayPool.Shared.Rent((asyncState.Buffer.Length < (int.MaxValue / 2)) ? asyncState.Buffer.Length * 2 : int.MaxValue); + byte[] oldBuffer = asyncState.Buffer; + int oldClearMax = asyncState.ClearMax; + byte[] newBuffer = ArrayPool.Shared.Rent((asyncState.Buffer.Length < (int.MaxValue / 2)) ? asyncState.Buffer.Length * 2 : int.MaxValue); // Copy the unprocessed data to the new buffer while shifting the processed bytes. - Buffer.BlockCopy(asyncState.Buffer, bytesConsumed + start, dest, 0, asyncState.BytesInBuffer); - - new Span(asyncState.Buffer, 0, asyncState.ClearMax).Clear(); - ArrayPool.Shared.Return(asyncState.Buffer); - + Buffer.BlockCopy(oldBuffer, bytesConsumed + start, newBuffer, 0, asyncState.BytesInBuffer); + asyncState.Buffer = newBuffer; asyncState.ClearMax = asyncState.BytesInBuffer; - asyncState.Buffer = dest; + + // Clear and return the old buffer + new Span(oldBuffer, 0, oldClearMax).Clear(); + ArrayPool.Shared.Return(oldBuffer); } else if (asyncState.BytesInBuffer != 0) { diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs index 2574ff0538c7f7..91f064c07f83ac 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs @@ -30,21 +30,12 @@ public ReadAsyncState(Type returnType, JsonSerializerOptions? options = null, Ca IsFirstIteration = true; } - private void Dispose(bool disposing) - { - if (disposing) - { - // Clear only what we used and return the buffer to the pool - new Span(Buffer, 0, ClearMax).Clear(); - ArrayPool.Shared.Return(Buffer); - Buffer = null!; - } - } - public void Dispose() { - Dispose(disposing: true); - GC.SuppressFinalize(this); + // Clear only what we used and return the buffer to the pool + new Span(Buffer, 0, ClearMax).Clear(); + ArrayPool.Shared.Return(Buffer); + Buffer = null!; } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs index 82a1a4814afe67..1c42892783725e 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs @@ -209,15 +209,12 @@ public void Pop(bool success) } // asynchronously await any pending stacks that resumable converters depend on. - public async ValueTask AwaitPendingTask() + public Task AwaitPendingTask() { Debug.Assert(PendingTask != null); - - if (!PendingTask.IsCompleted) - { - // wrap with Task.WhenAny to avoid surfacing any exceptions here - await Task.WhenAny(PendingTask).ConfigureAwait(false); - } + return PendingTask.IsCompleted ? + Task.CompletedTask : + Task.WhenAny(PendingTask); // wrap with Task.WhenAny to avoid surfacing any exceptions here // Do not clear the `PendingTask` field here since the result // will need to be consumed by a resumable converter. From 8ee1051b0e72bbf851f99580302092eed4cc8cde Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 7 Apr 2021 14:58:34 +0100 Subject: [PATCH 16/27] address feedback --- .../Text/Json/Serialization/WriteStack.cs | 33 +++++++------------ .../Text/Json/ThrowHelper.Serialization.cs | 16 +++++++++ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs index 1c42892783725e..0d5eb2116e2409 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs @@ -261,7 +261,7 @@ public void DisposePendingDisposablesOnException() List? exceptions = null; Debug.Assert(Current.AsyncEnumerator is null); - DisposeFrame(Current.CollectionEnumerator); + DisposeFrame(Current.CollectionEnumerator, ref exceptions); int stackSize = Math.Max(_count, _continuationCount); if (stackSize > 1) @@ -269,28 +269,22 @@ public void DisposePendingDisposablesOnException() for (int i = 0; i < stackSize - 1; i++) { Debug.Assert(_previous[i].AsyncEnumerator is null); - DisposeFrame(_previous[i].CollectionEnumerator); + DisposeFrame(_previous[i].CollectionEnumerator, ref exceptions); } } if (exceptions is not null) { - if (exceptions.Count == 1) - { - ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); - } - - throw new AggregateException(exceptions); + ThrowHelper.ThrowAggregateExceptions(exceptions); } - void DisposeFrame(IEnumerator? collectionEnumerator) + static void DisposeFrame(IEnumerator? collectionEnumerator, ref List? exceptions) { try { if (collectionEnumerator is IDisposable disposable) { disposable.Dispose(); - return; } } catch (Exception e) @@ -309,28 +303,23 @@ public async ValueTask DisposePendingDisposablesOnExceptionAsync() { List? exceptions = null; - await DisposeFrame(Current.CollectionEnumerator, Current.AsyncEnumerator).ConfigureAwait(false); + exceptions = await DisposeFrame(Current.CollectionEnumerator, Current.AsyncEnumerator, exceptions).ConfigureAwait(false); int stackSize = Math.Max(_count, _continuationCount); if (stackSize > 1) { for (int i = 0; i < stackSize - 1; i++) { - await DisposeFrame(_previous[i].CollectionEnumerator, _previous[i].AsyncEnumerator).ConfigureAwait(false); + exceptions = await DisposeFrame(_previous[i].CollectionEnumerator, _previous[i].AsyncEnumerator, exceptions).ConfigureAwait(false); } } if (exceptions is not null) { - if (exceptions.Count == 1) - { - ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); - } - - throw new AggregateException(exceptions); + ThrowHelper.ThrowAggregateExceptions(exceptions); } - async ValueTask DisposeFrame(IEnumerator? collectionEnumerator, IAsyncDisposable? asyncDisposable) + static async ValueTask?> DisposeFrame(IEnumerator? collectionEnumerator, IAsyncDisposable? asyncDisposable, List? exceptions) { Debug.Assert(!(collectionEnumerator is not null && asyncDisposable is not null)); @@ -339,10 +328,8 @@ async ValueTask DisposeFrame(IEnumerator? collectionEnumerator, IAsyncDisposable if (collectionEnumerator is IDisposable disposable) { disposable.Dispose(); - return; } - - if (asyncDisposable is not null) + else if (asyncDisposable is not null) { await asyncDisposable.DisposeAsync().ConfigureAwait(false); } @@ -352,6 +339,8 @@ async ValueTask DisposeFrame(IEnumerator? collectionEnumerator, IAsyncDisposable exceptions ??= new(); exceptions.Add(e); } + + return exceptions; } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs index 355cbd99ced919..4ebf6d65809b7b 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs @@ -2,10 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Reflection; using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; using System.Text.Json.Serialization; namespace System.Text.Json @@ -667,5 +669,19 @@ internal static void ThrowUnexpectedMetadataException( ThrowJsonException_MetadataInvalidPropertyWithLeadingDollarSign(propertyName, ref state, reader); } } + + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + internal static void ThrowAggregateExceptions(List exceptions) + { + Debug.Assert(exceptions?.Count > 0); + + if (exceptions.Count == 1) + { + ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); + } + + throw new AggregateException(exceptions); + } } } From 9a03fed1f8a49d4db608165f9da99e6a458e2eb4 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 7 Apr 2021 17:08:16 +0100 Subject: [PATCH 17/27] add test on exceptional IAsyncDisposable disposal --- .../CollectionTests.AsyncEnumerable.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs index 4b09bdf60d6ec7..7985d4b2d201f2 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs @@ -156,6 +156,23 @@ public static void WriteNestedAsyncEnumerableSync_ThrowsNotSupportedException() Assert.Throws(() => JsonSerializer.Serialize(new { Data = asyncEnumerable })); } + [Fact] + public static async Task WriteAsyncEnumerable_ElementSerializationThrows_ShouldDisposeEnumerator() + { + using var stream = new Utf8MemoryStream(); + var asyncEnumerable = new MockedAsyncEnumerable>(Enumerable.Repeat(ThrowingEnumerable(), 2)); + + await Assert.ThrowsAsync(() => JsonSerializer.SerializeAsync(stream, new { Data = asyncEnumerable })); + Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); + Assert.Equal(1, asyncEnumerable.TotalDisposedEnumerators); + + static IEnumerable ThrowingEnumerable() + { + yield return 0; + throw new DivideByZeroException(); + } + } + [Fact] public static async Task ReadRootLevelAsyncEnumerable() { From 537dc54da633ec5c04c82337e75be7cd84ee9286 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 7 Apr 2021 17:41:45 +0100 Subject: [PATCH 18/27] address feedback --- .../System.Text.Json/ref/System.Text.Json.cs | 2 +- .../src/System.Text.Json.csproj | 6 +-- .../IAsyncEnumerableOfTConverter.cs | 28 +++--------- .../JsonSerializer.Read.Stream.cs | 26 ++++++++++- .../JsonSerializer.Write.Stream.cs | 10 ++++- .../Text/Json/Serialization/WriteStack.cs | 44 +++++++------------ .../Text/Json/ThrowHelper.Serialization.cs | 14 ------ .../CollectionTests.AsyncEnumerable.cs | 2 +- .../Stream.DeserializeAsyncEnumerable.cs | 26 ++++++++++- 9 files changed, 82 insertions(+), 76 deletions(-) diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index b24167f6f07fca..48a4c8c5f9e519 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -192,7 +192,7 @@ public static partial class JsonSerializer public static object? Deserialize(ref System.Text.Json.Utf8JsonReader reader, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type returnType, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static System.Threading.Tasks.ValueTask DeserializeAsync(System.IO.Stream utf8Json, [System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] System.Type returnType, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.ValueTask DeserializeAsync<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public static System.Collections.Generic.IAsyncEnumerable DeserializeAsyncEnumerable<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } + public static System.Collections.Generic.IAsyncEnumerable DeserializeAsyncEnumerable<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.ReadOnlySpan utf8Json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(string json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } public static TValue? Deserialize<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] TValue>(System.ReadOnlySpan json, System.Text.Json.JsonSerializerOptions? options = null) { throw null; } diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 5ec3035a51e139..46c06a5dc352ff 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -1,4 +1,4 @@ - + true $(NetCoreAppCurrent);netstandard2.0;netcoreapp3.0;net461 @@ -69,12 +69,12 @@ - - + + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs index 08612cae8253cb..5176d3665a1170 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs @@ -112,33 +112,15 @@ private sealed class BufferedAsyncEnumerable : IAsyncEnumerable { public readonly List _buffer = new(); - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken _) => - new BufferedAsyncEnumerator(_buffer); - - private sealed class BufferedAsyncEnumerator : IAsyncEnumerator +#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken _) { - private readonly List _buffer; - private int _index; - - public BufferedAsyncEnumerator(List buffer) - { - _buffer = buffer; - _index = -1; - } - - public TElement Current => _index < 0 ? default! : _buffer[_index]; - public ValueTask DisposeAsync() => default; - public ValueTask MoveNextAsync() + foreach (TElement element in _buffer) { - if (_index == _buffer.Count - 1) - { - return new ValueTask(false); - } - - _index++; - return new ValueTask(true); + yield return element; } } +#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs index 97bd10ad8ccfe7..f9938c38b45712 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Runtime.CompilerServices; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; @@ -94,20 +95,41 @@ public static partial class JsonSerializer /// An representation of the provided JSON array. /// JSON data to parse. /// Options to control the behavior during reading. + /// The which may be used to cancel the read operation. /// An representation of the JSON value. /// /// is . /// public static IAsyncEnumerable DeserializeAsyncEnumerable<[DynamicallyAccessedMembers(MembersAccessedOnRead)] TValue>( Stream utf8Json, - JsonSerializerOptions? options = null) + JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) { if (utf8Json == null) { throw new ArgumentNullException(nameof(utf8Json)); } - return new AsyncEnumerableStreamingDeserializer(utf8Json, options); + return CreateAsyncEnumerableDeserializer(utf8Json, options, cancellationToken); + + static async IAsyncEnumerable CreateAsyncEnumerableDeserializer(Stream utf8Json, JsonSerializerOptions? options, [EnumeratorCancellation] CancellationToken cancellationToken) + { + using var asyncState = new ReadAsyncState(typeof(Queue), options, cancellationToken); + bool isFinalBlock = false; + do + { + isFinalBlock = await JsonSerializer.ReadFromStreamAsync(utf8Json, asyncState).ConfigureAwait(false); + JsonSerializer.ContinueDeserialize>(asyncState, isFinalBlock); + if (asyncState.ReadStack.Current.ReturnValue is Queue queue) + { + while (queue.Count > 0) + { + yield return queue.Dequeue(); + } + } + } + while (!isFinalBlock); + } } internal static async ValueTask ReadAllAsync( diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs index a44d66cc990514..015b136247c302 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs @@ -141,7 +141,15 @@ private static async Task WriteAsyncCore( if (state.PendingTask is not null) { - await state.AwaitPendingTask().ConfigureAwait(false); + try + { + await state.PendingTask.ConfigureAwait(false); + } + catch + { + // Exceptions will be propagated elsewhere + // TODO https://github.com/dotnet/runtime/issues/22144 + } } } while (!isFinalBlock); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs index 0d5eb2116e2409..feecd88783b2ae 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs @@ -208,18 +208,6 @@ public void Pop(bool success) } } - // asynchronously await any pending stacks that resumable converters depend on. - public Task AwaitPendingTask() - { - Debug.Assert(PendingTask != null); - return PendingTask.IsCompleted ? - Task.CompletedTask : - Task.WhenAny(PendingTask); // wrap with Task.WhenAny to avoid surfacing any exceptions here - - // Do not clear the `PendingTask` field here since the result - // will need to be consumed by a resumable converter. - } - // Asynchronously dispose of any AsyncDisposables that have been scheduled for disposal public async ValueTask DisposePendingAsyncDisposables() { @@ -258,10 +246,10 @@ public async ValueTask DisposePendingAsyncDisposables() /// public void DisposePendingDisposablesOnException() { - List? exceptions = null; + Exception? exception = null; Debug.Assert(Current.AsyncEnumerator is null); - DisposeFrame(Current.CollectionEnumerator, ref exceptions); + DisposeFrame(Current.CollectionEnumerator, ref exception); int stackSize = Math.Max(_count, _continuationCount); if (stackSize > 1) @@ -269,16 +257,16 @@ public void DisposePendingDisposablesOnException() for (int i = 0; i < stackSize - 1; i++) { Debug.Assert(_previous[i].AsyncEnumerator is null); - DisposeFrame(_previous[i].CollectionEnumerator, ref exceptions); + DisposeFrame(_previous[i].CollectionEnumerator, ref exception); } } - if (exceptions is not null) + if (exception is not null) { - ThrowHelper.ThrowAggregateExceptions(exceptions); + ExceptionDispatchInfo.Capture(exception).Throw(); } - static void DisposeFrame(IEnumerator? collectionEnumerator, ref List? exceptions) + static void DisposeFrame(IEnumerator? collectionEnumerator, ref Exception? exception) { try { @@ -289,8 +277,7 @@ static void DisposeFrame(IEnumerator? collectionEnumerator, ref List? } catch (Exception e) { - exceptions ??= new(); - exceptions.Add(e); + exception = e; } } } @@ -301,25 +288,25 @@ static void DisposeFrame(IEnumerator? collectionEnumerator, ref List? /// public async ValueTask DisposePendingDisposablesOnExceptionAsync() { - List? exceptions = null; + Exception? exception = null; - exceptions = await DisposeFrame(Current.CollectionEnumerator, Current.AsyncEnumerator, exceptions).ConfigureAwait(false); + exception = await DisposeFrame(Current.CollectionEnumerator, Current.AsyncEnumerator, exception).ConfigureAwait(false); int stackSize = Math.Max(_count, _continuationCount); if (stackSize > 1) { for (int i = 0; i < stackSize - 1; i++) { - exceptions = await DisposeFrame(_previous[i].CollectionEnumerator, _previous[i].AsyncEnumerator, exceptions).ConfigureAwait(false); + exception = await DisposeFrame(_previous[i].CollectionEnumerator, _previous[i].AsyncEnumerator, exception).ConfigureAwait(false); } } - if (exceptions is not null) + if (exception is not null) { - ThrowHelper.ThrowAggregateExceptions(exceptions); + ExceptionDispatchInfo.Capture(exception).Throw(); } - static async ValueTask?> DisposeFrame(IEnumerator? collectionEnumerator, IAsyncDisposable? asyncDisposable, List? exceptions) + static async ValueTask DisposeFrame(IEnumerator? collectionEnumerator, IAsyncDisposable? asyncDisposable, Exception? exception) { Debug.Assert(!(collectionEnumerator is not null && asyncDisposable is not null)); @@ -336,11 +323,10 @@ public async ValueTask DisposePendingDisposablesOnExceptionAsync() } catch (Exception e) { - exceptions ??= new(); - exceptions.Add(e); + exception = e; } - return exceptions; + return exception; } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs index 4ebf6d65809b7b..444d5b58d96a6f 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs @@ -669,19 +669,5 @@ internal static void ThrowUnexpectedMetadataException( ThrowJsonException_MetadataInvalidPropertyWithLeadingDollarSign(propertyName, ref state, reader); } } - - [DoesNotReturn] - [MethodImpl(MethodImplOptions.NoInlining)] - internal static void ThrowAggregateExceptions(List exceptions) - { - Debug.Assert(exceptions?.Count > 0); - - if (exceptions.Count == 1) - { - ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); - } - - throw new AggregateException(exceptions); - } } } diff --git a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs index 7985d4b2d201f2..eedfe5d475d44a 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs @@ -73,7 +73,7 @@ public static async Task WriteNestedAsyncEnumerable_DTO(IEnumerable( diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs index 75f16aadd9b028..27db96a7877b29 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.DeserializeAsyncEnumerable.cs @@ -91,11 +91,33 @@ public static async Task DeserializeAsyncEnumerable_CancellationToken_ThrowsOnCa var token = new CancellationToken(canceled: true); using var stream = new MemoryStream(data); - IAsyncEnumerable asyncEnumerable = JsonSerializer.DeserializeAsyncEnumerable(stream, options); + var cancellableAsyncEnumerable = JsonSerializer.DeserializeAsyncEnumerable(stream, options, token); await Assert.ThrowsAsync(async () => { - await foreach (int element in asyncEnumerable.WithCancellation(token)) + await foreach (int element in cancellableAsyncEnumerable) + { + } + }); + } + + [Fact] + public static async Task DeserializeAsyncEnumerable_EnumeratorWithCancellationToken_ThrowsOnCancellation() + { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = 1 + }; + + byte[] data = JsonSerializer.SerializeToUtf8Bytes(Enumerable.Range(1, 100)); + + var token = new CancellationToken(canceled: true); + using var stream = new MemoryStream(data); + var cancellableAsyncEnumerable = JsonSerializer.DeserializeAsyncEnumerable(stream, options).WithCancellation(token); + + await Assert.ThrowsAsync(async () => + { + await foreach (int element in cancellableAsyncEnumerable) { } }); From c5f57b492403683b7168ef6bf168c8d26d149f56 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 7 Apr 2021 17:52:38 +0100 Subject: [PATCH 19/27] Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs Co-authored-by: Layomi Akinrinade --- .../src/System/Text/Json/Serialization/ReadAsyncState.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs index 91f064c07f83ac..b62c58704f04ee 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs @@ -21,7 +21,7 @@ internal sealed class ReadAsyncState : IDisposable public ReadAsyncState(Type returnType, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) { - Options = options ??= JsonSerializerOptions.s_defaultOptions; + Options = options ?? JsonSerializerOptions.s_defaultOptions; Buffer = ArrayPool.Shared.Rent(Math.Max(Options.DefaultBufferSize, JsonConstants.Utf8Bom.Length)); ReadStack.Initialize(returnType, Options, supportContinuation: true); Converter = ReadStack.Current.JsonPropertyInfo!.ConverterBase; From 68f71c78e40129dd2a41e531a97c612e5b7f0bc3 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 7 Apr 2021 17:52:55 +0100 Subject: [PATCH 20/27] Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs Co-authored-by: Layomi Akinrinade --- .../src/System/Text/Json/Serialization/WriteStack.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs index feecd88783b2ae..f8cb95595fa004 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs @@ -197,7 +197,7 @@ public void Pop(bool success) { // we have completed serialization of an AsyncEnumerator, // pop from the stack and schedule for async disposal. - PendingAsyncDisposables ??= new(); + PendingAsyncDisposables ??= new List(); PendingAsyncDisposables.Add(Current.AsyncEnumerator); } } From bfc0e7b7f67e516882d350f13375e98d0f6aaba9 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 7 Apr 2021 17:53:45 +0100 Subject: [PATCH 21/27] fix build and remove dead code --- .../src/System.Text.Json.csproj | 4 +- .../AsyncEnumerableStreamingDeserializer.cs | 40 ------------------- .../Text/Json/Serialization/WriteStack.cs | 2 +- 3 files changed, 2 insertions(+), 44 deletions(-) delete mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 46c06a5dc352ff..cdb319490dcd5a 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -184,7 +184,6 @@ - @@ -239,8 +238,7 @@ - + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs deleted file mode 100644 index fbb549f8375674..00000000000000 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncEnumerableStreamingDeserializer.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.IO; -using System.Threading; - -namespace System.Text.Json.Serialization -{ - internal sealed class AsyncEnumerableStreamingDeserializer : IAsyncEnumerable - { - private readonly Stream _utf8Json; - private readonly JsonSerializerOptions? _options; - - public AsyncEnumerableStreamingDeserializer(Stream utf8Json, JsonSerializerOptions? options) - { - _utf8Json = utf8Json; - _options = options; - } - - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken) - { - using var asyncState = new ReadAsyncState(typeof(Queue), _options, cancellationToken); - bool isFinalBlock = false; - do - { - isFinalBlock = await JsonSerializer.ReadFromStreamAsync(_utf8Json, asyncState).ConfigureAwait(false); - JsonSerializer.ContinueDeserialize>(asyncState, isFinalBlock); - if (asyncState.ReadStack.Current.ReturnValue is Queue queue) - { - while (queue.Count > 0) - { - yield return queue.Dequeue(); - } - } - } - while (!isFinalBlock); - } - } -} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs index f8cb95595fa004..319b12aba712a1 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs @@ -197,7 +197,7 @@ public void Pop(bool success) { // we have completed serialization of an AsyncEnumerator, // pop from the stack and schedule for async disposal. - PendingAsyncDisposables ??= new List(); + PendingAsyncDisposables ??= new List(); PendingAsyncDisposables.Add(Current.AsyncEnumerator); } } From 65b2ec49fbf5d82ba05466c3a8e077d708eb842a Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 7 Apr 2021 19:35:23 +0100 Subject: [PATCH 22/27] address feedback --- .../IAsyncEnumerableConverterFactory.cs | 16 +------- .../IAsyncEnumerableOfTConverter.cs | 40 +++++++++---------- .../Text/Json/Serialization/WriteStack.cs | 16 +++----- 3 files changed, 27 insertions(+), 45 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs index ea4cae9662c9ef..abcae2a747ba00 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs @@ -27,26 +27,14 @@ public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializer internal static bool TryGetAsyncEnumerableInterface(Type type, [NotNullWhen(true)] out Type? asyncEnumerableInterface) { - if (type.IsInterface && IsAsyncEnumerableInterface(type)) + if (IEnumerableConverterFactoryHelpers.GetCompatibleGenericInterface(type, typeof(IAsyncEnumerable<>)) is Type interfaceTy) { - asyncEnumerableInterface = type; + asyncEnumerableInterface = interfaceTy; return true; } - foreach (Type interfaceTy in type.GetInterfaces()) - { - if (IsAsyncEnumerableInterface(interfaceTy)) - { - asyncEnumerableInterface = interfaceTy; - return true; - } - } - asyncEnumerableInterface = null; return false; - - static bool IsAsyncEnumerableInterface(Type interfaceTy) - => interfaceTy.IsGenericType && interfaceTy.GetGenericTypeDefinition() == typeof(IAsyncEnumerable<>); } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs index 5176d3665a1170..3deef03e10fbb4 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs @@ -12,6 +12,26 @@ internal sealed class IAsyncEnumerableOfTConverter : IEnumerableDefaultConverter where TAsyncEnumerable : IAsyncEnumerable { + internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, ref ReadStack state, out TAsyncEnumerable value) + { + if (!typeToConvert.IsAssignableFrom(typeof(IAsyncEnumerable))) + { + ThrowHelper.ThrowNotSupportedException_CannotPopulateCollection(TypeToConvert, ref reader, ref state); + } + + return base.OnTryRead(ref reader, typeToConvert, options, ref state, out value!); + } + + protected override void Add(in TElement value, ref ReadStack state) + { + ((BufferedAsyncEnumerable)state.Current.ReturnValue!)._buffer.Add(value); + } + + protected override void CreateCollection(ref Utf8JsonReader reader, ref ReadStack state, JsonSerializerOptions options) + { + state.Current.ReturnValue = new BufferedAsyncEnumerable(); + } + internal override bool OnTryWrite(Utf8JsonWriter writer, TAsyncEnumerable value, JsonSerializerOptions options, ref WriteStack state) { if (!state.SupportContinuation) @@ -88,26 +108,6 @@ protected override bool OnWriteResume(Utf8JsonWriter writer, TAsyncEnumerable va return false; } - internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, ref ReadStack state, out TAsyncEnumerable value) - { - if (!typeToConvert.IsAssignableFrom(typeof(IAsyncEnumerable))) - { - ThrowHelper.ThrowNotSupportedException_CannotPopulateCollection(TypeToConvert, ref reader, ref state); - } - - return base.OnTryRead(ref reader, typeToConvert, options, ref state, out value!); - } - - protected override void Add(in TElement value, ref ReadStack state) - { - ((BufferedAsyncEnumerable)state.Current.ReturnValue!)._buffer.Add(value); - } - - protected override void CreateCollection(ref Utf8JsonReader reader, ref ReadStack state, JsonSerializerOptions options) - { - state.Current.ReturnValue = new BufferedAsyncEnumerable(); - } - private sealed class BufferedAsyncEnumerable : IAsyncEnumerable { public readonly List _buffer = new(); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs index 319b12aba712a1..99b2ce37e3f3c2 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs @@ -212,7 +212,7 @@ public void Pop(bool success) public async ValueTask DisposePendingAsyncDisposables() { Debug.Assert(PendingAsyncDisposables?.Count > 0); - List? exceptions = null; + Exception? exception = null; foreach (IAsyncDisposable asyncDisposable in PendingAsyncDisposables) { @@ -220,21 +220,15 @@ public async ValueTask DisposePendingAsyncDisposables() { await asyncDisposable.DisposeAsync().ConfigureAwait(false); } - catch (Exception exn) + catch (Exception e) { - exceptions ??= new(); - exceptions.Add(exn); + exception = e; } } - if (exceptions is not null) + if (exception is not null) { - if (exceptions.Count == 1) - { - ExceptionDispatchInfo.Capture(exceptions[0]).Throw(); - } - - throw new AggregateException(exceptions); + ExceptionDispatchInfo.Capture(exception).Throw(); } PendingAsyncDisposables.Clear(); From 9ccf772c8434c56a598cb2bac33113e15e1b85cf Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 7 Apr 2021 19:54:34 +0100 Subject: [PATCH 23/27] Revert unneeded JsonClassInfo.ElementType workaround --- .../IAsyncEnumerableConverterFactory.cs | 17 ++++------------- .../Converters/Object/JsonObjectConverter.cs | 13 +------------ .../Text/Json/Serialization/JsonClassInfo.cs | 12 ++---------- 3 files changed, 7 insertions(+), 35 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs index abcae2a747ba00..d9ad1615bb4c28 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs @@ -13,11 +13,11 @@ namespace System.Text.Json.Serialization /// internal sealed class IAsyncEnumerableConverterFactory : JsonConverterFactory { - public override bool CanConvert(Type typeToConvert) => TryGetAsyncEnumerableInterface(typeToConvert, out _); + public override bool CanConvert(Type typeToConvert) => GetAsyncEnumerableInterface(typeToConvert) is not null; public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) { - TryGetAsyncEnumerableInterface(typeToConvert, out Type? asyncEnumerableInterface); + Type? asyncEnumerableInterface = GetAsyncEnumerableInterface(typeToConvert); Debug.Assert(asyncEnumerableInterface is not null, $"{typeToConvert} not supported by converter."); Type elementType = asyncEnumerableInterface.GetGenericArguments()[0]; @@ -25,16 +25,7 @@ public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializer return (JsonConverter)Activator.CreateInstance(converterType)!; } - internal static bool TryGetAsyncEnumerableInterface(Type type, [NotNullWhen(true)] out Type? asyncEnumerableInterface) - { - if (IEnumerableConverterFactoryHelpers.GetCompatibleGenericInterface(type, typeof(IAsyncEnumerable<>)) is Type interfaceTy) - { - asyncEnumerableInterface = interfaceTy; - return true; - } - - asyncEnumerableInterface = null; - return false; - } + private static Type? GetAsyncEnumerableInterface(Type type) + => IEnumerableConverterFactoryHelpers.GetCompatibleGenericInterface(type, typeof(IAsyncEnumerable<>)); } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/JsonObjectConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/JsonObjectConverter.cs index 9b5aaea790b07f..02752db31d549d 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/JsonObjectConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/JsonObjectConverter.cs @@ -9,18 +9,7 @@ namespace System.Text.Json.Serialization /// internal abstract class JsonObjectConverter : JsonResumableConverter { - internal JsonObjectConverter() - { - // Populate ElementType if the runtime type implements IAsyncEnumerable. - // Used to feed the (converter-agnostic) JsonClassInfo.ElementType instace - // which is subsequently consulted by custom enumerable converters fed via JsonConverterAttribute. - if (IAsyncEnumerableConverterFactory.TryGetAsyncEnumerableInterface(typeof(T), out Type? asyncEnumerableInterface)) - { - ElementType = asyncEnumerableInterface.GetGenericArguments()[0]; - } - } - internal sealed override ClassType ClassType => ClassType.Object; - internal sealed override Type? ElementType { get; } + internal sealed override Type? ElementType => null; } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonClassInfo.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonClassInfo.cs index b1d3460ac09d50..16374a3e180965 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonClassInfo.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonClassInfo.cs @@ -46,11 +46,8 @@ public JsonClassInfo? ElementClassInfo { if (_elementClassInfo == null && ElementType != null) { - Debug.Assert( - ClassType == ClassType.Enumerable || - ClassType == ClassType.Dictionary || - (ClassType == ClassType.Object && - IAsyncEnumerableConverterFactory.TryGetAsyncEnumerableInterface(Type, out _))); + Debug.Assert(ClassType == ClassType.Enumerable || + ClassType == ClassType.Dictionary); _elementClassInfo = Options.GetOrAddClass(ElementType); } @@ -249,11 +246,6 @@ public JsonClassInfo(Type type, JsonSerializerOptions options) { InitializeConstructorParameters(converter.ConstructorInfo!); } - - // Types like IAsyncEnumerable can default to the Object class type - // but still require the ElementType for other converters specified - // via JsonConverterAttribute. - ElementType = converter.ElementType; } break; case ClassType.Enumerable: From ac56fb396a85d48ccd963ac20832f4f42719da8b Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Thu, 8 Apr 2021 16:14:18 +0100 Subject: [PATCH 24/27] remove state allocation on async deserialization methods --- .../src/System.Text.Json.csproj | 2 +- .../JsonSerializer.Read.Stream.cs | 145 +++++++++++------- .../Serialization/ReadAsyncBufferState.cs | 34 ++++ .../Text/Json/Serialization/ReadAsyncState.cs | 41 ----- 4 files changed, 124 insertions(+), 98 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncBufferState.cs delete mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index cdb319490dcd5a..076bc715727857 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -175,7 +175,7 @@ - + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs index f9938c38b45712..5e4b09d6b3fa8a 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs @@ -110,25 +110,40 @@ public static partial class JsonSerializer throw new ArgumentNullException(nameof(utf8Json)); } + options ??= JsonSerializerOptions.s_defaultOptions; return CreateAsyncEnumerableDeserializer(utf8Json, options, cancellationToken); - static async IAsyncEnumerable CreateAsyncEnumerableDeserializer(Stream utf8Json, JsonSerializerOptions? options, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable CreateAsyncEnumerableDeserializer( + Stream utf8Json, + JsonSerializerOptions options, + [EnumeratorCancellation] CancellationToken cancellationToken) { - using var asyncState = new ReadAsyncState(typeof(Queue), options, cancellationToken); - bool isFinalBlock = false; - do + var bufferState = new ReadAsyncBufferState(options.DefaultBufferSize); + ReadStack readStack = default; + readStack.Initialize(typeof(Queue), options, supportContinuation: true); + JsonConverter converter = readStack.Current.JsonPropertyInfo!.ConverterBase; + var jsonReaderState = new JsonReaderState(options.GetReaderOptions()); + + try { - isFinalBlock = await JsonSerializer.ReadFromStreamAsync(utf8Json, asyncState).ConfigureAwait(false); - JsonSerializer.ContinueDeserialize>(asyncState, isFinalBlock); - if (asyncState.ReadStack.Current.ReturnValue is Queue queue) + do { - while (queue.Count > 0) + bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false); + ContinueDeserialize>(ref bufferState, ref jsonReaderState, ref readStack, converter, options); + if (readStack.Current.ReturnValue is Queue queue) { - yield return queue.Dequeue(); + while (queue.Count > 0) + { + yield return queue.Dequeue(); + } } } + while (!bufferState.IsFinalBlock); + } + finally + { + bufferState.Dispose(); } - while (!isFinalBlock); } } @@ -138,18 +153,30 @@ static async IAsyncEnumerable CreateAsyncEnumerableDeserializer(Stream u JsonSerializerOptions? options, CancellationToken cancellationToken) { - using var asyncState = new ReadAsyncState(inputType, options, cancellationToken); + options ??= JsonSerializerOptions.s_defaultOptions; + var asyncState = new ReadAsyncBufferState(options.DefaultBufferSize); + ReadStack readStack = default; + readStack.Initialize(inputType, options, supportContinuation: true); + JsonConverter converter = readStack.Current.JsonPropertyInfo!.ConverterBase; + var jsonReaderState = new JsonReaderState(options.GetReaderOptions()); - while (true) + try { - bool isFinalBlock = await ReadFromStreamAsync(utf8Json, asyncState).ConfigureAwait(false); - TValue value = ContinueDeserialize(asyncState, isFinalBlock); - - if (isFinalBlock) + while (true) { - return value!; + asyncState = await ReadFromStreamAsync(utf8Json, asyncState, cancellationToken).ConfigureAwait(false); + TValue value = ContinueDeserialize(ref asyncState, ref jsonReaderState, ref readStack, converter, options); + + if (asyncState.IsFinalBlock) + { + return value!; + } } } + finally + { + asyncState.Dispose(); + } } /// @@ -157,98 +184,104 @@ static async IAsyncEnumerable CreateAsyncEnumerableDeserializer(Stream u /// Calling ReadCore is relatively expensive, so we minimize the number of times /// we need to call it. /// - internal static async ValueTask ReadFromStreamAsync(Stream utf8Json, ReadAsyncState asyncState) + internal static async ValueTask ReadFromStreamAsync( + Stream utf8Json, + ReadAsyncBufferState bufferState, + CancellationToken cancellationToken) { - bool isFinalBlock = false; - while (!isFinalBlock) + while (true) { int bytesRead = await utf8Json.ReadAsync( #if BUILDING_INBOX_LIBRARY - asyncState.Buffer.AsMemory(asyncState.BytesInBuffer), + bufferState.Buffer.AsMemory(bufferState.BytesInBuffer), #else - asyncState.Buffer, asyncState.BytesInBuffer, asyncState.Buffer.Length - asyncState.BytesInBuffer, + bufferState.Buffer, bufferState.BytesInBuffer, bufferState.Buffer.Length - bufferState.BytesInBuffer, #endif - asyncState.CancellationToken).ConfigureAwait(false); + cancellationToken).ConfigureAwait(false); if (bytesRead == 0) { - isFinalBlock = true; + bufferState.IsFinalBlock = true; break; } - asyncState.TotalBytesRead += bytesRead; - asyncState.BytesInBuffer += bytesRead; + bufferState.BytesInBuffer += bytesRead; - if (asyncState.BytesInBuffer == asyncState.Buffer.Length) + if (bufferState.BytesInBuffer == bufferState.Buffer.Length) { break; } } - return isFinalBlock; + return bufferState; } - internal static TValue ContinueDeserialize(ReadAsyncState asyncState, bool isFinalBlock) + internal static TValue ContinueDeserialize( + ref ReadAsyncBufferState bufferState, + ref JsonReaderState jsonReaderState, + ref ReadStack readStack, + JsonConverter converter, + JsonSerializerOptions options) { - if (asyncState.BytesInBuffer > asyncState.ClearMax) + if (bufferState.BytesInBuffer > bufferState.ClearMax) { - asyncState.ClearMax = asyncState.BytesInBuffer; + bufferState.ClearMax = bufferState.BytesInBuffer; } int start = 0; - if (asyncState.IsFirstIteration) + if (bufferState.IsFirstIteration) { - asyncState.IsFirstIteration = false; + bufferState.IsFirstIteration = false; // Handle the UTF-8 BOM if present - Debug.Assert(asyncState.Buffer.Length >= JsonConstants.Utf8Bom.Length); - if (asyncState.Buffer.AsSpan().StartsWith(JsonConstants.Utf8Bom)) + Debug.Assert(bufferState.Buffer.Length >= JsonConstants.Utf8Bom.Length); + if (bufferState.Buffer.AsSpan().StartsWith(JsonConstants.Utf8Bom)) { start += JsonConstants.Utf8Bom.Length; - asyncState.BytesInBuffer -= JsonConstants.Utf8Bom.Length; + bufferState.BytesInBuffer -= JsonConstants.Utf8Bom.Length; } } // Process the data available TValue value = ReadCore( - ref asyncState.ReaderState, - isFinalBlock, - new ReadOnlySpan(asyncState.Buffer, start, asyncState.BytesInBuffer), - asyncState.Options, - ref asyncState.ReadStack, - asyncState.Converter); + ref jsonReaderState, + bufferState.IsFinalBlock, + new ReadOnlySpan(bufferState.Buffer, start, bufferState.BytesInBuffer), + options, + ref readStack, + converter); - Debug.Assert(asyncState.ReadStack.BytesConsumed <= asyncState.BytesInBuffer); - int bytesConsumed = checked((int)asyncState.ReadStack.BytesConsumed); + Debug.Assert(readStack.BytesConsumed <= bufferState.BytesInBuffer); + int bytesConsumed = checked((int)readStack.BytesConsumed); - asyncState.BytesInBuffer -= bytesConsumed; + bufferState.BytesInBuffer -= bytesConsumed; // The reader should have thrown if we have remaining bytes. - Debug.Assert(!isFinalBlock || asyncState.BytesInBuffer == 0); + Debug.Assert(!bufferState.IsFinalBlock || bufferState.BytesInBuffer == 0); - if (!isFinalBlock) + if (!bufferState.IsFinalBlock) { // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. - if ((uint)asyncState.BytesInBuffer > ((uint)asyncState.Buffer.Length / 2)) + if ((uint)bufferState.BytesInBuffer > ((uint)bufferState.Buffer.Length / 2)) { // We have less than half the buffer available, double the buffer size. - byte[] oldBuffer = asyncState.Buffer; - int oldClearMax = asyncState.ClearMax; - byte[] newBuffer = ArrayPool.Shared.Rent((asyncState.Buffer.Length < (int.MaxValue / 2)) ? asyncState.Buffer.Length * 2 : int.MaxValue); + byte[] oldBuffer = bufferState.Buffer; + int oldClearMax = bufferState.ClearMax; + byte[] newBuffer = ArrayPool.Shared.Rent((bufferState.Buffer.Length < (int.MaxValue / 2)) ? bufferState.Buffer.Length * 2 : int.MaxValue); // Copy the unprocessed data to the new buffer while shifting the processed bytes. - Buffer.BlockCopy(oldBuffer, bytesConsumed + start, newBuffer, 0, asyncState.BytesInBuffer); - asyncState.Buffer = newBuffer; - asyncState.ClearMax = asyncState.BytesInBuffer; + Buffer.BlockCopy(oldBuffer, bytesConsumed + start, newBuffer, 0, bufferState.BytesInBuffer); + bufferState.Buffer = newBuffer; + bufferState.ClearMax = bufferState.BytesInBuffer; // Clear and return the old buffer new Span(oldBuffer, 0, oldClearMax).Clear(); ArrayPool.Shared.Return(oldBuffer); } - else if (asyncState.BytesInBuffer != 0) + else if (bufferState.BytesInBuffer != 0) { // Shift the processed bytes to the beginning of buffer to make more room. - Buffer.BlockCopy(asyncState.Buffer, bytesConsumed + start, asyncState.Buffer, 0, asyncState.BytesInBuffer); + Buffer.BlockCopy(bufferState.Buffer, bytesConsumed + start, bufferState.Buffer, 0, bufferState.BytesInBuffer); } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncBufferState.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncBufferState.cs new file mode 100644 index 00000000000000..2fac132521fed8 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncBufferState.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Diagnostics; +using System.Threading; + +namespace System.Text.Json.Serialization +{ + internal struct ReadAsyncBufferState : IDisposable + { + public byte[] Buffer; + public int BytesInBuffer; + public int ClearMax; + public bool IsFirstIteration; + public bool IsFinalBlock; + + public ReadAsyncBufferState(int defaultBufferSize) + { + Buffer = ArrayPool.Shared.Rent(Math.Max(defaultBufferSize, JsonConstants.Utf8Bom.Length)); + BytesInBuffer = ClearMax = 0; + IsFirstIteration = true; + IsFinalBlock = false; + } + + public void Dispose() + { + // Clear only what we used and return the buffer to the pool + new Span(Buffer, 0, ClearMax).Clear(); + ArrayPool.Shared.Return(Buffer); + Buffer = null!; + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs deleted file mode 100644 index b62c58704f04ee..00000000000000 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Buffers; -using System.Threading; - -namespace System.Text.Json.Serialization -{ - internal sealed class ReadAsyncState : IDisposable - { - public readonly CancellationToken CancellationToken; - public byte[] Buffer; - public int BytesInBuffer; - public int ClearMax; - public JsonConverter Converter; - public bool IsFirstIteration; - public JsonReaderState ReaderState; - public ReadStack ReadStack; - public JsonSerializerOptions Options; - public long TotalBytesRead; - - public ReadAsyncState(Type returnType, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) - { - Options = options ?? JsonSerializerOptions.s_defaultOptions; - Buffer = ArrayPool.Shared.Rent(Math.Max(Options.DefaultBufferSize, JsonConstants.Utf8Bom.Length)); - ReadStack.Initialize(returnType, Options, supportContinuation: true); - Converter = ReadStack.Current.JsonPropertyInfo!.ConverterBase; - ReaderState = new JsonReaderState(Options.GetReaderOptions()); - CancellationToken = cancellationToken; - IsFirstIteration = true; - } - - public void Dispose() - { - // Clear only what we used and return the buffer to the pool - new Span(Buffer, 0, ClearMax).Clear(); - ArrayPool.Shared.Return(Buffer); - Buffer = null!; - } - } -} From d893083b061dfd4d2208e9d1d9dfc6636bf1d9c5 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Thu, 8 Apr 2021 17:19:10 +0100 Subject: [PATCH 25/27] remove tooling artifacts --- src/libraries/System.Text.Json/src/System.Text.Json.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 076bc715727857..c84780fae369a6 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -238,7 +238,7 @@ - + From 2a5b5f140fff8c692b09e55d0a9741091787a5ce Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Fri, 9 Apr 2021 16:46:01 +0100 Subject: [PATCH 26/27] address feedback --- .../src/System/Text/Json/ThrowHelper.Serialization.cs | 2 -- .../CollectionTests/CollectionTests.AsyncEnumerable.cs | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs index 444d5b58d96a6f..355cbd99ced919 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs @@ -2,12 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; -using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Reflection; using System.Runtime.CompilerServices; -using System.Runtime.ExceptionServices; using System.Text.Json.Serialization; namespace System.Text.Json diff --git a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs index eedfe5d475d44a..7c2b07d5203ed9 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/CollectionTests/CollectionTests.AsyncEnumerable.cs @@ -28,7 +28,7 @@ public static async Task WriteRootLevelAsyncEnumerable(IEnumerable(source, delayInterval); await JsonSerializer.SerializeAsync(stream, asyncEnumerable, options); - Assert.Equal(expectedJson, stream.ToString()); + JsonTestHelper.AssertJsonEqual(expectedJson, stream.ToString()); Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); Assert.Equal(1, asyncEnumerable.TotalDisposedEnumerators); } @@ -48,7 +48,7 @@ public static async Task WriteNestedAsyncEnumerable(IEnumerable(source, delayInterval); await JsonSerializer.SerializeAsync(stream, new { Data = asyncEnumerable }, options); - Assert.Equal(expectedJson, stream.ToString()); + JsonTestHelper.AssertJsonEqual(expectedJson, stream.ToString()); Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); Assert.Equal(1, asyncEnumerable.TotalDisposedEnumerators); } @@ -68,7 +68,7 @@ public static async Task WriteNestedAsyncEnumerable_DTO(IEnumerable(source, delayInterval); await JsonSerializer.SerializeAsync(stream, new AsyncEnumerableDto { Data = asyncEnumerable }, options); - Assert.Equal(expectedJson, stream.ToString()); + JsonTestHelper.AssertJsonEqual(expectedJson, stream.ToString()); Assert.Equal(1, asyncEnumerable.TotalCreatedEnumerators); Assert.Equal(1, asyncEnumerable.TotalDisposedEnumerators); } @@ -110,7 +110,7 @@ public static async Task WriteSequentialNestedAsyncEnumerables(IEnumer var asyncEnumerable = new MockedAsyncEnumerable(source, delayInterval); await JsonSerializer.SerializeAsync(stream, new { Data1 = asyncEnumerable, Data2 = asyncEnumerable }, options); - Assert.Equal(expectedJson, stream.ToString()); + JsonTestHelper.AssertJsonEqual(expectedJson, stream.ToString()); Assert.Equal(2, asyncEnumerable.TotalCreatedEnumerators); Assert.Equal(2, asyncEnumerable.TotalDisposedEnumerators); } @@ -135,7 +135,7 @@ public static async Task WriteAsyncEnumerableOfAsyncEnumerables(IEnume using var stream = new Utf8MemoryStream(); await JsonSerializer.SerializeAsync(stream, outerAsyncEnumerable, options); - Assert.Equal(expectedJson, stream.ToString()); + JsonTestHelper.AssertJsonEqual(expectedJson, stream.ToString()); Assert.Equal(1, outerAsyncEnumerable.TotalCreatedEnumerators); Assert.Equal(1, outerAsyncEnumerable.TotalDisposedEnumerators); Assert.Equal(OuterEnumerableCount, innerAsyncEnumerable.TotalCreatedEnumerators); From 77fc9028662f5865b1bff020feb2f5b59231ae28 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Fri, 9 Apr 2021 21:27:19 +0100 Subject: [PATCH 27/27] reset AsyncEnumeratorIsPendingCompletion field --- .../src/System/Text/Json/Serialization/WriteStackFrame.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStackFrame.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStackFrame.cs index bdb299c3b24372..c28e19c9bdd4a0 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStackFrame.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStackFrame.cs @@ -124,8 +124,9 @@ public JsonConverter InitializeReEntry(Type type, JsonSerializerOptions options) public void Reset() { CollectionEnumerator = null; - AsyncEnumerator = null; EnumeratorIndex = 0; + AsyncEnumerator = null; + AsyncEnumeratorIsPendingCompletion = false; IgnoreDictionaryKeyPolicy = false; JsonClassInfo = null!; OriginalDepth = 0;