From f27d98e545eafdfb5be903b70de82b61b864b81b Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Tue, 27 Jul 2021 09:12:45 -0700 Subject: [PATCH 01/32] Initial surgery --- .../src/IISocketConnectionContextFactory.cs | 22 +++ .../src/PublicAPI.Unshipped.txt | 10 +- .../src/SocketConnectionContextFactory.cs | 87 ++++++++++ .../src/SocketConnectionListener.cs | 157 +++++++++--------- .../src/SocketConnectionOptions.cs | 43 +++++ .../src/SocketTransportFactory.cs | 23 ++- .../src/WebHostBuilderSocketExtensions.cs | 1 + .../SocketTransportFactoryTests.cs | 2 +- ...anspotTests.cs => SocketTransportTests.cs} | 2 +- 9 files changed, 253 insertions(+), 94 deletions(-) create mode 100644 src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs create mode 100644 src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs create mode 100644 src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs rename src/Servers/Kestrel/test/Sockets.FunctionalTests/{SocketTranspotTests.cs => SocketTransportTests.cs} (97%) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs new file mode 100644 index 000000000000..073945d3028e --- /dev/null +++ b/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Sockets; +using Microsoft.AspNetCore.Connections; + +namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets +{ + /// + /// Defines an interface that provides the mechanisms to create a socket based . + /// + public interface ISocketConnectionContextFactory + { + /// + /// Create a for a socket. + /// + /// The socket for the connection. + /// The . + /// + ConnectionContext Create(Socket socket, SocketConnectionOptions options); + } +} diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index 5708e0985dfd..948da04ad045 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -3,10 +3,16 @@ *REMOVED*Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions options, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) -> void *REMOVED*static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder *REMOVED*static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder, System.Action configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.DisposeAsync() -> System.Threading.Tasks.ValueTask +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions! transportOptions, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! connectionOptions, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal.ISocketsTrace! trace) -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint! endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask -~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder! hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder! static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder! hostBuilder, System.Action! configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder! static Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateDefaultBoundListenSocket(System.Net.EndPoint! endpoint) -> System.Net.Sockets.Socket! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.get -> System.Func! -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.set -> void \ No newline at end of file +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.set -> void +~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! transportOptions, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory! contextFactory) -> void \ No newline at end of file diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs new file mode 100644 index 000000000000..d97695d32edc --- /dev/null +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Buffers; +using System.IO.Pipelines; +using System.Net.Sockets; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets +{ + /// + /// A factory for socket based connections contexts. + /// + internal sealed class SocketConnectionContextFactory : ISocketConnectionContextFactory, IAsyncDisposable + { + private readonly MemoryPool _memoryPool; + private readonly int _socketSendersCount; + private readonly SocketSenderPool[] _socketSenders; + private int _socketSenderIndex; + private readonly SocketTransportOptions _transportOptions; + private readonly PipeScheduler _transportScheduler; + private readonly ISocketsTrace _trace; + + public SocketConnectionContextFactory( + IOptions transportOptions, + ILoggerFactory loggerFactory) + { + _transportOptions = transportOptions.Value; + _memoryPool = _transportOptions.MemoryPoolFactory(); + + _transportScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); + // https://github.com/aspnet/KestrelHttpServer/issues/2573 + var awaiterScheduler = OperatingSystem.IsWindows() ? _transportScheduler : PipeScheduler.Inline; + + var ioQueueCount = _transportOptions.IOQueueCount; + _socketSendersCount = (ioQueueCount > 0) ? ioQueueCount : 1; + _socketSenders = new SocketSenderPool[_socketSendersCount]; + for (var i = 0; i < _socketSendersCount; i++) + { + _socketSenders[i] = new SocketSenderPool(awaiterScheduler); + } + + var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); + _trace = new SocketsTrace(logger); + } + + /// + /// Create a for a socket. + /// + /// The socket for the connection. + /// The . + /// + public ConnectionContext Create(Socket socket, SocketConnectionOptions options) + { + var setting = _socketSenders[_socketSenderIndex]; + + var connection = new SocketConnection(socket, + _memoryPool, + _transportScheduler, + _trace, + setting, + options.InputOptions, + options.OutputOptions, + waitForData: _transportOptions.WaitForDataBeforeAllocatingBuffer); + + _socketSenderIndex = (_socketSenderIndex + 1) % _socketSendersCount; + + return connection; + } + + public ValueTask DisposeAsync() + { + // Dispose any pooled senders + foreach (var sender in _socketSenders) + { + sender.Dispose(); + } + + return default; + } + } +} diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 45daa310ca1d..5cfd19e6e130 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -18,70 +18,82 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets internal sealed class SocketConnectionListener : IConnectionListener { private readonly MemoryPool _memoryPool; - private readonly int _settingsCount; - private readonly Settings[] _settings; + //private readonly int _settingsCount; + //private readonly Settings[] _settings; private readonly ISocketsTrace _trace; private Socket? _listenSocket; - private int _settingsIndex; - private readonly SocketTransportOptions _options; + private readonly SocketTransportOptions _transportOptions; + private readonly ISocketConnectionContextFactory _contextFactory; + private readonly PipeOptions _inputOptions; + private readonly PipeOptions _outputOptions; public EndPoint EndPoint { get; private set; } internal SocketConnectionListener( EndPoint endpoint, - SocketTransportOptions options, + SocketTransportOptions transportOptions, + ISocketConnectionContextFactory contextFactory, ISocketsTrace trace) { EndPoint = endpoint; _trace = trace; - _options = options; - _memoryPool = _options.MemoryPoolFactory(); - var ioQueueCount = options.IOQueueCount; - - var maxReadBufferSize = _options.MaxReadBufferSize ?? 0; - var maxWriteBufferSize = _options.MaxWriteBufferSize ?? 0; - var applicationScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; - - if (ioQueueCount > 0) - { - _settingsCount = ioQueueCount; - _settings = new Settings[_settingsCount]; - - for (var i = 0; i < _settingsCount; i++) - { - var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); - // https://github.com/aspnet/KestrelHttpServer/issues/2573 - var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; - - _settings[i] = new Settings - { - Scheduler = transportScheduler, - InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), - OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), - SocketSenderPool = new SocketSenderPool(awaiterScheduler) - }; - } - } - else - { - var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; - // https://github.com/aspnet/KestrelHttpServer/issues/2573 - var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; - - var directScheduler = new Settings[] - { - new Settings - { - Scheduler = transportScheduler, - InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), - OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), - SocketSenderPool = new SocketSenderPool(awaiterScheduler) - } - }; - - _settingsCount = directScheduler.Length; - _settings = directScheduler; - } + _transportOptions = transportOptions; + _contextFactory = contextFactory; + _memoryPool = _transportOptions.MemoryPoolFactory(); + //var ioQueueCount = transportOptions.IOQueueCount; + + var maxReadBufferSize = _transportOptions.MaxReadBufferSize ?? 0; + var maxWriteBufferSize = _transportOptions.MaxWriteBufferSize ?? 0; + var applicationScheduler = transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; + + var transportScheduler = transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); + // https://github.com/aspnet/KestrelHttpServer/issues/2573 + _inputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false); + _outputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false); + + //if (ioQueueCount > 0) + //{ + +// _settingsCount = ioQueueCount; +// _settings = new Settings[_settingsCount]; + +// for (var i = 0; i < _settingsCount; i++) +// { +// var transportScheduler = transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); +// // https://github.com/aspnet/KestrelHttpServer/issues/2573 +// var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; + +// _settings[i] = new Settings +// { +// Scheduler = transportScheduler, +//// TODO: use socket connection options here +// InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), +// OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), +// SocketSenderPool = new SocketSenderPool(awaiterScheduler) +// }; +// } +// } +// else +// { +// var transportScheduler = transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; +// // https://github.com/aspnet/KestrelHttpServer/issues/2573 +// var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; + +// var directScheduler = new Settings[] +// { +// new Settings +// { +// Scheduler = transportScheduler, +//// TODO: use socket connection options here +// InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), +// OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), +// SocketSenderPool = new SocketSenderPool(awaiterScheduler) +// } +// }; + +// _settingsCount = directScheduler.Length; +// _settings = directScheduler; +// } } internal void Bind() @@ -94,7 +106,7 @@ internal void Bind() Socket listenSocket; try { - listenSocket = _options.CreateBoundListenSocket(EndPoint); + listenSocket = _transportOptions.CreateBoundListenSocket(EndPoint); } catch (SocketException e) when (e.SocketErrorCode == SocketError.AddressAlreadyInUse) { @@ -104,7 +116,7 @@ internal void Bind() Debug.Assert(listenSocket.LocalEndPoint != null); EndPoint = listenSocket.LocalEndPoint; - listenSocket.Listen(_options.Backlog); + listenSocket.Listen(_transportOptions.Backlog); _listenSocket = listenSocket; } @@ -119,28 +131,15 @@ internal void Bind() var acceptSocket = await _listenSocket.AcceptAsync(cancellationToken); - // Only apply no delay to Tcp based endpoints - if (acceptSocket.LocalEndPoint is IPEndPoint) + var connectionOptions = new SocketConnectionOptions() { - acceptSocket.NoDelay = _options.NoDelay; - } - - var setting = _settings[_settingsIndex]; - - var connection = new SocketConnection(acceptSocket, - _memoryPool, - setting.Scheduler, - _trace, - setting.SocketSenderPool, - setting.InputOptions, - setting.OutputOptions, - waitForData: _options.WaitForDataBeforeAllocatingBuffer); - - connection.Start(); - - _settingsIndex = (_settingsIndex + 1) % _settingsCount; - - return connection; + // Only apply no delay to Tcp based endpoints + DelaySocketOperations = acceptSocket.LocalEndPoint is IPEndPoint, + InputOptions = _inputOptions, + OutputOptions = _outputOptions, + WaitForDataBeforeAllocatingBuffer = _transportOptions.WaitForDataBeforeAllocatingBuffer + }; + return _contextFactory.Create(acceptSocket, connectionOptions); } catch (ObjectDisposedException) { @@ -173,12 +172,6 @@ public ValueTask DisposeAsync() // Dispose the memory pool _memoryPool.Dispose(); - // Dispose any pooled senders - foreach (var setting in _settings) - { - setting.SocketSenderPool.Dispose(); - } - return default; } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs new file mode 100644 index 000000000000..60d0987b3b29 --- /dev/null +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using Microsoft.AspNetCore.Connections; + +namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets +{ + /// + /// Options for socket based connections. + /// + public class SocketConnectionOptions + { + /// + /// The for socket connections used for input. + /// + public PipeOptions InputOptions { get; init; } = new PipeOptions(); + + /// + /// The for socket connections used for output. + /// + public PipeOptions OutputOptions { get; init; } = new PipeOptions(); + + /// + /// Set to false to enable Nagle's algorithm for all socket connections. + /// + /// + /// Defaults to true. + /// + public bool DelaySocketOperations { get; init; } = false; + + /// + /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. + /// + /// + /// Defaults to true. + /// + public bool WaitForDataBeforeAllocatingBuffer { get; set; } = true; + } +} diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs index f67f2e41d27e..b18c5d18450a 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs @@ -8,7 +8,6 @@ using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets @@ -18,16 +17,23 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// public sealed class SocketTransportFactory : IConnectionListenerFactory { - private readonly SocketTransportOptions _options; + private readonly SocketTransportOptions _transportOptions; + private readonly ISocketConnectionContextFactory _contextFactory; private readonly SocketsTrace _trace; public SocketTransportFactory( - IOptions options, - ILoggerFactory loggerFactory) + IOptions transportOptions, + ILoggerFactory loggerFactory, + ISocketConnectionContextFactory contextFactory) { - if (options == null) + if (transportOptions == null) { - throw new ArgumentNullException(nameof(options)); + throw new ArgumentNullException(nameof(transportOptions)); + } + + if (contextFactory == null) + { + throw new ArgumentNullException(nameof(contextFactory)); } if (loggerFactory == null) @@ -35,14 +41,15 @@ public SocketTransportFactory( throw new ArgumentNullException(nameof(loggerFactory)); } - _options = options.Value; + _transportOptions = transportOptions.Value; + _contextFactory = contextFactory; var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); } public ValueTask BindAsync(EndPoint endpoint, CancellationToken cancellationToken = default) { - var transport = new SocketConnectionListener(endpoint, _options, _trace); + var transport = new SocketConnectionListener(endpoint, _transportOptions, _contextFactory, _trace); transport.Bind(); return new ValueTask(transport); } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs index 540dd07bce57..6c607638c80a 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs @@ -27,6 +27,7 @@ public static IWebHostBuilder UseSockets(this IWebHostBuilder hostBuilder) return hostBuilder.ConfigureServices(services => { services.AddSingleton(); + services.AddSingleton(); }); } diff --git a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs index d0ea2cad6917..c9739b1c6e5f 100644 --- a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs +++ b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs @@ -17,7 +17,7 @@ public class SocketTransportFactoryTests [Fact] public async Task ThrowsNotImplementedExceptionWhenBindingToUriEndPoint() { - var socketTransportFactory = new SocketTransportFactory(Options.Create(new SocketTransportOptions()), Mock.Of()); + var socketTransportFactory = new SocketTransportFactory(Options.Create(new SocketTransportOptions()), Options.Create(new SocketConnectionOptions()), Mock.Of()); await Assert.ThrowsAsync(async () => await socketTransportFactory.BindAsync(new UriEndPoint(new Uri("http://127.0.0.1:5554")))); } } diff --git a/src/Servers/Kestrel/test/Sockets.FunctionalTests/SocketTranspotTests.cs b/src/Servers/Kestrel/test/Sockets.FunctionalTests/SocketTransportTests.cs similarity index 97% rename from src/Servers/Kestrel/test/Sockets.FunctionalTests/SocketTranspotTests.cs rename to src/Servers/Kestrel/test/Sockets.FunctionalTests/SocketTransportTests.cs index acadeaff81ba..893babfb0cfd 100644 --- a/src/Servers/Kestrel/test/Sockets.FunctionalTests/SocketTranspotTests.cs +++ b/src/Servers/Kestrel/test/Sockets.FunctionalTests/SocketTransportTests.cs @@ -20,7 +20,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Sockets.FunctionalTests { - public class SocketTranspotTests : LoggedTestBase + public class SocketTransportTests : LoggedTestBase { [Fact] public async Task SocketTransportExposesSocketsFeature() From 18a1c7ee5f7c2e112cfa8645d6f265e632de0951 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Tue, 27 Jul 2021 10:00:55 -0700 Subject: [PATCH 02/32] Update SocketConnectionListener.cs --- .../src/SocketConnectionListener.cs | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 5cfd19e6e130..99039ade2f8c 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -24,8 +24,7 @@ internal sealed class SocketConnectionListener : IConnectionListener private Socket? _listenSocket; private readonly SocketTransportOptions _transportOptions; private readonly ISocketConnectionContextFactory _contextFactory; - private readonly PipeOptions _inputOptions; - private readonly PipeOptions _outputOptions; + public EndPoint EndPoint { get; private set; } @@ -42,15 +41,6 @@ internal SocketConnectionListener( _memoryPool = _transportOptions.MemoryPoolFactory(); //var ioQueueCount = transportOptions.IOQueueCount; - var maxReadBufferSize = _transportOptions.MaxReadBufferSize ?? 0; - var maxWriteBufferSize = _transportOptions.MaxWriteBufferSize ?? 0; - var applicationScheduler = transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; - - var transportScheduler = transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); - // https://github.com/aspnet/KestrelHttpServer/issues/2573 - _inputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false); - _outputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false); - //if (ioQueueCount > 0) //{ @@ -131,12 +121,19 @@ internal void Bind() var acceptSocket = await _listenSocket.AcceptAsync(cancellationToken); + var maxReadBufferSize = _transportOptions.MaxReadBufferSize ?? 0; + var maxWriteBufferSize = _transportOptions.MaxWriteBufferSize ?? 0; + var applicationScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; + + var transportScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); + // https://github.com/aspnet/KestrelHttpServer/issues/2573 + var connectionOptions = new SocketConnectionOptions() { // Only apply no delay to Tcp based endpoints DelaySocketOperations = acceptSocket.LocalEndPoint is IPEndPoint, - InputOptions = _inputOptions, - OutputOptions = _outputOptions, + InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), + OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), WaitForDataBeforeAllocatingBuffer = _transportOptions.WaitForDataBeforeAllocatingBuffer }; return _contextFactory.Create(acceptSocket, connectionOptions); From 259abf4adf90bee888429c0907882908521a1326 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Tue, 27 Jul 2021 12:29:35 -0700 Subject: [PATCH 03/32] Try to fix shutdown tests --- .../src/SocketConnectionContextFactory.cs | 33 +---- .../src/SocketConnectionListener.cs | 126 +++++++++--------- 2 files changed, 70 insertions(+), 89 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index d97695d32edc..c70e775039f2 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -19,9 +19,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets internal sealed class SocketConnectionContextFactory : ISocketConnectionContextFactory, IAsyncDisposable { private readonly MemoryPool _memoryPool; - private readonly int _socketSendersCount; - private readonly SocketSenderPool[] _socketSenders; - private int _socketSenderIndex; + private readonly SocketSenderPool _socketSender; private readonly SocketTransportOptions _transportOptions; private readonly PipeScheduler _transportScheduler; private readonly ISocketsTrace _trace; @@ -33,17 +31,11 @@ public SocketConnectionContextFactory( _transportOptions = transportOptions.Value; _memoryPool = _transportOptions.MemoryPoolFactory(); - _transportScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); + _transportScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : + (_transportOptions.IOQueueCount > 0) ? new IOQueue() : PipeScheduler.ThreadPool; // https://github.com/aspnet/KestrelHttpServer/issues/2573 var awaiterScheduler = OperatingSystem.IsWindows() ? _transportScheduler : PipeScheduler.Inline; - - var ioQueueCount = _transportOptions.IOQueueCount; - _socketSendersCount = (ioQueueCount > 0) ? ioQueueCount : 1; - _socketSenders = new SocketSenderPool[_socketSendersCount]; - for (var i = 0; i < _socketSendersCount; i++) - { - _socketSenders[i] = new SocketSenderPool(awaiterScheduler); - } + _socketSender = new SocketSenderPool(awaiterScheduler); var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); @@ -56,30 +48,19 @@ public SocketConnectionContextFactory( /// The . /// public ConnectionContext Create(Socket socket, SocketConnectionOptions options) - { - var setting = _socketSenders[_socketSenderIndex]; - - var connection = new SocketConnection(socket, + => new SocketConnection(socket, _memoryPool, _transportScheduler, _trace, - setting, + _socketSender, options.InputOptions, options.OutputOptions, waitForData: _transportOptions.WaitForDataBeforeAllocatingBuffer); - _socketSenderIndex = (_socketSenderIndex + 1) % _socketSendersCount; - - return connection; - } - public ValueTask DisposeAsync() { // Dispose any pooled senders - foreach (var sender in _socketSenders) - { - sender.Dispose(); - } + _socketSender.Dispose(); return default; } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 99039ade2f8c..3d58440c9ba0 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -18,11 +18,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets internal sealed class SocketConnectionListener : IConnectionListener { private readonly MemoryPool _memoryPool; - //private readonly int _settingsCount; - //private readonly Settings[] _settings; + private readonly int _settingsCount; + private readonly Settings[] _settings; + private int _settingsIndex; private readonly ISocketsTrace _trace; private Socket? _listenSocket; - private readonly SocketTransportOptions _transportOptions; + private readonly SocketTransportOptions _options; private readonly ISocketConnectionContextFactory _contextFactory; @@ -30,60 +31,61 @@ internal sealed class SocketConnectionListener : IConnectionListener internal SocketConnectionListener( EndPoint endpoint, - SocketTransportOptions transportOptions, + SocketTransportOptions options, ISocketConnectionContextFactory contextFactory, ISocketsTrace trace) { EndPoint = endpoint; _trace = trace; - _transportOptions = transportOptions; + _options = options; _contextFactory = contextFactory; - _memoryPool = _transportOptions.MemoryPoolFactory(); - //var ioQueueCount = transportOptions.IOQueueCount; - - //if (ioQueueCount > 0) - //{ - -// _settingsCount = ioQueueCount; -// _settings = new Settings[_settingsCount]; - -// for (var i = 0; i < _settingsCount; i++) -// { -// var transportScheduler = transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); -// // https://github.com/aspnet/KestrelHttpServer/issues/2573 -// var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; - -// _settings[i] = new Settings -// { -// Scheduler = transportScheduler, -//// TODO: use socket connection options here -// InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), -// OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), -// SocketSenderPool = new SocketSenderPool(awaiterScheduler) -// }; -// } -// } -// else -// { -// var transportScheduler = transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; -// // https://github.com/aspnet/KestrelHttpServer/issues/2573 -// var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; - -// var directScheduler = new Settings[] -// { -// new Settings -// { -// Scheduler = transportScheduler, -//// TODO: use socket connection options here -// InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), -// OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), -// SocketSenderPool = new SocketSenderPool(awaiterScheduler) -// } -// }; - -// _settingsCount = directScheduler.Length; -// _settings = directScheduler; -// } + _memoryPool = _options.MemoryPoolFactory(); + var ioQueueCount = options.IOQueueCount; + + var maxReadBufferSize = _options.MaxReadBufferSize ?? 0; + var maxWriteBufferSize = _options.MaxWriteBufferSize ?? 0; + var applicationScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; + + if (ioQueueCount > 0) + { + _settingsCount = ioQueueCount; + _settings = new Settings[_settingsCount]; + + for (var i = 0; i < _settingsCount; i++) + { + var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); + // https://github.com/aspnet/KestrelHttpServer/issues/2573 + var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; + + _settings[i] = new Settings + { + Scheduler = transportScheduler, + InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), + OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), + SocketSenderPool = new SocketSenderPool(awaiterScheduler) + }; + } + } + else + { + var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; + // https://github.com/aspnet/KestrelHttpServer/issues/2573 + var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; + + var directScheduler = new Settings[] + { + new Settings + { + Scheduler = transportScheduler, + InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), + OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), + SocketSenderPool = new SocketSenderPool(awaiterScheduler) + } + }; + + _settingsCount = directScheduler.Length; + _settings = directScheduler; + } } internal void Bind() @@ -96,7 +98,7 @@ internal void Bind() Socket listenSocket; try { - listenSocket = _transportOptions.CreateBoundListenSocket(EndPoint); + listenSocket = _options.CreateBoundListenSocket(EndPoint); } catch (SocketException e) when (e.SocketErrorCode == SocketError.AddressAlreadyInUse) { @@ -106,7 +108,7 @@ internal void Bind() Debug.Assert(listenSocket.LocalEndPoint != null); EndPoint = listenSocket.LocalEndPoint; - listenSocket.Listen(_transportOptions.Backlog); + listenSocket.Listen(_options.Backlog); _listenSocket = listenSocket; } @@ -119,23 +121,21 @@ internal void Bind() { Debug.Assert(_listenSocket != null, "Bind must be called first."); - var acceptSocket = await _listenSocket.AcceptAsync(cancellationToken); - - var maxReadBufferSize = _transportOptions.MaxReadBufferSize ?? 0; - var maxWriteBufferSize = _transportOptions.MaxWriteBufferSize ?? 0; - var applicationScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; + var setting = _settings[_settingsIndex]; - var transportScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); - // https://github.com/aspnet/KestrelHttpServer/issues/2573 + var acceptSocket = await _listenSocket.AcceptAsync(cancellationToken); var connectionOptions = new SocketConnectionOptions() { // Only apply no delay to Tcp based endpoints DelaySocketOperations = acceptSocket.LocalEndPoint is IPEndPoint, - InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), - OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), - WaitForDataBeforeAllocatingBuffer = _transportOptions.WaitForDataBeforeAllocatingBuffer + InputOptions = setting.InputOptions, + OutputOptions = setting.OutputOptions, + WaitForDataBeforeAllocatingBuffer = _options.WaitForDataBeforeAllocatingBuffer }; + + _settingsIndex = (_settingsIndex + 1) % _settingsCount; + return _contextFactory.Create(acceptSocket, connectionOptions); } catch (ObjectDisposedException) From 51818917bc5bbb016c3fa101c87e0b0c13116cb0 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Tue, 27 Jul 2021 12:40:42 -0700 Subject: [PATCH 04/32] Fix bind tests --- .../Transport.Sockets/src/PublicAPI.Unshipped.txt | 2 +- .../src/SocketConnectionListener.cs | 2 +- .../Transport.Sockets/src/SocketTransportFactory.cs | 12 ++++++------ .../Sockets.BindTests/SocketTransportFactoryTests.cs | 5 ++++- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index 948da04ad045..d170dddbd31d 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -15,4 +15,4 @@ static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(th static Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateDefaultBoundListenSocket(System.Net.EndPoint! endpoint) -> System.Net.Sockets.Socket! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.get -> System.Func! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.set -> void -~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! transportOptions, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory! contextFactory) -> void \ No newline at end of file +~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory! contextFactory) -> void \ No newline at end of file diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 3d58440c9ba0..9ce7edd439dd 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -20,9 +20,9 @@ internal sealed class SocketConnectionListener : IConnectionListener private readonly MemoryPool _memoryPool; private readonly int _settingsCount; private readonly Settings[] _settings; - private int _settingsIndex; private readonly ISocketsTrace _trace; private Socket? _listenSocket; + private int _settingsIndex; private readonly SocketTransportOptions _options; private readonly ISocketConnectionContextFactory _contextFactory; diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs index b18c5d18450a..f1eeebd08b08 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs @@ -17,18 +17,18 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// public sealed class SocketTransportFactory : IConnectionListenerFactory { - private readonly SocketTransportOptions _transportOptions; + private readonly SocketTransportOptions _options; private readonly ISocketConnectionContextFactory _contextFactory; private readonly SocketsTrace _trace; public SocketTransportFactory( - IOptions transportOptions, + IOptions options, ILoggerFactory loggerFactory, ISocketConnectionContextFactory contextFactory) { - if (transportOptions == null) + if (options == null) { - throw new ArgumentNullException(nameof(transportOptions)); + throw new ArgumentNullException(nameof(options)); } if (contextFactory == null) @@ -41,7 +41,7 @@ public SocketTransportFactory( throw new ArgumentNullException(nameof(loggerFactory)); } - _transportOptions = transportOptions.Value; + _options = options.Value; _contextFactory = contextFactory; var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); @@ -49,7 +49,7 @@ public SocketTransportFactory( public ValueTask BindAsync(EndPoint endpoint, CancellationToken cancellationToken = default) { - var transport = new SocketConnectionListener(endpoint, _transportOptions, _contextFactory, _trace); + var transport = new SocketConnectionListener(endpoint, _options, _contextFactory, _trace); transport.Bind(); return new ValueTask(transport); } diff --git a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs index c9739b1c6e5f..791cef79ad93 100644 --- a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs +++ b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs @@ -17,7 +17,10 @@ public class SocketTransportFactoryTests [Fact] public async Task ThrowsNotImplementedExceptionWhenBindingToUriEndPoint() { - var socketTransportFactory = new SocketTransportFactory(Options.Create(new SocketTransportOptions()), Options.Create(new SocketConnectionOptions()), Mock.Of()); + var options = Options.Create(new SocketTransportOptions()); + var logger = Mock.Of(); + var connectionFactory = new SocketConnectionContextFactory(options, logger); + var socketTransportFactory = new SocketTransportFactory(options, logger, connectionFactory); await Assert.ThrowsAsync(async () => await socketTransportFactory.BindAsync(new UriEndPoint(new Uri("http://127.0.0.1:5554")))); } } From e355dc5530af2b1a5459e95bddcb62a17730ff5d Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Tue, 27 Jul 2021 12:57:32 -0700 Subject: [PATCH 05/32] Update SocketConnectionListener.cs --- .../Transport.Sockets/src/SocketConnectionListener.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 9ce7edd439dd..63b2e44a3bf4 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -125,10 +125,15 @@ internal void Bind() var acceptSocket = await _listenSocket.AcceptAsync(cancellationToken); + // Only apply no delay to Tcp based endpoints + if (acceptSocket.LocalEndPoint is IPEndPoint) + { + acceptSocket.NoDelay = _options.NoDelay; + } + var connectionOptions = new SocketConnectionOptions() { - // Only apply no delay to Tcp based endpoints - DelaySocketOperations = acceptSocket.LocalEndPoint is IPEndPoint, + DelaySocketOperations = acceptSocket.NoDelay, InputOptions = setting.InputOptions, OutputOptions = setting.OutputOptions, WaitForDataBeforeAllocatingBuffer = _options.WaitForDataBeforeAllocatingBuffer From 9bfb03787962eb20f98643aa0e909819f8eb70a2 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 00:51:21 -0700 Subject: [PATCH 06/32] Fix lifetime issues with internal properties --- .../src/PublicAPI.Unshipped.txt | 11 +++++++++- .../src/SocketConnectionContextFactory.cs | 22 +++---------------- .../src/SocketConnectionListener.cs | 18 ++++++++++----- .../src/SocketConnectionOptions.cs | 15 ++++++++----- .../src/WebHostBuilderSocketExtensions.cs | 2 +- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index d170dddbd31d..39914148e63e 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -5,10 +5,19 @@ *REMOVED*static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder, System.Action configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.DisposeAsync() -> System.Threading.Tasks.ValueTask Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions! transportOptions, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! connectionOptions, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal.ISocketsTrace! trace) -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DelaySocketOperations.get -> bool +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DelaySocketOperations.init -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.InputOptions.get -> System.IO.Pipelines.PipeOptions! +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.InputOptions.init -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.OutputOptions.get -> System.IO.Pipelines.PipeOptions! +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.OutputOptions.init -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.SocketConnectionOptions() -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.WaitForDataBeforeAllocatingBuffer.get -> bool +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.WaitForDataBeforeAllocatingBuffer.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint! endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder! hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder! static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder! hostBuilder, System.Action! configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder! diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index c70e775039f2..8c0b9186004b 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -1,11 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Buffers; using System.IO.Pipelines; using System.Net.Sockets; -using System.Threading.Tasks; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; using Microsoft.Extensions.Logging; @@ -16,10 +13,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// /// A factory for socket based connections contexts. /// - internal sealed class SocketConnectionContextFactory : ISocketConnectionContextFactory, IAsyncDisposable + internal sealed class SocketConnectionContextFactory : ISocketConnectionContextFactory { - private readonly MemoryPool _memoryPool; - private readonly SocketSenderPool _socketSender; private readonly SocketTransportOptions _transportOptions; private readonly PipeScheduler _transportScheduler; private readonly ISocketsTrace _trace; @@ -29,13 +24,10 @@ public SocketConnectionContextFactory( ILoggerFactory loggerFactory) { _transportOptions = transportOptions.Value; - _memoryPool = _transportOptions.MemoryPoolFactory(); _transportScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : (_transportOptions.IOQueueCount > 0) ? new IOQueue() : PipeScheduler.ThreadPool; // https://github.com/aspnet/KestrelHttpServer/issues/2573 - var awaiterScheduler = OperatingSystem.IsWindows() ? _transportScheduler : PipeScheduler.Inline; - _socketSender = new SocketSenderPool(awaiterScheduler); var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); @@ -49,20 +41,12 @@ public SocketConnectionContextFactory( /// public ConnectionContext Create(Socket socket, SocketConnectionOptions options) => new SocketConnection(socket, - _memoryPool, + options.MemoryPool, _transportScheduler, _trace, - _socketSender, + options.SenderPool, options.InputOptions, options.OutputOptions, waitForData: _transportOptions.WaitForDataBeforeAllocatingBuffer); - - public ValueTask DisposeAsync() - { - // Dispose any pooled senders - _socketSender.Dispose(); - - return default; - } } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 63b2e44a3bf4..167218e64544 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -3,7 +3,6 @@ using System; using System.Buffers; -using System.ComponentModel; using System.Diagnostics; using System.IO.Pipelines; using System.Net; @@ -26,7 +25,6 @@ internal sealed class SocketConnectionListener : IConnectionListener private readonly SocketTransportOptions _options; private readonly ISocketConnectionContextFactory _contextFactory; - public EndPoint EndPoint { get; private set; } internal SocketConnectionListener( @@ -121,22 +119,26 @@ internal void Bind() { Debug.Assert(_listenSocket != null, "Bind must be called first."); - var setting = _settings[_settingsIndex]; var acceptSocket = await _listenSocket.AcceptAsync(cancellationToken); // Only apply no delay to Tcp based endpoints +// REVIEW: should this move to inside of _contextFactory.Create?? if (acceptSocket.LocalEndPoint is IPEndPoint) { acceptSocket.NoDelay = _options.NoDelay; } - + + var setting = _settings[_settingsIndex]; + var connectionOptions = new SocketConnectionOptions() { DelaySocketOperations = acceptSocket.NoDelay, InputOptions = setting.InputOptions, OutputOptions = setting.OutputOptions, - WaitForDataBeforeAllocatingBuffer = _options.WaitForDataBeforeAllocatingBuffer + WaitForDataBeforeAllocatingBuffer = _options.WaitForDataBeforeAllocatingBuffer, + MemoryPool = _memoryPool, + SenderPool = setting.SocketSenderPool }; _settingsIndex = (_settingsIndex + 1) % _settingsCount; @@ -174,6 +176,12 @@ public ValueTask DisposeAsync() // Dispose the memory pool _memoryPool.Dispose(); + // Dispose any pooled senders + foreach (var setting in _settings) + { + setting.SocketSenderPool.Dispose(); + } + return default; } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs index 60d0987b3b29..ada8adb572b9 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs @@ -3,9 +3,7 @@ using System.Buffers; using System.IO.Pipelines; -using System.Net; -using System.Net.Sockets; -using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets { @@ -25,12 +23,12 @@ public class SocketConnectionOptions public PipeOptions OutputOptions { get; init; } = new PipeOptions(); /// - /// Set to false to enable Nagle's algorithm for all socket connections. + /// Set to true to enable Nagle's algorithm for all socket connections. /// /// - /// Defaults to true. + /// Defaults to false. /// - public bool DelaySocketOperations { get; init; } = false; + public bool DelaySocketOperations { get; init; } /// /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. @@ -39,5 +37,10 @@ public class SocketConnectionOptions /// Defaults to true. /// public bool WaitForDataBeforeAllocatingBuffer { get; set; } = true; + + internal MemoryPool MemoryPool { get; init; } = default!; + + internal SocketSenderPool SenderPool { get; init; } = default!; + } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs index 6c607638c80a..41bb160f071f 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs @@ -27,7 +27,7 @@ public static IWebHostBuilder UseSockets(this IWebHostBuilder hostBuilder) return hostBuilder.ConfigureServices(services => { services.AddSingleton(); - services.AddSingleton(); + services.AddTransient(); }); } From 239fe902d3ce63386969cc3c1f1fe183b83ff75e Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 00:53:12 -0700 Subject: [PATCH 07/32] Update WebHostBuilderSocketExtensions.cs --- .../Transport.Sockets/src/WebHostBuilderSocketExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs index 41bb160f071f..6c607638c80a 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs @@ -27,7 +27,7 @@ public static IWebHostBuilder UseSockets(this IWebHostBuilder hostBuilder) return hostBuilder.ConfigureServices(services => { services.AddSingleton(); - services.AddTransient(); + services.AddSingleton(); }); } From 2dc98efde337963512e38be6e32b778323ce749c Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 09:12:06 -0700 Subject: [PATCH 08/32] Make context factory public and fix UseKestrel --- .../Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs | 1 + .../Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt | 4 ++-- .../Transport.Sockets/src/SocketConnectionContextFactory.cs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs index 0a008b0fd817..7c944dc69416 100644 --- a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs +++ b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs @@ -34,6 +34,7 @@ public static IWebHostBuilder UseKestrel(this IWebHostBuilder hostBuilder) { // Don't override an already-configured transport services.TryAddSingleton(); + services.TryAddSingleton(); services.AddTransient, KestrelServerOptionsSetup>(); services.AddSingleton(); diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index 39914148e63e..3ac3766314dd 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -5,9 +5,8 @@ *REMOVED*static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder, System.Action configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.DisposeAsync() -> System.Threading.Tasks.ValueTask -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions! transportOptions, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! connectionOptions, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal.ISocketsTrace! trace) -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DelaySocketOperations.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DelaySocketOperations.init -> void @@ -24,4 +23,5 @@ static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(th static Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateDefaultBoundListenSocket(System.Net.EndPoint! endpoint) -> System.Net.Sockets.Socket! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.get -> System.Func! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.set -> void +~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.Extensions.Options.IOptions! transportOptions, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory! contextFactory) -> void \ No newline at end of file diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index 8c0b9186004b..aa012e9bbdde 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -13,7 +13,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// /// A factory for socket based connections contexts. /// - internal sealed class SocketConnectionContextFactory : ISocketConnectionContextFactory + public sealed class SocketConnectionContextFactory : ISocketConnectionContextFactory { private readonly SocketTransportOptions _transportOptions; private readonly PipeScheduler _transportScheduler; From cc1328bccf1aa1ae0b5bc9501063ba552e8fd9e7 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 11:10:02 -0700 Subject: [PATCH 09/32] Fix DelaySocketOperations to match NoDelay defaults --- .../Kestrel/Transport.Sockets/src/SocketConnectionListener.cs | 2 +- .../Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 167218e64544..b9fba4a8b27a 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -133,7 +133,7 @@ internal void Bind() var connectionOptions = new SocketConnectionOptions() { - DelaySocketOperations = acceptSocket.NoDelay, + DelaySocketOperations = !acceptSocket.NoDelay, InputOptions = setting.InputOptions, OutputOptions = setting.OutputOptions, WaitForDataBeforeAllocatingBuffer = _options.WaitForDataBeforeAllocatingBuffer, diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs index ada8adb572b9..29598cc49542 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs @@ -26,9 +26,9 @@ public class SocketConnectionOptions /// Set to true to enable Nagle's algorithm for all socket connections. /// /// - /// Defaults to false. + /// Defaults to true. /// - public bool DelaySocketOperations { get; init; } + public bool DelaySocketOperations { get; init; } = true; /// /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. From 06116ebc49c3c8d1855925aaf267399ffc1bc83c Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 11:48:58 -0700 Subject: [PATCH 10/32] Don't access socket.NoDelay cuz unix sockets --- .../Transport.Sockets/src/SocketConnectionListener.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index b9fba4a8b27a..5a54cd94da82 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -123,17 +123,19 @@ internal void Bind() var acceptSocket = await _listenSocket.AcceptAsync(cancellationToken); // Only apply no delay to Tcp based endpoints + var noDelay = false; // REVIEW: should this move to inside of _contextFactory.Create?? if (acceptSocket.LocalEndPoint is IPEndPoint) { acceptSocket.NoDelay = _options.NoDelay; + noDelay = true; } var setting = _settings[_settingsIndex]; var connectionOptions = new SocketConnectionOptions() { - DelaySocketOperations = !acceptSocket.NoDelay, + DelaySocketOperations = !noDelay, InputOptions = setting.InputOptions, OutputOptions = setting.OutputOptions, WaitForDataBeforeAllocatingBuffer = _options.WaitForDataBeforeAllocatingBuffer, @@ -155,10 +157,10 @@ internal void Bind() // A call was made to UnbindAsync/DisposeAsync just return null which signals we're done return null; } - catch (SocketException) + catch (SocketException e) { // The connection got reset while it was in the backlog, so we try again. - _trace.ConnectionReset(connectionId: "(null)"); + _trace.ConnectionReset(connectionId: "(null)" + e.ToString()); } } } From fde7a12eaf443bd05a6ce9f77d782651a0f54436 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 11:50:15 -0700 Subject: [PATCH 11/32] Remove debugging --- .../Kestrel/Transport.Sockets/src/SocketConnectionListener.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 5a54cd94da82..10a83930242f 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -157,10 +157,10 @@ internal void Bind() // A call was made to UnbindAsync/DisposeAsync just return null which signals we're done return null; } - catch (SocketException e) + catch (SocketException) { // The connection got reset while it was in the backlog, so we try again. - _trace.ConnectionReset(connectionId: "(null)" + e.ToString()); + _trace.ConnectionReset(connectionId: "(null)"); } } } From 7868f4ae96ee704eebd49f9e1cba2284d9f1da6d Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 14:48:34 -0700 Subject: [PATCH 12/32] CR feeedback --- .../src/PublicAPI.Unshipped.txt | 6 ++--- .../src/SocketConnectionContextFactory.cs | 27 +++---------------- .../src/SocketConnectionListener.cs | 11 ++++---- .../src/SocketConnectionOptions.cs | 9 ++++--- 4 files changed, 17 insertions(+), 36 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index 3ac3766314dd..1c1c9b6fe0ca 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -7,9 +7,10 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFa Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory() -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DelaySocketOperations.get -> bool -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DelaySocketOperations.init -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DeferFirstOperation.get -> bool +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DeferFirstOperation.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.InputOptions.get -> System.IO.Pipelines.PipeOptions! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.InputOptions.init -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.OutputOptions.get -> System.IO.Pipelines.PipeOptions! @@ -23,5 +24,4 @@ static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(th static Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateDefaultBoundListenSocket(System.Net.EndPoint! endpoint) -> System.Net.Sockets.Socket! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.get -> System.Func! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.set -> void -~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.Extensions.Options.IOptions! transportOptions, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory! contextFactory) -> void \ No newline at end of file diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index aa012e9bbdde..1a250472e16a 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -1,12 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.IO.Pipelines; using System.Net.Sockets; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets { @@ -15,24 +12,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// public sealed class SocketConnectionContextFactory : ISocketConnectionContextFactory { - private readonly SocketTransportOptions _transportOptions; - private readonly PipeScheduler _transportScheduler; - private readonly ISocketsTrace _trace; - - public SocketConnectionContextFactory( - IOptions transportOptions, - ILoggerFactory loggerFactory) - { - _transportOptions = transportOptions.Value; - - _transportScheduler = _transportOptions.UnsafePreferInlineScheduling ? PipeScheduler.Inline : - (_transportOptions.IOQueueCount > 0) ? new IOQueue() : PipeScheduler.ThreadPool; - // https://github.com/aspnet/KestrelHttpServer/issues/2573 - - var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); - _trace = new SocketsTrace(logger); - } - /// /// Create a for a socket. /// @@ -42,11 +21,11 @@ public SocketConnectionContextFactory( public ConnectionContext Create(Socket socket, SocketConnectionOptions options) => new SocketConnection(socket, options.MemoryPool, - _transportScheduler, - _trace, + options.Scheduler, + options.Trace, options.SenderPool, options.InputOptions, options.OutputOptions, - waitForData: _transportOptions.WaitForDataBeforeAllocatingBuffer); + waitForData: options.WaitForDataBeforeAllocatingBuffer); } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 10a83930242f..50c429b66d11 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -119,28 +119,27 @@ internal void Bind() { Debug.Assert(_listenSocket != null, "Bind must be called first."); - var acceptSocket = await _listenSocket.AcceptAsync(cancellationToken); // Only apply no delay to Tcp based endpoints - var noDelay = false; -// REVIEW: should this move to inside of _contextFactory.Create?? if (acceptSocket.LocalEndPoint is IPEndPoint) { acceptSocket.NoDelay = _options.NoDelay; - noDelay = true; } var setting = _settings[_settingsIndex]; var connectionOptions = new SocketConnectionOptions() { - DelaySocketOperations = !noDelay, + Scheduler = setting.Scheduler, +//TODO: once https://github.com/dotnet/aspnetcore/pull/34639 is merged + // DeferFirstOperation = _options.DeferFirstOperation, InputOptions = setting.InputOptions, OutputOptions = setting.OutputOptions, WaitForDataBeforeAllocatingBuffer = _options.WaitForDataBeforeAllocatingBuffer, MemoryPool = _memoryPool, - SenderPool = setting.SocketSenderPool + SenderPool = setting.SocketSenderPool, + Trace = _trace }; _settingsIndex = (_settingsIndex + 1) % _settingsCount; diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs index 29598cc49542..fd63a826c4aa 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs @@ -23,12 +23,12 @@ public class SocketConnectionOptions public PipeOptions OutputOptions { get; init; } = new PipeOptions(); /// - /// Set to true to enable Nagle's algorithm for all socket connections. + /// Delay socket read/write operations until the transport pipe is read from or written to. /// /// - /// Defaults to true. + /// Defaults to false. /// - public bool DelaySocketOperations { get; init; } = true; + public bool DeferFirstOperation { get; set; } /// /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. @@ -42,5 +42,8 @@ public class SocketConnectionOptions internal SocketSenderPool SenderPool { get; init; } = default!; + internal PipeScheduler Scheduler { get; init; } = default!; + + internal ISocketsTrace Trace { get; init; } = default!; } } From 2e98b461c19debd2783e4a18caf7eb9e416102ae Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 14:51:20 -0700 Subject: [PATCH 13/32] Cleanup --- .../Transport.Sockets/src/SocketConnectionContextFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index 1a250472e16a..a5921c0ccd40 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -26,6 +26,6 @@ public ConnectionContext Create(Socket socket, SocketConnectionOptions options) options.SenderPool, options.InputOptions, options.OutputOptions, - waitForData: options.WaitForDataBeforeAllocatingBuffer); + options.WaitForDataBeforeAllocatingBuffer); } } From 93ab48b2919facf2a73757f06c695b840c5ea64d Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 16:27:12 -0700 Subject: [PATCH 14/32] Fix test --- .../test/Sockets.BindTests/SocketTransportFactoryTests.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs index 791cef79ad93..39f702029082 100644 --- a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs +++ b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs @@ -17,10 +17,7 @@ public class SocketTransportFactoryTests [Fact] public async Task ThrowsNotImplementedExceptionWhenBindingToUriEndPoint() { - var options = Options.Create(new SocketTransportOptions()); - var logger = Mock.Of(); - var connectionFactory = new SocketConnectionContextFactory(options, logger); - var socketTransportFactory = new SocketTransportFactory(options, logger, connectionFactory); + var socketTransportFactory = new SocketTransportFactory(Options.Create(new SocketTransportOptions()), Mock.Of(), new SocketConnectionContextFactory()); await Assert.ThrowsAsync(async () => await socketTransportFactory.BindAsync(new UriEndPoint(new Uri("http://127.0.0.1:5554")))); } } From 6da8c403592ff55402af50883b512cde059e639d Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 17:25:38 -0700 Subject: [PATCH 15/32] Remove interface --- .../src/WebHostBuilderKestrelExtensions.cs | 2 +- .../src/IISocketConnectionContextFactory.cs | 22 ------------------- .../src/PublicAPI.Unshipped.txt | 2 +- .../src/SocketConnectionContextFactory.cs | 2 +- .../src/SocketConnectionListener.cs | 4 ++-- .../src/SocketTransportFactory.cs | 4 ++-- .../src/WebHostBuilderSocketExtensions.cs | 2 +- 7 files changed, 8 insertions(+), 30 deletions(-) delete mode 100644 src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs diff --git a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs index 7c944dc69416..2f7d493e05ab 100644 --- a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs +++ b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs @@ -34,7 +34,7 @@ public static IWebHostBuilder UseKestrel(this IWebHostBuilder hostBuilder) { // Don't override an already-configured transport services.TryAddSingleton(); - services.TryAddSingleton(); + services.TryAddSingleton(); services.AddTransient, KestrelServerOptionsSetup>(); services.AddSingleton(); diff --git a/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs deleted file mode 100644 index 073945d3028e..000000000000 --- a/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Net.Sockets; -using Microsoft.AspNetCore.Connections; - -namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets -{ - /// - /// Defines an interface that provides the mechanisms to create a socket based . - /// - public interface ISocketConnectionContextFactory - { - /// - /// Create a for a socket. - /// - /// The socket for the connection. - /// The . - /// - ConnectionContext Create(Socket socket, SocketConnectionOptions options); - } -} diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index 1c1c9b6fe0ca..ee64f5c47aef 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -24,4 +24,4 @@ static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(th static Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateDefaultBoundListenSocket(System.Net.EndPoint! endpoint) -> System.Net.Sockets.Socket! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.get -> System.Func! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.set -> void -~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory! contextFactory) -> void \ No newline at end of file +~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory! contextFactory) -> void \ No newline at end of file diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index a5921c0ccd40..54fdfea3e392 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -10,7 +10,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// /// A factory for socket based connections contexts. /// - public sealed class SocketConnectionContextFactory : ISocketConnectionContextFactory + public sealed class SocketConnectionContextFactory { /// /// Create a for a socket. diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 50c429b66d11..5487fe12b072 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -23,14 +23,14 @@ internal sealed class SocketConnectionListener : IConnectionListener private Socket? _listenSocket; private int _settingsIndex; private readonly SocketTransportOptions _options; - private readonly ISocketConnectionContextFactory _contextFactory; + private readonly SocketConnectionContextFactory _contextFactory; public EndPoint EndPoint { get; private set; } internal SocketConnectionListener( EndPoint endpoint, SocketTransportOptions options, - ISocketConnectionContextFactory contextFactory, + SocketConnectionContextFactory contextFactory, ISocketsTrace trace) { EndPoint = endpoint; diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs index f1eeebd08b08..48a48f2526f9 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs @@ -18,13 +18,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets public sealed class SocketTransportFactory : IConnectionListenerFactory { private readonly SocketTransportOptions _options; - private readonly ISocketConnectionContextFactory _contextFactory; + private readonly SocketConnectionContextFactory _contextFactory; private readonly SocketsTrace _trace; public SocketTransportFactory( IOptions options, ILoggerFactory loggerFactory, - ISocketConnectionContextFactory contextFactory) + SocketConnectionContextFactory contextFactory) { if (options == null) { diff --git a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs index 6c607638c80a..df5bc6c9fa69 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs @@ -27,7 +27,7 @@ public static IWebHostBuilder UseSockets(this IWebHostBuilder hostBuilder) return hostBuilder.ConfigureServices(services => { services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); }); } From bc570550864ed1cc82552471791e5c9bf1c75319 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Tue, 27 Jul 2021 09:12:45 -0700 Subject: [PATCH 16/32] Initial surgery --- .../src/IISocketConnectionContextFactory.cs | 22 +++++++++++++++++++ src/submodules/googletest | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs diff --git a/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs new file mode 100644 index 000000000000..073945d3028e --- /dev/null +++ b/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Sockets; +using Microsoft.AspNetCore.Connections; + +namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets +{ + /// + /// Defines an interface that provides the mechanisms to create a socket based . + /// + public interface ISocketConnectionContextFactory + { + /// + /// Create a for a socket. + /// + /// The socket for the connection. + /// The . + /// + ConnectionContext Create(Socket socket, SocketConnectionOptions options); + } +} diff --git a/src/submodules/googletest b/src/submodules/googletest index aefb45469ee7..2d924d7a971e 160000 --- a/src/submodules/googletest +++ b/src/submodules/googletest @@ -1 +1 @@ -Subproject commit aefb45469ee7e6bde0cd1d2c18412046c30e7bb6 +Subproject commit 2d924d7a971e9667d76ad09727fb2402b4f8a1e3 From 4fff4d32f9c804efe815e9422953bbf2509b28c6 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 09:12:06 -0700 Subject: [PATCH 17/32] Make context factory public and fix UseKestrel --- .../Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs index 2f7d493e05ab..4702fa6de600 100644 --- a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs +++ b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs @@ -29,12 +29,11 @@ public static class WebHostBuilderKestrelExtensions /// public static IWebHostBuilder UseKestrel(this IWebHostBuilder hostBuilder) { - hostBuilder.UseQuic(); return hostBuilder.ConfigureServices(services => { // Don't override an already-configured transport services.TryAddSingleton(); - services.TryAddSingleton(); + services.TryAddSingleton(); services.AddTransient, KestrelServerOptionsSetup>(); services.AddSingleton(); From 52c8d11c9c715e141f24fc8e106cc612b3f6422a Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 28 Jul 2021 17:25:38 -0700 Subject: [PATCH 18/32] Remove interface --- .../src/WebHostBuilderKestrelExtensions.cs | 2 +- .../src/IISocketConnectionContextFactory.cs | 22 ------------------- 2 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs diff --git a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs index 4702fa6de600..6267d4a641a7 100644 --- a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs +++ b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs @@ -33,7 +33,7 @@ public static IWebHostBuilder UseKestrel(this IWebHostBuilder hostBuilder) { // Don't override an already-configured transport services.TryAddSingleton(); - services.TryAddSingleton(); + services.TryAddSingleton(); services.AddTransient, KestrelServerOptionsSetup>(); services.AddSingleton(); diff --git a/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs deleted file mode 100644 index 073945d3028e..000000000000 --- a/src/Servers/Kestrel/Transport.Sockets/src/IISocketConnectionContextFactory.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Net.Sockets; -using Microsoft.AspNetCore.Connections; - -namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets -{ - /// - /// Defines an interface that provides the mechanisms to create a socket based . - /// - public interface ISocketConnectionContextFactory - { - /// - /// Create a for a socket. - /// - /// The socket for the connection. - /// The . - /// - ConnectionContext Create(Socket socket, SocketConnectionOptions options); - } -} From 26cba395d1d35715cb82e6e6f8c06c6c0dc262f0 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Mon, 9 Aug 2021 16:21:11 -0700 Subject: [PATCH 19/32] Updates --- .../src/PublicAPI.Unshipped.txt | 4 +-- .../src/SocketConnectionContextFactory.cs | 34 +++++++++++++------ .../src/SocketConnectionListener.cs | 9 ++--- .../src/SocketTransportFactory.cs | 12 ++----- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index ee64f5c47aef..a4ce84f2df50 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -6,8 +6,8 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory() -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket) -> Microsoft.AspNetCore.Connections.ConnectionContext! +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerfactory) -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DeferFirstOperation.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DeferFirstOperation.set -> void diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index 54fdfea3e392..399718420abe 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -4,28 +4,42 @@ using System.Net.Sockets; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; +using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets { /// /// A factory for socket based connections contexts. /// - public sealed class SocketConnectionContextFactory + public sealed class SocketConnectionContextFactory : IDisposable { + private readonly SocketConnectionOptions _options; + + /// + /// Creates the . + /// + /// The options. + /// The logger factory. + public SocketConnectionContextFactory(SocketConnectionOptions options, ILoggerFactory loggerfactory) + { + _options = options; + } + /// /// Create a for a socket. /// /// The socket for the connection. - /// The . /// - public ConnectionContext Create(Socket socket, SocketConnectionOptions options) + public ConnectionContext Create(Socket socket) => new SocketConnection(socket, - options.MemoryPool, - options.Scheduler, - options.Trace, - options.SenderPool, - options.InputOptions, - options.OutputOptions, - options.WaitForDataBeforeAllocatingBuffer); + _options.MemoryPool, + _options.Scheduler, + _options.Trace, + _options.SenderPool, + _options.InputOptions, + _options.OutputOptions, + _options.WaitForDataBeforeAllocatingBuffer); + + public void Dispose() { } } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 5487fe12b072..682807ed2360 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -23,20 +23,17 @@ internal sealed class SocketConnectionListener : IConnectionListener private Socket? _listenSocket; private int _settingsIndex; private readonly SocketTransportOptions _options; - private readonly SocketConnectionContextFactory _contextFactory; public EndPoint EndPoint { get; private set; } internal SocketConnectionListener( EndPoint endpoint, SocketTransportOptions options, - SocketConnectionContextFactory contextFactory, ISocketsTrace trace) { EndPoint = endpoint; _trace = trace; _options = options; - _contextFactory = contextFactory; _memoryPool = _options.MemoryPoolFactory(); var ioQueueCount = options.IOQueueCount; @@ -142,9 +139,12 @@ internal void Bind() Trace = _trace }; + + setting.SocketFactory = new SocketConnectionContextFactory(connectionOptions, loggerfactory: null /* TODO */); + _settingsIndex = (_settingsIndex + 1) % _settingsCount; - return _contextFactory.Create(acceptSocket, connectionOptions); + return setting.SocketFactory.Create(acceptSocket); } catch (ObjectDisposedException) { @@ -192,6 +192,7 @@ private class Settings public PipeOptions InputOptions { get; init; } = default!; public PipeOptions OutputOptions { get; init; } = default!; public SocketSenderPool SocketSenderPool { get; init; } = default!; + public SocketConnectionContextFactory SocketFactory { get; set; } = default!; } } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs index 48a48f2526f9..817fc142d708 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs @@ -18,38 +18,30 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets public sealed class SocketTransportFactory : IConnectionListenerFactory { private readonly SocketTransportOptions _options; - private readonly SocketConnectionContextFactory _contextFactory; private readonly SocketsTrace _trace; public SocketTransportFactory( IOptions options, - ILoggerFactory loggerFactory, - SocketConnectionContextFactory contextFactory) + ILoggerFactory loggerFactory) { if (options == null) { throw new ArgumentNullException(nameof(options)); } - if (contextFactory == null) - { - throw new ArgumentNullException(nameof(contextFactory)); - } - if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } _options = options.Value; - _contextFactory = contextFactory; var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); } public ValueTask BindAsync(EndPoint endpoint, CancellationToken cancellationToken = default) { - var transport = new SocketConnectionListener(endpoint, _options, _contextFactory, _trace); + var transport = new SocketConnectionListener(endpoint, _options, _trace); transport.Bind(); return new ValueTask(transport); } From 66b244baf472401aaef514098dd88a5e8f4e0d86 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 11 Aug 2021 15:13:28 -0700 Subject: [PATCH 20/32] Update to new API --- .../src/Internal/SocketConnection.cs | 13 +++ .../src/PublicAPI.Shipped.txt | 2 +- .../src/PublicAPI.Unshipped.txt | 6 +- .../src/SocketConnectionContextFactory.cs | 39 ++++++--- .../src/SocketConnectionListener.cs | 84 ++++++------------- .../src/SocketConnectionOptions.cs | 10 --- .../src/SocketTransportFactory.cs | 8 +- .../SocketTransportFactoryTests.cs | 2 +- 8 files changed, 72 insertions(+), 92 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs b/src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs index 5c076a38b2ce..c825970ff8e5 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs @@ -34,6 +34,19 @@ internal sealed partial class SocketConnection : TransportConnection private bool _connectionClosed; private readonly bool _waitForData; + internal SocketConnection(Socket socket, + SocketConnectionOptions options, + PipeScheduler transportScheduler, + SocketSenderPool socketSenderPool, + ISocketsTrace trace) + : this(socket, options.InputOptions.Pool, transportScheduler, trace, socketSenderPool, options.InputOptions, options.OutputOptions, options.WaitForDataBeforeAllocatingBuffer) + { + if (options.InputOptions.Pool != options.OutputOptions.Pool) + { + throw new InvalidOperationException("SocketConnection expects options.InputOptions.Pool to be the same as options.OutputOptions.Pool"); + } + } + internal SocketConnection(Socket socket, MemoryPool memoryPool, PipeScheduler transportScheduler, diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt index f0fc2fa03cbe..9eb7ae6f719d 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt @@ -18,6 +18,6 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.Uns Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.set -> void ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask -~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions options, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) -> void +~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void ~static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder ~static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder, System.Action configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index a4ce84f2df50..66be617080a5 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -3,11 +3,10 @@ *REMOVED*Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions options, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) -> void *REMOVED*static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder *REMOVED*static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder, System.Action configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.ISocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options) -> Microsoft.AspNetCore.Connections.ConnectionContext! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket) -> Microsoft.AspNetCore.Connections.ConnectionContext! -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerfactory) -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Dispose() -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DeferFirstOperation.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DeferFirstOperation.set -> void @@ -24,4 +23,3 @@ static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(th static Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateDefaultBoundListenSocket(System.Net.EndPoint! endpoint) -> System.Net.Sockets.Socket! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.get -> System.Func! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateBoundListenSocket.set -> void -~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory! contextFactory) -> void \ No newline at end of file diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index 399718420abe..d8033ac2d695 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.IO.Pipelines; using System.Net.Sockets; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; @@ -14,15 +15,34 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets public sealed class SocketConnectionContextFactory : IDisposable { private readonly SocketConnectionOptions _options; + private readonly ISocketsTrace _trace; + private readonly PipeScheduler _scheduler; + private readonly SocketSenderPool _senderPool; /// /// Creates the . /// /// The options. - /// The logger factory. - public SocketConnectionContextFactory(SocketConnectionOptions options, ILoggerFactory loggerfactory) + /// The logger factory. + public SocketConnectionContextFactory(SocketConnectionOptions options, ILoggerFactory loggerFactory) { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + if (loggerFactory == null) + { + throw new ArgumentNullException(nameof(loggerFactory)); + } + _options = options; + var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); + _trace = new SocketsTrace(logger); + _scheduler = options.InputOptions.WriterScheduler; + + // https://github.com/aspnet/KestrelHttpServer/issues/2573 + _senderPool = new SocketSenderPool(OperatingSystem.IsWindows() ? _scheduler : PipeScheduler.Inline); } /// @@ -31,15 +51,10 @@ public SocketConnectionContextFactory(SocketConnectionOptions options, ILoggerFa /// The socket for the connection. /// public ConnectionContext Create(Socket socket) - => new SocketConnection(socket, - _options.MemoryPool, - _options.Scheduler, - _options.Trace, - _options.SenderPool, - _options.InputOptions, - _options.OutputOptions, - _options.WaitForDataBeforeAllocatingBuffer); - - public void Dispose() { } + => new SocketConnection(socket, _options, _scheduler, _senderPool, _trace); + + /// + public void Dispose() + => _senderPool.Dispose(); } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 682807ed2360..91d53c5610f7 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -1,27 +1,25 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Buffers; using System.Diagnostics; using System.IO.Pipelines; using System.Net; using System.Net.Sockets; -using System.Threading; -using System.Threading.Tasks; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; +using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets { internal sealed class SocketConnectionListener : IConnectionListener { private readonly MemoryPool _memoryPool; - private readonly int _settingsCount; - private readonly Settings[] _settings; + private readonly int _factoryCount; + private readonly SocketConnectionContextFactory[] _factories; private readonly ISocketsTrace _trace; private Socket? _listenSocket; - private int _settingsIndex; + private int _factoryIndex; private readonly SocketTransportOptions _options; public EndPoint EndPoint { get; private set; } @@ -29,10 +27,9 @@ internal sealed class SocketConnectionListener : IConnectionListener internal SocketConnectionListener( EndPoint endpoint, SocketTransportOptions options, - ISocketsTrace trace) + ILoggerFactory loggerFactory) { EndPoint = endpoint; - _trace = trace; _options = options; _memoryPool = _options.MemoryPoolFactory(); var ioQueueCount = options.IOQueueCount; @@ -41,45 +38,37 @@ internal SocketConnectionListener( var maxWriteBufferSize = _options.MaxWriteBufferSize ?? 0; var applicationScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; + var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); + _trace = new SocketsTrace(logger); + if (ioQueueCount > 0) { - _settingsCount = ioQueueCount; - _settings = new Settings[_settingsCount]; + _factoryCount = ioQueueCount; + _factories = new SocketConnectionContextFactory[_factoryCount]; - for (var i = 0; i < _settingsCount; i++) + for (var i = 0; i < _factoryCount; i++) { var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); - // https://github.com/aspnet/KestrelHttpServer/issues/2573 - var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; - - _settings[i] = new Settings + _factories[i] = new SocketConnectionContextFactory(new SocketConnectionOptions { - Scheduler = transportScheduler, InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), - SocketSenderPool = new SocketSenderPool(awaiterScheduler) - }; + + }, loggerFactory); } } else { var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; - // https://github.com/aspnet/KestrelHttpServer/issues/2573 - var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; - - var directScheduler = new Settings[] + _factories = new SocketConnectionContextFactory[] { - new Settings + new SocketConnectionContextFactory(new SocketConnectionOptions { - Scheduler = transportScheduler, InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), - SocketSenderPool = new SocketSenderPool(awaiterScheduler) - } + }, loggerFactory) }; - - _settingsCount = directScheduler.Length; - _settings = directScheduler; + _factoryCount = _factories.Length; } } @@ -124,27 +113,11 @@ internal void Bind() acceptSocket.NoDelay = _options.NoDelay; } - var setting = _settings[_settingsIndex]; + var factory = _factories[_factoryIndex]; - var connectionOptions = new SocketConnectionOptions() - { - Scheduler = setting.Scheduler, -//TODO: once https://github.com/dotnet/aspnetcore/pull/34639 is merged - // DeferFirstOperation = _options.DeferFirstOperation, - InputOptions = setting.InputOptions, - OutputOptions = setting.OutputOptions, - WaitForDataBeforeAllocatingBuffer = _options.WaitForDataBeforeAllocatingBuffer, - MemoryPool = _memoryPool, - SenderPool = setting.SocketSenderPool, - Trace = _trace - }; + _factoryIndex = (_factoryIndex + 1) % _factoryCount; - - setting.SocketFactory = new SocketConnectionContextFactory(connectionOptions, loggerfactory: null /* TODO */); - - _settingsIndex = (_settingsIndex + 1) % _settingsCount; - - return setting.SocketFactory.Create(acceptSocket); + return factory.Create(acceptSocket); } catch (ObjectDisposedException) { @@ -177,22 +150,13 @@ public ValueTask DisposeAsync() // Dispose the memory pool _memoryPool.Dispose(); - // Dispose any pooled senders - foreach (var setting in _settings) + // Dispose any pooled senders in the factories + foreach (var factory in _factories) { - setting.SocketSenderPool.Dispose(); + factory.Dispose(); } return default; } - - private class Settings - { - public PipeScheduler Scheduler { get; init; } = default!; - public PipeOptions InputOptions { get; init; } = default!; - public PipeOptions OutputOptions { get; init; } = default!; - public SocketSenderPool SocketSenderPool { get; init; } = default!; - public SocketConnectionContextFactory SocketFactory { get; set; } = default!; - } } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs index fd63a826c4aa..8f89d53dbb2e 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs @@ -1,9 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers; using System.IO.Pipelines; -using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.Internal; namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets { @@ -37,13 +35,5 @@ public class SocketConnectionOptions /// Defaults to true. /// public bool WaitForDataBeforeAllocatingBuffer { get; set; } = true; - - internal MemoryPool MemoryPool { get; init; } = default!; - - internal SocketSenderPool SenderPool { get; init; } = default!; - - internal PipeScheduler Scheduler { get; init; } = default!; - - internal ISocketsTrace Trace { get; init; } = default!; } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs index 817fc142d708..71abfc650b2f 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportFactory.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.IO.Pipelines; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -18,7 +19,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets public sealed class SocketTransportFactory : IConnectionListenerFactory { private readonly SocketTransportOptions _options; - private readonly SocketsTrace _trace; + private readonly ILoggerFactory _logger; public SocketTransportFactory( IOptions options, @@ -35,13 +36,12 @@ public SocketTransportFactory( } _options = options.Value; - var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); - _trace = new SocketsTrace(logger); + _logger = loggerFactory; } public ValueTask BindAsync(EndPoint endpoint, CancellationToken cancellationToken = default) { - var transport = new SocketConnectionListener(endpoint, _options, _trace); + var transport = new SocketConnectionListener(endpoint, _options, _logger); transport.Bind(); return new ValueTask(transport); } diff --git a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs index 39f702029082..d0ea2cad6917 100644 --- a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs +++ b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs @@ -17,7 +17,7 @@ public class SocketTransportFactoryTests [Fact] public async Task ThrowsNotImplementedExceptionWhenBindingToUriEndPoint() { - var socketTransportFactory = new SocketTransportFactory(Options.Create(new SocketTransportOptions()), Mock.Of(), new SocketConnectionContextFactory()); + var socketTransportFactory = new SocketTransportFactory(Options.Create(new SocketTransportOptions()), Mock.Of()); await Assert.ThrowsAsync(async () => await socketTransportFactory.BindAsync(new UriEndPoint(new Uri("http://127.0.0.1:5554")))); } } From 3f4e88bb595350457857e1a056bfbab2b05395be Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 11 Aug 2021 15:21:14 -0700 Subject: [PATCH 21/32] Undo --- .../Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs index 6267d4a641a7..0a008b0fd817 100644 --- a/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs +++ b/src/Servers/Kestrel/Kestrel/src/WebHostBuilderKestrelExtensions.cs @@ -29,11 +29,11 @@ public static class WebHostBuilderKestrelExtensions /// public static IWebHostBuilder UseKestrel(this IWebHostBuilder hostBuilder) { + hostBuilder.UseQuic(); return hostBuilder.ConfigureServices(services => { // Don't override an already-configured transport services.TryAddSingleton(); - services.TryAddSingleton(); services.AddTransient, KestrelServerOptionsSetup>(); services.AddSingleton(); From e03c29c6a6ac8266cd35a9d33940931fe4af0162 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 11 Aug 2021 15:24:04 -0700 Subject: [PATCH 22/32] Revert --- .../Transport.Sockets/src/WebHostBuilderSocketExtensions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs index df5bc6c9fa69..540dd07bce57 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/WebHostBuilderSocketExtensions.cs @@ -27,7 +27,6 @@ public static IWebHostBuilder UseSockets(this IWebHostBuilder hostBuilder) return hostBuilder.ConfigureServices(services => { services.AddSingleton(); - services.AddSingleton(); }); } From c327e9cbe0c614311910732b5215b0dace2025b2 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 11 Aug 2021 15:38:16 -0700 Subject: [PATCH 23/32] Undo submodule update --- src/submodules/googletest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/submodules/googletest b/src/submodules/googletest index 2d924d7a971e..aefb45469ee7 160000 --- a/src/submodules/googletest +++ b/src/submodules/googletest @@ -1 +1 @@ -Subproject commit 2d924d7a971e9667d76ad09727fb2402b4f8a1e3 +Subproject commit aefb45469ee7e6bde0cd1d2c18412046c30e7bb6 From 218fae62dec90e194351e877e7e8a703049eab84 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 11 Aug 2021 15:42:53 -0700 Subject: [PATCH 24/32] Undo api changes --- src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt | 1 - .../Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt index 9eb7ae6f719d..5c1dfb419174 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt @@ -18,6 +18,5 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.Uns Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.set -> void ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask -~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void ~static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder ~static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder, System.Action configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index 66be617080a5..72d1dea61962 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -18,6 +18,7 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.So Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.WaitForDataBeforeAllocatingBuffer.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.WaitForDataBeforeAllocatingBuffer.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint! endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask +~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder! hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder! static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder! hostBuilder, System.Action! configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder! static Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.CreateDefaultBoundListenSocket(System.Net.EndPoint! endpoint) -> System.Net.Sockets.Socket! From 776ad32865024b415022d7281ab4fa66163f439b Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 11 Aug 2021 15:43:43 -0700 Subject: [PATCH 25/32] Update PublicAPI.Shipped.txt --- src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt index 5c1dfb419174..f0fc2fa03cbe 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt @@ -18,5 +18,6 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.Uns Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.set -> void ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask +~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions options, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) -> void ~static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder ~static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder, System.Action configureOptions) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder From 401322fff9365094365961fc7fdb573a402caad9 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 11 Aug 2021 16:17:50 -0700 Subject: [PATCH 26/32] Remove DeferFirstOperation --- .../Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt | 2 -- .../Transport.Sockets/src/SocketConnectionOptions.cs | 8 -------- 2 files changed, 10 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index 72d1dea61962..a0b2f271d4ea 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -8,8 +8,6 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFac Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Dispose() -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DeferFirstOperation.get -> bool -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.DeferFirstOperation.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.InputOptions.get -> System.IO.Pipelines.PipeOptions! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.InputOptions.init -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.OutputOptions.get -> System.IO.Pipelines.PipeOptions! diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs index 8f89d53dbb2e..07ca98376786 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs @@ -20,14 +20,6 @@ public class SocketConnectionOptions /// public PipeOptions OutputOptions { get; init; } = new PipeOptions(); - /// - /// Delay socket read/write operations until the transport pipe is read from or written to. - /// - /// - /// Defaults to false. - /// - public bool DeferFirstOperation { get; set; } - /// /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. /// From 407c2c5965d8ce98f51e5cb9ef7d059d5e5e9a60 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Thu, 12 Aug 2021 00:09:14 -0700 Subject: [PATCH 27/32] Start connection in Create --- .../Transport.Sockets/src/SocketConnectionContextFactory.cs | 6 +++++- .../Transport.Sockets/src/SocketConnectionListener.cs | 6 ++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index d8033ac2d695..e81fdaa8904c 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -51,7 +51,11 @@ public SocketConnectionContextFactory(SocketConnectionOptions options, ILoggerFa /// The socket for the connection. /// public ConnectionContext Create(Socket socket) - => new SocketConnection(socket, _options, _scheduler, _senderPool, _trace); + { + var connection = new SocketConnection(socket, _options, _scheduler, _senderPool, _trace); + connection.Start(); + return connection; + } /// public void Dispose() diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index 91d53c5610f7..c55996a52087 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -113,11 +113,9 @@ internal void Bind() acceptSocket.NoDelay = _options.NoDelay; } - var factory = _factories[_factoryIndex]; - + var connection = _factories[_factoryIndex].Create(acceptSocket); _factoryIndex = (_factoryIndex + 1) % _factoryCount; - - return factory.Create(acceptSocket); + return connection; } catch (ObjectDisposedException) { From 7399f8bf3050bee979b68892934ea483c957bebc Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Thu, 12 Aug 2021 12:41:45 -0700 Subject: [PATCH 28/32] Move sender pool queues into Factory --- .../src/Internal/SocketConnection.cs | 16 ---- .../src/PublicAPI.Shipped.txt | 10 --- .../src/PublicAPI.Unshipped.txt | 22 +++-- .../src/SocketConnectionContextFactory.cs | 90 +++++++++++++++++-- .../src/SocketConnectionFactoryOptions.cs | 53 +++++++++++ .../src/SocketConnectionListener.cs | 58 +----------- .../src/SocketConnectionOptions.cs | 31 ------- .../src/SocketTransportOptions.cs | 44 +-------- 8 files changed, 153 insertions(+), 171 deletions(-) create mode 100644 src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionFactoryOptions.cs delete mode 100644 src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs diff --git a/src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs b/src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs index c825970ff8e5..f43c2e1454f1 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs @@ -1,13 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Buffers; using System.Diagnostics; using System.IO.Pipelines; using System.Net.Sockets; -using System.Threading; -using System.Threading.Tasks; using Microsoft.AspNetCore.Connections; using Microsoft.Extensions.Logging; @@ -34,19 +31,6 @@ internal sealed partial class SocketConnection : TransportConnection private bool _connectionClosed; private readonly bool _waitForData; - internal SocketConnection(Socket socket, - SocketConnectionOptions options, - PipeScheduler transportScheduler, - SocketSenderPool socketSenderPool, - ISocketsTrace trace) - : this(socket, options.InputOptions.Pool, transportScheduler, trace, socketSenderPool, options.InputOptions, options.OutputOptions, options.WaitForDataBeforeAllocatingBuffer) - { - if (options.InputOptions.Pool != options.OutputOptions.Pool) - { - throw new InvalidOperationException("SocketConnection expects options.InputOptions.Pool to be the same as options.OutputOptions.Pool"); - } - } - internal SocketConnection(Socket socket, MemoryPool memoryPool, PipeScheduler transportScheduler, diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt index f0fc2fa03cbe..572a4830b56c 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt @@ -4,19 +4,9 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.Backlog.get -> int Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.Backlog.set -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.IOQueueCount.get -> int -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.IOQueueCount.set -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.MaxReadBufferSize.get -> long? -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.MaxReadBufferSize.set -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.MaxWriteBufferSize.get -> long? -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.MaxWriteBufferSize.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.NoDelay.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.NoDelay.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.SocketTransportOptions() -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.UnsafePreferInlineScheduling.get -> bool -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.UnsafePreferInlineScheduling.set -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.get -> bool -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.set -> void ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions options, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) -> void ~static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index a0b2f271d4ea..645b36537874 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -6,15 +6,19 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket) -> Microsoft.AspNetCore.Connections.ConnectionContext! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Dispose() -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.InputOptions.get -> System.IO.Pipelines.PipeOptions! -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.InputOptions.init -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.OutputOptions.get -> System.IO.Pipelines.PipeOptions! -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.OutputOptions.init -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.SocketConnectionOptions() -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.WaitForDataBeforeAllocatingBuffer.get -> bool -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionOptions.WaitForDataBeforeAllocatingBuffer.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.IOQueueCount.get -> int +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.IOQueueCount.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.MaxReadBufferSize.get -> long? +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.MaxReadBufferSize.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.MaxWriteBufferSize.get -> long? +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.MaxWriteBufferSize.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.SocketConnectionFactoryOptions() -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.UnsafePreferInlineScheduling.get -> bool +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.UnsafePreferInlineScheduling.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.WaitForDataBeforeAllocatingBuffer.get -> bool +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.WaitForDataBeforeAllocatingBuffer.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint! endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder! hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder! diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index e81fdaa8904c..c1ac26eade4b 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.IO.Pipelines; using System.Net.Sockets; using Microsoft.AspNetCore.Connections; @@ -14,17 +15,19 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// public sealed class SocketConnectionContextFactory : IDisposable { - private readonly SocketConnectionOptions _options; + private readonly MemoryPool _memoryPool; + private readonly SocketConnectionFactoryOptions _options; private readonly ISocketsTrace _trace; - private readonly PipeScheduler _scheduler; - private readonly SocketSenderPool _senderPool; + private readonly int _ioQueueCount; + private readonly QueueSettings[] _settings; + private int _settingsIndex; /// /// Creates the . /// /// The options. /// The logger factory. - public SocketConnectionContextFactory(SocketConnectionOptions options, ILoggerFactory loggerFactory) + public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, ILoggerFactory loggerFactory) { if (options == null) { @@ -39,10 +42,54 @@ public SocketConnectionContextFactory(SocketConnectionOptions options, ILoggerFa _options = options; var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); - _scheduler = options.InputOptions.WriterScheduler; + //_scheduler = options.InputOptions.WriterScheduler; // https://github.com/aspnet/KestrelHttpServer/issues/2573 - _senderPool = new SocketSenderPool(OperatingSystem.IsWindows() ? _scheduler : PipeScheduler.Inline); + //_senderPool = new SocketSenderPool(OperatingSystem.IsWindows() ? _scheduler : PipeScheduler.Inline); + + _memoryPool = _options.MemoryPoolFactory(); + _ioQueueCount = _options.IOQueueCount; + + var maxReadBufferSize = _options.MaxReadBufferSize ?? 0; + var maxWriteBufferSize = _options.MaxWriteBufferSize ?? 0; + var applicationScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; + + if (_ioQueueCount > 0) + { + _settings = new QueueSettings[_ioQueueCount]; + + for (var i = 0; i < _ioQueueCount; i++) + { + var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); + // https://github.com/aspnet/KestrelHttpServer/issues/2573 + var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; + + _settings[i] = new QueueSettings() + { + Scheduler = transportScheduler, + InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), + OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), + SocketSenderPool = new SocketSenderPool(awaiterScheduler) + }; + } + } + else + { + var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; + // https://github.com/aspnet/KestrelHttpServer/issues/2573 + var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline; + _settings = new QueueSettings[] + { + new QueueSettings() + { + Scheduler = transportScheduler, + InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), + OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), + SocketSenderPool = new SocketSenderPool(awaiterScheduler) + } + }; + _ioQueueCount = 1; + } } /// @@ -52,13 +99,40 @@ public SocketConnectionContextFactory(SocketConnectionOptions options, ILoggerFa /// public ConnectionContext Create(Socket socket) { - var connection = new SocketConnection(socket, _options, _scheduler, _senderPool, _trace); + var setting = _settings[Interlocked.Increment(ref _settingsIndex) % _ioQueueCount]; + + var connection = new SocketConnection(socket, + _memoryPool, + setting.Scheduler, + _trace, + setting.SocketSenderPool, + setting.InputOptions, + setting.OutputOptions, + waitForData: _options.WaitForDataBeforeAllocatingBuffer); + connection.Start(); return connection; } /// public void Dispose() - => _senderPool.Dispose(); + { + // Dispose the memory pool + _memoryPool.Dispose(); + + // Dispose any pooled senders + foreach (var setting in _settings) + { + setting.SocketSenderPool.Dispose(); + } + } + + private class QueueSettings + { + public PipeScheduler Scheduler { get; init; } = default!; + public PipeOptions InputOptions { get; init; } = default!; + public PipeOptions OutputOptions { get; init; } = default!; + public SocketSenderPool SocketSenderPool { get; init; } = default!; + } } } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionFactoryOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionFactoryOptions.cs new file mode 100644 index 000000000000..3bde195c9259 --- /dev/null +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionFactoryOptions.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; + +namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets +{ + /// + /// Options for . + /// + public class SocketConnectionFactoryOptions + { + /// + /// The number of I/O queues used to process requests. Set to 0 to directly schedule I/O to the ThreadPool. + /// + /// + /// Defaults to rounded down and clamped between 1 and 16. + /// + public int IOQueueCount { get; set; } = Math.Min(Environment.ProcessorCount, 16); + + /// + /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. + /// + /// + /// Defaults to true. + /// + public bool WaitForDataBeforeAllocatingBuffer { get; set; } = true; + + /// + /// Gets or sets the maximum unconsumed incoming bytes the transport will buffer. + /// + public long? MaxReadBufferSize { get; set; } = 1024 * 1024; + + /// + /// Gets or sets the maximum outgoing bytes the transport will buffer before applying write backpressure. + /// + public long? MaxWriteBufferSize { get; set; } = 64 * 1024; + + /// + /// Inline application and transport continuations instead of dispatching to the threadpool. + /// + /// + /// This will run application code on the IO thread which is why this is unsafe. + /// It is recommended to set the DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS environment variable to '1' when using this setting to also inline the completions + /// at the runtime layer as well. + /// This setting can make performance worse if there is expensive work that will end up holding onto the IO thread for longer than needed. + /// Test to make sure this setting helps performance. + /// + public bool UnsafePreferInlineScheduling { get; set; } + + internal Func> MemoryPoolFactory { get; set; } = PinnedBlockMemoryPoolFactory.Create; + } +} diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index c55996a52087..b946eddc7ec6 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -1,9 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers; using System.Diagnostics; -using System.IO.Pipelines; using System.Net; using System.Net.Sockets; using Microsoft.AspNetCore.Connections; @@ -14,12 +12,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets { internal sealed class SocketConnectionListener : IConnectionListener { - private readonly MemoryPool _memoryPool; - private readonly int _factoryCount; - private readonly SocketConnectionContextFactory[] _factories; + private readonly SocketConnectionContextFactory _factory; private readonly ISocketsTrace _trace; private Socket? _listenSocket; - private int _factoryIndex; private readonly SocketTransportOptions _options; public EndPoint EndPoint { get; private set; } @@ -31,45 +26,9 @@ internal SocketConnectionListener( { EndPoint = endpoint; _options = options; - _memoryPool = _options.MemoryPoolFactory(); - var ioQueueCount = options.IOQueueCount; - - var maxReadBufferSize = _options.MaxReadBufferSize ?? 0; - var maxWriteBufferSize = _options.MaxWriteBufferSize ?? 0; - var applicationScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; - var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); - - if (ioQueueCount > 0) - { - _factoryCount = ioQueueCount; - _factories = new SocketConnectionContextFactory[_factoryCount]; - - for (var i = 0; i < _factoryCount; i++) - { - var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); - _factories[i] = new SocketConnectionContextFactory(new SocketConnectionOptions - { - InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), - OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), - - }, loggerFactory); - } - } - else - { - var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; - _factories = new SocketConnectionContextFactory[] - { - new SocketConnectionContextFactory(new SocketConnectionOptions - { - InputOptions = new PipeOptions(_memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false), - OutputOptions = new PipeOptions(_memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false), - }, loggerFactory) - }; - _factoryCount = _factories.Length; - } + _factory = new SocketConnectionContextFactory(options, loggerFactory); } internal void Bind() @@ -113,9 +72,7 @@ internal void Bind() acceptSocket.NoDelay = _options.NoDelay; } - var connection = _factories[_factoryIndex].Create(acceptSocket); - _factoryIndex = (_factoryIndex + 1) % _factoryCount; - return connection; + return _factory.Create(acceptSocket); } catch (ObjectDisposedException) { @@ -145,14 +102,7 @@ public ValueTask DisposeAsync() { _listenSocket?.Dispose(); - // Dispose the memory pool - _memoryPool.Dispose(); - - // Dispose any pooled senders in the factories - foreach (var factory in _factories) - { - factory.Dispose(); - } + _factory.Dispose(); return default; } diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs deleted file mode 100644 index 07ca98376786..000000000000 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionOptions.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.IO.Pipelines; - -namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets -{ - /// - /// Options for socket based connections. - /// - public class SocketConnectionOptions - { - /// - /// The for socket connections used for input. - /// - public PipeOptions InputOptions { get; init; } = new PipeOptions(); - - /// - /// The for socket connections used for output. - /// - public PipeOptions OutputOptions { get; init; } = new PipeOptions(); - - /// - /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. - /// - /// - /// Defaults to true. - /// - public bool WaitForDataBeforeAllocatingBuffer { get; set; } = true; - } -} diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportOptions.cs index 3434e066b390..b82c558093f1 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportOptions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportOptions.cs @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Buffers; using System.Net; using System.Net.Sockets; using Microsoft.AspNetCore.Connections; @@ -12,24 +10,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// /// Options for socket based transports. /// - public class SocketTransportOptions + public class SocketTransportOptions : SocketConnectionFactoryOptions { - /// - /// The number of I/O queues used to process requests. Set to 0 to directly schedule I/O to the ThreadPool. - /// - /// - /// Defaults to rounded down and clamped between 1 and 16. - /// - public int IOQueueCount { get; set; } = Math.Min(Environment.ProcessorCount, 16); - - /// - /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. - /// - /// - /// Defaults to true. - /// - public bool WaitForDataBeforeAllocatingBuffer { get; set; } = true; - /// /// Set to false to enable Nagle's algorithm for all connections. /// @@ -46,28 +28,6 @@ public class SocketTransportOptions /// public int Backlog { get; set; } = 512; - /// - /// Gets or sets the maximum unconsumed incoming bytes the transport will buffer. - /// - public long? MaxReadBufferSize { get; set; } = 1024 * 1024; - - /// - /// Gets or sets the maximum outgoing bytes the transport will buffer before applying write backpressure. - /// - public long? MaxWriteBufferSize { get; set; } = 64 * 1024; - - /// - /// Inline application and transport continuations instead of dispatching to the threadpool. - /// - /// - /// This will run application code on the IO thread which is why this is unsafe. - /// It is recommended to set the DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS environment variable to '1' when using this setting to also inline the completions - /// at the runtime layer as well. - /// This setting can make performance worse if there is expensive work that will end up holding onto the IO thread for longer than needed. - /// Test to make sure this setting helps performance. - /// - public bool UnsafePreferInlineScheduling { get; set; } - /// /// A function used to create a new to listen with. If /// not set, is used. @@ -139,7 +99,5 @@ public static Socket CreateDefaultBoundListenSocket(EndPoint endpoint) return listenSocket; } - - internal Func> MemoryPoolFactory { get; set; } = System.Buffers.PinnedBlockMemoryPoolFactory.Create; } } From 0ae6a8270580c49cf8ec222fcc44fcdaccaa6308 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Thu, 12 Aug 2021 12:43:17 -0700 Subject: [PATCH 29/32] Cleanup --- .../Transport.Sockets/src/SocketConnectionContextFactory.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index c1ac26eade4b..46dfe6d4aa66 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -42,11 +42,6 @@ public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, IL _options = options; var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); - //_scheduler = options.InputOptions.WriterScheduler; - - // https://github.com/aspnet/KestrelHttpServer/issues/2573 - //_senderPool = new SocketSenderPool(OperatingSystem.IsWindows() ? _scheduler : PipeScheduler.Inline); - _memoryPool = _options.MemoryPoolFactory(); _ioQueueCount = _options.IOQueueCount; From e0494d2c71bdac553fcdb6663ff9c7c078a9e7c2 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Thu, 12 Aug 2021 12:49:52 -0700 Subject: [PATCH 30/32] Rename field --- .../src/SocketConnectionContextFactory.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index 46dfe6d4aa66..eaf850fdf142 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -18,7 +18,7 @@ public sealed class SocketConnectionContextFactory : IDisposable private readonly MemoryPool _memoryPool; private readonly SocketConnectionFactoryOptions _options; private readonly ISocketsTrace _trace; - private readonly int _ioQueueCount; + private readonly int _settingsCount; private readonly QueueSettings[] _settings; private int _settingsIndex; @@ -43,17 +43,17 @@ public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, IL var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); _memoryPool = _options.MemoryPoolFactory(); - _ioQueueCount = _options.IOQueueCount; + _settingsCount = _options.IOQueueCount; var maxReadBufferSize = _options.MaxReadBufferSize ?? 0; var maxWriteBufferSize = _options.MaxWriteBufferSize ?? 0; var applicationScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool; - if (_ioQueueCount > 0) + if (_settingsCount > 0) { - _settings = new QueueSettings[_ioQueueCount]; + _settings = new QueueSettings[_settingsCount]; - for (var i = 0; i < _ioQueueCount; i++) + for (var i = 0; i < _settingsCount; i++) { var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue(); // https://github.com/aspnet/KestrelHttpServer/issues/2573 @@ -83,7 +83,7 @@ public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, IL SocketSenderPool = new SocketSenderPool(awaiterScheduler) } }; - _ioQueueCount = 1; + _settingsCount = 1; } } @@ -94,7 +94,7 @@ public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, IL /// public ConnectionContext Create(Socket socket) { - var setting = _settings[Interlocked.Increment(ref _settingsIndex) % _ioQueueCount]; + var setting = _settings[Interlocked.Increment(ref _settingsIndex) % _settingsCount]; var connection = new SocketConnection(socket, _memoryPool, From b60a92f586a4c79f5e487d7c8cac884adb43a4bf Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Thu, 12 Aug 2021 14:06:37 -0700 Subject: [PATCH 31/32] CR feedback --- .../src/PublicAPI.Shipped.txt | 10 +++++ .../src/PublicAPI.Unshipped.txt | 2 +- .../src/SocketConnectionContextFactory.cs | 9 ++-- .../src/SocketConnectionFactoryOptions.cs | 15 +++++++ .../src/SocketConnectionListener.cs | 2 +- .../src/SocketTransportOptions.cs | 43 ++++++++++++++++++- 6 files changed, 73 insertions(+), 8 deletions(-) diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt index 572a4830b56c..f0fc2fa03cbe 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Shipped.txt @@ -4,9 +4,19 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.Backlog.get -> int Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.Backlog.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.IOQueueCount.get -> int +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.IOQueueCount.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.MaxReadBufferSize.get -> long? +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.MaxReadBufferSize.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.MaxWriteBufferSize.get -> long? +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.MaxWriteBufferSize.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.NoDelay.get -> bool Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.NoDelay.set -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.SocketTransportOptions() -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.UnsafePreferInlineScheduling.get -> bool +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.UnsafePreferInlineScheduling.set -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.get -> bool +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportOptions.WaitForDataBeforeAllocatingBuffer.set -> void ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.BindAsync(System.Net.EndPoint endpoint, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask ~Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketTransportFactory.SocketTransportFactory(Microsoft.Extensions.Options.IOptions options, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) -> void ~static Microsoft.AspNetCore.Hosting.WebHostBuilderSocketExtensions.UseSockets(this Microsoft.AspNetCore.Hosting.IWebHostBuilder hostBuilder) -> Microsoft.AspNetCore.Hosting.IWebHostBuilder diff --git a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt index 645b36537874..4ca1d1a846e2 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt +++ b/src/Servers/Kestrel/Transport.Sockets/src/PublicAPI.Unshipped.txt @@ -6,7 +6,7 @@ Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Create(System.Net.Sockets.Socket! socket) -> Microsoft.AspNetCore.Connections.ConnectionContext! Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.Dispose() -> void -Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void +Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionContextFactory.SocketConnectionContextFactory(Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions! options, Microsoft.Extensions.Logging.ILogger! logger) -> void Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.IOQueueCount.get -> int Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.SocketConnectionFactoryOptions.IOQueueCount.set -> void diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs index eaf850fdf142..b096b7bcf26c 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs @@ -26,21 +26,20 @@ public sealed class SocketConnectionContextFactory : IDisposable /// Creates the . /// /// The options. - /// The logger factory. - public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, ILoggerFactory loggerFactory) + /// The logger. + public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, ILogger logger) { if (options == null) { throw new ArgumentNullException(nameof(options)); } - if (loggerFactory == null) + if (logger == null) { - throw new ArgumentNullException(nameof(loggerFactory)); + throw new ArgumentNullException(nameof(logger)); } _options = options; - var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); _memoryPool = _options.MemoryPoolFactory(); _settingsCount = _options.IOQueueCount; diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionFactoryOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionFactoryOptions.cs index 3bde195c9259..d0c88c0535f3 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionFactoryOptions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionFactoryOptions.cs @@ -10,6 +10,21 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// public class SocketConnectionFactoryOptions { + /// + /// Create a new instance. + /// + public SocketConnectionFactoryOptions() { } + + internal SocketConnectionFactoryOptions(SocketTransportOptions transportOptions) + { + IOQueueCount = transportOptions.IOQueueCount; + WaitForDataBeforeAllocatingBuffer = transportOptions.WaitForDataBeforeAllocatingBuffer; + MaxReadBufferSize = transportOptions.MaxReadBufferSize; + MaxWriteBufferSize = transportOptions.MaxWriteBufferSize; + UnsafePreferInlineScheduling = transportOptions.UnsafePreferInlineScheduling; + MemoryPoolFactory = transportOptions.MemoryPoolFactory; + } + /// /// The number of I/O queues used to process requests. Set to 0 to directly schedule I/O to the ThreadPool. /// diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs index b946eddc7ec6..3094a7c192e3 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionListener.cs @@ -28,7 +28,7 @@ internal SocketConnectionListener( _options = options; var logger = loggerFactory.CreateLogger("Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets"); _trace = new SocketsTrace(logger); - _factory = new SocketConnectionContextFactory(options, loggerFactory); + _factory = new SocketConnectionContextFactory(new SocketConnectionFactoryOptions(options), logger); } internal void Bind() diff --git a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportOptions.cs b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportOptions.cs index b82c558093f1..9130cb980454 100644 --- a/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportOptions.cs +++ b/src/Servers/Kestrel/Transport.Sockets/src/SocketTransportOptions.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Net; using System.Net.Sockets; using Microsoft.AspNetCore.Connections; @@ -10,8 +11,24 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets /// /// Options for socket based transports. /// - public class SocketTransportOptions : SocketConnectionFactoryOptions + public class SocketTransportOptions { + /// + /// The number of I/O queues used to process requests. Set to 0 to directly schedule I/O to the ThreadPool. + /// + /// + /// Defaults to rounded down and clamped between 1 and 16. + /// + public int IOQueueCount { get; set; } = Math.Min(Environment.ProcessorCount, 16); + + /// + /// Wait until there is data available to allocate a buffer. Setting this to false can increase throughput at the cost of increased memory usage. + /// + /// + /// Defaults to true. + /// + public bool WaitForDataBeforeAllocatingBuffer { get; set; } = true; + /// /// Set to false to enable Nagle's algorithm for all connections. /// @@ -28,6 +45,28 @@ public class SocketTransportOptions : SocketConnectionFactoryOptions /// public int Backlog { get; set; } = 512; + /// + /// Gets or sets the maximum unconsumed incoming bytes the transport will buffer. + /// + public long? MaxReadBufferSize { get; set; } = 1024 * 1024; + + /// + /// Gets or sets the maximum outgoing bytes the transport will buffer before applying write backpressure. + /// + public long? MaxWriteBufferSize { get; set; } = 64 * 1024; + + /// + /// Inline application and transport continuations instead of dispatching to the threadpool. + /// + /// + /// This will run application code on the IO thread which is why this is unsafe. + /// It is recommended to set the DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS environment variable to '1' when using this setting to also inline the completions + /// at the runtime layer as well. + /// This setting can make performance worse if there is expensive work that will end up holding onto the IO thread for longer than needed. + /// Test to make sure this setting helps performance. + /// + public bool UnsafePreferInlineScheduling { get; set; } + /// /// A function used to create a new to listen with. If /// not set, is used. @@ -99,5 +138,7 @@ public static Socket CreateDefaultBoundListenSocket(EndPoint endpoint) return listenSocket; } + + internal Func> MemoryPoolFactory { get; set; } = System.Buffers.PinnedBlockMemoryPoolFactory.Create; } } From ebbca3d01736841077d59a2dbef4f8ce50ae7a36 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Thu, 12 Aug 2021 19:08:16 -0700 Subject: [PATCH 32/32] Fix test --- .../test/Sockets.BindTests/SocketTransportFactoryTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs index d0ea2cad6917..6dcfb1e6acce 100644 --- a/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs +++ b/src/Servers/Kestrel/test/Sockets.BindTests/SocketTransportFactoryTests.cs @@ -17,7 +17,7 @@ public class SocketTransportFactoryTests [Fact] public async Task ThrowsNotImplementedExceptionWhenBindingToUriEndPoint() { - var socketTransportFactory = new SocketTransportFactory(Options.Create(new SocketTransportOptions()), Mock.Of()); + var socketTransportFactory = new SocketTransportFactory(Options.Create(new SocketTransportOptions()), new LoggerFactory()); await Assert.ThrowsAsync(async () => await socketTransportFactory.BindAsync(new UriEndPoint(new Uri("http://127.0.0.1:5554")))); } }