[WIP] Add Configuration references

src/libraries/Microsoft.Extensions.Configuration/src/ConfigurationBuilder.cs

@@ -44,11 +44,10 @@ public IConfigurationBuilder Add(IConfigurationSource source)
         /// <returns>An <see cref="IConfigurationRoot"/> with keys and values from the registered providers.</returns>
         public IConfigurationRoot Build()
         {
-            var providers = new List<IConfigurationProvider>();
+            var providers = new List<IConfigurationProvider>(_sources.Count);
             foreach (IConfigurationSource source in _sources)
             {
-                IConfigurationProvider provider = source.Build(this);
-                providers.Add(provider);
+                providers.Add(source.Build(this));
             }
             return new ConfigurationRoot(providers);
         }

src/libraries/Microsoft.Extensions.Configuration/src/ConfigurationManager.cs

@@ -55,12 +55,22 @@ public string? this[string key]
             get
             {
                 using ReferenceCountedProviders reference = _providerManager.GetReference();
-                return ConfigurationRoot.GetConfiguration(reference.Providers, key);
+                if (ReferenceEngine.Disabled)
+                {
+                    return ConfigurationRoot.GetConfiguration(reference.Providers, key);
+                }
+
+                // Read the engine once and resolve against its own providers. AddProvider swaps this generation's
+                // provider list and engine in place, so reading the two separately could pair an old list with a newer
+                // engine (or the reverse) and resolve the key against the wrong providers.
+                reference.ReferenceEngine!.TryRead(key, out string? value);
+                return value;
             }
             set
             {
                 using ReferenceCountedProviders reference = _providerManager.GetReference();
                 ConfigurationRoot.SetConfiguration(reference.Providers, key, value);
+                _providerManager.ResetReferenceEngine();
             }
         }
 
@@ -116,6 +126,7 @@ void IConfigurationRoot.Reload()
 
         private void RaiseChanged()
         {
+            _providerManager.ResetReferenceEngine();
             var previousToken = Interlocked.Exchange(ref _changeToken, new ConfigurationReloadToken());
             previousToken.OnReload();
         }

src/libraries/Microsoft.Extensions.Configuration/src/ConfigurationProvider.cs

@@ -29,6 +29,10 @@ protected ConfigurationProvider()
         /// </summary>
         protected IDictionary<string, string?> Data { get; set; }
 
+        // Exposes this provider's key/value pairs so the reference engine can build its reference index by scanning the
+        // already-loaded backing dictionary directly, instead of going through the O(n^2) GetChildKeys reconstruction.
+        internal IEnumerable<KeyValuePair<string, string?>> GetEntriesForReferenceScan() => Data;
+
         /// <summary>
         /// Attempts to find a value with the given key.
         /// </summary>

src/libraries/Microsoft.Extensions.Configuration/src/ConfigurationRoot.cs

@@ -20,6 +20,11 @@ public class ConfigurationRoot : IConfigurationRoot, IDisposable
         private readonly List<IDisposable> _changeTokenRegistrations;
         private ConfigurationReloadToken _changeToken = new ConfigurationReloadToken();
 
+        // Per-root reference state, bound to the (fixed) provider list and swapped for a fresh instance on every reload
+        // (see RaiseChanged) so a reload can never surface stale reference information. When no provider declares any
+        // $ref the index is empty, so reads take the plain provider path with no reference overhead.
+        private volatile ReferenceEngine? _referenceEngine;
+
         /// <summary>
         /// Initializes a Configuration root with a list of providers.
         /// </summary>
@@ -29,6 +34,7 @@ public ConfigurationRoot(IList<IConfigurationProvider> providers)
             ArgumentNullException.ThrowIfNull(providers);
 
             _providers = providers;
+            _referenceEngine = ReferenceEngine.Create(providers);
             _changeTokenRegistrations = new List<IDisposable>(providers.Count);
             foreach (IConfigurationProvider p in providers)
             {
@@ -49,8 +55,21 @@ public ConfigurationRoot(IList<IConfigurationProvider> providers)
         /// <returns>The configuration value.</returns>
         public string? this[string key]
         {
-            get => GetConfiguration(_providers, key);
-            set => SetConfiguration(_providers, key, value);
+            get
+            {
+                if (ReferenceEngine.Disabled)
+                {
+                    return GetConfiguration(_providers, key);
+                }
+
+                _referenceEngine!.TryRead(key, out string? value);
+                return value;
+            }
+            set
+            {
+                SetConfiguration(_providers, key, value);
+                _referenceEngine = ReferenceEngine.Create(_providers);
+            }
         }
 
         /// <summary>
@@ -91,10 +110,13 @@ public void Reload()
 
         private void RaiseChanged()
         {
+            _referenceEngine = ReferenceEngine.Create(_providers);
             ConfigurationReloadToken previousToken = Interlocked.Exchange(ref _changeToken, new ConfigurationReloadToken());
             previousToken.OnReload();
         }
 
+        internal ReferenceEngine? ReferenceEngine => _referenceEngine;
+
         /// <inheritdoc />
         public void Dispose()
         {

src/libraries/Microsoft.Extensions.Configuration/src/InternalConfigurationRootExtensions.cs

@@ -20,13 +20,31 @@ internal static class InternalConfigurationRootExtensions
         /// <returns>Immediate children sub-sections of section specified by key.</returns>
         internal static IEnumerable<IConfigurationSection> GetChildrenImplementation(this IConfigurationRoot root, string? path)
         {
+            // For ConfigurationManager the engine is obtained from the pinned provider generation, so the engine and the
+            // providers it resolves against are always the same generation. A plain ConfigurationRoot exposes its engine
+            // directly (its provider list is fixed).
             using ReferenceCountedProviders? reference = (root as ConfigurationManager)?.GetProvidersReference();
-            IEnumerable<IConfigurationProvider> providers = reference?.Providers ?? root.Providers;
+            ReferenceEngine? engine = reference?.ReferenceEngine ?? (root as ConfigurationRoot)?.ReferenceEngine;
 
-            IEnumerable<IConfigurationSection> children = providers
-                .Aggregate(Enumerable.Empty<string>(),
-                    (seed, source) => source.GetChildKeys(seed, path))
-                .Distinct(StringComparer.OrdinalIgnoreCase)
+            IEnumerable<string> childKeys;
+            if (ReferenceEngine.Disabled || engine is null || engine.IndexIsEmpty)
+            {
+                // No engine (references disabled, or a third-party root), or this generation declares no references:
+                // enumerate children the plain way, with nothing to resolve, merge or hide, and allocate exactly like a
+                // reference-free configuration. Use the engine's provider list when present so it matches the generation.
+                IEnumerable<IConfigurationProvider> providers = engine?.Providers ?? reference?.Providers ?? root.Providers;
+                childKeys = providers
+                    .Aggregate(Enumerable.Empty<string>(), (seed, source) => source.GetChildKeys(seed, path))
+                    .Distinct(StringComparer.OrdinalIgnoreCase);
+            }
+            else
+            {
+                // References merge into the children: a redirected section lists its target's children unioned with any
+                // keys a higher provider defines under the reference, at every hop of the chain.
+                childKeys = engine.ChildKeys(path);
+            }
+
+            IEnumerable<IConfigurationSection> children = childKeys
                 .Select(key => root.GetSection(path == null ? key : path + ConfigurationPath.KeyDelimiter + key));
 
             if (reference is null)
@@ -42,19 +60,37 @@ internal static IEnumerable<IConfigurationSection> GetChildrenImplementation(thi
 
         internal static bool TryGetConfiguration(this IConfigurationRoot root, string key, out string? value)
         {
-            // common cases Providers is IList<IConfigurationProvider> in ConfigurationRoot
+            // For ConfigurationManager, take a counted reference and use the engine from that pinned generation
+            // so a concurrent source mutation can neither dispose the providers mid-read nor pair them with a different
+            // generation's index. The resolved key is then read like any other.
+            if (!ReferenceEngine.Disabled)
+            {
+                if (root is ConfigurationManager cm)
+                {
+                    using ReferenceCountedProviders reference = cm.GetProvidersReference();
+                    return reference.ReferenceEngine!.TryRead(key, out value);
+                }
+                else if (root is ConfigurationRoot cr)
+                {
+                    return cr.ReferenceEngine!.TryRead(key, out value);
+                }
+            }
+
+            // Plain path: references globally disabled, or a third-party root implementation.
+            // Commonly Providers is already IList<IConfigurationProvider> in ConfigurationRoot.
             IList<IConfigurationProvider> providers = root.Providers is IList<IConfigurationProvider> list
                 ? list
                 : root.Providers.ToList();
+            return TryGet(providers, key, out value);
+        }
 
-            // ensure looping in the reverse order
+        private static bool TryGet(IList<IConfigurationProvider> providers, string key, out string? value)
+        {
             for (int i = providers.Count - 1; i >= 0; i--)
             {
-                IConfigurationProvider provider = providers[i];
-
                 try
                 {
-                    if (provider.TryGet(key, out value))
+                    if (providers[i].TryGet(key, out value))
                     {
                         return true;
                     }
@@ -76,6 +112,5 @@ internal static bool TryGetConfiguration(this IConfigurationRoot root, string ke
             value = null;
             return false;
         }
-
     }
 }

src/libraries/Microsoft.Extensions.Configuration/src/Microsoft.Extensions.Configuration.csproj

@@ -14,6 +14,11 @@
              Link="Common\src\Extensions\EmptyDisposable.cs" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(TargetFrameworkIdentifier)' != '.NETCoreApp' or $([MSBuild]::VersionLessThan('$(TargetFrameworkVersion)', '9.0'))">
+    <Compile Include="$(CoreLibSharedDir)System\Diagnostics\CodeAnalysis\FeatureSwitchDefinitionAttribute.cs"
+             Link="Common\src\System\Diagnostics\CodeAnalysis\FeatureSwitchDefinitionAttribute.cs" />
+  </ItemGroup>
+
   <ItemGroup Condition="'$(TargetFramework)' != '$(NetCoreAppCurrent)'">
     <ProjectReference Include="$(LibrariesProjectRoot)Microsoft.Extensions.Configuration.Abstractions\src\Microsoft.Extensions.Configuration.Abstractions.csproj" />
     <ProjectReference Include="$(LibrariesProjectRoot)Microsoft.Extensions.Primitives\src\Microsoft.Extensions.Primitives.csproj" />

src/libraries/Microsoft.Extensions.Configuration/src/ReferenceCountedProviders.cs

@@ -29,16 +29,22 @@ internal abstract class ReferenceCountedProviders : IDisposable
         // This is Dispose() rather than RemoveReference() so we can conveniently release a reference at the end of a using block.
         public abstract void Dispose();
 
+        public abstract ReferenceEngine? ReferenceEngine { get; }
+
+        public abstract void ResetReferenceEngine();
+
         private sealed class ActiveReferenceCountedProviders : ReferenceCountedProviders
         {
             private long _refCount = 1;
             // volatile is not strictly necessary because the runtime adds a barrier either way, but volatile indicates that this field has
             // unsynchronized readers meaning the all writes initializing the list must be published before updating the _providers reference.
             private volatile List<IConfigurationProvider> _providers;
+            private volatile ReferenceEngine? _referenceEngine;
 
             public ActiveReferenceCountedProviders(List<IConfigurationProvider> providers)
             {
                 _providers = providers;
+                _referenceEngine = ReferenceEngine.Create(providers);
             }
 
             public override List<IConfigurationProvider> Providers
@@ -52,11 +58,16 @@ public override List<IConfigurationProvider> Providers
                 {
                     Debug.Assert(_refCount > 0);
                     _providers = value;
+                    _referenceEngine = ReferenceEngine.Create(value);
                 }
             }
 
             public override List<IConfigurationProvider> NonReferenceCountedProviders => _providers;
 
+            public override ReferenceEngine? ReferenceEngine => _referenceEngine;
+
+            public override void ResetReferenceEngine() => _referenceEngine = ReferenceEngine.Create(_providers);
+
             public override void AddReference()
             {
                 // AddReference() is always called with a lock to ensure _refCount hasn't already decremented to zero.
@@ -78,14 +89,21 @@ public override void Dispose()
 
         private sealed class DisposedReferenceCountedProviders : ReferenceCountedProviders
         {
+            private volatile ReferenceEngine? _referenceEngine;
+
             public DisposedReferenceCountedProviders(List<IConfigurationProvider> providers)
             {
                 Providers = providers;
+                _referenceEngine = ReferenceEngine.Create(providers);
             }
 
             public override List<IConfigurationProvider> Providers { get; set; }
             public override List<IConfigurationProvider> NonReferenceCountedProviders => Providers;
 
+            public override ReferenceEngine? ReferenceEngine => _referenceEngine;
+
+            public override void ResetReferenceEngine() => _referenceEngine = ReferenceEngine.Create(Providers);
+
             public override void AddReference() { }
             public override void Dispose() { }
         }

src/libraries/Microsoft.Extensions.Configuration/src/ReferenceCountedProvidersManager.cs

@@ -75,6 +75,17 @@ public void AddProvider(IConfigurationProvider provider)
             }
         }
 
+        public void ResetReferenceEngine()
+        {
+            lock (_replaceProvidersLock)
+            {
+                if (!_disposed)
+                {
+                    _refCountedProviders.ResetReferenceEngine();
+                }
+            }
+        }
+
         public void Dispose()
         {
             ReferenceCountedProviders oldRefCountedProviders = _refCountedProviders;