diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ae6c8e..35ddf22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,10 @@ jobs: run: dotnet format SharedMemoryStore.slnx --verify-no-changes --no-restore - name: Test - run: dotnet test SharedMemoryStore.slnx -c Release --no-build + # Keep the projects sequential: individual integration scenarios retain + # their real 8-12 process concurrency without unrelated test assemblies + # oversubscribing the hosted runner's one-second cold-open budget. + run: dotnet test SharedMemoryStore.slnx -c Release --no-build --maxcpucount:1 - name: Validate documentation shell: pwsh diff --git a/.specify/feature.json b/.specify/feature.json index df1504a..e5d0c74 100644 --- a/.specify/feature.json +++ b/.specify/feature.json @@ -1,3 +1,3 @@ { - "feature_directory": "specs/008-cpp-python-implementations" + "feature_directory": "specs/009-lock-free-publish-read" } diff --git a/AGENTS.md b/AGENTS.md index 9f9c3fe..cb5fb29 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,5 @@ For additional context about technologies to be used, project structure, shell commands, and other important information, read the current plan -at specs/008-cpp-python-implementations/plan.md +at specs/009-lock-free-publish-read/plan.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 65a6054..7c196d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,37 @@ chronological order. ## Unreleased +## 2.0.0 - 2026-07-13 + +### Added + +- Added an explicitly selected C# lock-free key-value profile using mapped + layout `2.0` and resource protocol `2`, with direct reservation publication, + shared zero-copy leases, generation-fenced removal/reuse, participant + incarnations, explicit recovery, and bounded diagnostics. +- Added `StoreProfile`, `CreateLockFree`, profile-aware sizing, + `ParticipantRecordCount`, immutable `StoreProtocolInfo`, and the appended + `ParticipantTableFull` open status. + +### Changed + +- NuGet `SharedMemoryStore` advances to `2.0.0`. Lock-free wait options bound + local retry, revalidation, helping, and backoff; they do not acquire a named + operation lock or wait for keys/capacity to appear. +- Documented reservation tokens as exclusive single-producer lifetimes and + clarified logical removal, `RemovePending`, borrowed-view, recovery, and + local-handle disposal semantics. + +### Compatibility + +- The legacy profile remains the default and preserves layout `1.2`, resource + protocol `1`, public workflow signatures, and existing status numbers. +- Layout `1.2` and `2.0` are never reinterpreted in place or used by mixed + synchronization participants. Same-name upgrade and rollback require draining + handles, recreating the mapping, and republishing application-owned values. +- C++ and Python `0.1.0` remain layout-1.2-only and reject layout `2.0`; their + independently versioned packages and C ABI `1.0` are unchanged. + ## 1.0.2 - 2026-07-10 ### Added diff --git a/README.md b/README.md index 9f4b765..25867d5 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,21 @@ payloads through a broker process. Distribution identities: -- NuGet: `SharedMemoryStore` `1.0.2`, targeting `net10.0` with .NET BCL - runtime dependencies only. +- NuGet: `SharedMemoryStore` `2.0.0`, targeting `net10.0` with .NET BCL + runtime dependencies only. The legacy layout remains the default; C# callers + opt in to the lock-free layout explicitly. - CMake: `SharedMemoryStore` `0.1.0`, exposing a C++20 RAII API and fixed-width C ABI `1.0` over the native shared library. - Python: `shared-memory-store` `0.1.0`, requiring Python 3.10 or newer and using standard-library `ctypes` with the packaged native library. -- Shared protocol: mapped layout `1.2`, resource naming `1`, little-endian - 64-bit Windows and Linux targets. +- Shared protocol: C# supports mapped layouts `1.2` and `2.0`; C++ and Python + remain layout-`1.2` clients and reject layout 2.0. Resource naming versions + are `1` and `2` respectively. - License: MIT, see the [license file](LICENSE). -The managed `1.0.0` line establishes the production .NET API contract. The +The managed `1.0.0` line established the original production .NET API contract; +the `2.0.0` line adds the explicit lock-free profile while retaining the legacy +profile and its layout. The native and Python `0.1.0` lines are independently versioned initial distributions; they do not change or ship inside the NuGet package. Linux and Windows are implementation targets. Same-host Linux Docker containers require @@ -56,6 +60,41 @@ The store does not parse frame headers, own application schemas, provide a cross-host cache, persist data beyond process and mapping lifetime, or turn Docker into distributed storage. +## Explicit lock-free C# profile + +`SharedMemoryStoreOptions.CreateLockFree(...)` creates or opens mapped layout +2.0. Existing helpers and manually constructed options remain on the legacy +layout unless `StoreProfile.LockFree` is selected explicitly. Processes using +different profiles cannot participate in the same live mapping; incompatible +opens fail before payload projection. + +The lock-free profile is still a bounded key-value store. An application broker +may load-balance work by sending keys to 6-12 workers, while observers and other +processes independently acquire the same immutable values. The store does not +enqueue keys, choose workers, acknowledge work, or turn a read lease into an +exclusive claim. See the runnable +[`LockFreeBrokerKeys`](samples/LockFreeBrokerKeys/Program.cs) sample. + +Steady-state layout-2.0 publish, acquire, release, remove, and helping paths do +not enter the named cross-process lifecycle lock. Progress is system-wide +lock-free, not wait-free: a paused or terminated participant cannot own a +store-wide data-path lock, but one caller can still exhaust its bounded retry +budget under sustained contention and receive `StoreBusy`. Reservations belong +to one producer; leases are shared exact-generation protections; recovery is an +explicit caller-controlled operation rather than a background worker. + +The trust boundary is same-host cooperating processes with write access to the +mapping. Layout 2.0 is neither cross-host storage nor protection against a +malicious mapped writer. Performance depends on payload size, key distribution, +contention, lease duration, capacity pressure, process placement, and recovery; +qualification reports separate those workloads instead of treating one +throughput number as universal. + +To migrate under the same public name, drain and close every legacy handle, +recreate the mapping as layout 2.0, and republish application-owned values. A +side-by-side cutover uses a distinct store name. Rollback recreates layout 1.2; +neither direction reinterprets mapped bytes in place. + ## First Use Start with [Getting started](docs/getting-started.md). It separates the NuGet, @@ -205,6 +244,7 @@ Runnable samples: - [Docker shared-memory sample](samples/DockerSharedMemory/README.md) - [C++ basic usage sample](samples/CppBasicUsage/README.md) - [Python basic usage sample](samples/PythonBasicUsage/README.md) +- [Lock-free broker-key sample](samples/LockFreeBrokerKeys/Program.cs) ## Project Policies @@ -248,5 +288,5 @@ pwsh ./scripts/validate-interoperability.ps1 -Configuration Release -Stress Documentation changes must keep package metadata, README content, release notes, support policy, security policy, compatibility metadata, and contract links -aligned across managed `1.0.2`, native `0.1.0`, Python `0.1.0`, ABI `1.0`, and -layout `1.2`. +aligned across managed `2.0.0`, native `0.1.0`, Python `0.1.0`, ABI `1.0`, and +layouts `1.2` and `2.0`. diff --git a/SharedMemoryStore.slnx b/SharedMemoryStore.slnx index 4e6cac9..67ff985 100644 --- a/SharedMemoryStore.slnx +++ b/SharedMemoryStore.slnx @@ -6,12 +6,14 @@ + + @@ -23,6 +25,8 @@ + + diff --git a/benchmarks/SharedMemoryStore.Benchmarks/LockFreeProfileBenchmarks.cs b/benchmarks/SharedMemoryStore.Benchmarks/LockFreeProfileBenchmarks.cs new file mode 100644 index 0000000..d32976a --- /dev/null +++ b/benchmarks/SharedMemoryStore.Benchmarks/LockFreeProfileBenchmarks.cs @@ -0,0 +1,194 @@ +using BenchmarkDotNet.Attributes; + +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.Benchmarks; + +/// +/// Side-by-side profile benchmarks for the v1.2 synchronized engine and the +/// v2 lock-free engine. Keys and buffers are prepared during setup so the +/// MemoryDiagnoser reports library-path allocations rather than harness noise. +/// +[MemoryDiagnoser] +public class LockFreeProfileBenchmarks +{ + private const int SlotCount = 64; + private const int PayloadBytes = 256; + private const int PrimaryBucketCount = 64; + + private Store _store = null!; + private byte[][] _keys = null!; + private byte[][] _collidingKeys = null!; + private byte[] _payload = null!; + private int _cursor; + + [Params(StoreProfile.Legacy, StoreProfile.LockFree)] + public StoreProfile Profile { get; set; } + + [GlobalSetup] + public void Setup() + { + string name = $"sms-profile-benchmark-{Profile}-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions options = Profile == StoreProfile.LockFree + ? SharedMemoryStoreOptions.CreateLockFree( + name, + SlotCount, + PayloadBytes, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount: 128, + participantRecordCount: 16, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true) + : SharedMemoryStoreOptions.Create( + name, + SlotCount, + PayloadBytes, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount: 128, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + + StoreOpenStatus open = Store.TryCreateOrOpen(options, out Store? store); + if (open != StoreOpenStatus.Success || store is null) + { + throw new InvalidOperationException($"Cannot open {Profile} benchmark store: {open}."); + } + + _store = store; + _keys = Enumerable.Range(0, SlotCount).Select(BitConverter.GetBytes).ToArray(); + _payload = new byte[PayloadBytes]; + for (var index = 0; index < _payload.Length; index++) + { + _payload[index] = unchecked((byte)(index * 31)); + } + + _collidingKeys = FindCollidingKeys(9); + for (var index = 0; index < 8; index++) + { + Ensure(_store.TryPublish(_collidingKeys[index], _payload)); + } + + Ensure(_store.TryPublish(_keys[0], _payload)); + } + + [GlobalCleanup] + public void Cleanup() + { + _store.Dispose(); + } + + [Benchmark(Baseline = true)] + public StoreStatus AcquireProjectRelease() + { + StoreStatus acquire = _store.TryAcquire(_keys[0], out ValueLease lease); + if (acquire != StoreStatus.Success) + { + return acquire; + } + + _ = lease.ValueSpan[PayloadBytes - 1]; + return lease.Release(); + } + + [Benchmark] + public StoreStatus PublishRemove() + { + int index = 1 + (Interlocked.Increment(ref _cursor) & 31); + byte[] key = _keys[index]; + StoreStatus publish = _store.TryPublish(key, _payload); + return publish == StoreStatus.Success ? _store.TryRemove(key) : publish; + } + + [Benchmark] + public StoreStatus ZeroCopyReserveCommitRemove() + { + int index = 33 + (Interlocked.Increment(ref _cursor) & 15); + byte[] key = _keys[index]; + StoreStatus reserve = _store.TryReserve(key, PayloadBytes, [], out ValueReservation reservation); + if (reserve != StoreStatus.Success) + { + return reserve; + } + + _payload.CopyTo(reservation.GetSpan()); + StoreStatus advance = reservation.Advance(PayloadBytes); + if (advance != StoreStatus.Success) + { + _ = reservation.Abort(); + return advance; + } + + StoreStatus commit = reservation.Commit(); + return commit == StoreStatus.Success ? _store.TryRemove(key) : commit; + } + + [Benchmark] + public StoreStatus CollisionOverflowPublishRemove() + { + byte[] key = _collidingKeys[8]; + StoreStatus publish = _store.TryPublish(key, _payload); + return publish == StoreStatus.Success ? _store.TryRemove(key) : publish; + } + + [Benchmark] + public StoreStatus CurrentOwnerLeaseRecovery() + { + int index = 49 + (Interlocked.Increment(ref _cursor) & 7); + byte[] key = _keys[index]; + StoreStatus publish = _store.TryPublish(key, _payload); + if (publish != StoreStatus.Success) + { + return publish; + } + + StoreStatus acquire = _store.TryAcquire(key, out _); + if (acquire != StoreStatus.Success) + { + _ = _store.TryRemove(key); + return acquire; + } + + StoreStatus recovery = _store.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: true), + out _); + StoreStatus remove = _store.TryRemove(key); + return recovery == StoreStatus.Success ? remove : recovery; + } + + private static byte[][] FindCollidingKeys(int count) + { + var keys = new List(count); + for (long candidate = 0; keys.Count < count; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + if ((Hash(key) & (PrimaryBucketCount - 1)) == 0) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static ulong Hash(ReadOnlySpan key) + { + ulong hash = 14_695_981_039_346_656_037UL; + foreach (byte value in key) + { + hash ^= value; + hash *= 1_099_511_628_211UL; + } + + return hash; + } + + private static void Ensure(StoreStatus status) + { + if (status != StoreStatus.Success) + { + throw new InvalidOperationException("Benchmark setup failed: " + status); + } + } +} diff --git a/benchmarks/SharedMemoryStore.Benchmarks/Program.cs b/benchmarks/SharedMemoryStore.Benchmarks/Program.cs index 3e2a6eb..4fba1ca 100644 --- a/benchmarks/SharedMemoryStore.Benchmarks/Program.cs +++ b/benchmarks/SharedMemoryStore.Benchmarks/Program.cs @@ -47,7 +47,10 @@ return; } -BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); +// Assembly discovery intentionally includes LockFreeProfileBenchmarks alongside +// the existing reliability suites; use --filter *LockFreeProfileBenchmarks* for +// the v1.2-versus-v2 allocation/collision/recovery comparison. +BenchmarkSwitcher.FromAssembly(typeof(LockFreeProfileBenchmarks).Assembly).Run(args); static FrameThroughputValidationResult RunSimplePublishSustained() { diff --git a/benchmarks/SharedMemoryStore.SyncProbe/BenchmarkProtocol.cs b/benchmarks/SharedMemoryStore.SyncProbe/BenchmarkProtocol.cs new file mode 100644 index 0000000..1842e7f --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/BenchmarkProtocol.cs @@ -0,0 +1,864 @@ +using System.Buffers.Binary; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using SharedMemoryStore; + +internal enum OperationKind +{ + Acquire, + Release, + Publish, + Remove, + Reserve, + Advance, + Commit, + RecoverLeases, + RecoverReservations +} + +internal sealed class StatusCounters +{ + private readonly long[,] _counts = new long[ + Enum.GetValues().Length, + Enum.GetValues().Length]; + private long _checksumFailures; + private long _operations; + private readonly Dictionary _corruptReasons = new(StringComparer.Ordinal); + + internal long TotalOperations => _operations; + + internal void Record(OperationKind operation, StoreStatus status) + { + _counts[(int)operation, (int)status]++; + _operations++; + } + + internal void RecordChecksumFailure() + { + _checksumFailures++; + } + + internal void RecordCorruptReason(string reason) + { + _corruptReasons.TryGetValue(reason, out long count); + _corruptReasons[reason] = count + 1; + } + + internal SortedDictionary ToHistogram() + { + var histogram = new SortedDictionary(StringComparer.Ordinal); + foreach (var operation in Enum.GetValues()) + { + foreach (var status in Enum.GetValues()) + { + long count = _counts[(int)operation, (int)status]; + if (count != 0) + { + histogram[operation + "." + status] = count; + } + } + } + + if (_checksumFailures != 0) + { + histogram["Validation.ChecksumMismatch"] = _checksumFailures; + } + + foreach ((string reason, long count) in _corruptReasons) + { + histogram["CorruptReason." + reason] = count; + } + + return histogram; + } +} + +internal enum ProbeScenarioKind +{ + Autonomous, + MixedChurn, + BrokerDirected, + LargeIngest, + StickyOverflow +} + +internal sealed record ScenarioPlan( + string Name, + ProbeScenarioKind Kind, + int[] ProcessCounts, + int PublisherCount = 0, + int ObserverCount = 0); + +internal static class ProbeScenarioCatalog +{ + private static readonly ScenarioPlan[] Sync = + [ + new("acquire-release", ProbeScenarioKind.Autonomous, [1, 2, 4, 8, 12]), + new("publish-remove", ProbeScenarioKind.Autonomous, [1, 2, 4, 8, 12]) + ]; + + private static readonly ScenarioPlan[] Readers = + [ + new("same-key-read", ProbeScenarioKind.Autonomous, [1, 2, 4, 6, 8, 12]), + new("distributed-key-read", ProbeScenarioKind.Autonomous, [1, 2, 4, 6, 8, 12]) + ]; + + private static readonly ScenarioPlan Broker = + new("broker-directed", ProbeScenarioKind.BrokerDirected, [1, 12], ObserverCount: 1); + + private static readonly ScenarioPlan Churn = + new("mixed-churn", ProbeScenarioKind.MixedChurn, [12], PublisherCount: 2); + + private static readonly ScenarioPlan Large = + new("large-ingest", ProbeScenarioKind.LargeIngest, [1, 12]); + + private static readonly ScenarioPlan StickyOverflow = + new("sticky-overflow-miss", ProbeScenarioKind.StickyOverflow, [1]); + + internal static ScenarioPlan[] Select(string mode) => mode switch + { + "sync" => Sync, + "readers" => Readers, + "broker" => [Broker], + "churn" => [Churn], + "large" => [Large], + "overflow" => [StickyOverflow], + "all" => [.. Sync, .. Readers], + "full" => [.. Sync, .. Readers, Broker, Churn, Large, StickyOverflow], + _ => throw new ArgumentException( + "--mode must be sync, readers, broker, churn, large, overflow, all, or full.") + }; +} + +internal enum BrokerMessageKind +{ + Key = 1, + Stop = 2, + Reset = 3 +} + +internal readonly record struct BrokerKeyMessage( + BrokerMessageKind Kind, + string KeyHex, + int KeyIndex, + long Generation, + int PayloadLength, + long PublishedTimestamp); + +internal readonly record struct BrokerAcknowledgement( + int WorkerId, + string Role, + int KeyIndex, + long Generation, + StoreStatus AcquireStatus, + StoreStatus ReleaseStatus, + bool DescriptorValid, + bool PayloadValid, + int BytesObserved, + double EndToEndMicroseconds); + +internal readonly record struct BucketPairCollisionSet( + byte[][] Keys, + long CandidatesExamined); + +internal static class BenchmarkProtocol +{ + private const ulong FnvOffsetBasis = 14695981039346656037UL; + private const ulong FnvPrime = 1099511628211UL; + private const int PatternBlockBytes = 256; + + internal static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = false + }; + + internal static byte[] Key(int keyIndex) + { + var key = new byte[sizeof(long)]; + BinaryPrimitives.WriteInt64LittleEndian(key, keyIndex + 1L); + return key; + } + + internal static BenchmarkKeyCatalog CreateKeyCatalog(int count) => new(count); + + internal static BenchmarkKeyCatalog CreateCanonicalBucketKeyCatalog( + int keysPerWorker, + int maximumWorkerCount, + int canonicalBucketCount) + { + if (keysPerWorker <= 0 + || maximumWorkerCount <= 0 + || maximumWorkerCount > canonicalBucketCount + || canonicalBucketCount <= 0 + || (canonicalBucketCount & (canonicalBucketCount - 1)) != 0) + { + throw new ArgumentOutOfRangeException(nameof(keysPerWorker)); + } + + var keys = new byte[checked(keysPerWorker * maximumWorkerCount)][]; + Span candidateKey = stackalloc byte[sizeof(long)]; + long candidate = 1; + for (var workerId = 0; workerId < maximumWorkerCount; workerId++) + { + for (var keyOrdinal = 0; keyOrdinal < keysPerWorker;) + { + BinaryPrimitives.WriteInt64LittleEndian(candidateKey, candidate++); + if (GetCanonicalBucket(candidateKey, canonicalBucketCount) != workerId) + { + continue; + } + + keys[(workerId * keysPerWorker) + keyOrdinal] = candidateKey.ToArray(); + keyOrdinal++; + } + } + + return new BenchmarkKeyCatalog(keys); + } + + internal static byte[][] CreateCollisionKeys(int count, int canonicalBucketCount) + { + if (count <= 0 || canonicalBucketCount <= 0 || (canonicalBucketCount & (canonicalBucketCount - 1)) != 0) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + var keys = new byte[count][]; + var found = 0; + long candidate = 1; + while (found < count) + { + var key = new byte[sizeof(long)]; + BinaryPrimitives.WriteInt64LittleEndian(key, candidate++); + if ((Mix(Hash(key)) & (uint)(canonicalBucketCount - 1)) == 0) + { + keys[found++] = key; + } + } + + return keys; + } + + internal static BucketPairCollisionSet CreateBucketPairCollisionKeys( + int count, + int bucketCount, + int firstBucket, + int secondBucket) + { + if (count <= 0 + || bucketCount <= 1 + || (bucketCount & (bucketCount - 1)) != 0 + || (uint)firstBucket >= (uint)bucketCount + || (uint)secondBucket >= (uint)bucketCount + || firstBucket == secondBucket) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + var keys = new byte[count][]; + Span candidateKey = stackalloc byte[sizeof(long)]; + var found = 0; + long candidate = 1; + while (found < count) + { + BinaryPrimitives.WriteInt64LittleEndian(candidateKey, candidate++); + (int actualFirst, int actualSecond) = GetBucketPair(candidateKey, bucketCount); + + if (actualFirst == firstBucket && actualSecond == secondBucket) + { + keys[found++] = candidateKey.ToArray(); + } + } + + return new BucketPairCollisionSet(keys, candidate - 1); + } + + internal static (int First, int Second) GetBucketPair(ReadOnlySpan key, int bucketCount) + { + if (bucketCount <= 1 || (bucketCount & (bucketCount - 1)) != 0) + { + throw new ArgumentOutOfRangeException(nameof(bucketCount)); + } + + uint mask = (uint)(bucketCount - 1); + ulong hash = Hash(key); + int first = (int)(Mix(hash) & mask); + int second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & mask); + if (second == first) + { + second = (first + 1) & (int)mask; + } + + return (first, second); + } + + internal static int GetCanonicalBucket(ReadOnlySpan key, int bucketCount) + { + if (bucketCount <= 0 || (bucketCount & (bucketCount - 1)) != 0) + { + throw new ArgumentOutOfRangeException(nameof(bucketCount)); + } + + return (int)(Mix(Hash(key)) & (uint)(bucketCount - 1)); + } + + internal static int CalculatePrimaryBucketCount(int slotCount) + { + ArgumentOutOfRangeException.ThrowIfLessThan(slotCount, 1); + int laneTarget = Math.Max(32, checked(slotCount * 4)); + var primaryLaneCount = 1; + while (primaryLaneCount < laneTarget) + { + primaryLaneCount = checked(primaryLaneCount << 1); + } + + // LayoutV2Constants.PrimaryLanesPerBucket is protocol-fixed at eight. + return primaryLaneCount / 8; + } + + internal static void WriteDescriptor(Span descriptor, int keyIndex, long generation, int payloadLength) + { + if (descriptor.Length != 16) + { + throw new ArgumentException("The benchmark descriptor is exactly 16 bytes.", nameof(descriptor)); + } + + BinaryPrimitives.WriteInt64LittleEndian(descriptor, generation); + BinaryPrimitives.WriteInt32LittleEndian(descriptor[8..], keyIndex); + BinaryPrimitives.WriteInt32LittleEndian(descriptor[12..], payloadLength); + } + + internal static bool ValidateDescriptor( + ReadOnlySpan descriptor, + int keyIndex, + long generation, + int payloadLength) => + descriptor.Length == 16 + && BinaryPrimitives.ReadInt64LittleEndian(descriptor) == generation + && BinaryPrimitives.ReadInt32LittleEndian(descriptor[8..]) == keyIndex + && BinaryPrimitives.ReadInt32LittleEndian(descriptor[12..]) == payloadLength; + + internal static void FillGenerationPayload(Span payload, int keyIndex, long generation) + { + Span pattern = stackalloc byte[PatternBlockBytes]; + FillPatternBlock(pattern, keyIndex, generation); + var offset = 0; + while (offset < payload.Length) + { + int length = Math.Min(pattern.Length, payload.Length - offset); + pattern[..length].CopyTo(payload[offset..]); + offset += length; + } + } + + internal static bool ValidateGenerationPayload(ReadOnlySpan payload, int keyIndex, long generation) + { + Span pattern = stackalloc byte[PatternBlockBytes]; + FillPatternBlock(pattern, keyIndex, generation); + var offset = 0; + while (offset < payload.Length) + { + int length = Math.Min(pattern.Length, payload.Length - offset); + if (!payload.Slice(offset, length).SequenceEqual(pattern[..length])) + { + return false; + } + + offset += length; + } + + return true; + } + + internal static ulong Hash(ReadOnlySpan key) + { + ulong hash = FnvOffsetBasis; + foreach (byte value in key) + { + hash ^= value; + hash *= FnvPrime; + } + + return hash; + } + + private static void FillPatternBlock(Span pattern, int keyIndex, long generation) + { + BinaryPrimitives.WriteInt64LittleEndian(pattern, generation); + BinaryPrimitives.WriteInt64LittleEndian(pattern[8..], ~generation); + BinaryPrimitives.WriteInt32LittleEndian(pattern[16..], keyIndex); + BinaryPrimitives.WriteInt32LittleEndian(pattern[20..], ~keyIndex); + for (var index = 24; index < pattern.Length; index++) + { + pattern[index] = unchecked((byte)( + (generation * 131) + + (keyIndex * 17L) + + (index * 29L) + + ((generation >> (index & 7)) * 7))); + } + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } +} + +/// +/// Process-local, read-only benchmark keys. Keeping the backing arrays private +/// prevents the measured loop from allocating one short-lived byte array per +/// lookup while still passing the library a normal . +/// +internal sealed class BenchmarkKeyCatalog +{ + private readonly ReadOnlyMemory[] _keys; + private readonly string[] _hexKeys; + + internal BenchmarkKeyCatalog(int count) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(count); + _keys = new ReadOnlyMemory[count]; + _hexKeys = new string[count]; + for (var index = 0; index < count; index++) + { + byte[] key = BenchmarkProtocol.Key(index); + _keys[index] = key; + _hexKeys[index] = Convert.ToHexString(key); + } + } + + internal BenchmarkKeyCatalog(byte[][] keys) + { + ArgumentNullException.ThrowIfNull(keys); + ArgumentOutOfRangeException.ThrowIfZero(keys.Length); + _keys = new ReadOnlyMemory[keys.Length]; + _hexKeys = new string[keys.Length]; + for (var index = 0; index < keys.Length; index++) + { + byte[] key = keys[index] + ?? throw new ArgumentException("The key catalog cannot contain null entries.", nameof(keys)); + _keys[index] = key; + _hexKeys[index] = Convert.ToHexString(key); + } + } + + internal int Count => _keys.Length; + + internal ReadOnlyMemory this[int index] => _keys[index]; + + internal string Hex(int index) => _hexKeys[index]; + + internal string CalculateSha256() + { + using IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + foreach (ReadOnlyMemory key in _keys) + { + hash.AppendData(key.Span); + } + + return Convert.ToHexString(hash.GetHashAndReset()); + } +} + +internal struct DeterministicXorShift64 +{ + private ulong _state; + + internal DeterministicXorShift64(ulong seed) + { + _state = seed == 0 ? 0x9e37_79b9_7f4a_7c15UL : seed; + } + + internal ulong NextUInt64() + { + ulong value = _state; + value ^= value >> 12; + value ^= value << 25; + value ^= value >> 27; + _state = value; + return value * 0x2545_f491_4f6c_dd1dUL; + } + + internal ulong NextUInt64(ulong exclusiveUpperBound) + { + if (exclusiveUpperBound == 0) + { + throw new ArgumentOutOfRangeException(nameof(exclusiveUpperBound)); + } + + ulong rejectionThreshold = unchecked(0UL - exclusiveUpperBound) % exclusiveUpperBound; + ulong candidate; + do + { + candidate = NextUInt64(); + } + while (candidate < rejectionThreshold); + + return candidate % exclusiveUpperBound; + } +} + +internal struct GeometricCycleSampler +{ + private const double UnitIntervalScale = 1.0 / 9_007_199_254_740_992.0; + private const double InverseLogSurvivalProbability = -63.498_687_642_344; + + private DeterministicXorShift64 _random; + private long _nextCandidateCycle; + + internal GeometricCycleSampler(ulong seed) + { + _random = new DeterministicXorShift64(seed); + _nextCandidateCycle = NextGap() - 1L; + } + + internal bool ShouldSample(long cycle) + { + if (cycle < _nextCandidateCycle) + { + return false; + } + + int gap = NextGap(); + _nextCandidateCycle = cycle > long.MaxValue - gap ? long.MaxValue : cycle + gap; + return true; + } + + private int NextGap() + { + // A success probability of 1/64 gives an exact geometric mean of 64 + // cycles. One 53-bit uniform variate avoids a per-cycle RNG cost while + // keeping candidate selection deterministic and allocation-free. + ulong mantissa = (_random.NextUInt64() >> 11) + 1; + double unitInterval = mantissa * UnitIntervalScale; + double gap = Math.Floor(Math.Log(unitInterval) * InverseLogSurvivalProbability) + 1; + return gap >= int.MaxValue ? int.MaxValue : (int)gap; + } +} + +internal sealed class LatencyReservoir +{ + private readonly double[] _samples; + private DeterministicXorShift64 _random; + private ulong _observedCount; + private int _sampleCount; + private double _maximumObserved; + + internal LatencyReservoir(int capacity, ulong seed) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity); + _samples = new double[capacity]; + _random = new DeterministicXorShift64(seed); + } + + internal int Count => _sampleCount; + + internal double MaximumObserved => _maximumObserved; + + internal void Add(double sample) + { + _maximumObserved = Math.Max(_maximumObserved, sample); + _observedCount++; + if (_sampleCount < _samples.Length) + { + _samples[_sampleCount++] = sample; + return; + } + + ulong replacement = _random.NextUInt64(_observedCount); + if (replacement < (ulong)_samples.Length) + { + _samples[(int)replacement] = sample; + } + } + + internal double[] ToArray() => _samples[.._sampleCount]; +} + +internal static class ProcessorAffinityPlanner +{ + private const int CpuSetInformationType = 0; + + internal static bool TryApply(int ordinal, out int assignedProcessor, out string strategy) + { + try + { + using Process process = Process.GetCurrentProcess(); + return TryApply(process, ordinal, out assignedProcessor, out strategy); + } + catch + { + assignedProcessor = -1; + strategy = "logical-order-fallback-apply-failed"; + return false; + } + } + + internal static bool TryApply( + Process process, + int ordinal, + out int assignedProcessor, + out string strategy) + { + ArgumentNullException.ThrowIfNull(process); + assignedProcessor = -1; + strategy = ordinal < 0 ? "not-requested" : "logical-order-fallback"; + if (ordinal < 0) + { + return false; + } + + if (!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + { + strategy = "unsupported-platform"; + return false; + } + + try + { + ulong available = unchecked((ulong)(nuint)process.ProcessorAffinity); + int[] order = TryGetPhysicalCoreFirstOrder(available, out strategy); + if (order.Length == 0) + { + return false; + } + + assignedProcessor = order[ordinal % order.Length]; + process.ProcessorAffinity = unchecked((nint)(1UL << assignedProcessor)); + return true; + } + catch + { + assignedProcessor = -1; + strategy += "-apply-failed"; + return false; + } + } + + private static int[] TryGetPhysicalCoreFirstOrder(ulong available, out string strategy) + { + if (OperatingSystem.IsWindows() + && TryGetWindowsTopology(available, out int[] windowsOrder)) + { + strategy = "windows-physical-core-first"; + return windowsOrder; + } + + if (OperatingSystem.IsLinux() + && TryGetLinuxTopology(available, out int[] linuxOrder)) + { + strategy = "linux-physical-core-first"; + return linuxOrder; + } + + strategy = "logical-order-fallback"; + return Enumerable.Range(0, Math.Min(64, IntPtr.Size * 8)) + .Where(cpu => (available & (1UL << cpu)) != 0) + .ToArray(); + } + + private static bool TryGetWindowsTopology(ulong available, out int[] order) + { + order = []; + if (!GetSystemCpuSetInformation(IntPtr.Zero, 0, out uint requiredBytes, IntPtr.Zero, 0) + && Marshal.GetLastWin32Error() != 122) + { + return false; + } + + if (requiredBytes < 24 || requiredBytes > 16 * 1024 * 1024) + { + return false; + } + + IntPtr buffer = Marshal.AllocHGlobal(checked((int)requiredBytes)); + try + { + if (!GetSystemCpuSetInformation(buffer, requiredBytes, out uint returnedBytes, IntPtr.Zero, 0)) + { + return false; + } + + var entries = new List<(int Cpu, int Core, int Efficiency)>(); + int offset = 0; + while (offset + 8 <= returnedBytes) + { + int size = Marshal.ReadInt32(buffer, offset); + int type = Marshal.ReadInt32(buffer, offset + 4); + if (size < 8 || offset + size > returnedBytes) + { + return false; + } + + if (type == CpuSetInformationType && size >= 21) + { + short group = Marshal.ReadInt16(buffer, offset + 12); + int logicalProcessor = Marshal.ReadByte(buffer, offset + 14); + int coreIndex = Marshal.ReadByte(buffer, offset + 15); + int efficiencyClass = Marshal.ReadByte(buffer, offset + 18); + if (group == 0 + && logicalProcessor < Math.Min(64, IntPtr.Size * 8) + && (available & (1UL << logicalProcessor)) != 0) + { + entries.Add((logicalProcessor, coreIndex, efficiencyClass)); + } + } + + offset += size; + } + + order = PhysicalFirst(entries.Select(static entry => + (entry.Cpu, Package: 0, entry.Core, entry.Efficiency))); + return order.Length != 0; + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + + private static bool TryGetLinuxTopology(ulong available, out int[] order) + { + order = []; + const string CpuRoot = "/sys/devices/system/cpu"; + if (!Directory.Exists(CpuRoot)) + { + return false; + } + + var entries = new List<(int Cpu, int Package, int Core)>(); + foreach (string directory in Directory.EnumerateDirectories(CpuRoot, "cpu*")) + { + string suffix = Path.GetFileName(directory).AsSpan(3).ToString(); + if (!int.TryParse(suffix, out int cpu) + || cpu < 0 + || cpu >= Math.Min(64, IntPtr.Size * 8) + || (available & (1UL << cpu)) == 0) + { + continue; + } + + string packagePath = Path.Combine(directory, "topology", "physical_package_id"); + string corePath = Path.Combine(directory, "topology", "core_id"); + if (!int.TryParse(File.ReadAllText(packagePath).Trim(), out int package) + || !int.TryParse(File.ReadAllText(corePath).Trim(), out int core)) + { + return false; + } + + entries.Add((cpu, package, core)); + } + + order = PhysicalFirst(entries.Select(static entry => + (entry.Cpu, entry.Package, entry.Core, Efficiency: 0))); + return order.Length != 0; + } + + private static int[] PhysicalFirst( + IEnumerable<(int Cpu, int Package, int Core, int Efficiency)> source) + { + var entries = source + .DistinctBy(static entry => entry.Cpu) + .OrderByDescending(static entry => entry.Efficiency) + .ThenBy(static entry => entry.Package) + .ThenBy(static entry => entry.Core) + .ThenBy(static entry => entry.Cpu) + .ToArray(); + var firstThreads = entries + .GroupBy(static entry => (entry.Package, entry.Core)) + .OrderByDescending(static group => group.Max(static entry => entry.Efficiency)) + .ThenBy(static group => group.Key.Package) + .ThenBy(static group => group.Key.Core) + .Select(static group => group.First().Cpu); + var siblings = entries + .GroupBy(static entry => (entry.Package, entry.Core)) + .OrderByDescending(static group => group.Max(static entry => entry.Efficiency)) + .ThenBy(static group => group.Key.Package) + .ThenBy(static group => group.Key.Core) + .SelectMany(static group => group.Skip(1).Select(static entry => entry.Cpu)); + return firstThreads.Concat(siblings).ToArray(); + } + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GetSystemCpuSetInformation( + IntPtr information, + uint bufferLength, + out uint returnedLength, + IntPtr process, + uint flags); +} + +internal sealed class TemporaryProcessAffinity : IDisposable +{ + private readonly Process? _process; + private readonly nint _originalAffinity; + + private TemporaryProcessAffinity( + Process? process, + nint originalAffinity, + bool applied, + int assignedProcessor, + string strategy) + { + _process = process; + _originalAffinity = originalAffinity; + Applied = applied; + AssignedProcessor = assignedProcessor; + Strategy = strategy; + } + + internal bool Applied { get; } + + internal int AssignedProcessor { get; } + + internal string Strategy { get; } + + internal static TemporaryProcessAffinity Apply(int ordinal) + { + if (ordinal < 0 || (!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux())) + { + string unavailableStrategy = ordinal < 0 ? "not-requested" : "unsupported-platform"; + return new TemporaryProcessAffinity(null, 0, false, -1, unavailableStrategy); + } + + Process process = Process.GetCurrentProcess(); + nint originalAffinity = process.ProcessorAffinity; + bool applied = ProcessorAffinityPlanner.TryApply( + ordinal, + out int assignedProcessor, + out string strategy); + return new TemporaryProcessAffinity( + process, + originalAffinity, + applied, + assignedProcessor, + strategy); + } + + public void Dispose() + { + if (_process is null) + { + return; + } + + try + { + if (Applied && (OperatingSystem.IsWindows() || OperatingSystem.IsLinux())) + { + _process.ProcessorAffinity = _originalAffinity; + } + } + finally + { + _process.Dispose(); + } + } +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/BenchmarkResults.cs b/benchmarks/SharedMemoryStore.SyncProbe/BenchmarkResults.cs new file mode 100644 index 0000000..8d1f7f0 --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/BenchmarkResults.cs @@ -0,0 +1,256 @@ +internal sealed record WorkerResult( + int WorkerId, + string Role, + long Cycles, + long Operations, + long BytesProcessed, + long Failures, + long MeasuredThreadAllocatedBytes, + double ElapsedSeconds, + bool AffinityApplied, + int AssignedProcessor, + string AffinityStrategy, + SortedDictionary StatusHistogram, + double[] SamplesMicroseconds, + double[] EarlySamplesMicroseconds, + double[] LateSamplesMicroseconds, + double MaximumSampleMicroseconds); + +internal sealed record BrokerWorkerSummary( + int WorkerId, + string Role, + long Frames, + long Operations, + long BytesProcessed, + long Failures, + double ElapsedSeconds, + bool AffinityApplied, + int AssignedProcessor, + string AffinityStrategy, + SortedDictionary StatusHistogram); + +internal readonly record struct PendingBrokerFrame( + long FrameNumber, + int KeyIndex, + long Generation, + long PublishedTimestamp, + int ReaderId, + bool ObserverSampled); + +internal readonly record struct BrokerMeasuredResult( + long Frames, + long Failures, + TimeSpan Elapsed, + long MeasuredThreadAllocatedBytes, + long ProducerStoreOperationAllocatedBytes, + StatusCounters Counters, + double[] SamplesMicroseconds, + double[] EarlySamplesMicroseconds, + double[] LateSamplesMicroseconds); + +internal sealed record StickyOverflowEvidence( + int SlotCount, + int PrimaryBucketCount, + int ExactBucketPairCollisionKeyCount, + long CollisionCandidatesExamined, + int ChurnCycles, + int MissingSamplesPerWindow, + int SpilledBucketCountBeforeChurn, + int SpilledBucketCountDuringChurn, + int OverflowDirectoryOccupancyDuringChurn, + int SpilledBucketCountAfterFirstCleanup, + int OverflowDirectoryOccupancyAfterFirstCleanup, + int SpilledBucketCountAfterChurn, + int OverflowDirectoryOccupancyAfterChurn, + long OverflowScanCountBeforeFirstCleanup, + long OverflowScanCountAfterFirstCleanup, + int MaxObservedOverflowScanLengthAfterFirstCleanup, + long OverflowScanCountBeforeLateWindow, + long OverflowScanCountAfterLateWindow, + int MaxObservedOverflowScanLength, + double[] EarlyMissingSamplesMicroseconds, + double[] LateMissingSamplesMicroseconds, + double LateToEarlyP99Gate, + bool DiagnosticsGatePassed, + bool LatencyGatePassed); + +internal sealed record RoleLatencyResult( + string Role, + int SampleCount, + double EarlyP99Microseconds, + double LateP99Microseconds, + double LateToEarlyP99Ratio); + +internal sealed record RunResult( + string Profile, + string Scenario, + int ProcessCount, + int ReaderProcessCount, + int PublisherProcessCount, + int ObserverProcessCount, + int Trial, + long Cycles, + long Operations, + double ApiCallsPerSecond, + double P50Microseconds, + double P95Microseconds, + double P99Microseconds, + double MaxMicroseconds, + double EarlyP99Microseconds, + double LateP99Microseconds, + double LateToEarlyP99Ratio, + RoleLatencyResult[] RoleLatency, + long Frames, + double FramesPerSecond, + long BytesWritten, + long BytesRead, + double BytesPerSecond, + long FullPayloadCopies, + long MeasuredThreadAllocatedBytes, + long Failures, + double MeasuredSeconds, + double WallSeconds, + int SampleCount, + double FairnessIndex, + double MinWorkerApiCallsPerSecond, + double MaxWorkerApiCallsPerSecond, + double WorstWorkerP99Microseconds, + int AffinityAppliedCount, + int[] AssignedProcessors, + string[] AffinityStrategies, + bool Oversubscribed, + string Qualification, + SortedDictionary StatusHistogram, + long[] WorkerCycles, + // Schema v7 is additive. FullPayloadCopies is retained above for schema + // compatibility, but this tag states whether it is an instrumented event + // counter or structural evidence and prevents a literal zero from being + // presented as a measurement. + bool FullPayloadCopyCountIsInstrumented = false, + string FullPayloadCopyEvidenceKind = "not-applicable", + long ProducerStoreOperationAllocatedBytes = 0, + string AllocationMeasurementScope = "not-applicable", + StickyOverflowEvidence? StickyOverflow = null, + int EarlySampleCount = 0, + int LateSampleCount = 0, + long OperationTarget = 0, + long FrameTarget = 0); + +internal sealed record SummaryResult( + string Profile, + string Scenario, + int ProcessCount, + double MedianApiCallsPerSecond, + double MedianP50Microseconds, + double MedianP95Microseconds, + double MedianP99Microseconds, + double MedianMaxMicroseconds, + double MedianEarlyP99Microseconds, + double MedianLateP99Microseconds, + double MedianLateToEarlyP99Ratio, + double MedianFramesPerSecond, + double MedianBytesPerSecond, + double MedianFairnessIndex, + double MedianWorstWorkerP99Microseconds, + long TotalFrames, + long TotalBytesWritten, + long TotalFullPayloadCopies, + long TotalMeasuredThreadAllocatedBytes, + long TotalFailures, + SortedDictionary StatusHistogram, + bool FullPayloadCopyCountsAreInstrumented, + string[] FullPayloadCopyEvidenceKinds, + long TotalProducerStoreOperationAllocatedBytes, + string[] AllocationMeasurementScopes); + +internal sealed record ProbeEnvironment( + string RepositoryCommit, + string RepositoryWorkingTreeState, + string SharedMemoryStoreAssemblySha256, + string ProbeAssemblySha256, + string OperatingSystem, + string OperatingSystemArchitecture, + string ProcessArchitecture, + string Framework, + string RuntimeVersion, + int LogicalProcessorCount, + int PhysicalCoreCount, + long TotalMemoryBytes, + string ProcessorModel, + string ProcessorIdentifier, + bool ServerGarbageCollection, + long StopwatchFrequency); + +internal sealed record ProbeStoreDimensions( + int SlotCount, + int MaxValueBytes, + int MaxDescriptorBytes, + int MaxKeyBytes, + int LeaseRecordCount, + int LockFreeParticipantRecordCount); + +internal sealed record ProbeConfiguration( + string Mode, + int DurationSeconds, + int DurationBoundGraceSeconds, + int Trials, + string[] Profiles, + string[] CountBoundProfiles, + string[] Scenarios, + SortedDictionary ScenarioProcessCounts, + SortedDictionary ScenarioStoreDimensions, + int ReaderKeyCount, + int ReaderPayloadBytes, + int BrokerRotatingKeyCount, + int LargeFrameBytes, + long LargeFrames, + long MixedOperationTarget, + int MixedCollisionKeyCount, + int MixedPrimaryBucketCount, + int WarmupCycles, + int WarmupSeconds, + int BrokerObserverSamplingInterval, + int SamplingInterval, + int MaxLatencySamplesPerWorker, + bool AffinityRequested, + string AffinityPolicy, + int StickyOverflowSlotCount, + int StickyOverflowChurnCycles, + int StickyOverflowMissingSamplesPerWindow, + string LegacyFullPayloadCopiesFieldSemantics, + int SyncKeysPerWorker, + int SyncMaximumWorkerCount, + int SyncCanonicalBucketCount, + string SyncKeyCatalogSha256, + int[] SyncKeyCanonicalBucketAssignments); + +internal sealed record ProbeReport( + int SchemaVersion, + DateTimeOffset TimestampUtc, + ProbeEnvironment Environment, + ProbeConfiguration Configuration, + IReadOnlyList Runs, + IReadOnlyList Summary, + int MinimumCompatibleSchemaVersion, + string SchemaCompatibility); + +internal static class ProbeReportSchema +{ + internal const int CurrentVersion = 8; + internal const int MinimumCompatibleVersion = 8; + internal const string Compatibility = + "Schema v8 requires a v8-aware qualification reader because count-bound profiles and " + + "per-run operation/frame targets distinguish durability targets from duration-bound " + + "comparison rows. Property names retained from v3-v7 support archival parsing only; " + + "their former global completion-target meaning is not qualification-compatible. " + + "New evidence tags disambiguate structural assertions from measured counters, and " + + "overflow qualification fields expose the spill/cleanup/late-window transitions; " + + "sync topology fields identify the deterministic key catalog, and early/late sample " + + "counts identify the autonomous latency reservoirs; autonomous MaxMicroseconds " + + "retains every sampled candidate even when reservoir replacement discards it; portable " + + "hardware metadata and exact per-scenario store dimensions bind benchmark topology."; + + internal const string LegacyFullPayloadCopiesFieldSemantics = + "Retained for v3-v5 readers. Consult FullPayloadCopyCountIsInstrumented and " + + "FullPayloadCopyEvidenceKind before interpreting the value as a measured event count."; +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/BoundedProcessRunner.cs b/benchmarks/SharedMemoryStore.SyncProbe/BoundedProcessRunner.cs new file mode 100644 index 0000000..07dd4ca --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/BoundedProcessRunner.cs @@ -0,0 +1,223 @@ +using System.Diagnostics; + +internal readonly record struct BoundedProcessResult( + bool Started, + bool TimedOut, + int? ExitCode, + string StandardOutput, + string StandardError) +{ + internal bool Succeeded => Started && !TimedOut && ExitCode == 0; +} + +internal static class BoundedProcessRunner +{ + private static readonly TimeSpan CleanupGrace = TimeSpan.FromSeconds(2); + + internal static BoundedProcessResult Run(ProcessStartInfo startInfo, TimeSpan timeout) + { + ArgumentNullException.ThrowIfNull(startInfo); + if (timeout <= TimeSpan.Zero || timeout.TotalMilliseconds > int.MaxValue) + { + throw new ArgumentOutOfRangeException( + nameof(timeout), + "The process timeout must be positive and no greater than Int32.MaxValue milliseconds."); + } + + if (startInfo.UseShellExecute + || !startInfo.RedirectStandardOutput + || !startInfo.RedirectStandardError) + { + throw new ArgumentException( + "Bounded processes must disable shell execution and redirect stdout and stderr.", + nameof(startInfo)); + } + + try + { + using Process? process = Process.Start(startInfo); + if (process is null) + { + return NotStarted("Process.Start returned null."); + } + + // Start both drains before waiting. A child can otherwise block on a + // full stdout or stderr pipe and never reach process exit. + Task standardOutput = process.StandardOutput.ReadToEndAsync(); + Task standardError = process.StandardError.ReadToEndAsync(); + + long operationDeadline = CreateDeadline(timeout); + bool exited = WaitForExit(process, operationDeadline); + bool drained = exited + && WaitForDrains(standardOutput, standardError, operationDeadline); + bool timedOut = !exited || !drained; + if (timedOut) + { + long cleanupDeadline = CreateDeadline(CleanupGrace); + Terminate(process); + exited = WaitForExit(process, cleanupDeadline); + drained = WaitForDrains(standardOutput, standardError, cleanupDeadline); + } + + string output = ReadCompletedDrain(standardOutput, out bool outputCompleted); + string error = ReadCompletedDrain(standardError, out bool errorCompleted); + drained &= outputCompleted && errorCompleted; + if (!drained) + { + CloseIncompleteDrain(process.StandardOutput, standardOutput); + CloseIncompleteDrain(process.StandardError, standardError); + } + + int? exitCode = exited ? TryGetExitCode(process) : null; + return new BoundedProcessResult( + Started: true, + TimedOut: timedOut || !exited || !drained, + ExitCode: exitCode, + StandardOutput: output, + StandardError: error); + } + catch (Exception exception) when ( + exception is InvalidOperationException + or System.ComponentModel.Win32Exception + or IOException + or UnauthorizedAccessException) + { + return NotStarted(exception.Message); + } + } + + private static void Terminate(Process process) + { + TryKill(process, entireProcessTree: true); + TryKill(process, entireProcessTree: false); + } + + private static void TryKill(Process process, bool entireProcessTree) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree); + } + } + catch (InvalidOperationException) + { + // The process exited between the timeout and the kill attempt. + } + catch (System.ComponentModel.Win32Exception) + { + // A whole-tree kill can be unavailable even when terminating the + // root process remains possible. The caller always attempts both. + } + } + + private static bool WaitForExit(Process process, long deadline) + { + try + { + if (process.HasExited) + { + return true; + } + + int remainingMilliseconds = GetRemainingMilliseconds(deadline); + return remainingMilliseconds > 0 + && process.WaitForExit(remainingMilliseconds); + } + catch (InvalidOperationException) + { + return false; + } + catch (System.ComponentModel.Win32Exception) + { + return false; + } + } + + private static bool WaitForDrains( + Task standardOutput, + Task standardError, + long deadline) + { + Task drains = Task.WhenAll(standardOutput, standardError); + if (!drains.IsCompleted) + { + int remainingMilliseconds = GetRemainingMilliseconds(deadline); + if (remainingMilliseconds <= 0 + || Task.WaitAny([drains], remainingMilliseconds) != 0) + { + return false; + } + } + + return standardOutput.IsCompletedSuccessfully + && standardError.IsCompletedSuccessfully; + } + + private static string ReadCompletedDrain(Task drain, out bool completed) + { + completed = drain.IsCompletedSuccessfully; + return completed ? drain.Result : string.Empty; + } + + private static void CloseIncompleteDrain(StreamReader reader, Task drain) + { + if (drain.IsCompleted) + { + return; + } + + _ = drain.ContinueWith( + static completed => _ = completed.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + try + { + reader.Dispose(); + } + catch (Exception exception) when (exception is IOException or ObjectDisposedException) + { + // The result is already fail-closed. Disposal is only best-effort + // cancellation of a descendant-held redirected pipe. + } + } + + private static int? TryGetExitCode(Process process) + { + try + { + return process.ExitCode; + } + catch (InvalidOperationException) + { + return null; + } + } + + private static long CreateDeadline(TimeSpan duration) + { + double timestampTicks = duration.TotalSeconds * Stopwatch.Frequency; + return checked(Stopwatch.GetTimestamp() + (long)Math.Ceiling(timestampTicks)); + } + + private static int GetRemainingMilliseconds(long deadline) + { + long remainingTicks = deadline - Stopwatch.GetTimestamp(); + if (remainingTicks <= 0) + { + return 0; + } + + double milliseconds = remainingTicks * 1000d / Stopwatch.Frequency; + return Math.Max(1, checked((int)Math.Ceiling(milliseconds))); + } + + private static BoundedProcessResult NotStarted(string error) => new( + Started: false, + TimedOut: false, + ExitCode: null, + StandardOutput: string.Empty, + StandardError: error); +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/HostEnvironmentProbe.cs b/benchmarks/SharedMemoryStore.SyncProbe/HostEnvironmentProbe.cs new file mode 100644 index 0000000..eafa3b3 --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/HostEnvironmentProbe.cs @@ -0,0 +1,364 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using Microsoft.Win32; + +internal readonly record struct HostHardwareInfo( + string ProcessorModel, + int LogicalProcessorCount, + int PhysicalCoreCount, + long TotalMemoryBytes); + +internal static class HostEnvironmentProbe +{ + private const int RelationProcessorCore = 0; + private const int ErrorInsufficientBuffer = 122; + + internal static HostHardwareInfo Capture() + { + int logicalProcessorCount = Environment.ProcessorCount; + string processorModel = GetProcessorModel(); + int physicalCoreCount = GetPhysicalCoreCount(); + if (physicalCoreCount <= 0 || physicalCoreCount > logicalProcessorCount) + { + physicalCoreCount = 0; + } + long totalMemoryBytes = GetTotalMemoryBytes(); + return new HostHardwareInfo( + processorModel, + logicalProcessorCount, + physicalCoreCount, + totalMemoryBytes); + } + + private static string GetProcessorModel() + { + string? value = null; + if (OperatingSystem.IsWindows()) + { + try + { + value = Registry.GetValue( + @"HKEY_LOCAL_MACHINE\HARDWARE\DESCRIPTION\System\CentralProcessor\0", + "ProcessorNameString", + null) as string; + } + catch + { + // The process environment still provides a stable processor identifier. + } + + value ??= Environment.GetEnvironmentVariable("PROCESSOR_IDENTIFIER"); + } + else if (OperatingSystem.IsLinux()) + { + value = ReadLinuxCpuModel(); + } + else if (OperatingSystem.IsMacOS()) + { + value = RunAndRead("sysctl", "-n", "machdep.cpu.brand_string") + ?? RunAndRead("sysctl", "-n", "hw.model"); + } + + return string.IsNullOrWhiteSpace(value) ? "unknown" : value.Trim(); + } + + private static string? ReadLinuxCpuModel() + { + const string CpuInfoPath = "/proc/cpuinfo"; + if (!File.Exists(CpuInfoPath)) + { + return null; + } + + foreach (string line in File.ReadLines(CpuInfoPath)) + { + int separator = line.IndexOf(':'); + if (separator <= 0) + { + continue; + } + + string key = line[..separator].Trim(); + if (!key.Equals("model name", StringComparison.OrdinalIgnoreCase) + && !key.Equals("hardware", StringComparison.OrdinalIgnoreCase) + && !key.Equals("cpu model", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + string model = line[(separator + 1)..].Trim(); + if (!string.IsNullOrWhiteSpace(model)) + { + return model; + } + } + + return null; + } + + private static int GetPhysicalCoreCount() + { + try + { + if (OperatingSystem.IsWindows()) + { + int windowsCount = GetWindowsPhysicalCoreCount(); + return windowsCount; + } + + if (OperatingSystem.IsLinux()) + { + int linuxCount = GetLinuxPhysicalCoreCount(); + return linuxCount; + } + + if (OperatingSystem.IsMacOS() + && int.TryParse( + RunAndRead("sysctl", "-n", "hw.physicalcpu"), + NumberStyles.None, + CultureInfo.InvariantCulture, + out int macCount) + && macCount > 0) + { + return macCount; + } + } + catch + { + // Qualification must fail closed rather than label a logical- + // processor fallback as a measured physical-core count. + } + + return 0; + } + + private static int GetWindowsPhysicalCoreCount() + { + uint bytes = 0; + if (GetLogicalProcessorInformationEx(RelationProcessorCore, IntPtr.Zero, ref bytes) + || Marshal.GetLastWin32Error() != ErrorInsufficientBuffer + || bytes < 8) + { + return 0; + } + + IntPtr buffer = Marshal.AllocHGlobal(checked((int)bytes)); + try + { + if (!GetLogicalProcessorInformationEx(RelationProcessorCore, buffer, ref bytes)) + { + return 0; + } + + int count = 0; + int offset = 0; + while (offset + 8 <= bytes) + { + int relationship = Marshal.ReadInt32(buffer, offset); + int size = Marshal.ReadInt32(buffer, offset + 4); + if (size < 8 || offset + size > bytes) + { + return 0; + } + + if (relationship == RelationProcessorCore) + { + count++; + } + + offset += size; + } + + return offset == bytes ? count : 0; + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + + private static int GetLinuxPhysicalCoreCount() + { + const string CpuRoot = "/sys/devices/system/cpu"; + if (!Directory.Exists(CpuRoot)) + { + return 0; + } + + HashSet? available = ReadLinuxAllowedCpuSet(); + var cores = new HashSet<(int Package, int Core)>(); + foreach (string directory in Directory.EnumerateDirectories(CpuRoot, "cpu*")) + { + string suffix = Path.GetFileName(directory).AsSpan(3).ToString(); + if (!int.TryParse(suffix, NumberStyles.None, CultureInfo.InvariantCulture, out int cpu) + || cpu < 0 + || (available is not null && !available.Contains(cpu))) + { + continue; + } + + string packagePath = Path.Combine(directory, "topology", "physical_package_id"); + string corePath = Path.Combine(directory, "topology", "core_id"); + if (!int.TryParse(File.ReadAllText(packagePath).Trim(), out int package) + || !int.TryParse(File.ReadAllText(corePath).Trim(), out int core)) + { + return 0; + } + + cores.Add((package, core)); + } + + return cores.Count; + } + + private static HashSet? ReadLinuxAllowedCpuSet() + { + const string StatusPath = "/proc/self/status"; + try + { + string? value = File.ReadLines(StatusPath) + .FirstOrDefault(static line => line.StartsWith("Cpus_allowed_list:", StringComparison.Ordinal)); + if (value is null) + { + return null; + } + + var cpus = new HashSet(); + foreach (string item in value[(value.IndexOf(':') + 1)..].Split(',', StringSplitOptions.TrimEntries)) + { + string[] limits = item.Split('-', 2, StringSplitOptions.TrimEntries); + if (!int.TryParse(limits[0], NumberStyles.None, CultureInfo.InvariantCulture, out int first) + || first < 0 + || (limits.Length == 2 + && (!int.TryParse(limits[1], NumberStyles.None, CultureInfo.InvariantCulture, out int last) + || last < first))) + { + return null; + } + + int final = limits.Length == 1 + ? first + : int.Parse(limits[1], NumberStyles.None, CultureInfo.InvariantCulture); + for (int cpu = first; cpu <= final; cpu++) + { + cpus.Add(cpu); + if (cpu == int.MaxValue) + { + break; + } + } + } + + return cpus.Count == 0 ? null : cpus; + } + catch + { + return null; + } + } + + private static long GetTotalMemoryBytes() + { + try + { + if (OperatingSystem.IsWindows()) + { + var status = new MemoryStatusEx + { + Length = (uint)Marshal.SizeOf() + }; + if (GlobalMemoryStatusEx(ref status) && status.TotalPhysical <= long.MaxValue) + { + return (long)status.TotalPhysical; + } + } + else if (OperatingSystem.IsLinux()) + { + foreach (string line in File.ReadLines("/proc/meminfo")) + { + if (!line.StartsWith("MemTotal:", StringComparison.Ordinal)) + { + continue; + } + + string digits = new(line.Where(char.IsDigit).ToArray()); + if (long.TryParse(digits, NumberStyles.None, CultureInfo.InvariantCulture, out long kibibytes) + && kibibytes > 0) + { + return checked(kibibytes * 1024); + } + } + } + else if (OperatingSystem.IsMacOS() + && long.TryParse( + RunAndRead("sysctl", "-n", "hw.memsize"), + NumberStyles.None, + CultureInfo.InvariantCulture, + out long macBytes) + && macBytes > 0) + { + return macBytes; + } + } + catch + { + // Qualification must fail closed rather than label a process + // memory bound as total host memory. + } + + return 0; + } + + private static string? RunAndRead(string fileName, params string[] arguments) + { + try + { + var start = new ProcessStartInfo(fileName) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + foreach (string argument in arguments) + { + start.ArgumentList.Add(argument); + } + + BoundedProcessResult result = BoundedProcessRunner.Run( + start, + TimeSpan.FromSeconds(2)); + return result.Succeeded ? result.StandardOutput.Trim() : null; + } + catch + { + return null; + } + } + + [StructLayout(LayoutKind.Sequential)] + private struct MemoryStatusEx + { + internal uint Length; + internal uint MemoryLoad; + internal ulong TotalPhysical; + internal ulong AvailablePhysical; + internal ulong TotalPageFile; + internal ulong AvailablePageFile; + internal ulong TotalVirtual; + internal ulong AvailableVirtual; + internal ulong AvailableExtendedVirtual; + } + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GetLogicalProcessorInformationEx( + int relationshipType, + IntPtr buffer, + ref uint returnedLength); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GlobalMemoryStatusEx(ref MemoryStatusEx buffer); +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/ProbeCompletionTargetPolicy.cs b/benchmarks/SharedMemoryStore.SyncProbe/ProbeCompletionTargetPolicy.cs new file mode 100644 index 0000000..a4467c4 --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/ProbeCompletionTargetPolicy.cs @@ -0,0 +1,33 @@ +using SharedMemoryStore; + +internal readonly record struct ProbeRunTargets(long OperationTarget, long FrameTarget) +{ + internal bool HasCountTarget => OperationTarget > 0 || FrameTarget > 0; +} + +internal static class ProbeCompletionTargetPolicy +{ + internal static ProbeRunTargets Resolve( + StoreProfile profile, + ProbeScenarioKind scenarioKind, + IReadOnlyCollection countBoundProfiles, + long mixedOperationTarget, + long largeFrameTarget) + { + ArgumentNullException.ThrowIfNull(countBoundProfiles); + ArgumentOutOfRangeException.ThrowIfNegative(mixedOperationTarget); + ArgumentOutOfRangeException.ThrowIfNegative(largeFrameTarget); + + if (!countBoundProfiles.Contains(profile)) + { + return default; + } + + return scenarioKind switch + { + ProbeScenarioKind.MixedChurn => new ProbeRunTargets(mixedOperationTarget, 0), + ProbeScenarioKind.LargeIngest => new ProbeRunTargets(0, largeFrameTarget), + _ => default + }; + } +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/ProbeProcessRegistry.cs b/benchmarks/SharedMemoryStore.SyncProbe/ProbeProcessRegistry.cs new file mode 100644 index 0000000..6bcda1e --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/ProbeProcessRegistry.cs @@ -0,0 +1,33 @@ +using System.Diagnostics; + +internal sealed class ProbeProcessRegistry +{ + private readonly object _gate = new(); + private readonly List _processes = []; + private bool _accepting = true; + + internal Process Start(Func start) + { + ArgumentNullException.ThrowIfNull(start); + lock (_gate) + { + if (!_accepting) + { + throw new InvalidOperationException("The probe trial is already terminating."); + } + + Process process = start(); + _processes.Add(process); + return process; + } + } + + internal Process[] StopAcceptingAndSnapshot() + { + lock (_gate) + { + _accepting = false; + return _processes.ToArray(); + } + } +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/ProbeTrialWatchdog.cs b/benchmarks/SharedMemoryStore.SyncProbe/ProbeTrialWatchdog.cs new file mode 100644 index 0000000..f187f44 --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/ProbeTrialWatchdog.cs @@ -0,0 +1,132 @@ +using System.Diagnostics; + +internal sealed class ProbeTrialWatchdog : IAsyncDisposable +{ + private const int Armed = 0; + private const int Completing = 1; + private const int Completed = 2; + private const int TimedOut = 3; + + private readonly long _deadlineTimestamp; + private readonly Action? _afterCompletionLatch; + private readonly Action _timeoutAction; + private readonly Timer _timer; + private int _state; + + internal ProbeTrialWatchdog( + TimeSpan deadline, + Action timeoutAction, + TimeSpan? timerDueTime = null, + Action? afterCompletionLatch = null) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(deadline, TimeSpan.Zero); + ArgumentNullException.ThrowIfNull(timeoutAction); + TimeSpan dueTime = timerDueTime ?? deadline; + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(dueTime, TimeSpan.Zero); + + long deadlineStopwatchTicks = checked((long)Math.Ceiling( + deadline.TotalSeconds * Stopwatch.Frequency)); + _deadlineTimestamp = checked(Stopwatch.GetTimestamp() + deadlineStopwatchTicks); + _afterCompletionLatch = afterCompletionLatch; + _timeoutAction = timeoutAction; + _timer = new Timer( + static state => ((ProbeTrialWatchdog)state!).OnTimer(), + this, + dueTime, + Timeout.InfiniteTimeSpan); + } + + internal async ValueTask CompleteAsync() + { + int previous = Interlocked.CompareExchange(ref _state, Completing, Armed); + if (previous == Armed) + { + _afterCompletionLatch?.Invoke(); + if (Stopwatch.GetTimestamp() < _deadlineTimestamp) + { + if (Interlocked.CompareExchange(ref _state, Completed, Completing) == Completing) + { + await _timer.DisposeAsync().ConfigureAwait(false); + return; + } + } + else + { + _ = Interlocked.CompareExchange(ref _state, TimedOut, Completing); + } + } + else if (previous == Completed) + { + await _timer.DisposeAsync().ConfigureAwait(false); + return; + } + + await FailTimedOutCompletionAsync().ConfigureAwait(false); + } + + public ValueTask DisposeAsync() => _timer.DisposeAsync(); + + private async ValueTask FailTimedOutCompletionAsync() + { + try + { + // Invoke on the completing thread too. This closes the race where the + // timer committed TimedOut but its ThreadPool callback was preempted + // before reaching the process-termination action. + _timeoutAction(); + } + finally + { + await _timer.DisposeAsync().ConfigureAwait(false); + } + + throw new TimeoutException("The probe trial timeout action returned without terminating the process."); + } + + private void OnTimer() + { + long now = Stopwatch.GetTimestamp(); + if (now < _deadlineTimestamp) + { + int state = Volatile.Read(ref _state); + if (state is Completed or TimedOut) + { + return; + } + + long remainingStopwatchTicks = _deadlineTimestamp - now; + double remainingSeconds = (double)remainingStopwatchTicks / Stopwatch.Frequency; + try + { + _timer.Change(TimeSpan.FromSeconds(remainingSeconds), Timeout.InfiniteTimeSpan); + } + catch (ObjectDisposedException) + { + // Normal completion won the race and disposed the timer. + } + return; + } + + if (TryBeginTimeout()) + { + _timeoutAction(); + } + } + + private bool TryBeginTimeout() + { + while (true) + { + int state = Volatile.Read(ref _state); + if (state is Completed or TimedOut) + { + return false; + } + + if (Interlocked.CompareExchange(ref _state, TimedOut, state) == state) + { + return true; + } + } + } +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/Program.cs b/benchmarks/SharedMemoryStore.SyncProbe/Program.cs new file mode 100644 index 0000000..e109918 --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/Program.cs @@ -0,0 +1,3068 @@ +using System.Buffers.Binary; +using System.Diagnostics; +using System.Globalization; +using System.Runtime; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using SharedMemoryStore; +using SharedMemoryStore.LockFree; + +using Store = SharedMemoryStore.MemoryStore; + +const int SyncSlotCount = 32; +const int ReaderSlotCount = 256; +const int MixedSlotCount = 768; +const int BrokerSlotCount = 256; +const int DefaultStickyOverflowSlotCount = 4_096; +const int SyncValueBytes = 8; +const int ReaderPayloadBytes = 256; +const int MixedPayloadBytes = 256; +const int DefaultLargeFrameBytes = 1_363_148; +const int BrokerRotatingKeyCount = 256; +const int MixedCollisionKeyCount = 512; +const int StickyOverflowPublishedCollisionKeyCount = 17; +const int StickyOverflowProbeKeyCount = StickyOverflowPublishedCollisionKeyCount + 1; +const int DefaultStickyOverflowChurnCycles = 10_000; +const int DefaultStickyOverflowMissingSamplesPerWindow = 16_384; +const double StickyOverflowLateToEarlyP99Gate = 2.0; +const int BenchmarkDescriptorBytes = 16; +const int MaxKeyBytes = 8; +const int DefaultLeaseRecordCount = 64; +const int MixedLeaseRecordCount = 128; +const int ParticipantRecordCount = 64; +const int ReaderKeyCount = 256; +const int SyncKeysPerWorker = 2; +const int SyncMaximumWorkerCount = 12; +const int DefaultShortWarmupSeconds = 2; +const int ReleaseWarmupSeconds = 10; +const int BrokerObserverSamplingInterval = 16; +const int SamplingInterval = 64; +const int MaxLatencySamplesPerWorker = 65_536; +const int LatencyReservoirCapacityPerWindow = MaxLatencySamplesPerWorker / 2; +const int DefaultDurationSeconds = 3; +const int DefaultDurationBoundGraceSeconds = 60; +const int DefaultTrials = 3; +const int TrialHeartbeatSeconds = 30; +const int WatchdogChildKillBudgetMilliseconds = 100; + +if (args.Length > 0 && string.Equals(args[0], "worker", StringComparison.Ordinal)) +{ + return RunAutonomousWorker(args); +} + +if (args.Length > 0 && string.Equals(args[0], "broker-worker", StringComparison.Ordinal)) +{ + return await RunBrokerWorker(args); +} + +if (args.Length > 0 && string.Equals(args[0], "suspension-worker", StringComparison.Ordinal)) +{ + return SuspensionQualification.RunWorker(args); +} + +return await RunController(args); + +static async Task RunController(string[] args) +{ + AssertLatencyReservoirMaximumSemantics(); + int durationSeconds = ReadPositiveIntOption(args, "--duration", DefaultDurationSeconds); + int durationBoundGraceSeconds = ReadPositiveIntOption( + args, + "--duration-bound-grace", + DefaultDurationBoundGraceSeconds); + int trials = ReadPositiveIntOption(args, "--trials", DefaultTrials); + string? outputPath = ReadStringOption(args, "--output"); + string mode = (ReadStringOption(args, "--mode") ?? "sync").ToLowerInvariant(); + if (mode == "suspension") + { + return await SuspensionQualification.RunControllerAsync(args); + } + int warmupSeconds = ReadNonNegativeIntOption( + args, + "--warmup", + mode == "full" ? ReleaseWarmupSeconds : DefaultShortWarmupSeconds); + StoreProfile[] profiles = ParseProfiles( + ReadStringOption(args, "--profile") ?? (mode == "overflow" ? "v2" : "legacy"), + "--profile"); + StoreProfile[] countBoundProfiles = ParseProfiles( + ReadStringOption(args, "--count-bound-profiles") ?? "both", + "--count-bound-profiles"); + bool affinityRequested = !args.Contains("--no-affinity", StringComparer.Ordinal); + ScenarioPlan[] plans = ProbeScenarioCatalog.Select(mode); + string? scenarioFilter = ReadStringOption(args, "--scenario"); + if (!string.IsNullOrWhiteSpace(scenarioFilter)) + { + string[] requestedScenarios = scenarioFilter + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + plans = plans + .Where(plan => requestedScenarios.Contains(plan.Name, StringComparer.OrdinalIgnoreCase)) + .ToArray(); + if (plans.Length == 0) + { + throw new ArgumentException("--scenario did not match a scenario in the selected mode."); + } + } + + int[]? requestedProcessCounts = ParsePositiveIntListOption(args, "--process-counts"); + if (requestedProcessCounts is not null) + { + plans = plans + .Select(plan => plan with + { + ProcessCounts = plan.ProcessCounts + .Where(requestedProcessCounts.Contains) + .ToArray() + }) + .Where(static plan => plan.ProcessCounts.Length != 0) + .ToArray(); + if (plans.Length == 0) + { + throw new ArgumentException("--process-counts did not match the selected workload matrix."); + } + } + int largeFrameBytes = ReadPositiveIntOption( + args, + "--large-frame-bytes", + ReadPositiveIntOption(args, "--frame-bytes", DefaultLargeFrameBytes)); + long defaultLargeFrames = mode == "full" ? 100_000L : 256L; + long largeFrames = ReadPositiveLongOption( + args, + "--large-frames", + ReadPositiveLongEnvironment("SMS_LOCK_FREE_LARGE_FRAMES", defaultLargeFrames)); + long defaultMixedTarget = mode == "full" ? 100_000_000L : 0L; + long mixedOperationTarget = ReadNonNegativeLongOption( + args, + "--mixed-operations", + ReadNonNegativeLongOption( + args, + "--churn-operations", + ReadNonNegativeLongEnvironment("SMS_LOCK_FREE_CHURN_OPERATIONS", defaultMixedTarget))); + int stickyOverflowSlotCount = ReadPositivePowerOfTwoOption( + args, + "--overflow-slot-count", + DefaultStickyOverflowSlotCount); + int stickyOverflowChurnCycles = ReadPositiveIntOption( + args, + "--overflow-churn-cycles", + DefaultStickyOverflowChurnCycles); + int stickyOverflowMissingSamples = ReadPositiveIntOption( + args, + "--overflow-missing-samples", + DefaultStickyOverflowMissingSamplesPerWindow); + bool includesStickyOverflow = plans.Any( + static plan => plan.Kind == ProbeScenarioKind.StickyOverflow); + if (includesStickyOverflow && !profiles.Contains(StoreProfile.LockFree)) + { + throw new ArgumentException("The sticky-overflow qualification requires --profile v2 or both."); + } + + BucketPairCollisionSet stickyOverflowKeys = includesStickyOverflow + ? BenchmarkProtocol.CreateBucketPairCollisionKeys( + StickyOverflowProbeKeyCount, + BenchmarkProtocol.CalculatePrimaryBucketCount(stickyOverflowSlotCount), + firstBucket: 0, + secondBucket: 1) + : default; + + RepositoryProvenanceSnapshot repositoryProvenance = RepositoryEnvironmentProbe.Capture(args); + ProbeEnvironment probeEnvironment = CaptureEnvironment(repositoryProvenance); + + var runs = new List(); + foreach (StoreProfile profile in profiles) + { + foreach (ScenarioPlan plan in plans) + { + if (plan.Kind == ProbeScenarioKind.StickyOverflow && profile != StoreProfile.LockFree) + { + continue; + } + + foreach (int processCount in plan.ProcessCounts) + { + for (var trial = 1; trial <= trials; trial++) + { + ProbeRunTargets targets = ProbeCompletionTargetPolicy.Resolve( + profile, + plan.Kind, + countBoundProfiles, + mixedOperationTarget, + largeFrames); + long runOperationTarget = targets.OperationTarget; + long runFrameTarget = targets.FrameTarget; + Console.Error.WriteLine( + $"trial-start profile={profile} scenario={plan.Name} processes={processCount} " + + $"trial={trial}/{trials} limit=" + + (plan.Kind == ProbeScenarioKind.StickyOverflow + ? $"fixed-work:churn-cycles:{stickyOverflowChurnCycles}" + : runOperationTarget > 0 + ? $"operations:{runOperationTarget}" + : runFrameTarget > 0 + ? $"frames:{runFrameTarget}" + : $"duration-seconds:{durationSeconds}")); + using var heartbeatCancellation = new CancellationTokenSource(); + Task heartbeat = ReportTrialHeartbeats( + profile, + plan.Name, + processCount, + trial, + trials, + plan.Kind, + targets, + heartbeatCancellation.Token); + RunResult result; + try + { + result = plan.Kind switch + { + ProbeScenarioKind.Autonomous => await RunAutonomousTrial( + profile, + plan, + processCount, + trial, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds, + affinityRequested, + operationTarget: 0), + ProbeScenarioKind.MixedChurn => await RunMixedTrial( + profile, + plan, + processCount, + trial, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds, + affinityRequested, + runOperationTarget), + ProbeScenarioKind.BrokerDirected => await RunBrokerTrial( + profile, + plan, + processCount, + trial, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds, + affinityRequested, + largeFrameBytes, + frameTarget: 0), + ProbeScenarioKind.LargeIngest => await RunBrokerTrial( + profile, + plan, + processCount, + trial, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds, + affinityRequested, + largeFrameBytes, + runFrameTarget), + ProbeScenarioKind.StickyOverflow => RunStickyOverflowTrial( + profile, + plan, + trial, + affinityRequested, + stickyOverflowSlotCount, + stickyOverflowChurnCycles, + stickyOverflowMissingSamples, + stickyOverflowKeys), + _ => throw new ArgumentOutOfRangeException(nameof(plan.Kind)) + }; + } + finally + { + await heartbeatCancellation.CancelAsync(); + await heartbeat; + } + + runs.Add(result); + Console.Error.WriteLine( + $"{profile,-8} {plan.Name,-20} processes={processCount,2} trial={trial} " + + $"calls/s={result.ApiCallsPerSecond:N0} frames/s={result.FramesPerSecond:N2} " + + $"p50={result.P50Microseconds:N3}us p95={result.P95Microseconds:N3}us " + + $"p99={result.P99Microseconds:N3}us late/early={result.LateToEarlyP99Ratio:N3} " + + $"failures={result.Failures} affinity={result.AffinityAppliedCount}/" + + $"{result.ReaderProcessCount + result.PublisherProcessCount + result.ObserverProcessCount} " + + $"qualification={result.Qualification}"); + } + } + } + } + + var scenarioCounts = new SortedDictionary(StringComparer.Ordinal); + foreach (ScenarioPlan plan in plans) + { + scenarioCounts[plan.Name] = plan.ProcessCounts; + } + + int syncCanonicalBucketCount = BenchmarkProtocol.CalculatePrimaryBucketCount(SyncSlotCount); + BenchmarkKeyCatalog syncKeyCatalog = BenchmarkProtocol.CreateCanonicalBucketKeyCatalog( + SyncKeysPerWorker, + SyncMaximumWorkerCount, + syncCanonicalBucketCount); + int[] syncKeyCanonicalBucketAssignments = Enumerable.Range(0, syncKeyCatalog.Count) + .Select(index => BenchmarkProtocol.GetCanonicalBucket( + syncKeyCatalog[index].Span, + syncCanonicalBucketCount)) + .ToArray(); + + var report = new ProbeReport( + SchemaVersion: ProbeReportSchema.CurrentVersion, + TimestampUtc: DateTimeOffset.UtcNow, + Environment: probeEnvironment, + Configuration: new ProbeConfiguration( + mode, + durationSeconds, + durationBoundGraceSeconds, + trials, + profiles.Select(static profile => profile.ToString()).ToArray(), + countBoundProfiles.Select(static profile => profile.ToString()).ToArray(), + plans.Select(static plan => plan.Name).ToArray(), + scenarioCounts, + CreateScenarioStoreDimensions(plans, largeFrameBytes, stickyOverflowSlotCount), + ReaderKeyCount, + ReaderPayloadBytes, + BrokerRotatingKeyCount, + largeFrameBytes, + largeFrames, + mixedOperationTarget, + MixedCollisionKeyCount, + BenchmarkProtocol.CalculatePrimaryBucketCount(MixedSlotCount), + WarmupCycles: 0, + warmupSeconds, + BrokerObserverSamplingInterval, + SamplingInterval, + MaxLatencySamplesPerWorker, + affinityRequested, + "physical-core-first-then-siblings", + stickyOverflowSlotCount, + stickyOverflowChurnCycles, + stickyOverflowMissingSamples, + ProbeReportSchema.LegacyFullPayloadCopiesFieldSemantics, + SyncKeysPerWorker, + SyncMaximumWorkerCount, + syncCanonicalBucketCount, + syncKeyCatalog.CalculateSha256(), + syncKeyCanonicalBucketAssignments), + Runs: runs, + Summary: Summarize(runs), + MinimumCompatibleSchemaVersion: ProbeReportSchema.MinimumCompatibleVersion, + SchemaCompatibility: ProbeReportSchema.Compatibility); + + string json = JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true }); + if (!string.IsNullOrWhiteSpace(outputPath)) + { + string fullPath = Path.GetFullPath(outputPath); + string? directory = Path.GetDirectoryName(fullPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + await File.WriteAllTextAsync(fullPath, json); + Console.Error.WriteLine($"report={fullPath}"); + } + else + { + Console.WriteLine(json); + } + + if (runs.Any(static run => run.Failures != 0)) + { + return 2; + } + + if (runs.Any(static run => run.StickyOverflow is { DiagnosticsGatePassed: false })) + { + return 3; + } + + return runs.Any(static run => run.StickyOverflow is { LatencyGatePassed: false }) ? 4 : 0; +} + +static async Task RunAutonomousTrial( + StoreProfile profile, + ScenarioPlan plan, + int processCount, + int trial, + int durationSeconds, + int warmupSeconds, + int durationBoundGraceSeconds, + bool affinityRequested, + long operationTarget) +{ + string name = $"sms-sync-{Guid.NewGuid():N}"; + var workers = new List(processCount); + var processRegistry = new ProbeProcessRegistry(); + ProbeTrialWatchdog? trialWatchdog = CreateTrialWatchdog( + plan.Name, + profile, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds, + operationTarget, + frameTarget: 0, + processRegistry); + try + { + StoreOpenStatus openStatus = Store.TryCreateOrOpen( + Options(name, OpenMode.CreateNew, profile, plan.Name, payloadBytes: 0), + out Store? owner); + if (openStatus != StoreOpenStatus.Success || owner is null) + { + throw new InvalidOperationException($"Owner open failed for {profile}: {openStatus}"); + } + + using (owner) + { + Seed(owner, plan.Name, processCount); + try + { + for (var workerId = 0; workerId < processCount; workerId++) + { + workers.Add(processRegistry.Start(() => StartAutonomousWorker( + plan.Name, + profile, + name, + workerId, + durationSeconds, + warmupSeconds, + affinityRequested ? workerId : -1, + operationTarget, + payloadBytes: 0))); + } + + await AwaitReady(workers, CancellationToken.None); + var wall = Stopwatch.StartNew(); + await SignalGo(workers, CancellationToken.None); + List results = await CollectWorkerResults(workers, CancellationToken.None); + wall.Stop(); + return AggregateAutonomousRun( + profile, + plan, + processCount, + trial, + warmupSeconds, + wall.Elapsed, + results, + operationTarget); + } + finally + { + DisposeProcesses(workers); + } + } + } + finally + { + if (trialWatchdog is not null) + { + await trialWatchdog.CompleteAsync(); + } + } +} + +static async Task RunMixedTrial( + StoreProfile profile, + ScenarioPlan plan, + int readerCount, + int trial, + int durationSeconds, + int warmupSeconds, + int durationBoundGraceSeconds, + bool affinityRequested, + long totalOperationTarget) +{ + string name = $"sms-churn-{Guid.NewGuid():N}"; + int participantCount = readerCount + plan.PublisherCount; + var workers = new List(participantCount); + var processRegistry = new ProbeProcessRegistry(); + ProbeTrialWatchdog? trialWatchdog = CreateTrialWatchdog( + plan.Name, + profile, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds, + totalOperationTarget, + frameTarget: 0, + processRegistry); + try + { + StoreOpenStatus openStatus = Store.TryCreateOrOpen( + Options(name, OpenMode.CreateNew, profile, plan.Name, MixedPayloadBytes), + out Store? owner); + if (openStatus != StoreOpenStatus.Success || owner is null) + { + throw new InvalidOperationException($"Mixed-churn owner open failed for {profile}: {openStatus}"); + } + + using (owner) + { + SeedMixed(owner); + long perWorkerOperationTarget = totalOperationTarget <= 0 + ? 0 + : checked((totalOperationTarget + participantCount - 1) / participantCount); + try + { + for (var readerId = 0; readerId < readerCount; readerId++) + { + workers.Add(processRegistry.Start(() => StartAutonomousWorker( + "mixed-churn-reader", + profile, + name, + readerId, + durationSeconds, + warmupSeconds, + affinityRequested ? readerId : -1, + perWorkerOperationTarget, + MixedPayloadBytes))); + } + + for (var publisherId = 0; publisherId < plan.PublisherCount; publisherId++) + { + workers.Add(processRegistry.Start(() => StartAutonomousWorker( + "mixed-churn-writer", + profile, + name, + publisherId, + durationSeconds, + warmupSeconds, + affinityRequested ? readerCount + publisherId : -1, + perWorkerOperationTarget, + MixedPayloadBytes))); + } + + await AwaitReady(workers, CancellationToken.None); + var wall = Stopwatch.StartNew(); + await SignalGo(workers, CancellationToken.None); + List results = await CollectWorkerResults(workers, CancellationToken.None); + wall.Stop(); + RunResult aggregate = AggregateAutonomousRun( + profile, + plan, + readerCount, + trial, + warmupSeconds, + wall.Elapsed, + results, + totalOperationTarget); + if (totalOperationTarget > 0 && aggregate.Operations < totalOperationTarget) + { + throw new InvalidOperationException( + $"Mixed-churn operation target was not met: {aggregate.Operations:N0} < {totalOperationTarget:N0}."); + } + + return aggregate; + } + finally + { + DisposeProcesses(workers); + } + } + } + finally + { + if (trialWatchdog is not null) + { + await trialWatchdog.CompleteAsync(); + } + } +} + +static RunResult RunStickyOverflowTrial( + StoreProfile profile, + ScenarioPlan plan, + int trial, + bool affinityRequested, + int slotCount, + int churnCycles, + int missingSamplesPerWindow, + BucketPairCollisionSet collisionSet) +{ + if (profile != StoreProfile.LockFree + || collisionSet.Keys.Length != StickyOverflowProbeKeyCount) + { + throw new InvalidOperationException("Sticky-overflow qualification requires the lock-free profile and exact collision keys."); + } + + string name = $"sms-sticky-overflow-{Guid.NewGuid():N}"; + StoreOpenStatus openStatus = Store.TryCreateOrOpen( + StickyOverflowOptions(name, slotCount), + out Store? store); + if (openStatus != StoreOpenStatus.Success || store is null) + { + throw new InvalidOperationException($"Sticky-overflow store open failed: {openStatus}"); + } + + using TemporaryProcessAffinity affinity = TemporaryProcessAffinity.Apply( + affinityRequested ? 0 : -1); + using (store) + { + byte[][] publishedKeys = collisionSet.Keys[..StickyOverflowPublishedCollisionKeyCount]; + byte[] missingKey = collisionSet.Keys[^1]; + var counters = new StatusCounters(); + var earlySamples = new double[missingSamplesPerWindow]; + var lateSamples = new double[missingSamplesPerWindow]; + Span payload = stackalloc byte[1]; + long failures = 0; + + // JIT and warm the exact normal-miss path before either latency window. + for (var index = 0; index < Math.Min(1_024, missingSamplesPerWindow); index++) + { + StoreStatus warmStatus = store.TryAcquire( + missingKey, + StoreWaitOptions.Infinite, + out ValueLease warmLease); + if (warmStatus == StoreStatus.Success) + { + failures++; + _ = warmLease.Release(StoreWaitOptions.Infinite); + } + else if (warmStatus != StoreStatus.NotFound) + { + failures++; + } + } + + StoreStatus beforeChurnStatus = store.TryGetDiagnostics( + StoreWaitOptions.Infinite, + out DiagnosticsSnapshot beforeChurn); + if (beforeChurnStatus != StoreStatus.Success) + { + failures++; + } + long allocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + var measured = Stopwatch.StartNew(); + failures += MeasureMissingWindow(store, missingKey, earlySamples, counters); + + DiagnosticsSnapshot duringFirstSpill = default; + DiagnosticsSnapshot afterFirstCleanup = default; + for (var cycle = 0; cycle < churnCycles; cycle++) + { + for (var keyIndex = 0; keyIndex < publishedKeys.Length; keyIndex++) + { + payload[0] = unchecked((byte)(cycle + keyIndex)); + StoreStatus publish = PublishWithRetry( + store, + publishedKeys[keyIndex], + payload, + counters); + if (publish != StoreStatus.Success) + { + failures++; + } + } + + if (cycle == 0) + { + StoreStatus diagnosticsStatus = store.TryGetDiagnostics( + StoreWaitOptions.Infinite, + out DiagnosticsSnapshot duringChurn); + if (diagnosticsStatus != StoreStatus.Success) + { + failures++; + } + else + { + duringFirstSpill = duringChurn; + } + } + + for (var keyIndex = 0; keyIndex < publishedKeys.Length; keyIndex++) + { + StoreStatus remove = RemoveWithRetry(store, publishedKeys[keyIndex], counters); + if (remove is not (StoreStatus.Success or StoreStatus.RemovePending)) + { + failures++; + } + } + + if (cycle == 0) + { + StoreStatus diagnosticsStatus = store.TryGetDiagnostics( + StoreWaitOptions.Infinite, + out DiagnosticsSnapshot afterCleanup); + if (diagnosticsStatus != StoreStatus.Success) + { + failures++; + } + else + { + afterFirstCleanup = afterCleanup; + } + } + } + + StoreStatus beforeLateStatus = store.TryGetDiagnostics( + StoreWaitOptions.Infinite, + out DiagnosticsSnapshot beforeLate); + if (beforeLateStatus != StoreStatus.Success) + { + failures++; + } + + failures += MeasureMissingWindow(store, missingKey, lateSamples, counters); + measured.Stop(); + long measuredThreadAllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - allocatedBefore; + + StoreStatus afterLateStatus = store.TryGetDiagnostics( + StoreWaitOptions.Infinite, + out DiagnosticsSnapshot afterLate); + if (afterLateStatus != StoreStatus.Success) + { + failures++; + } + + double[] sortedEarly = earlySamples.Order().ToArray(); + double[] sortedLate = lateSamples.Order().ToArray(); + double earlyP99 = Percentile(sortedEarly, 0.99); + double lateP99 = Percentile(sortedLate, 0.99); + double lateToEarlyRatio = earlyP99 == 0 ? 0 : lateP99 / earlyP99; + bool diagnosticsGatePassed = beforeChurn.SpilledBucketCount == 0 + && duringFirstSpill.SpilledBucketCount > 0 + && duringFirstSpill.OverflowDirectoryOccupancy > 0 + && afterFirstCleanup.SpilledBucketCount == 0 + && afterFirstCleanup.OverflowDirectoryOccupancy == 0 + && afterFirstCleanup.OverflowScanCount > duringFirstSpill.OverflowScanCount + && afterFirstCleanup.MaxObservedOverflowScanLength >= slotCount + && beforeLate.SpilledBucketCount == 0 + && beforeLate.OverflowDirectoryOccupancy == 0 + && beforeLate.OverflowScanCount > beforeChurn.OverflowScanCount + && afterLate.OverflowScanCount == beforeLate.OverflowScanCount; + bool latencyGatePassed = earlyP99 > 0 + && lateP99 > 0 + && lateToEarlyRatio <= StickyOverflowLateToEarlyP99Gate; + string qualification = !diagnosticsGatePassed + ? "qualification-failed-overflow-diagnostics" + : latencyGatePassed + ? "qualification-passed-versioned-overflow-cleanup" + : "qualification-failed-versioned-overflow-latency"; + double measuredSeconds = Math.Max(measured.Elapsed.TotalSeconds, 0.000_001); + double[] allSamples = [.. earlySamples, .. lateSamples]; + Array.Sort(allSamples); + var evidence = new StickyOverflowEvidence( + SlotCount: slotCount, + PrimaryBucketCount: BenchmarkProtocol.CalculatePrimaryBucketCount(slotCount), + ExactBucketPairCollisionKeyCount: StickyOverflowPublishedCollisionKeyCount, + CollisionCandidatesExamined: collisionSet.CandidatesExamined, + ChurnCycles: churnCycles, + MissingSamplesPerWindow: missingSamplesPerWindow, + SpilledBucketCountBeforeChurn: beforeChurn.SpilledBucketCount, + SpilledBucketCountDuringChurn: duringFirstSpill.SpilledBucketCount, + OverflowDirectoryOccupancyDuringChurn: duringFirstSpill.OverflowDirectoryOccupancy, + SpilledBucketCountAfterFirstCleanup: afterFirstCleanup.SpilledBucketCount, + OverflowDirectoryOccupancyAfterFirstCleanup: afterFirstCleanup.OverflowDirectoryOccupancy, + SpilledBucketCountAfterChurn: beforeLate.SpilledBucketCount, + OverflowDirectoryOccupancyAfterChurn: beforeLate.OverflowDirectoryOccupancy, + OverflowScanCountBeforeFirstCleanup: duringFirstSpill.OverflowScanCount, + OverflowScanCountAfterFirstCleanup: afterFirstCleanup.OverflowScanCount, + MaxObservedOverflowScanLengthAfterFirstCleanup: afterFirstCleanup.MaxObservedOverflowScanLength, + OverflowScanCountBeforeLateWindow: beforeLate.OverflowScanCount, + OverflowScanCountAfterLateWindow: afterLate.OverflowScanCount, + MaxObservedOverflowScanLength: afterLate.MaxObservedOverflowScanLength, + EarlyMissingSamplesMicroseconds: earlySamples, + LateMissingSamplesMicroseconds: lateSamples, + LateToEarlyP99Gate: StickyOverflowLateToEarlyP99Gate, + DiagnosticsGatePassed: diagnosticsGatePassed, + LatencyGatePassed: latencyGatePassed); + + long bytesWritten = checked((long)churnCycles * publishedKeys.Length); + long operations = counters.TotalOperations; + return new RunResult( + profile.ToString(), + plan.Name, + ProcessCount: 1, + ReaderProcessCount: 0, + PublisherProcessCount: 1, + ObserverProcessCount: 0, + trial, + Cycles: churnCycles, + operations, + operations / measuredSeconds, + Percentile(allSamples, 0.50), + Percentile(allSamples, 0.95), + Percentile(allSamples, 0.99), + allSamples.Length == 0 ? 0 : allSamples[^1], + earlyP99, + lateP99, + lateToEarlyRatio, + [new RoleLatencyResult( + "missing-key", + earlySamples.Length + lateSamples.Length, + earlyP99, + lateP99, + lateToEarlyRatio)], + Frames: 0, + FramesPerSecond: 0, + bytesWritten, + BytesRead: 0, + bytesWritten / measuredSeconds, + FullPayloadCopies: 0, + measuredThreadAllocatedBytes, + failures, + measuredSeconds, + measuredSeconds, + allSamples.Length, + FairnessIndex: 1, + MinWorkerApiCallsPerSecond: operations / measuredSeconds, + MaxWorkerApiCallsPerSecond: operations / measuredSeconds, + WorstWorkerP99Microseconds: lateP99, + AffinityAppliedCount: affinity.Applied ? 1 : 0, + AssignedProcessors: [affinity.AssignedProcessor], + AffinityStrategies: [affinity.Strategy], + Oversubscribed: false, + qualification, + counters.ToHistogram(), + WorkerCycles: [churnCycles], + FullPayloadCopyCountIsInstrumented: false, + FullPayloadCopyEvidenceKind: "not-applicable-no-large-payload-path", + ProducerStoreOperationAllocatedBytes: 0, + AllocationMeasurementScope: "controller-thread-entire-overflow-measured-window", + StickyOverflow: evidence, + EarlySampleCount: earlySamples.Length, + LateSampleCount: lateSamples.Length); + } +} + +static long MeasureMissingWindow( + Store store, + ReadOnlySpan missingKey, + Span samples, + StatusCounters counters) +{ + long failures = 0; + for (var index = 0; index < samples.Length; index++) + { + long started = Stopwatch.GetTimestamp(); + StoreStatus status = store.TryAcquire( + missingKey, + StoreWaitOptions.Infinite, + out ValueLease lease); + samples[index] = Stopwatch.GetElapsedTime(started).TotalMicroseconds; + counters.Record(OperationKind.Acquire, status); + if (status == StoreStatus.Success) + { + failures++; + StoreStatus release = lease.Release(StoreWaitOptions.Infinite); + counters.Record(OperationKind.Release, release); + if (release != StoreStatus.Success) + { + failures++; + } + } + else if (status != StoreStatus.NotFound) + { + failures++; + } + } + + return failures; +} + +static async Task RunBrokerTrial( + StoreProfile profile, + ScenarioPlan plan, + int readerCount, + int trial, + int durationSeconds, + int warmupSeconds, + int durationBoundGraceSeconds, + bool affinityRequested, + int frameBytes, + long frameTarget) +{ + string name = $"sms-broker-{Guid.NewGuid():N}"; + var readers = new List(readerCount); + var observers = new List(plan.ObserverCount); + var allProcesses = new List(readerCount + plan.ObserverCount); + var processRegistry = new ProbeProcessRegistry(); + ProbeTrialWatchdog? trialWatchdog = CreateTrialWatchdog( + plan.Name, + profile, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds, + operationTarget: 0, + frameTarget, + processRegistry); + try + { + StoreOpenStatus openStatus = Store.TryCreateOrOpen( + Options(name, OpenMode.CreateNew, profile, plan.Name, frameBytes), + out Store? producer); + if (openStatus != StoreOpenStatus.Success || producer is null) + { + throw new InvalidOperationException($"Broker producer open failed for {profile}: {openStatus}"); + } + + using TemporaryProcessAffinity producerAffinity = TemporaryProcessAffinity.Apply( + affinityRequested ? 0 : -1); + using (producer) + { + try + { + for (var readerId = 0; readerId < readerCount; readerId++) + { + Process process = processRegistry.Start(() => StartBrokerWorker( + plan.Name, + profile, + name, + readerId, + "reader", + affinityRequested ? readerId + 1 : -1, + frameBytes)); + readers.Add(process); + allProcesses.Add(process); + } + + for (var observerId = 0; observerId < plan.ObserverCount; observerId++) + { + Process process = processRegistry.Start(() => StartBrokerWorker( + plan.Name, + profile, + name, + observerId, + "observer", + affinityRequested ? readerCount + observerId + 1 : -1, + frameBytes)); + observers.Add(process); + allProcesses.Add(process); + } + + await AwaitReady(allProcesses, CancellationToken.None); + BenchmarkKeyCatalog keys = BenchmarkProtocol.CreateKeyCatalog(BrokerRotatingKeyCount); + await Task.Run( + () => WarmBrokerWorkers( + producer, + readers, + observers, + keys, + frameBytes, + warmupSeconds)); + await Task.Run(() => ResetBrokerWorkers(allProcesses)); + var readerFrames = new long[readerCount]; + BrokerMeasuredResult measuredResult = await RunBrokerMeasuredOnDedicatedThread( + producer, + readers, + observers, + keys, + readerFrames, + frameBytes, + durationSeconds, + frameTarget); + StatusCounters counters = measuredResult.Counters; + long failures = measuredResult.Failures; + long frames = measuredResult.Frames; + var stop = new BrokerKeyMessage(BrokerMessageKind.Stop, string.Empty, 0, 0, 0, 0); + string stopLine = JsonSerializer.Serialize(stop, BenchmarkProtocol.JsonOptions); + foreach (Process process in allProcesses) + { + await process.StandardInput.WriteLineAsync(stopLine.AsMemory(), CancellationToken.None); + await process.StandardInput.FlushAsync(CancellationToken.None); + } + + var summaries = new List(allProcesses.Count); + foreach (Process process in allProcesses) + { + string? line = await process.StandardOutput.ReadLineAsync(CancellationToken.None); + string error = await process.StandardError.ReadToEndAsync(CancellationToken.None); + await process.WaitForExitAsync(CancellationToken.None); + if (process.ExitCode != 0 || line is null) + { + throw new InvalidOperationException($"Broker worker exited {process.ExitCode}: {error}"); + } + + summaries.Add(JsonSerializer.Deserialize(line, BenchmarkProtocol.JsonOptions) + ?? throw new InvalidOperationException("Broker worker returned invalid summary JSON.")); + } + + failures += summaries.Sum(static summary => summary.Failures); + long operations = counters.TotalOperations + summaries.Sum(static summary => summary.Operations); + long bytesWritten = checked(frames * frameBytes); + long bytesRead = summaries.Sum(static summary => summary.BytesProcessed); + double measuredSeconds = Math.Max(measuredResult.Elapsed.TotalSeconds, 0.000_001); + double[] sortedSamples = measuredResult.SamplesMicroseconds.Order().ToArray(); + double[] sortedEarly = measuredResult.EarlySamplesMicroseconds.Order().ToArray(); + double[] sortedLate = measuredResult.LateSamplesMicroseconds.Order().ToArray(); + double earlyP99 = Percentile(sortedEarly, 0.99); + double lateP99 = Percentile(sortedLate, 0.99); + double[] readerRates = readerFrames.Select(count => count / measuredSeconds).ToArray(); + double fairness = JainFairness(readerRates); + int participantProcesses = readerCount + plan.ObserverCount + 1; + bool oversubscribed = participantProcesses > Environment.ProcessorCount; + var histograms = summaries.Select(static summary => summary.StatusHistogram).Append(counters.ToHistogram()); + + return new RunResult( + profile.ToString(), + plan.Name, + readerCount, + readerCount, + PublisherProcessCount: 1, + plan.ObserverCount, + trial, + frames, + operations, + operations / measuredSeconds, + Percentile(sortedSamples, 0.50), + Percentile(sortedSamples, 0.95), + Percentile(sortedSamples, 0.99), + sortedSamples.Length == 0 ? 0 : sortedSamples[^1], + earlyP99, + lateP99, + earlyP99 == 0 ? 0 : lateP99 / earlyP99, + [new RoleLatencyResult( + "broker-end-to-end", + sortedSamples.Length, + earlyP99, + lateP99, + earlyP99 == 0 ? 0 : lateP99 / earlyP99)], + frames, + frames / measuredSeconds, + bytesWritten, + bytesRead, + (bytesWritten + bytesRead) / measuredSeconds, + FullPayloadCopies: 0, + MeasuredThreadAllocatedBytes: measuredResult.MeasuredThreadAllocatedBytes, + failures, + measuredSeconds, + measuredSeconds, + sortedSamples.Length, + fairness, + readerRates.Length == 0 ? 0 : readerRates.Min(), + readerRates.Length == 0 ? 0 : readerRates.Max(), + summaries.Where(static summary => summary.Role == "reader") + .Select(static summary => summary.ElapsedSeconds == 0 ? 0 : summary.Frames / summary.ElapsedSeconds) + .DefaultIfEmpty() + .Max(), + summaries.Count(static summary => summary.AffinityApplied) + (producerAffinity.Applied ? 1 : 0), + [producerAffinity.AssignedProcessor, .. summaries.Select(static summary => summary.AssignedProcessor)], + [producerAffinity.Strategy, .. summaries + .Select(static summary => summary.AffinityStrategy) + .Where(strategy => !string.Equals(strategy, producerAffinity.Strategy, StringComparison.Ordinal)) + .Distinct(StringComparer.Ordinal)], + oversubscribed, + Qualification( + oversubscribed, + durationSeconds, + warmupSeconds, + operationTarget: 0, + frameTarget), + MergeHistograms(histograms), + readerFrames, + FullPayloadCopyCountIsInstrumented: false, + FullPayloadCopyEvidenceKind: "structural-direct-reservation-write-and-borrowed-lease-read", + ProducerStoreOperationAllocatedBytes: measuredResult.ProducerStoreOperationAllocatedBytes, + AllocationMeasurementScope: + "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + EarlySampleCount: sortedEarly.Length, + LateSampleCount: sortedLate.Length, + OperationTarget: 0, + FrameTarget: frameTarget); + } + finally + { + DisposeProcesses(allProcesses); + } + } + } + finally + { + if (trialWatchdog is not null) + { + await trialWatchdog.CompleteAsync(); + } + } +} + +static RunResult AggregateAutonomousRun( + StoreProfile profile, + ScenarioPlan plan, + int processCount, + int trial, + int warmupSeconds, + TimeSpan wall, + IReadOnlyList workerResults, + long operationTarget) +{ + double[] samples = workerResults.SelectMany(static result => result.SamplesMicroseconds).Order().ToArray(); + double[] early = workerResults.SelectMany(static result => result.EarlySamplesMicroseconds).Order().ToArray(); + double[] late = workerResults.SelectMany(static result => result.LateSamplesMicroseconds).Order().ToArray(); + long totalCycles = workerResults.Sum(static result => result.Cycles); + long totalOperations = workerResults.Sum(static result => result.Operations); + long failures = workerResults.Sum(static result => result.Failures); + double measuredSeconds = workerResults.Max(static result => result.ElapsedSeconds); + double[] workerRates = workerResults + .Select(static result => result.Operations / Math.Max(result.ElapsedSeconds, 0.000_001)) + .ToArray(); + double fairness = JainFairness(workerRates); + double worstWorkerP99 = workerResults.Max(result => + Percentile(result.SamplesMicroseconds.Order().ToArray(), 0.99)); + bool mixed = plan.Kind == ProbeScenarioKind.MixedChurn; + int readerCount = mixed + ? workerResults.Count(static result => result.Role == "reader") + : plan.Name == "publish-remove" ? 0 : processCount; + int publisherCount = mixed + ? workerResults.Count(static result => result.Role == "publisher") + : plan.Name == "publish-remove" ? processCount : 0; + int participantProcesses = workerResults.Count; + bool oversubscribed = participantProcesses > Environment.ProcessorCount; + double earlyP99 = Percentile(early, 0.99); + double lateP99 = Percentile(late, 0.99); + long bytesRead = workerResults + .Where(static result => result.Role == "reader") + .Sum(static result => result.BytesProcessed); + long bytesWritten = workerResults + .Where(static result => result.Role == "publisher") + .Sum(static result => result.BytesProcessed); + RoleLatencyResult[] roleLatency = workerResults + .GroupBy(static result => result.Role, StringComparer.Ordinal) + .Select(group => + { + double[] roleEarly = group.SelectMany(static result => result.EarlySamplesMicroseconds).Order().ToArray(); + double[] roleLate = group.SelectMany(static result => result.LateSamplesMicroseconds).Order().ToArray(); + double roleEarlyP99 = Percentile(roleEarly, 0.99); + double roleLateP99 = Percentile(roleLate, 0.99); + return new RoleLatencyResult( + group.Key, + roleEarly.Length + roleLate.Length, + roleEarlyP99, + roleLateP99, + roleEarlyP99 == 0 ? 0 : roleLateP99 / roleEarlyP99); + }) + .OrderBy(static result => result.Role, StringComparer.Ordinal) + .ToArray(); + + return new RunResult( + profile.ToString(), + plan.Name, + processCount, + readerCount, + publisherCount, + ObserverProcessCount: 0, + trial, + totalCycles, + totalOperations, + totalOperations / measuredSeconds, + Percentile(samples, 0.50), + Percentile(samples, 0.95), + Percentile(samples, 0.99), + workerResults.Max(static result => result.MaximumSampleMicroseconds), + earlyP99, + lateP99, + earlyP99 == 0 ? 0 : lateP99 / earlyP99, + roleLatency, + Frames: 0, + FramesPerSecond: 0, + bytesWritten, + bytesRead, + (bytesWritten + bytesRead) / measuredSeconds, + FullPayloadCopies: 0, + workerResults.Sum(static result => result.MeasuredThreadAllocatedBytes), + failures, + measuredSeconds, + wall.TotalSeconds, + samples.Length, + fairness, + workerRates.Min(), + workerRates.Max(), + worstWorkerP99, + workerResults.Count(static result => result.AffinityApplied), + workerResults.Select(static result => result.AssignedProcessor).ToArray(), + workerResults.Select(static result => result.AffinityStrategy).Distinct(StringComparer.Ordinal).ToArray(), + oversubscribed, + Qualification( + oversubscribed, + (int)Math.Floor(measuredSeconds), + warmupSeconds, + operationTarget, + frameTarget: 0), + MergeHistograms(workerResults.Select(static result => result.StatusHistogram)), + workerResults.Select(static result => result.Cycles).ToArray(), + FullPayloadCopyCountIsInstrumented: false, + FullPayloadCopyEvidenceKind: "not-instrumented-legacy-field-do-not-interpret-as-count", + ProducerStoreOperationAllocatedBytes: 0, + AllocationMeasurementScope: "sum-of-dedicated-worker-thread-measured-regions", + EarlySampleCount: early.Length, + LateSampleCount: late.Length, + OperationTarget: operationTarget, + FrameTarget: 0); +} + +static async Task AwaitReady(IReadOnlyList workers, CancellationToken cancellationToken) +{ + foreach (Process worker in workers) + { + string? ready = await worker.StandardOutput.ReadLineAsync(cancellationToken); + if (ready != "READY") + { + string error = await worker.StandardError.ReadToEndAsync(cancellationToken); + throw new InvalidOperationException($"Worker failed to become ready: {ready}; {error}"); + } + } +} + +static async Task SignalGo(IEnumerable workers, CancellationToken cancellationToken) +{ + foreach (Process worker in workers) + { + await worker.StandardInput.WriteLineAsync("GO".AsMemory(), cancellationToken); + await worker.StandardInput.FlushAsync(cancellationToken); + } +} + +static async Task> CollectWorkerResults( + IReadOnlyList workers, + CancellationToken cancellationToken) +{ + var results = new List(workers.Count); + foreach (Process worker in workers) + { + string? line = await worker.StandardOutput.ReadLineAsync(cancellationToken); + string error = await worker.StandardError.ReadToEndAsync(cancellationToken); + await worker.WaitForExitAsync(cancellationToken); + if (worker.ExitCode != 0 || line is null) + { + throw new InvalidOperationException($"Worker exited {worker.ExitCode}: {error}"); + } + + results.Add(JsonSerializer.Deserialize(line, BenchmarkProtocol.JsonOptions) + ?? throw new InvalidOperationException("Worker returned invalid JSON.")); + } + + return results; +} + +static ProbeTrialWatchdog? CreateTrialWatchdog( + string scenario, + StoreProfile profile, + int durationSeconds, + int warmupSeconds, + int durationBoundGraceSeconds, + long operationTarget, + long frameTarget, + ProbeProcessRegistry processRegistry) +{ + if (operationTarget > 0 || frameTarget > 0) + { + return null; + } + + long timeoutSeconds = checked( + (long)durationSeconds + warmupSeconds + durationBoundGraceSeconds); + return new ProbeTrialWatchdog( + TimeSpan.FromSeconds(timeoutSeconds), + () => FailFastDurationBoundTrial( + scenario, + profile, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds, + processRegistry)); +} + +static TimeoutException DurationBoundTrialTimeout( + string scenario, + StoreProfile profile, + int durationSeconds, + int warmupSeconds, + int durationBoundGraceSeconds) => + new( + $"Duration-bound trial timed out: profile={profile}; scenario={scenario}; " + + $"warmupSeconds={warmupSeconds}; durationSeconds={durationSeconds}; " + + $"graceSeconds={durationBoundGraceSeconds}."); + +static void FailFastDurationBoundTrial( + string scenario, + StoreProfile profile, + int durationSeconds, + int warmupSeconds, + int durationBoundGraceSeconds, + ProbeProcessRegistry processRegistry) +{ + TimeoutException timeout = DurationBoundTrialTimeout( + scenario, + profile, + durationSeconds, + warmupSeconds, + durationBoundGraceSeconds); + try + { + var killer = new Thread( + () => KillProcesses(processRegistry.StopAcceptingAndSnapshot())) + { + IsBackground = true, + Name = "SyncProbe timeout child cleanup" + }; + killer.Start(); + _ = killer.Join(WatchdogChildKillBudgetMilliseconds); + } + catch + { + // Process termination below is unconditional; child cleanup is best effort. + } + finally + { + // Store calls can be blocked in native/shared-memory coordination and cannot be + // safely abandoned or unwound. This executable is the runner's child-process + // isolation boundary, so a missed duration deadline must terminate it directly. + Environment.FailFast(timeout.Message, timeout); + } +} + +static async Task ReportTrialHeartbeats( + StoreProfile profile, + string scenario, + int processCount, + int trial, + int trials, + ProbeScenarioKind scenarioKind, + ProbeRunTargets targets, + CancellationToken cancellationToken) +{ + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(TrialHeartbeatSeconds)); + var elapsed = Stopwatch.StartNew(); + try + { + while (await timer.WaitForNextTickAsync(cancellationToken)) + { + Console.Error.WriteLine( + $"trial-progress profile={profile} scenario={scenario} processes={processCount} " + + $"trial={trial}/{trials} elapsed-seconds={elapsed.Elapsed.TotalSeconds:F0} " + + (scenarioKind == ProbeScenarioKind.StickyOverflow + ? "fixed-work=true" + : targets.OperationTarget > 0 + ? $"operation-target={targets.OperationTarget}" + : targets.FrameTarget > 0 + ? $"frame-target={targets.FrameTarget}" + : "duration-bound=true")); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } +} + +static void Seed(Store owner, string scenario, int processCount) +{ + if (scenario == "acquire-release") + { + if (processCount > SyncMaximumWorkerCount) + { + throw new ArgumentOutOfRangeException(nameof(processCount)); + } + + BenchmarkKeyCatalog keys = CreateSyncKeyCatalog(); + for (var workerId = 0; workerId < processCount; workerId++) + { + for (var keyOrdinal = 0; keyOrdinal < SyncKeysPerWorker; keyOrdinal++) + { + int keyIndex = checked((workerId * SyncKeysPerWorker) + keyOrdinal); + Ensure(owner.TryPublish(keys[keyIndex].Span, [(byte)workerId]), "seed publish"); + } + } + + return; + } + + if (scenario == "same-key-read") + { + Ensure(owner.TryPublish(BenchmarkProtocol.Key(0), ReaderPayload(0)), "same-key seed publish"); + return; + } + + if (scenario == "distributed-key-read") + { + for (var keyIndex = 0; keyIndex < ReaderKeyCount; keyIndex++) + { + Ensure(owner.TryPublish(BenchmarkProtocol.Key(keyIndex), ReaderPayload(keyIndex)), "distributed seed publish"); + } + } +} + +static BenchmarkKeyCatalog CreateSyncKeyCatalog() => + BenchmarkProtocol.CreateCanonicalBucketKeyCatalog( + SyncKeysPerWorker, + SyncMaximumWorkerCount, + BenchmarkProtocol.CalculatePrimaryBucketCount(SyncSlotCount)); + +static bool IsSyncScenario(string scenario) => + scenario is "acquire-release" or "publish-remove"; + +static int SyncKeyIndex(int workerId, long cycle) +{ + if ((uint)workerId >= SyncMaximumWorkerCount) + { + throw new ArgumentOutOfRangeException(nameof(workerId)); + } + + return checked((workerId * SyncKeysPerWorker) + (int)(cycle & 1)); +} + +static ulong AutonomousSamplingSeed(string scenario, int workerId) +{ + ulong scenarioSeed = scenario switch + { + "acquire-release" => 0x243f_6a88_85a3_08d3UL, + "publish-remove" => 0x1319_8a2e_0370_7344UL, + "same-key-read" => 0xa409_3822_299f_31d0UL, + "distributed-key-read" => 0x082e_fa98_ec4e_6c89UL, + "mixed-churn-reader" => 0x4528_21e6_38d0_1377UL, + "mixed-churn-writer" => 0xbe54_66cf_34e9_0c6cUL, + _ => throw new ArgumentOutOfRangeException(nameof(scenario)) + }; + + return scenarioSeed ^ (unchecked((ulong)(workerId + 1)) * 0x9e37_79b9_7f4a_7c15UL); +} + +static void SeedMixed(Store owner) +{ + int primaryBucketCount = BenchmarkProtocol.CalculatePrimaryBucketCount(MixedSlotCount); + byte[][] keys = BenchmarkProtocol.CreateCollisionKeys(MixedCollisionKeyCount, primaryBucketCount); + var counters = new StatusCounters(); + for (var keyIndex = 0; keyIndex < keys.Length; keyIndex++) + { + if (!PublishGeneration(owner, keys[keyIndex], keyIndex, generation: 0, MixedPayloadBytes, counters)) + { + throw new InvalidOperationException($"Mixed-churn seed failed for collision key {keyIndex}."); + } + } +} + +static Process StartAutonomousWorker( + string scenario, + StoreProfile profile, + string name, + int workerId, + int durationSeconds, + int warmupSeconds, + int affinityOrdinal, + long operationTarget, + int payloadBytes) +{ + ProcessStartInfo start = CreateChildStartInfo(); + start.ArgumentList.Add("worker"); + start.ArgumentList.Add(scenario); + start.ArgumentList.Add(profile.ToString()); + start.ArgumentList.Add(name); + start.ArgumentList.Add(workerId.ToString(CultureInfo.InvariantCulture)); + start.ArgumentList.Add(durationSeconds.ToString(CultureInfo.InvariantCulture)); + start.ArgumentList.Add(warmupSeconds.ToString(CultureInfo.InvariantCulture)); + start.ArgumentList.Add(affinityOrdinal.ToString(CultureInfo.InvariantCulture)); + start.ArgumentList.Add(operationTarget.ToString(CultureInfo.InvariantCulture)); + start.ArgumentList.Add(payloadBytes.ToString(CultureInfo.InvariantCulture)); + return Process.Start(start) ?? throw new InvalidOperationException("Failed to start worker process."); +} + +static Process StartBrokerWorker( + string scenario, + StoreProfile profile, + string name, + int workerId, + string role, + int affinityOrdinal, + int payloadBytes) +{ + ProcessStartInfo start = CreateChildStartInfo(); + start.ArgumentList.Add("broker-worker"); + start.ArgumentList.Add(scenario); + start.ArgumentList.Add(profile.ToString()); + start.ArgumentList.Add(name); + start.ArgumentList.Add(workerId.ToString(CultureInfo.InvariantCulture)); + start.ArgumentList.Add(role); + start.ArgumentList.Add(affinityOrdinal.ToString(CultureInfo.InvariantCulture)); + start.ArgumentList.Add(payloadBytes.ToString(CultureInfo.InvariantCulture)); + return Process.Start(start) ?? throw new InvalidOperationException("Failed to start broker worker process."); +} + +static ProcessStartInfo CreateChildStartInfo() +{ + string processPath = Environment.ProcessPath + ?? throw new InvalidOperationException("Cannot determine executable path."); + var start = new ProcessStartInfo(processPath) + { + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + + if (string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase)) + { + start.ArgumentList.Add(typeof(Program).Assembly.Location); + } + + return start; +} + +static int RunAutonomousWorker(string[] args) +{ + if (args.Length != 10) + { + Console.Error.WriteLine( + "Worker requires: worker " + + " ."); + return 3; + } + + string scenario = args[1]; + StoreProfile profile = Enum.Parse(args[2], ignoreCase: true); + string name = args[3]; + int workerId = int.Parse(args[4], CultureInfo.InvariantCulture); + int durationSeconds = int.Parse(args[5], CultureInfo.InvariantCulture); + int warmupSeconds = int.Parse(args[6], CultureInfo.InvariantCulture); + int affinityOrdinal = int.Parse(args[7], CultureInfo.InvariantCulture); + long operationTarget = long.Parse(args[8], CultureInfo.InvariantCulture); + int payloadBytes = int.Parse(args[9], CultureInfo.InvariantCulture); + bool affinityApplied = ProcessorAffinityPlanner.TryApply( + affinityOrdinal, + out int assignedProcessor, + out string affinityStrategy); + StoreOpenStatus openStatus = Store.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting, profile, scenario, payloadBytes), + out Store? store); + if (openStatus != StoreOpenStatus.Success || store is null) + { + Console.Error.WriteLine($"Worker open failed: {openStatus}"); + return 4; + } + + using (store) + { + BenchmarkKeyCatalog stableKeys = IsSyncScenario(scenario) + ? CreateSyncKeyCatalog() + : BenchmarkProtocol.CreateKeyCatalog(ReaderKeyCount); + byte[][]? collisionKeys = scenario.StartsWith("mixed-churn", StringComparison.Ordinal) + ? BenchmarkProtocol.CreateCollisionKeys( + MixedCollisionKeyCount, + BenchmarkProtocol.CalculatePrimaryBucketCount(MixedSlotCount)) + : null; + var warmupCounters = new StatusCounters(); + long warmupCycle = 0; + var warmup = Stopwatch.StartNew(); + while (warmup.Elapsed.TotalSeconds < warmupSeconds) + { + RunCycle( + store, + scenario, + workerId, + warmupCycle++, + stableKeys, + collisionKeys, + warmupCounters, + out int warmupFailures, + out _); + if (warmupFailures != 0) + { + Console.Error.WriteLine( + "Warm-up correctness failure: " + JsonSerializer.Serialize(warmupCounters.ToHistogram())); + return 6; + } + } + + Console.WriteLine("READY"); + if (Console.ReadLine() != "GO") + { + return 5; + } + + ulong samplingSeed = AutonomousSamplingSeed(scenario, workerId); + var candidateSampler = new GeometricCycleSampler(samplingSeed); + var earlySamples = new LatencyReservoir( + LatencyReservoirCapacityPerWindow, + samplingSeed ^ 0xa076_1d64_78bd_642fUL); + var lateSamples = new LatencyReservoir( + LatencyReservoirCapacityPerWindow, + samplingSeed ^ 0xe703_7ed1_a0b4_28dbUL); + var counters = new StatusCounters(); + long cycles = 0; + long bytesProcessed = 0; + long failures = 0; + var elapsed = Stopwatch.StartNew(); + long allocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + while (operationTarget > 0 + ? counters.TotalOperations < operationTarget + : elapsed.Elapsed.TotalSeconds < durationSeconds) + { + bool sample = candidateSampler.ShouldSample(cycles); + long started = sample ? Stopwatch.GetTimestamp() : 0; + RunCycle( + store, + scenario, + workerId, + cycles, + stableKeys, + collisionKeys, + counters, + out int cycleFailures, + out long cycleBytes); + if (sample) + { + double microseconds = Stopwatch.GetElapsedTime(started).TotalMicroseconds; + bool early = operationTarget > 0 + ? counters.TotalOperations < operationTarget / 2 + : elapsed.Elapsed.TotalSeconds < durationSeconds / 2.0; + (early ? earlySamples : lateSamples).Add(microseconds); + } + + bytesProcessed += cycleBytes; + failures += cycleFailures; + cycles++; + } + + elapsed.Stop(); + long measuredThreadAllocatedBytes = Math.Max( + 0, + GC.GetAllocatedBytesForCurrentThread() - allocatedBefore); + double[] earlySnapshot = earlySamples.ToArray(); + double[] lateSnapshot = lateSamples.ToArray(); + double[] samples = [.. earlySnapshot, .. lateSnapshot]; + Console.WriteLine(JsonSerializer.Serialize(new WorkerResult( + workerId, + ScenarioRole(scenario), + cycles, + counters.TotalOperations, + bytesProcessed, + failures, + measuredThreadAllocatedBytes, + elapsed.Elapsed.TotalSeconds, + affinityApplied, + assignedProcessor, + affinityStrategy, + counters.ToHistogram(), + samples, + earlySnapshot, + lateSnapshot, + Math.Max(earlySamples.MaximumObserved, lateSamples.MaximumObserved)), BenchmarkProtocol.JsonOptions)); + } + + return 0; +} + +static void AssertLatencyReservoirMaximumSemantics() +{ + const double outlier = 12_345.0; + var reservoir = new LatencyReservoir(capacity: 1, seed: 1); + reservoir.Add(outlier); + for (var index = 0; index < 100_000; index++) + { + reservoir.Add(1.0); + } + + double[] retained = reservoir.ToArray(); + if (retained.Length != 1 || retained[0] == outlier || reservoir.MaximumObserved != outlier) + { + throw new InvalidOperationException( + "Latency reservoir maximum self-test did not preserve an evicted sampled outlier."); + } +} + +static async Task RunBrokerWorker(string[] args) +{ + if (args.Length != 8) + { + Console.Error.WriteLine( + "Broker worker requires: broker-worker " + + " ."); + return 3; + } + + string scenario = args[1]; + StoreProfile profile = Enum.Parse(args[2], ignoreCase: true); + string name = args[3]; + int workerId = int.Parse(args[4], CultureInfo.InvariantCulture); + string role = args[5]; + int affinityOrdinal = int.Parse(args[6], CultureInfo.InvariantCulture); + int payloadBytes = int.Parse(args[7], CultureInfo.InvariantCulture); + bool affinityApplied = ProcessorAffinityPlanner.TryApply( + affinityOrdinal, + out int assignedProcessor, + out string affinityStrategy); + StoreOpenStatus openStatus = Store.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting, profile, scenario, payloadBytes), + out Store? store); + if (openStatus != StoreOpenStatus.Success || store is null) + { + Console.Error.WriteLine($"Broker worker open failed: {openStatus}"); + return 4; + } + + using (store) + { + BenchmarkKeyCatalog keys = BenchmarkProtocol.CreateKeyCatalog(BrokerRotatingKeyCount); + Console.WriteLine("READY"); + var counters = new StatusCounters(); + long frames = 0; + long failures = 0; + long bytesProcessed = 0; + var elapsed = Stopwatch.StartNew(); + while (await Console.In.ReadLineAsync() is { } line) + { + BrokerKeyMessage message = JsonSerializer.Deserialize( + line, + BenchmarkProtocol.JsonOptions); + if (message.Kind == BrokerMessageKind.Stop) + { + break; + } + + if (message.Kind == BrokerMessageKind.Reset) + { + counters = new StatusCounters(); + frames = 0; + failures = 0; + bytesProcessed = 0; + elapsed.Restart(); + Console.WriteLine("RESET"); + await Console.Out.FlushAsync(); + continue; + } + + bool keyMessageValid = (uint)message.KeyIndex < (uint)keys.Count + && string.Equals(message.KeyHex, keys.Hex(message.KeyIndex), StringComparison.Ordinal); + ReadOnlyMemory key = keyMessageValid ? keys[message.KeyIndex] : ReadOnlyMemory.Empty; + ValueLease lease = default; + StoreStatus acquire; + if (keyMessageValid) + { + acquire = AcquireWithRetry(store, key.Span, counters, out lease); + } + else + { + acquire = StoreStatus.InvalidKey; + counters.Record(OperationKind.Acquire, acquire); + } + StoreStatus release = StoreStatus.InvalidLease; + bool descriptorValid = false; + bool payloadValid = false; + int bytesObserved = 0; + if (acquire == StoreStatus.Success) + { + descriptorValid = BenchmarkProtocol.ValidateDescriptor( + lease.DescriptorSpan, + message.KeyIndex, + message.Generation, + message.PayloadLength); + bytesObserved = lease.ValueSpan.Length; + payloadValid = bytesObserved == message.PayloadLength + && BenchmarkProtocol.ValidateGenerationPayload( + lease.ValueSpan, + message.KeyIndex, + message.Generation); + bytesProcessed += bytesObserved; + release = ReleaseWithRetry(lease, counters); + } + + if (acquire != StoreStatus.Success + || release != StoreStatus.Success + || !keyMessageValid + || !descriptorValid + || !payloadValid) + { + failures++; + if (!descriptorValid || !payloadValid) + { + counters.RecordChecksumFailure(); + } + } + + frames++; + var acknowledgement = new BrokerAcknowledgement( + workerId, + role, + message.KeyIndex, + message.Generation, + acquire, + release, + descriptorValid, + payloadValid, + bytesObserved, + Stopwatch.GetElapsedTime(message.PublishedTimestamp).TotalMicroseconds); + Console.WriteLine(JsonSerializer.Serialize(acknowledgement, BenchmarkProtocol.JsonOptions)); + await Console.Out.FlushAsync(); + } + + elapsed.Stop(); + Console.WriteLine(JsonSerializer.Serialize(new BrokerWorkerSummary( + workerId, + role, + frames, + counters.TotalOperations, + bytesProcessed, + failures, + elapsed.Elapsed.TotalSeconds, + affinityApplied, + assignedProcessor, + affinityStrategy, + counters.ToHistogram()), BenchmarkProtocol.JsonOptions)); + } + + return 0; +} + +static void RunCycle( + Store store, + string scenario, + int workerId, + long cycle, + BenchmarkKeyCatalog stableKeys, + byte[][]? collisionKeys, + StatusCounters counters, + out int failures, + out long bytesProcessed) +{ + failures = 0; + bytesProcessed = 0; + if (scenario is "acquire-release" or "same-key-read" or "distributed-key-read") + { + int lookupKeyIndex = scenario switch + { + "same-key-read" => 0, + "distributed-key-read" => (int)((cycle + workerId * 17L) % ReaderKeyCount), + _ => SyncKeyIndex(workerId, cycle) + }; + StoreStatus acquire = AcquireWithRetry( + store, + stableKeys[lookupKeyIndex].Span, + counters, + out ValueLease lease); + if (acquire != StoreStatus.Success) + { + failures++; + return; + } + + bool valid = scenario == "acquire-release" + ? lease.ValueSpan.Length == 1 && lease.ValueSpan[0] == unchecked((byte)workerId) + : ValidateReaderPayload(lease.ValueSpan, lookupKeyIndex); + bytesProcessed = lease.ValueSpan.Length; + if (!valid) + { + counters.RecordChecksumFailure(); + failures++; + } + + StoreStatus release = ReleaseWithRetry(lease, counters); + if (release != StoreStatus.Success) + { + failures++; + } + + return; + } + + if (scenario == "publish-remove") + { + int syncKeyIndex = SyncKeyIndex(workerId, cycle); + StoreStatus publish = PublishWithRetry( + store, + stableKeys[syncKeyIndex].Span, + [unchecked((byte)cycle)], + counters); + if (publish != StoreStatus.Success) + { + failures++; + return; + } + + bytesProcessed = 1; + StoreStatus remove = RemoveWithRetry(store, stableKeys[syncKeyIndex].Span, counters); + if (remove != StoreStatus.Success) + { + failures++; + } + + return; + } + + if (scenario == "mixed-churn-reader") + { + byte[][] keys = collisionKeys + ?? throw new InvalidOperationException("Mixed-churn keys were not initialized."); + int readKeyIndex = (int)((cycle * 17 + workerId * 43L) % keys.Length); + StoreStatus acquire = AcquireWithRetry(store, keys[readKeyIndex], counters, out ValueLease lease); + if (acquire == StoreStatus.NotFound) + { + return; + } + + if (acquire != StoreStatus.Success) + { + failures++; + return; + } + + bool descriptorValid = lease.DescriptorSpan.Length == BenchmarkDescriptorBytes; + long generation = descriptorValid + ? BinaryPrimitives.ReadInt64LittleEndian(lease.DescriptorSpan) + : long.MinValue; + descriptorValid = descriptorValid + && BenchmarkProtocol.ValidateDescriptor( + lease.DescriptorSpan, + readKeyIndex, + generation, + MixedPayloadBytes); + bool payloadValid = descriptorValid + && BenchmarkProtocol.ValidateGenerationPayload(lease.ValueSpan, readKeyIndex, generation); + bytesProcessed = lease.ValueSpan.Length; + if (!descriptorValid || !payloadValid) + { + counters.RecordChecksumFailure(); + failures++; + } + + StoreStatus release = ReleaseWithRetry(lease, counters); + if (release != StoreStatus.Success) + { + failures++; + } + + return; + } + + if (scenario != "mixed-churn-writer") + { + throw new ArgumentOutOfRangeException(nameof(scenario), scenario, "Unknown sync-probe scenario."); + } + + byte[][] writerKeys = collisionKeys + ?? throw new InvalidOperationException("Mixed-churn keys were not initialized."); + const int ChurnKeyStart = MixedCollisionKeyCount / 2; + int localKeyCount = (MixedCollisionKeyCount - ChurnKeyStart) / 2; + int keyIndex = ChurnKeyStart + workerId + (int)((cycle % localKeyCount) * 2); + byte[] writerKey = writerKeys[keyIndex]; + StoreStatus removeStatus = RemoveWithRetry(store, writerKey, counters); + if (removeStatus is not (StoreStatus.Success or StoreStatus.RemovePending)) + { + failures++; + return; + } + + long writerGeneration = checked(((long)(workerId + 1) << 56) | (cycle + 1)); + if (!PublishGeneration( + store, + writerKey, + keyIndex, + writerGeneration, + MixedPayloadBytes, + counters, + retryDuplicate: true)) + { + failures++; + return; + } + + bytesProcessed = MixedPayloadBytes; + if ((cycle & 1023) == 1023) + { + StoreStatus leaseRecovery = store.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: false), + StoreWaitOptions.Infinite, + out _); + counters.Record(OperationKind.RecoverLeases, leaseRecovery); + StoreStatus reservationRecovery = store.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: false), + StoreWaitOptions.Infinite, + out _); + counters.Record(OperationKind.RecoverReservations, reservationRecovery); + if (leaseRecovery != StoreStatus.Success || reservationRecovery != StoreStatus.Success) + { + failures++; + } + } +} + +static bool PublishGeneration( + Store store, + ReadOnlySpan key, + int keyIndex, + long generation, + int payloadBytes, + StatusCounters counters, + bool retryDuplicate = false) +{ + Span descriptor = stackalloc byte[BenchmarkDescriptorBytes]; + BenchmarkProtocol.WriteDescriptor(descriptor, keyIndex, generation, payloadBytes); + StoreStatus reserve; + ValueReservation reservation; + var attempt = 0; + do + { + reserve = store.TryReserve( + key, + payloadBytes, + descriptor, + StoreWaitOptions.Infinite, + out reservation); + counters.Record(OperationKind.Reserve, reserve); + RetryPause(attempt++); + } + // A pending-removal lifecycle continues to own its key until its final + // reader releases and reclamation completes. Only mixed remove/republish + // calls opt into retrying the contractually expected DuplicateKey result. + while ((reserve == StoreStatus.StoreBusy || (retryDuplicate && reserve == StoreStatus.DuplicateKey)) + && attempt < 4096); + if (reserve != StoreStatus.Success) + { + return false; + } + + using (reservation) + { + BenchmarkProtocol.FillGenerationPayload(reservation.GetSpan(payloadBytes), keyIndex, generation); + StoreStatus advance; + attempt = 0; + do + { + advance = reservation.Advance(payloadBytes, StoreWaitOptions.Infinite); + counters.Record(OperationKind.Advance, advance); + RetryPause(attempt++); + } + while (advance == StoreStatus.StoreBusy && attempt < 4096); + if (advance != StoreStatus.Success) + { + return false; + } + + StoreStatus commit; + attempt = 0; + do + { + commit = reservation.Commit(StoreWaitOptions.Infinite); + counters.Record(OperationKind.Commit, commit); + RetryPause(attempt++); + } + while (commit == StoreStatus.StoreBusy && attempt < 4096); + return commit == StoreStatus.Success; + } +} + +static StoreStatus AcquireWithRetry( + Store store, + ReadOnlySpan key, + StatusCounters counters, + out ValueLease lease) +{ + StoreStatus status; + var attempt = 0; + do + { + status = store.TryAcquire(key, StoreWaitOptions.Infinite, out lease); + counters.Record(OperationKind.Acquire, status); + RetryPause(attempt++); + } + while (status == StoreStatus.StoreBusy && attempt < 4096); + + return status; +} + +static StoreStatus ReleaseWithRetry(ValueLease lease, StatusCounters counters) +{ + StoreStatus status; + var attempt = 0; + do + { + status = lease.Release(StoreWaitOptions.Infinite); + counters.Record(OperationKind.Release, status); + RetryPause(attempt++); + } + while (status == StoreStatus.StoreBusy && attempt < 4096); + + return status; +} + +static StoreStatus PublishWithRetry( + Store store, + ReadOnlySpan key, + ReadOnlySpan value, + StatusCounters counters) +{ + StoreStatus status; + var attempt = 0; + do + { + status = store.TryPublish(key, value, default, StoreWaitOptions.Infinite); + counters.Record(OperationKind.Publish, status); + if (status == StoreStatus.CorruptStore) + { + counters.RecordCorruptReason( + LockFreeCorruptionTrace.Consume() ?? "untraced"); + } + RetryPause(attempt++); + } + while (status == StoreStatus.StoreBusy && attempt < 4096); + + return status; +} + +static StoreStatus RemoveWithRetry(Store store, ReadOnlySpan key, StatusCounters counters) +{ + StoreStatus status; + var attempt = 0; + var observedBusy = false; + do + { + status = store.TryRemove(key, StoreWaitOptions.Infinite); + counters.Record(OperationKind.Remove, status); + if (status == StoreStatus.CorruptStore) + { + counters.RecordCorruptReason( + LockFreeCorruptionTrace.Consume() ?? "untraced"); + } + observedBusy |= status == StoreStatus.StoreBusy; + RetryPause(attempt++); + } + while (status == StoreStatus.StoreBusy && attempt < 4096); + + // A bounded helper may report StoreBusy after another participant completes + // this exact unlink. A retry observing NotFound proves the requested key is + // now logically absent and is therefore the successful final state. + return observedBusy && status == StoreStatus.NotFound + ? StoreStatus.Success + : status; +} + +static void RetryPause(int attempt) +{ + if (attempt > 0) + { + Thread.SpinWait(4 << Math.Min(attempt, 10)); + } +} + +static SharedMemoryStoreOptions Options( + string name, + OpenMode mode, + StoreProfile profile, + string scenario, + int payloadBytes) +{ + bool reader = scenario is "same-key-read" or "distributed-key-read"; + bool mixed = scenario.StartsWith("mixed-churn", StringComparison.Ordinal); + bool broker = scenario is "broker-directed" or "large-ingest"; + int slotCount = mixed + ? MixedSlotCount + : broker + ? BrokerSlotCount + : reader ? ReaderSlotCount : SyncSlotCount; + int valueBytes = mixed + ? MixedPayloadBytes + : broker + ? payloadBytes + : reader ? ReaderPayloadBytes : SyncValueBytes; + int descriptorBytes = mixed || broker ? BenchmarkDescriptorBytes : 0; + int leaseRecords = mixed ? MixedLeaseRecordCount : DefaultLeaseRecordCount; + return profile == StoreProfile.LockFree + ? SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + valueBytes, + descriptorBytes, + MaxKeyBytes, + leaseRecords, + ParticipantRecordCount, + mode, + enableLeaseRecovery: true) + : SharedMemoryStoreOptions.Create( + name, + slotCount, + valueBytes, + descriptorBytes, + MaxKeyBytes, + leaseRecords, + mode, + enableLeaseRecovery: true); +} + +static SharedMemoryStoreOptions StickyOverflowOptions(string name, int slotCount) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + 1, + 0, + MaxKeyBytes, + DefaultLeaseRecordCount, + ParticipantRecordCount, + OpenMode.CreateNew, + enableLeaseRecovery: true); + +static byte[] ReaderPayload(int keyIndex) +{ + var payload = new byte[ReaderPayloadBytes]; + for (var index = 0; index < payload.Length; index++) + { + payload[index] = ExpectedReaderByte(keyIndex, index); + } + + return payload; +} + +static bool ValidateReaderPayload(ReadOnlySpan payload, int keyIndex) +{ + if (payload.Length != ReaderPayloadBytes) + { + return false; + } + + for (var index = 0; index < payload.Length; index++) + { + if (payload[index] != ExpectedReaderByte(keyIndex, index)) + { + return false; + } + } + + return true; +} + +static byte ExpectedReaderByte(int keyIndex, int byteIndex) => + unchecked((byte)(keyIndex * 31 + byteIndex * 17 + 0x5A)); + +static Task RunBrokerMeasuredOnDedicatedThread( + Store producer, + IReadOnlyList readers, + IReadOnlyList observers, + BenchmarkKeyCatalog keys, + long[] readerFrames, + int frameBytes, + int durationSeconds, + long frameTarget) +{ + var completion = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var thread = new Thread(() => + { + try + { + completion.SetResult(RunBrokerMeasuredRegion( + producer, + readers, + observers, + keys, + readerFrames, + frameBytes, + durationSeconds, + frameTarget)); + } + catch (Exception error) + { + completion.SetException(error); + } + }) + { + IsBackground = true, + Name = "SharedMemoryStore benchmark producer" + }; + thread.Start(); + return completion.Task; +} + +static BrokerMeasuredResult RunBrokerMeasuredRegion( + Store producer, + IReadOnlyList readers, + IReadOnlyList observers, + BenchmarkKeyCatalog keys, + long[] readerFrames, + int frameBytes, + int durationSeconds, + long frameTarget) +{ + WarmBrokerProducerCoordinatorThread(producer, readers, observers, keys, frameBytes); + ResetBrokerWorkersSync(readers, observers); + + var counters = new StatusCounters(); + var samples = new List(MaxLatencySamplesPerWorker); + var earlySamples = new List(MaxLatencySamplesPerWorker); + var lateSamples = new List(MaxLatencySamplesPerWorker); + var pending = new PendingBrokerFrame[readers.Count]; + var measured = new Stopwatch(); + long failures = 0; + long frames = 0; + long producerStoreOperationAllocatedBytes = 0; + + // This is one contiguous interval on an explicitly created thread. It + // intentionally includes test-broker serialization and pipe coordination; + // ProducerStoreOperationAllocatedBytes separately isolates the synchronous + // store calls made by the same thread. + long measuredAllocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + measured.Start(); + while ((frameTarget > 0 && frames < frameTarget) + || (frameTarget == 0 && (frames == 0 || measured.Elapsed.TotalSeconds < durationSeconds))) + { + var pendingCount = 0; + bool batchFailed = false; + int batchSize = frameTarget > 0 + ? checked((int)Math.Min(readers.Count, frameTarget - frames)) + : readers.Count; + for (var batchOffset = 0; batchOffset < batchSize; batchOffset++) + { + long frameNumber = frames; + int keyIndex = (int)(frameNumber % BrokerRotatingKeyCount); + ReadOnlyMemory key = keys[keyIndex]; + if (frameNumber >= BrokerRotatingKeyCount) + { + long operationAllocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + StoreStatus remove = RemoveWithRetry(producer, key.Span, counters); + producerStoreOperationAllocatedBytes += + GC.GetAllocatedBytesForCurrentThread() - operationAllocatedBefore; + if (remove is not (StoreStatus.Success or StoreStatus.RemovePending)) + { + failures++; + batchFailed = true; + break; + } + } + + long generation = frameNumber + 1; + long started = Stopwatch.GetTimestamp(); + long publishAllocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + bool published = PublishGeneration( + producer, + key.Span, + keyIndex, + generation, + frameBytes, + counters); + producerStoreOperationAllocatedBytes += + GC.GetAllocatedBytesForCurrentThread() - publishAllocatedBefore; + if (!published) + { + failures++; + batchFailed = true; + break; + } + + var message = new BrokerKeyMessage( + BrokerMessageKind.Key, + keys.Hex(keyIndex), + keyIndex, + generation, + frameBytes, + started); + string encoded = JsonSerializer.Serialize(message, BenchmarkProtocol.JsonOptions); + int readerId = (int)(frameNumber % readers.Count); + Process assignedReader = readers[readerId]; + assignedReader.StandardInput.WriteLine(encoded); + assignedReader.StandardInput.Flush(); + + bool observerSampled = frameNumber % BrokerObserverSamplingInterval == 0; + if (observerSampled) + { + foreach (Process observer in observers) + { + observer.StandardInput.WriteLine(encoded); + observer.StandardInput.Flush(); + } + } + + pending[pendingCount++] = new PendingBrokerFrame( + frameNumber, + keyIndex, + generation, + started, + readerId, + observerSampled); + frames++; + } + + for (var index = 0; index < pendingCount; index++) + { + PendingBrokerFrame frame = pending[index]; + BrokerAcknowledgement assignedAck = ReadBrokerAcknowledgementSync(readers[frame.ReaderId]); + if (assignedAck.WorkerId != frame.ReaderId + || !IsValidAcknowledgement( + assignedAck, + frame.KeyIndex, + frame.Generation, + frameBytes, + "reader")) + { + failures++; + } + + readerFrames[frame.ReaderId]++; + if (frame.ObserverSampled) + { + foreach (Process observer in observers) + { + BrokerAcknowledgement observerAck = ReadBrokerAcknowledgementSync(observer); + if (!IsValidAcknowledgement( + observerAck, + frame.KeyIndex, + frame.Generation, + frameBytes, + "observer")) + { + failures++; + } + } + } + + if (frame.FrameNumber % SamplingInterval == 0 + && samples.Count < MaxLatencySamplesPerWorker) + { + double latency = Stopwatch.GetElapsedTime(frame.PublishedTimestamp).TotalMicroseconds; + samples.Add(latency); + bool early = frameTarget > 0 + ? frame.FrameNumber < frameTarget / 2 + : measured.Elapsed.TotalSeconds < durationSeconds / 2.0; + (early ? earlySamples : lateSamples).Add(latency); + } + } + + if (batchFailed) + { + break; + } + } + + measured.Stop(); + long measuredThreadAllocatedBytes = + GC.GetAllocatedBytesForCurrentThread() - measuredAllocatedBefore; + return new BrokerMeasuredResult( + frames, + failures, + measured.Elapsed, + measuredThreadAllocatedBytes, + producerStoreOperationAllocatedBytes, + counters, + samples.ToArray(), + earlySamples.ToArray(), + lateSamples.ToArray()); +} + +static void WarmBrokerProducerCoordinatorThread( + Store producer, + IReadOnlyList readers, + IReadOnlyList observers, + BenchmarkKeyCatalog keys, + int frameBytes) +{ + var counters = new StatusCounters(); + int warmFrames = Math.Max(readers.Count, BrokerObserverSamplingInterval); + for (var frame = 0; frame < warmFrames; frame++) + { + int keyIndex = frame % keys.Count; + ReadOnlyMemory key = keys[keyIndex]; + long generation = -frame - 1L; + long started = Stopwatch.GetTimestamp(); + if (!PublishGeneration(producer, key.Span, keyIndex, generation, frameBytes, counters)) + { + throw new InvalidOperationException($"Dedicated producer-thread warm-up publication {frame} failed."); + } + + var message = new BrokerKeyMessage( + BrokerMessageKind.Key, + keys.Hex(keyIndex), + keyIndex, + generation, + frameBytes, + started); + string encoded = JsonSerializer.Serialize(message, BenchmarkProtocol.JsonOptions); + int readerId = frame % readers.Count; + Process reader = readers[readerId]; + reader.StandardInput.WriteLine(encoded); + reader.StandardInput.Flush(); + bool observerSampled = frame % BrokerObserverSamplingInterval == 0; + if (observerSampled) + { + foreach (Process observer in observers) + { + observer.StandardInput.WriteLine(encoded); + observer.StandardInput.Flush(); + } + } + + BrokerAcknowledgement readerAck = ReadBrokerAcknowledgementSync(reader); + if (readerAck.WorkerId != readerId + || !IsValidAcknowledgement(readerAck, keyIndex, generation, frameBytes, "reader")) + { + throw new InvalidOperationException($"Dedicated producer-thread reader warm-up {frame} failed validation."); + } + + if (observerSampled) + { + foreach (Process observer in observers) + { + BrokerAcknowledgement observerAck = ReadBrokerAcknowledgementSync(observer); + if (!IsValidAcknowledgement(observerAck, keyIndex, generation, frameBytes, "observer")) + { + throw new InvalidOperationException( + $"Dedicated producer-thread observer warm-up {frame} failed validation."); + } + } + } + + StoreStatus remove = RemoveWithRetry(producer, key.Span, counters); + if (remove is not (StoreStatus.Success or StoreStatus.RemovePending)) + { + throw new InvalidOperationException( + $"Dedicated producer-thread warm-up removal {frame} failed: {remove}."); + } + } +} + +static void ResetBrokerWorkersSync( + IReadOnlyList readers, + IReadOnlyList observers) +{ + var reset = new BrokerKeyMessage(BrokerMessageKind.Reset, string.Empty, 0, 0, 0, 0); + string encoded = JsonSerializer.Serialize(reset, BenchmarkProtocol.JsonOptions); + foreach (Process process in readers) + { + process.StandardInput.WriteLine(encoded); + process.StandardInput.Flush(); + } + + foreach (Process process in observers) + { + process.StandardInput.WriteLine(encoded); + process.StandardInput.Flush(); + } + + foreach (Process process in readers) + { + if (process.StandardOutput.ReadLine() != "RESET") + { + throw new InvalidOperationException("Broker reader did not reset after dedicated-thread warm-up."); + } + } + + foreach (Process process in observers) + { + if (process.StandardOutput.ReadLine() != "RESET") + { + throw new InvalidOperationException("Broker observer did not reset after dedicated-thread warm-up."); + } + } +} + +static bool IsValidAcknowledgement( + BrokerAcknowledgement acknowledgement, + int keyIndex, + long generation, + int payloadBytes, + string role) => + acknowledgement.Role == role + && acknowledgement.KeyIndex == keyIndex + && acknowledgement.Generation == generation + && acknowledgement.AcquireStatus == StoreStatus.Success + && acknowledgement.ReleaseStatus == StoreStatus.Success + && acknowledgement.DescriptorValid + && acknowledgement.PayloadValid + && acknowledgement.BytesObserved == payloadBytes; + +static BrokerAcknowledgement ReadBrokerAcknowledgementSync(Process process) +{ + string? line = process.StandardOutput.ReadLine(); + if (line is null) + { + string error = process.StandardError.ReadToEnd(); + throw new InvalidOperationException($"Broker worker ended before acknowledgement: {error}"); + } + + return JsonSerializer.Deserialize(line, BenchmarkProtocol.JsonOptions); +} + +static async Task ReadBrokerAcknowledgement(Process process) +{ + string? line = await process.StandardOutput.ReadLineAsync(); + if (line is null) + { + string error = await process.StandardError.ReadToEndAsync(); + throw new InvalidOperationException($"Broker worker ended before acknowledgement: {error}"); + } + + return JsonSerializer.Deserialize(line, BenchmarkProtocol.JsonOptions); +} + +static async Task WarmBrokerWorkers( + Store producer, + IReadOnlyList readers, + IReadOnlyList observers, + BenchmarkKeyCatalog keys, + int frameBytes, + int warmupSeconds) +{ + var counters = new StatusCounters(); + long frame = 0; + var elapsed = Stopwatch.StartNew(); + while (frame < readers.Count || elapsed.Elapsed.TotalSeconds < warmupSeconds) + { + int keyIndex = (int)(frame % keys.Count); + ReadOnlyMemory key = keys[keyIndex]; + long generation = -frame - 1L; + long started = Stopwatch.GetTimestamp(); + if (!PublishGeneration(producer, key.Span, keyIndex, generation, frameBytes, counters)) + { + throw new InvalidOperationException($"Broker warm-up publication {frame} failed."); + } + + var message = new BrokerKeyMessage( + BrokerMessageKind.Key, + keys.Hex(keyIndex), + keyIndex, + generation, + frameBytes, + started); + string encoded = JsonSerializer.Serialize(message, BenchmarkProtocol.JsonOptions); + int readerId = (int)(frame % readers.Count); + Process reader = readers[readerId]; + await reader.StandardInput.WriteLineAsync(encoded); + await reader.StandardInput.FlushAsync(); + bool observerSampled = frame % BrokerObserverSamplingInterval == 0; + if (observerSampled) + { + foreach (Process observer in observers) + { + await observer.StandardInput.WriteLineAsync(encoded); + await observer.StandardInput.FlushAsync(); + } + } + + BrokerAcknowledgement readerAck = await ReadBrokerAcknowledgement(reader); + if (readerAck.WorkerId != readerId + || !IsValidAcknowledgement(readerAck, keyIndex, generation, frameBytes, "reader")) + { + throw new InvalidOperationException($"Broker reader warm-up {frame} failed validation."); + } + + if (observerSampled) + { + foreach (Process observer in observers) + { + BrokerAcknowledgement observerAck = await ReadBrokerAcknowledgement(observer); + if (!IsValidAcknowledgement(observerAck, keyIndex, generation, frameBytes, "observer")) + { + throw new InvalidOperationException($"Broker observer warm-up {frame} failed validation."); + } + } + } + + StoreStatus remove = RemoveWithRetry(producer, key.Span, counters); + if (remove is not (StoreStatus.Success or StoreStatus.RemovePending)) + { + throw new InvalidOperationException($"Broker warm-up removal {frame} failed: {remove}."); + } + + frame++; + } +} + +static async Task ResetBrokerWorkers(IReadOnlyList processes) +{ + var reset = new BrokerKeyMessage(BrokerMessageKind.Reset, string.Empty, 0, 0, 0, 0); + string encoded = JsonSerializer.Serialize(reset, BenchmarkProtocol.JsonOptions); + foreach (Process process in processes) + { + await process.StandardInput.WriteLineAsync(encoded); + await process.StandardInput.FlushAsync(); + } + + foreach (Process process in processes) + { + string? acknowledgement = await process.StandardOutput.ReadLineAsync(); + if (acknowledgement != "RESET") + { + string error = await process.StandardError.ReadToEndAsync(); + throw new InvalidOperationException( + $"Broker worker did not reset its measured counters: {acknowledgement}; {error}"); + } + } +} + +static string ScenarioRole(string scenario) => scenario switch +{ + "publish-remove" or "mixed-churn-writer" => "publisher", + _ => "reader" +}; + +static void Ensure(StoreStatus status, string operation) +{ + if (status != StoreStatus.Success) + { + throw new InvalidOperationException($"{operation} failed: {status}"); + } +} + +static double Percentile(double[] sorted, double percentile) +{ + if (sorted.Length == 0) + { + return 0; + } + + int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1); + return sorted[index]; +} + +static double JainFairness(double[] rates) +{ + if (rates.Length == 0) + { + return 0; + } + + double sum = rates.Sum(); + double squareSum = rates.Sum(static rate => rate * rate); + return squareSum == 0 ? 0 : sum * sum / (rates.Length * squareSum); +} + +static IReadOnlyList Summarize(IReadOnlyList runs) +{ + return runs + .GroupBy(static run => new { run.Profile, run.Scenario, run.ProcessCount }) + .Select(group => new SummaryResult( + group.Key.Profile, + group.Key.Scenario, + group.Key.ProcessCount, + Median(group.Select(static run => run.ApiCallsPerSecond)), + Median(group.Select(static run => run.P50Microseconds)), + Median(group.Select(static run => run.P95Microseconds)), + Median(group.Select(static run => run.P99Microseconds)), + Median(group.Select(static run => run.MaxMicroseconds)), + Median(group.Select(static run => run.EarlyP99Microseconds)), + Median(group.Select(static run => run.LateP99Microseconds)), + Median(group.Select(static run => run.LateToEarlyP99Ratio)), + Median(group.Select(static run => run.FramesPerSecond)), + Median(group.Select(static run => run.BytesPerSecond)), + Median(group.Select(static run => run.FairnessIndex)), + Median(group.Select(static run => run.WorstWorkerP99Microseconds)), + group.Sum(static run => run.Frames), + group.Sum(static run => run.BytesWritten), + group.Sum(static run => run.FullPayloadCopies), + group.Sum(static run => run.MeasuredThreadAllocatedBytes), + group.Sum(static run => run.Failures), + MergeHistograms(group.Select(static run => run.StatusHistogram)), + group.All(static run => run.FullPayloadCopyCountIsInstrumented), + group.Select(static run => run.FullPayloadCopyEvidenceKind) + .Distinct(StringComparer.Ordinal) + .ToArray(), + group.Sum(static run => run.ProducerStoreOperationAllocatedBytes), + group.Select(static run => run.AllocationMeasurementScope) + .Distinct(StringComparer.Ordinal) + .ToArray())) + .OrderBy(static result => result.Profile, StringComparer.Ordinal) + .ThenBy(static result => result.Scenario, StringComparer.Ordinal) + .ThenBy(static result => result.ProcessCount) + .ToArray(); +} + +static SortedDictionary MergeHistograms(IEnumerable> histograms) +{ + var merged = new SortedDictionary(StringComparer.Ordinal); + foreach (IReadOnlyDictionary histogram in histograms) + { + foreach ((string key, long value) in histogram) + { + merged[key] = merged.GetValueOrDefault(key) + value; + } + } + + return merged; +} + +static double Median(IEnumerable values) +{ + double[] sorted = values.Order().ToArray(); + return sorted.Length % 2 == 0 + ? (sorted[sorted.Length / 2 - 1] + sorted[sorted.Length / 2]) / 2.0 + : sorted[sorted.Length / 2]; +} + +static string Qualification( + bool oversubscribed, + int durationSeconds, + int warmupSeconds, + long operationTarget, + long frameTarget) +{ + if (oversubscribed) + { + return "not-qualified-oversubscribed"; + } + + if (warmupSeconds < ReleaseWarmupSeconds) + { + return "smoke-only-insufficient-warmup"; + } + + return durationSeconds >= 60 || operationTarget >= 100_000_000 || frameTarget >= 100_000 + ? "qualification-measurement" + : "smoke-only"; +} + +static void KillProcesses(IReadOnlyList processes) +{ + for (var index = 0; index < processes.Count; index++) + { + try + { + Process process = processes[index]; + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch + { + // Best effort only. Fail-fast remains the hard isolation boundary. + } + } +} + +static void DisposeProcesses(IReadOnlyList processes) +{ + foreach (Process process in processes) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + _ = process.WaitForExit(milliseconds: 5_000); + } + } + catch + { + // Preserve cleanup of every remaining child and the original trial result. + } + finally + { + try + { + process.Dispose(); + } + catch + { + // Process disposal is idempotent from the controller's perspective. + } + } + } +} + +static StoreProfile[] ParseProfiles(string value, string optionName) => value.ToLowerInvariant() switch +{ + "legacy" => [StoreProfile.Legacy], + "v2" or "lockfree" or "lock-free" => [StoreProfile.LockFree], + "both" => [StoreProfile.Legacy, StoreProfile.LockFree], + _ => throw new ArgumentException($"{optionName} must be legacy, v2, or both.") +}; + +static int ReadPositiveIntOption(string[] args, string name, int fallback) +{ + int index = Array.IndexOf(args, name); + if (index < 0) + { + return fallback; + } + + if (index + 1 >= args.Length + || !int.TryParse(args[index + 1], NumberStyles.None, CultureInfo.InvariantCulture, out int value) + || value <= 0) + { + throw new ArgumentException($"{name} requires a positive integer."); + } + + return value; +} + +static int ReadPositivePowerOfTwoOption(string[] args, string name, int fallback) +{ + int value = ReadPositiveIntOption(args, name, fallback); + if (value < 32 || (value & (value - 1)) != 0) + { + throw new ArgumentException($"{name} requires a power of two greater than or equal to 32."); + } + + return value; +} + +static int ReadNonNegativeIntOption(string[] args, string name, int fallback) +{ + int index = Array.IndexOf(args, name); + if (index < 0) + { + return fallback; + } + + if (index + 1 >= args.Length + || !int.TryParse(args[index + 1], NumberStyles.None, CultureInfo.InvariantCulture, out int value) + || value < 0) + { + throw new ArgumentException($"{name} requires a non-negative integer."); + } + + return value; +} + +static long ReadPositiveLongOption(string[] args, string name, long fallback) +{ + int index = Array.IndexOf(args, name); + if (index < 0) + { + return fallback; + } + + if (index + 1 >= args.Length + || !long.TryParse(args[index + 1], NumberStyles.None, CultureInfo.InvariantCulture, out long value) + || value <= 0) + { + throw new ArgumentException($"{name} requires a positive integer."); + } + + return value; +} + +static long ReadNonNegativeLongOption(string[] args, string name, long fallback) +{ + int index = Array.IndexOf(args, name); + if (index < 0) + { + return fallback; + } + + if (index + 1 >= args.Length + || !long.TryParse(args[index + 1], NumberStyles.None, CultureInfo.InvariantCulture, out long value) + || value < 0) + { + throw new ArgumentException($"{name} requires a non-negative integer."); + } + + return value; +} + +static string? ReadStringOption(string[] args, string name) +{ + int index = Array.IndexOf(args, name); + return index >= 0 && index + 1 < args.Length ? args[index + 1] : null; +} + +static int[]? ParsePositiveIntListOption(string[] args, string name) +{ + string? raw = ReadStringOption(args, name); + if (raw is null) + { + return null; + } + + string[] parts = raw.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length == 0) + { + throw new ArgumentException($"{name} requires one or more positive integers."); + } + + var values = new int[parts.Length]; + for (var index = 0; index < parts.Length; index++) + { + if (!int.TryParse(parts[index], NumberStyles.None, CultureInfo.InvariantCulture, out int value) + || value <= 0) + { + throw new ArgumentException($"{name} requires a comma-separated list of positive integers."); + } + + values[index] = value; + } + + return values.Distinct().ToArray(); +} + +static long ReadPositiveLongEnvironment(string name, long fallback) => + long.TryParse(Environment.GetEnvironmentVariable(name), NumberStyles.None, CultureInfo.InvariantCulture, out long value) + && value > 0 + ? value + : fallback; + +static long ReadNonNegativeLongEnvironment(string name, long fallback) => + long.TryParse(Environment.GetEnvironmentVariable(name), NumberStyles.None, CultureInfo.InvariantCulture, out long value) + && value >= 0 + ? value + : fallback; + +static ProbeEnvironment CaptureEnvironment(RepositoryProvenanceSnapshot repositoryProvenance) +{ + HostHardwareInfo hardware = HostEnvironmentProbe.Capture(); + return new ProbeEnvironment( + repositoryProvenance.Commit, + repositoryProvenance.WorkingTreeState, + TryGetFileSha256(typeof(Store).Assembly.Location), + TryGetFileSha256(typeof(BenchmarkProtocol).Assembly.Location), + RuntimeInformation.OSDescription, + RuntimeInformation.OSArchitecture.ToString(), + RuntimeInformation.ProcessArchitecture.ToString(), + RuntimeInformation.FrameworkDescription, + Environment.Version.ToString(), + hardware.LogicalProcessorCount, + hardware.PhysicalCoreCount, + hardware.TotalMemoryBytes, + hardware.ProcessorModel, + hardware.ProcessorModel, + GCSettings.IsServerGC, + Stopwatch.Frequency); +} + +static SortedDictionary CreateScenarioStoreDimensions( + IReadOnlyList plans, + int largeFrameBytes, + int stickyOverflowSlotCount) +{ + var dimensions = new SortedDictionary(StringComparer.Ordinal); + foreach (ScenarioPlan plan in plans) + { + dimensions[plan.Name] = plan.Name switch + { + "acquire-release" or "publish-remove" => new ProbeStoreDimensions( + SyncSlotCount, + SyncValueBytes, + MaxDescriptorBytes: 0, + MaxKeyBytes, + DefaultLeaseRecordCount, + ParticipantRecordCount), + "same-key-read" or "distributed-key-read" => new ProbeStoreDimensions( + ReaderSlotCount, + ReaderPayloadBytes, + MaxDescriptorBytes: 0, + MaxKeyBytes, + DefaultLeaseRecordCount, + ParticipantRecordCount), + "broker-directed" or "large-ingest" => new ProbeStoreDimensions( + BrokerSlotCount, + largeFrameBytes, + BenchmarkDescriptorBytes, + MaxKeyBytes, + DefaultLeaseRecordCount, + ParticipantRecordCount), + "mixed-churn" => new ProbeStoreDimensions( + MixedSlotCount, + MixedPayloadBytes, + BenchmarkDescriptorBytes, + MaxKeyBytes, + MixedLeaseRecordCount, + ParticipantRecordCount), + "sticky-overflow-miss" => new ProbeStoreDimensions( + stickyOverflowSlotCount, + MaxValueBytes: 1, + MaxDescriptorBytes: 0, + MaxKeyBytes, + DefaultLeaseRecordCount, + ParticipantRecordCount), + _ => throw new InvalidOperationException( + $"Scenario '{plan.Name}' does not declare benchmark store dimensions.") + }; + } + + return dimensions; +} + +static string TryGetFileSha256(string path) +{ + try + { + using FileStream stream = File.OpenRead(path); + return Convert.ToHexString(SHA256.HashData(stream)); + } + catch + { + return "unknown"; + } +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/README.md b/benchmarks/SharedMemoryStore.SyncProbe/README.md new file mode 100644 index 0000000..9925d3c --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/README.md @@ -0,0 +1,107 @@ +# SharedMemoryStore synchronization probe + +This executable is a process-level correctness and performance probe. It emits +raw JSON; it is not a BenchmarkDotNet microbenchmark. + +## Evidence semantics + +Report schema v8 requires a v8-aware reader for qualification. Existing JSON +property names remain present so older readers may parse or archive the report, +but schemas v3-v7 interpreted completion targets globally and therefore cannot +safely qualify the profile-aware rows. `MinimumCompatibleSchemaVersion` is 8, +and `SchemaCompatibility` states this contract in every report. + +`Environment` records the repository commit and clean/dirty state plus SHA-256 +digests for the exact SharedMemoryStore and probe assemblies that produced the +report. Assembly digests are the authoritative trace when a qualification is +run from an intentionally dirty development tree. Schema v7 records a +portable processor model, logical and physical processor counts, and total host +memory; `Configuration.ScenarioStoreDimensions` binds every selected scenario +to its exact slot/value/descriptor/key/lease/participant dimensions. + +Schema v8 records `Configuration.CountBoundProfiles` and each run's +`OperationTarget` and `FrameTarget`. Qualification uses duration-bound Legacy +comparison rows and applies the 100-million mixed-operation and 100,000-frame +durability targets only to the lock-free profile. Standalone probes preserve the +previous behavior by defaulting `--count-bound-profiles` to `both`. +`Configuration.DurationBoundGraceSeconds` records the controller watchdog grace; +`--duration-bound-grace` configures it. A duration-bound trial gets one hard +deadline covering store creation, warm-up, measurement, worker collection, and +cleanup: warm-up plus measurement plus grace. An atomic monotonic-deadline latch +rejects late completion even if timer dispatch is delayed. Missing that deadline +gives child-tree termination a bounded 100 ms best-effort budget and then +unconditionally fail-fast terminates the probe process because an in-flight +infinite store wait cannot be safely unwound. Count-bound trials and the +fixed-work sticky-overflow scenario do not use that duration watchdog. + +Qualification orchestrators pass `--repository-commit` and +`--repository-working-tree-state` as one validated pair from their independently +captured start provenance. The probe captures that pair, host metadata, and +assembly hashes once before it starts any workload. Standalone runs may omit +both options and use bounded, fail-closed Git discovery instead; supplying only +one option or an invalid commit/state is an error. Qualification still compares +the report with clean start and completion provenance and the fresh assembly +manifest, so the injected pair is not accepted on trust. + +`FullPayloadCopies` is retained for schema compatibility. A zero in that legacy +field is not, by itself, a measured copy count. Consumers must also inspect: + +- `FullPayloadCopyCountIsInstrumented` +- `FullPayloadCopyEvidenceKind` + +The broker scenarios use direct reservation writes and borrowed lease reads, so +their copy evidence is structural and the instrumentation flag is `false`. + +Broker allocation evidence has two scopes: + +- `MeasuredThreadAllocatedBytes` covers the complete measurement interval on an + explicitly created producer/coordinator thread. That same thread first runs an + unmeasured warm-up and resets worker counters. The measured value includes JSON + and pipe coordination performed by the benchmark harness. +- `ProducerStoreOperationAllocatedBytes` sums current-thread allocation deltas + immediately around producer store calls in that same interval. + +`AllocationMeasurementScope` identifies the applicable scope for every run. + +## Tiny-operation topology and latency sampling + +The synchronization scenarios use one deterministic catalog for both store +profiles. Each of the 12 supported workers owns two keys and alternates between +them by cycle. Both keys for worker `i` have canonical bucket `i` in the +16-bucket directory used by the 32-slot synchronization store. Configuration +records the key count per worker, maximum worker count, canonical bucket count, +SHA-256 of the ordered key bytes, and the ordered per-key bucket assignments. + +Autonomous workers select latency candidates with deterministic xorshift-driven +geometric gaps whose mean is `SamplingInterval` cycles. Separate early and late +Algorithm-R reservoirs retain at most 32,768 samples each across their complete +measurement windows. Overall percentiles use the merged reservoirs; +`EarlySampleCount`, `LateSampleCount`, and `SampleCount` expose the retained +sample topology for machine validation. Reservoir storage is allocated before +the measured region and candidate selection and replacement are allocation-free. +`MaxMicroseconds` is tracked separately across every sampled candidate, so an +outlier remains visible to the stall gate even if Algorithm R later evicts it +from a percentile reservoir. + +## Sticky-overflow qualification + +The `overflow` mode constructs 17 exact two-choice bucket-pair collisions in a +large lock-free store. It measures a missing key before churn, repeatedly +publishes/removes the colliding set, and measures the same missing key after the +overflow directory is empty. The report includes raw early and late samples plus +spill/occupancy/scan diagnostics in `StickyOverflow`. + +```powershell +dotnet run -c Release --project benchmarks/SharedMemoryStore.SyncProbe -- ` + --mode overflow --profile v2 --overflow-slot-count 4096 ` + --overflow-churn-cycles 10000 --overflow-missing-samples 16384 ` + --trials 3 --output artifacts/sticky-overflow.json +``` + +The diagnostics gate requires a real spill and occupied overflow cell during the +first cycle, a full cleanup scan and logical `Empty` summary immediately after +that cycle, no residual spill or overflow occupancy after churn, and exactly zero +new overflow scans in the late missing-key window. The latency gate is late +missing-key p99 no greater than 2x early p99. Exit code `4` means the raw report +was written but that performance gate failed. Exit code `3` means the diagnostics +gate failed; exit code `2` means a correctness failure occurred. diff --git a/benchmarks/SharedMemoryStore.SyncProbe/RepositoryEnvironmentProbe.cs b/benchmarks/SharedMemoryStore.SyncProbe/RepositoryEnvironmentProbe.cs new file mode 100644 index 0000000..4e1265b --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/RepositoryEnvironmentProbe.cs @@ -0,0 +1,197 @@ +using System.Diagnostics; + +internal readonly record struct RepositoryProvenanceSnapshot( + string Commit, + string WorkingTreeState); + +internal static class RepositoryEnvironmentProbe +{ + private const string CommitOption = "--repository-commit"; + private const string WorkingTreeStateOption = "--repository-working-tree-state"; + private const string Unknown = "unknown"; + private static readonly TimeSpan GitTimeout = TimeSpan.FromSeconds(30); + + internal static RepositoryProvenanceSnapshot Capture(string[] args) => + Capture(args, Environment.CurrentDirectory, BoundedProcessRunner.Run); + + internal static RepositoryProvenanceSnapshot Capture( + IReadOnlyList args, + string currentDirectory, + Func runProcess) + { + ArgumentNullException.ThrowIfNull(args); + ArgumentException.ThrowIfNullOrWhiteSpace(currentDirectory); + ArgumentNullException.ThrowIfNull(runProcess); + + string? injectedCommit = ReadUniqueOption(args, CommitOption); + string? injectedWorkingTreeState = ReadUniqueOption(args, WorkingTreeStateOption); + if ((injectedCommit is null) != (injectedWorkingTreeState is null)) + { + throw new ArgumentException( + $"{CommitOption} and {WorkingTreeStateOption} must be supplied together."); + } + + if (injectedCommit is not null) + { + ValidateCommit(injectedCommit, CommitOption); + ValidateWorkingTreeState(injectedWorkingTreeState!, WorkingTreeStateOption); + return new RepositoryProvenanceSnapshot(injectedCommit, injectedWorkingTreeState!); + } + + return CaptureFromGit(currentDirectory, runProcess); + } + + private static RepositoryProvenanceSnapshot CaptureFromGit( + string currentDirectory, + Func runProcess) + { + try + { + string? repositoryRoot = FindRepositoryRoot(currentDirectory); + if (repositoryRoot is null) + { + return UnknownSnapshot(); + } + + BoundedProcessResult commit = runProcess( + CreateGitStartInfo(repositoryRoot, "rev-parse", "HEAD"), + GitTimeout); + if (!commit.Succeeded) + { + return UnknownSnapshot(); + } + + string commitValue = commit.StandardOutput.Trim(); + if (!IsValidCommit(commitValue)) + { + return UnknownSnapshot(); + } + + BoundedProcessResult status = runProcess( + CreateGitStartInfo( + repositoryRoot, + "status", + "--porcelain=v2", + "--untracked-files=normal"), + GitTimeout); + if (!status.Succeeded) + { + return UnknownSnapshot(); + } + + return new RepositoryProvenanceSnapshot( + commitValue, + string.IsNullOrWhiteSpace(status.StandardOutput) ? "clean" : "dirty"); + } + catch (Exception exception) when ( + exception is IOException + or UnauthorizedAccessException + or ArgumentException + or NotSupportedException) + { + return UnknownSnapshot(); + } + } + + internal static string? FindRepositoryRoot(string currentDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(currentDirectory); + + var directory = new DirectoryInfo(Path.GetFullPath(currentDirectory)); + while (directory is not null) + { + string marker = Path.Combine(directory.FullName, ".git"); + if (Directory.Exists(marker) || File.Exists(marker)) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } + + private static ProcessStartInfo CreateGitStartInfo( + string repositoryRoot, + params string[] commandArguments) + { + var startInfo = new ProcessStartInfo("git") + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + startInfo.Environment["GIT_OPTIONAL_LOCKS"] = "0"; + startInfo.ArgumentList.Add("-c"); + startInfo.ArgumentList.Add("core.autocrlf=true"); + startInfo.ArgumentList.Add("-c"); + startInfo.ArgumentList.Add("core.safecrlf=false"); + startInfo.ArgumentList.Add("-C"); + startInfo.ArgumentList.Add(repositoryRoot); + foreach (string argument in commandArguments) + { + startInfo.ArgumentList.Add(argument); + } + + return startInfo; + } + + private static string? ReadUniqueOption(IReadOnlyList args, string option) + { + string? value = null; + for (var index = 0; index < args.Count; index++) + { + if (!string.Equals(args[index], option, StringComparison.Ordinal)) + { + continue; + } + + if (value is not null) + { + throw new ArgumentException($"{option} may be supplied only once.", option); + } + + if (index + 1 >= args.Count + || string.IsNullOrWhiteSpace(args[index + 1]) + || args[index + 1].StartsWith("--", StringComparison.Ordinal)) + { + throw new ArgumentException($"{option} requires a value.", option); + } + + value = args[index + 1]; + index++; + } + + return value; + } + + private static void ValidateCommit(string commit, string option) + { + if (!IsValidCommit(commit)) + { + throw new ArgumentException( + $"{option} must be exactly 40 or 64 hexadecimal characters.", + option); + } + } + + private static bool IsValidCommit(string commit) => + (commit.Length == 40 || commit.Length == 64) + && commit.All(static character => + character is >= '0' and <= '9' + or >= 'a' and <= 'f' + or >= 'A' and <= 'F'); + + private static void ValidateWorkingTreeState(string state, string option) + { + if (!string.Equals(state, "clean", StringComparison.Ordinal) + && !string.Equals(state, "dirty", StringComparison.Ordinal)) + { + throw new ArgumentException($"{option} must be 'clean' or 'dirty'.", option); + } + } + + private static RepositoryProvenanceSnapshot UnknownSnapshot() => new(Unknown, Unknown); +} diff --git a/benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj b/benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj new file mode 100644 index 0000000..ca6c1d0 --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj @@ -0,0 +1,20 @@ + + + + + + + + + Exe + net10.0 + enable + enable + + + + + + + diff --git a/benchmarks/SharedMemoryStore.SyncProbe/SuspensionQualification.cs b/benchmarks/SharedMemoryStore.SyncProbe/SuspensionQualification.cs new file mode 100644 index 0000000..e2a7f2f --- /dev/null +++ b/benchmarks/SharedMemoryStore.SyncProbe/SuspensionQualification.cs @@ -0,0 +1,1528 @@ +using System.Diagnostics; +using System.Globalization; +using System.Numerics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using SharedMemoryStore; +using SharedMemoryStore.LockFree; + +using Store = SharedMemoryStore.MemoryStore; + +internal static class SuspensionQualification +{ + private const int SlotCount = 512; + private const int MaxValueBytes = 256; + private const int MaxDescriptorBytes = 16; + private const int MaxKeyBytes = 8; + private const int LeaseRecordCount = 128; + private const int ParticipantRecordCount = 64; + private const int StableKeyCount = 256; + private const int ReadKeyCount = 128; + private const int KeyBase = 1_000_000; + private const int TokenKeyIndex = 900_000; + private const int ExistingKeyIndex = 900_001; + private const int OperationKeyBase = 910_000; + private const int RecoveryKeyBase = 920_000; + private const double DefaultMinimumRatio = 0.90; + private const int DefaultBaselineSeconds = 1; + private const int DefaultPauseSeconds = 1; + private const int DefaultWarmupSeconds = 1; + private static readonly TimeSpan ChildStartupTimeout = TimeSpan.FromSeconds(30); + private static readonly TimeSpan ChildExitTimeout = TimeSpan.FromSeconds(30); + + internal static async Task RunControllerAsync(string[] arguments) + { + int baselineSeconds = ReadPositiveInt(arguments, "--suspension-baseline-seconds", DefaultBaselineSeconds); + int pauseSeconds = ReadPositiveInt(arguments, "--suspension-pause-seconds", DefaultPauseSeconds); + int warmupSeconds = ReadNonNegativeInt(arguments, "--warmup", DefaultWarmupSeconds); + double minimumRatio = ReadRatio(arguments, "--suspension-minimum-ratio", DefaultMinimumRatio); + bool affinityRequested = !arguments.Contains("--no-affinity", StringComparer.Ordinal); + string? outputPath = ReadString(arguments, "--output"); + string[] workloadFilter = ParseList(arguments, "--suspension-workloads"); + string[] checkpointFilter = ParseList(arguments, "--suspension-checkpoints"); + + string agentPath = FindAgentAssembly(); + CheckpointCatalogEntry[] catalog = await ReadCheckpointCatalog(agentPath); + CheckpointCatalogEntry[] checkpoints = catalog + .Where(static checkpoint => checkpoint.Family is not ("Participant" or "Disposal")) + .Where(checkpoint => checkpointFilter.Length == 0 + || checkpointFilter.Contains(checkpoint.Id.ToString(CultureInfo.InvariantCulture), StringComparer.OrdinalIgnoreCase) + || checkpointFilter.Contains(checkpoint.Name, StringComparer.OrdinalIgnoreCase)) + .OrderBy(static checkpoint => checkpoint.Id) + .ToArray(); + if (checkpoints.Length == 0) + { + throw new ArgumentException("No steady-state checkpoints matched --suspension-checkpoints."); + } + + SuspensionWorkload[] workloads = Workloads + .Where(workload => workloadFilter.Length == 0 + || workloadFilter.Contains(workload.Name, StringComparer.OrdinalIgnoreCase)) + .ToArray(); + if (workloads.Length == 0) + { + throw new ArgumentException("No workload matched --suspension-workloads."); + } + + int availableProcessors = GetAvailableProcessorCount(); + var results = new List(checkpoints.Length * workloads.Length); + foreach (SuspensionWorkload workload in workloads) + { + foreach (CheckpointCatalogEntry checkpoint in checkpoints) + { + int requiredProcessors = workload.HealthyProcessCount + 1; + if (affinityRequested && availableProcessors < requiredProcessors) + { + results.Add(NotQualifiedForProcessors( + checkpoint, + workload, + baselineSeconds, + pauseSeconds, + minimumRatio, + availableProcessors, + requiredProcessors)); + Console.Error.WriteLine( + $"suspension {workload.Name} checkpoint={checkpoint.Id}:{checkpoint.Name} " + + $"qualification=not-qualified-insufficient-processors available={availableProcessors} " + + $"required={requiredProcessors}"); + continue; + } + + SuspensionCheckpointResult result; + try + { + result = await RunCheckpointAsync( + agentPath, + checkpoint, + workload, + baselineSeconds, + pauseSeconds, + warmupSeconds, + minimumRatio, + affinityRequested, + availableProcessors, + requiredProcessors); + } + catch (Exception exception) + { + result = HarnessFailure( + checkpoint, + workload, + baselineSeconds, + pauseSeconds, + minimumRatio, + availableProcessors, + requiredProcessors, + exception); + } + + results.Add(result); + Console.Error.WriteLine( + $"suspension {workload.Name} checkpoint={checkpoint.Id}:{checkpoint.Name} " + + $"baseline={result.BaselineCompletedCyclesPerSecond:N0}/s " + + $"paused={result.SuspendedCompletedCyclesPerSecond:N0}/s ratio={result.ThroughputRatio:N3} " + + $"healthy={result.HealthyProcessCount} failures={result.CorrectnessFailureCount} " + + $"qualification={result.Qualification}"); + } + } + + var report = new SuspensionQualificationReport( + SchemaVersion: 1, + TimestampUtc: DateTimeOffset.UtcNow, + Environment: CaptureEnvironment(availableProcessors), + Configuration: new SuspensionConfiguration( + baselineSeconds, + pauseSeconds, + warmupSeconds, + minimumRatio, + affinityRequested, + "physical-core-first-then-siblings", + checkpoints.Length, + workloads.Select(static workload => workload.Name).ToArray(), + catalog.Length, + ["Participant", "Disposal"], + "Same persistent healthy process set is measured immediately before and while one external participant is blocked inside a production checkpoint."), + Results: results, + RequiredResultCount: checkpoints.Length * workloads.Length, + QualifiedPassCount: results.Count(static result => result.Qualification == "qualified-pass"), + SmokePassCount: results.Count(static result => result.Qualification == "smoke-pass"), + FailCount: results.Count(static result => result.Qualification.EndsWith("-fail", StringComparison.Ordinal)), + NotQualifiedCount: results.Count(static result => result.Qualification.StartsWith("not-qualified-", StringComparison.Ordinal)), + AllRequiredQualifiedAndPassed: results.Count == checkpoints.Length * workloads.Length + && results.All(static result => result.Qualification == "qualified-pass")); + string json = JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true }); + if (string.IsNullOrWhiteSpace(outputPath)) + { + Console.WriteLine(json); + } + else + { + string fullPath = Path.GetFullPath(outputPath); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + await File.WriteAllTextAsync(fullPath, json); + Console.Error.WriteLine("report=" + fullPath); + } + + return report.FailCount == 0 ? 0 : 2; + } + + internal static int RunWorker(string[] arguments) + { + if (arguments.Length != 8 + || !int.TryParse(arguments[4], NumberStyles.None, CultureInfo.InvariantCulture, out int workerId) + || !int.TryParse(arguments[6], NumberStyles.Integer, CultureInfo.InvariantCulture, out int affinityOrdinal) + || !int.TryParse(arguments[7], NumberStyles.None, CultureInfo.InvariantCulture, out int warmupSeconds) + || warmupSeconds < 0) + { + Console.Error.WriteLine( + "suspension-worker requires ."); + return 64; + } + + string workload = arguments[1]; + string storeName = arguments[2]; + string role = arguments[3]; + bool affinityApplied = ProcessorAffinityPlanner.TryApply( + affinityOrdinal, + out int assignedProcessor, + out string affinityStrategy); + StoreOpenStatus open = Store.TryCreateOrOpen( + CreateOptions(storeName, OpenMode.OpenExisting), + out Store? store); + if (open != StoreOpenStatus.Success || store is null) + { + Console.Error.WriteLine("Suspension worker open failed: " + open); + return 65; + } + + using (store) + { + var state = new WorkerState(workload, role, workerId); + if (!RunWarmup(store, state, warmupSeconds)) + { + return 66; + } + + Console.WriteLine(JsonSerializer.Serialize(new SuspensionWorkerReady( + workerId, + role, + Environment.ProcessId, + affinityApplied, + assignedProcessor, + affinityStrategy))); + Console.Out.Flush(); + while (Console.ReadLine() is { } command) + { + if (string.Equals(command, "STOP", StringComparison.Ordinal)) + { + return 0; + } + + string[] fields = command.Split('|'); + if (fields.Length != 3 + || fields[0] != "MEASURE" + || !int.TryParse(fields[1], NumberStyles.None, CultureInfo.InvariantCulture, out int durationMilliseconds) + || durationMilliseconds <= 0) + { + return 67; + } + + SuspensionWindowResult result = MeasureWindow( + store, + state, + fields[2], + durationMilliseconds, + affinityApplied, + assignedProcessor, + affinityStrategy); + Console.WriteLine(JsonSerializer.Serialize(result)); + Console.Out.Flush(); + } + } + + return 68; + } + + private static readonly SuspensionWorkload[] Workloads = + [ + new("distributed-key", ReaderCount: 6, WriterCount: 0), + new("mixed-churn", ReaderCount: 12, WriterCount: 2) + ]; + + private static async Task RunCheckpointAsync( + string agentPath, + CheckpointCatalogEntry checkpoint, + SuspensionWorkload workload, + int baselineSeconds, + int pauseSeconds, + int warmupSeconds, + double minimumRatio, + bool affinityRequested, + int availableProcessors, + int requiredProcessors) + { + string storeName = $"sms-suspend-{Guid.NewGuid():N}"; + StoreOpenStatus open = Store.TryCreateOrOpen( + CreateOptions(storeName, OpenMode.CreateNew), + out Store? owner); + if (open != StoreOpenStatus.Success || owner is null) + { + throw new InvalidOperationException("Suspension owner open failed: " + open); + } + + using (owner) + { + Seed(owner, workload.Name); + (int spillFirstBucket, int spillSecondBucket) = SelectUnusedSpillBucketPair(workload.Name); + var workers = new List(workload.HealthyProcessCount); + Process? pausedAgent = null; + bool agentReleased = false; + try + { + for (var readerId = 0; readerId < workload.ReaderCount; readerId++) + { + workers.Add(StartWorker( + workload.Name, + storeName, + "reader", + readerId, + affinityRequested ? readerId : -1, + warmupSeconds)); + } + + for (var writerId = 0; writerId < workload.WriterCount; writerId++) + { + workers.Add(StartWorker( + workload.Name, + storeName, + "writer", + writerId, + affinityRequested ? workload.ReaderCount + writerId : -1, + warmupSeconds)); + } + + SuspensionWorkerReady[] ready = await AwaitWorkersReady(workers); + DiagnosticsSnapshot beforeBaseline = GetDiagnostics(owner); + SuspensionWindowResult[] baseline = await MeasureWorkers( + workers, + baselineSeconds, + "baseline"); + DiagnosticsSnapshot afterBaseline = GetDiagnostics(owner); + + pausedAgent = StartPausedAgent( + agentPath, + storeName, + checkpoint.Id, + workload.Name, + spillFirstBucket, + spillSecondBucket); + string signalLine = await ReadLine(pausedAgent, ChildStartupTimeout) + ?? throw new InvalidOperationException( + "Paused checkpoint agent exited before signaling: " + + (await pausedAgent.StandardError.ReadToEndAsync()).Trim()); + if (!signalLine.StartsWith("CHECKPOINT ", StringComparison.Ordinal)) + { + throw new InvalidOperationException("Unexpected checkpoint signal: " + signalLine); + } + + CheckpointSignal signal = JsonSerializer.Deserialize(signalLine[11..]) + ?? throw new InvalidOperationException("Checkpoint agent emitted invalid signal JSON."); + if (signal.Id != checkpoint.Id) + { + throw new InvalidOperationException( + $"Checkpoint agent paused at {signal.Id}, expected {checkpoint.Id}."); + } + + int pausedProcessor = -1; + string pausedAffinityStrategy = "not-requested"; + bool pausedAffinityApplied = affinityRequested + && ProcessorAffinityPlanner.TryApply( + pausedAgent, + workload.HealthyProcessCount, + out pausedProcessor, + out pausedAffinityStrategy); + + if (IsStoreFullProofCheckpoint(checkpoint.Id)) + { + RemoveStoreFullFillers(owner, checkpoint.Id); + } + + DiagnosticsSnapshot beforeSuspended = GetDiagnostics(owner); + SuspensionWindowResult[] suspended = await MeasureWorkers( + workers, + pauseSeconds, + "suspended"); + DiagnosticsSnapshot afterSuspended = GetDiagnostics(owner); + + await pausedAgent.StandardInput.WriteLineAsync("CONTINUE"); + await pausedAgent.StandardInput.FlushAsync(); + agentReleased = true; + Task agentOutput = pausedAgent.StandardOutput.ReadToEndAsync(); + Task agentError = pausedAgent.StandardError.ReadToEndAsync(); + await pausedAgent.WaitForExitAsync().WaitAsync(ChildExitTimeout); + string trailingOutput = await agentOutput; + string trailingError = await agentError; + int agentExitCode = pausedAgent.ExitCode; + DiagnosticsSnapshot afterResume = GetDiagnostics(owner); + + long baselineFailures = baseline.Sum(static result => result.Failures); + long suspendedFailures = suspended.Sum(static result => result.Failures); + long correctnessFailures = baselineFailures + suspendedFailures; + var correctnessErrors = new List(); + if (agentExitCode != 0) + { + correctnessErrors.Add( + $"checkpoint-agent-exit={agentExitCode}; stderr={trailingError.Trim()}; stdout={trailingOutput.Trim()}"); + } + + if (IsStoreFullProofCheckpoint(checkpoint.Id) + && (beforeSuspended.ActiveReservationCount != 0 + || beforeSuspended.InitializingSlotCount != 0 + || beforeSuspended.ReservedSlotCount != 0 + || beforeSuspended.ReclaimingSlotCount != 0)) + { + correctnessErrors.Add( + "store-full-proof-paused-ownership=" + + $"reservations:{beforeSuspended.ActiveReservationCount}," + + $"initializing:{beforeSuspended.InitializingSlotCount}," + + $"reserved:{beforeSuspended.ReservedSlotCount}," + + $"reclaiming:{beforeSuspended.ReclaimingSlotCount}"); + } + + if (afterResume.ActiveLeaseCount != 0) + { + correctnessErrors.Add("active-leases-after-resume=" + afterResume.ActiveLeaseCount); + } + + if (afterResume.ActiveReservationCount != 0) + { + correctnessErrors.Add("active-reservations-after-resume=" + afterResume.ActiveReservationCount); + } + + int transitionalSlots = afterResume.InitializingSlotCount + + afterResume.ReservedSlotCount + + afterResume.ReclaimingSlotCount; + if (transitionalSlots != 0) + { + correctnessErrors.Add("transitional-slots-after-resume=" + transitionalSlots); + } + + int transitionalLeases = afterResume.ClaimingLeaseCount + afterResume.RecoveringLeaseCount; + if (transitionalLeases != 0) + { + correctnessErrors.Add("transitional-leases-after-resume=" + transitionalLeases); + } + + int transitionalParticipants = afterResume.RegisteringParticipantCount + + afterResume.ClosingParticipantCount + + afterResume.RecoveringParticipantCount + + afterResume.ReclaimingParticipantCount; + if (transitionalParticipants != 0) + { + correctnessErrors.Add("transitional-participants-after-resume=" + transitionalParticipants); + } + + correctnessFailures += correctnessErrors.Count; + double baselineThroughput = baseline.Sum(static result => result.CompletedCyclesPerSecond); + double suspendedThroughput = suspended.Sum(static result => result.CompletedCyclesPerSecond); + long baselineAttemptedCycles = baseline.Sum(static result => result.AttemptedCycles); + long baselineCompletedCycles = baseline.Sum(static result => result.CompletedCycles); + long baselineApiCalls = baseline.Sum(static result => result.ApiCalls); + long suspendedAttemptedCycles = suspended.Sum(static result => result.AttemptedCycles); + long suspendedCompletedCycles = suspended.Sum(static result => result.CompletedCycles); + long suspendedApiCalls = suspended.Sum(static result => result.ApiCalls); + double baselineApiCallsPerSecond = baseline.Sum(static result => result.ApiCallsPerSecond); + double suspendedApiCallsPerSecond = suspended.Sum(static result => result.ApiCallsPerSecond); + double ratio = baselineThroughput <= 0 ? 0 : suspendedThroughput / baselineThroughput; + bool capacityPermits = CapacityPermits(afterBaseline, workload.HealthyProcessCount) + && CapacityPermits(beforeSuspended, workload.HealthyProcessCount) + && CapacityPermits(afterSuspended, workload.HealthyProcessCount); + int healthyAffinityCount = ready.Count(static result => result.AffinityApplied); + bool affinityQualified = !affinityRequested + || (healthyAffinityCount == workload.HealthyProcessCount && pausedAffinityApplied); + bool releaseDurationEligible = pauseSeconds >= 30; + string passLabel = releaseDurationEligible ? "qualified-pass" : "smoke-pass"; + string failLabel = releaseDurationEligible ? "qualified-fail" : "smoke-fail"; + string qualification = correctnessFailures != 0 + ? failLabel + : !capacityPermits + ? "not-qualified-capacity-pressure" + : !affinityQualified + ? "not-qualified-affinity" + : !releaseDurationEligible || ratio >= minimumRatio + ? passLabel + : failLabel; + + return new SuspensionCheckpointResult( + checkpoint.Id, + checkpoint.Name, + checkpoint.Family, + checkpoint.Position, + checkpoint.Pause, + checkpoint.Crash, + checkpoint.Race, + checkpoint.IsPublicOrderingPoint, + workload.Name, + workload.ReaderCount, + workload.WriterCount, + workload.HealthyProcessCount, + baselineSeconds, + pauseSeconds, + baselineThroughput, + suspendedThroughput, + baselineAttemptedCycles, + baselineCompletedCycles, + baselineApiCalls, + suspendedAttemptedCycles, + suspendedCompletedCycles, + suspendedApiCalls, + baselineApiCallsPerSecond, + suspendedApiCallsPerSecond, + ratio, + minimumRatio, + capacityPermits, + correctnessFailures, + correctnessErrors.ToArray(), + healthyAffinityCount, + pausedAffinityApplied, + pausedProcessor, + pausedAffinityStrategy, + spillFirstBucket, + spillSecondBucket, + availableProcessors, + requiredProcessors, + signal.ProcessId, + agentExitCode, + qualification, + qualification.EndsWith("-pass", StringComparison.Ordinal), + baseline, + suspended, + CapacityEvidence(beforeBaseline), + CapacityEvidence(afterBaseline), + CapacityEvidence(beforeSuspended), + CapacityEvidence(afterSuspended), + CapacityEvidence(afterResume)); + } + finally + { + if (pausedAgent is not null) + { + try + { + if (!pausedAgent.HasExited && !agentReleased) + { + await pausedAgent.StandardInput.WriteLineAsync("CONTINUE"); + await pausedAgent.StandardInput.FlushAsync(); + } + + if (!pausedAgent.HasExited) + { + using var timeout = new CancellationTokenSource(ChildExitTimeout); + await pausedAgent.WaitForExitAsync(timeout.Token); + } + } + catch + { + if (!pausedAgent.HasExited) + { + pausedAgent.Kill(entireProcessTree: true); + } + } + finally + { + pausedAgent.Dispose(); + } + } + + await StopWorkers(workers); + } + } + } + + private static bool RunWarmup(Store store, WorkerState state, int seconds) + { + var counters = new WindowCounters(); + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed.TotalSeconds < seconds) + { + ExecuteCycle(store, state, counters); + } + + return counters.Failures == 0; + } + + private static SuspensionWindowResult MeasureWindow( + Store store, + WorkerState state, + string window, + int durationMilliseconds, + bool affinityApplied, + int assignedProcessor, + string affinityStrategy) + { + var counters = new WindowCounters(); + var elapsed = Stopwatch.StartNew(); + while (elapsed.ElapsedMilliseconds < durationMilliseconds) + { + ExecuteCycle(store, state, counters); + } + + elapsed.Stop(); + double seconds = elapsed.Elapsed.TotalSeconds; + return new SuspensionWindowResult( + state.WorkerId, + state.Role, + Environment.ProcessId, + window, + counters.AttemptedCycles, + counters.CompletedCycles, + counters.ApiCalls, + seconds, + seconds <= 0 ? 0 : counters.CompletedCycles / seconds, + seconds <= 0 ? 0 : counters.ApiCalls / seconds, + counters.Failures, + affinityApplied, + assignedProcessor, + affinityStrategy, + counters.ToHistogram()); + } + + private static void ExecuteCycle(Store store, WorkerState state, WindowCounters counters) + { + counters.AttemptedCycles++; + if (state.Role == "reader") + { + int keyCount = StableKeyCount; + int keyIndex = (int)((state.Cycle + state.WorkerId * 17L) % keyCount); + byte[] key = state.Keys[keyIndex]; + StoreStatus acquire = store.TryAcquire(key, out ValueLease lease); + counters.Record(OperationKind.Acquire, acquire); + if (state.Workload == "mixed-churn" && acquire == StoreStatus.NotFound) + { + counters.CompletedCycles++; + state.Cycle++; + return; + } + + if (acquire != StoreStatus.Success) + { + counters.Failures++; + state.Cycle++; + return; + } + + bool descriptorValid = lease.DescriptorSpan.Length == MaxDescriptorBytes; + long readerGeneration = descriptorValid + ? System.Buffers.Binary.BinaryPrimitives.ReadInt64LittleEndian(lease.DescriptorSpan) + : long.MinValue; + descriptorValid = descriptorValid + && BenchmarkProtocol.ValidateDescriptor( + lease.DescriptorSpan, + keyIndex, + readerGeneration, + MaxValueBytes); + bool payloadValid = descriptorValid + && BenchmarkProtocol.ValidateGenerationPayload(lease.ValueSpan, keyIndex, readerGeneration); + if (!descriptorValid || !payloadValid) + { + counters.Failures++; + counters.RecordChecksumFailure(); + } + + StoreStatus release = lease.Release(); + counters.Record(OperationKind.Release, release); + if (release != StoreStatus.Success) + { + counters.Failures++; + } + else if (descriptorValid && payloadValid) + { + counters.CompletedCycles++; + } + + state.Cycle++; + return; + } + + int writerKeyIndex = ReadKeyCount + + state.WorkerId + + (int)((state.Cycle % ((StableKeyCount - ReadKeyCount) / 2)) * 2); + byte[] writerKey = state.Keys[writerKeyIndex]; + if (!RemoveForRewrite(store, writerKey, counters)) + { + counters.Failures++; + state.Cycle++; + return; + } + + long writerGeneration = checked(((long)(state.WorkerId + 1) << 56) | (state.Cycle + 1)); + if (!ReserveAndCommit(store, writerKey, writerKeyIndex, writerGeneration, counters)) + { + counters.Failures++; + } + else + { + counters.CompletedCycles++; + } + + state.Cycle++; + } + + private static bool RemoveForRewrite(Store store, byte[] key, WindowCounters counters) + { + for (var attempt = 0; attempt < 4096; attempt++) + { + StoreStatus status = store.TryRemove(key); + counters.Record(OperationKind.Remove, status); + if (status is StoreStatus.Success or StoreStatus.NotFound) + { + return true; + } + + if (status is not (StoreStatus.RemovePending or StoreStatus.StoreBusy)) + { + return false; + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + } + + return false; + } + + private static bool ReserveAndCommit( + Store store, + byte[] key, + int keyIndex, + long generation, + WindowCounters counters) + { + Span descriptor = stackalloc byte[MaxDescriptorBytes]; + BenchmarkProtocol.WriteDescriptor(descriptor, keyIndex, generation, MaxValueBytes); + for (var attempt = 0; attempt < 4096; attempt++) + { + StoreStatus reserve = store.TryReserve( + key, + MaxValueBytes, + descriptor, + out ValueReservation reservation); + counters.Record(OperationKind.Reserve, reserve); + if (reserve == StoreStatus.Success) + { + BenchmarkProtocol.FillGenerationPayload( + reservation.GetSpan(MaxValueBytes), + keyIndex, + generation); + StoreStatus advance = reservation.Advance(MaxValueBytes); + counters.Record(OperationKind.Advance, advance); + if (advance != StoreStatus.Success) + { + _ = reservation.Abort(); + return false; + } + + StoreStatus commit = reservation.Commit(); + counters.Record(OperationKind.Commit, commit); + return commit == StoreStatus.Success; + } + + if (reserve is not (StoreStatus.DuplicateKey or StoreStatus.StoreBusy)) + { + return false; + } + + _ = RemoveForRewrite(store, key, counters); + Thread.SpinWait(4 << Math.Min(attempt, 10)); + } + + return false; + } + + private static void Seed(Store store, string workload) + { + byte[][] keys = CreateWorkloadKeys(workload); + for (var keyIndex = 0; keyIndex < StableKeyCount; keyIndex++) + { + PublishSeed(store, keys[keyIndex], keyIndex, generation: 0); + } + + PublishSeed(store, BenchmarkProtocol.Key(TokenKeyIndex), StableKeyCount, generation: 0); + PublishSeed(store, BenchmarkProtocol.Key(ExistingKeyIndex), StableKeyCount + 1, generation: 0); + } + + private static byte[][] CreateWorkloadKeys(string workload) + { + if (workload == "distributed-key") + { + return Enumerable.Range(0, StableKeyCount) + .Select(static index => BenchmarkProtocol.Key(KeyBase + index)) + .ToArray(); + } + + return BenchmarkProtocol.CreateCollisionKeys( + StableKeyCount, + BenchmarkProtocol.CalculatePrimaryBucketCount(SlotCount)); + } + + private static (int First, int Second) SelectUnusedSpillBucketPair(string workload) + { + int bucketCount = BenchmarkProtocol.CalculatePrimaryBucketCount(SlotCount); + var occupied = new bool[bucketCount]; + IEnumerable existingKeys = CreateWorkloadKeys(workload).Concat( + [ + BenchmarkProtocol.Key(TokenKeyIndex), + BenchmarkProtocol.Key(ExistingKeyIndex) + ]); + foreach (byte[] key in existingKeys) + { + (int first, int second) = BenchmarkProtocol.GetBucketPair(key, bucketCount); + occupied[first] = true; + occupied[second] = true; + } + + int[] unused = Enumerable.Range(0, bucketCount) + .Where(index => !occupied[index]) + .Take(2) + .ToArray(); + if (unused.Length != 2) + { + throw new InvalidOperationException( + "No isolated directory bucket pair is available for the paused spill transition."); + } + + return (unused[0], unused[1]); + } + + private static void PublishSeed(Store store, byte[] key, int keyIndex, long generation) + { + byte[] value = new byte[MaxValueBytes]; + BenchmarkProtocol.FillGenerationPayload(value, keyIndex, generation); + Span descriptor = stackalloc byte[MaxDescriptorBytes]; + BenchmarkProtocol.WriteDescriptor(descriptor, keyIndex, generation, value.Length); + StoreStatus status = store.TryPublish(key, value, descriptor); + if (status != StoreStatus.Success) + { + throw new InvalidOperationException("Suspension seed publish failed: " + status); + } + } + + private static Process StartWorker( + string workload, + string storeName, + string role, + int workerId, + int affinityOrdinal, + int warmupSeconds) + { + ProcessStartInfo start = CreateSelfStartInfo(); + foreach (string argument in new[] + { + "suspension-worker", + workload, + storeName, + role, + workerId.ToString(CultureInfo.InvariantCulture), + "reserved", + affinityOrdinal.ToString(CultureInfo.InvariantCulture), + warmupSeconds.ToString(CultureInfo.InvariantCulture) + }) + { + start.ArgumentList.Add(argument); + } + + return Process.Start(start) ?? throw new InvalidOperationException("Failed to start suspension worker."); + } + + private static async Task AwaitWorkersReady(IReadOnlyList workers) + { + Task[] tasks = workers.Select(async worker => + { + string? line = await ReadLine(worker, ChildStartupTimeout); + if (line is null) + { + throw new InvalidOperationException("Suspension worker exited before READY."); + } + + return JsonSerializer.Deserialize(line) + ?? throw new InvalidOperationException("Suspension worker emitted invalid READY JSON."); + }).ToArray(); + return await Task.WhenAll(tasks); + } + + private static async Task MeasureWorkers( + IReadOnlyList workers, + int durationSeconds, + string window) + { + string command = $"MEASURE|{checked(durationSeconds * 1000).ToString(CultureInfo.InvariantCulture)}|{window}"; + foreach (Process worker in workers) + { + await worker.StandardInput.WriteLineAsync(command); + await worker.StandardInput.FlushAsync(); + } + + TimeSpan timeout = TimeSpan.FromSeconds(durationSeconds + 30); + Task[] tasks = workers.Select(async worker => + { + string? line = await ReadLine(worker, timeout); + if (line is null) + { + throw new InvalidOperationException("Suspension worker exited during " + window + "."); + } + + return JsonSerializer.Deserialize(line) + ?? throw new InvalidOperationException("Suspension worker emitted invalid window JSON."); + }).ToArray(); + return await Task.WhenAll(tasks); + } + + private static async Task StopWorkers(IEnumerable workers) + { + foreach (Process worker in workers) + { + try + { + if (!worker.HasExited) + { + await worker.StandardInput.WriteLineAsync("STOP"); + await worker.StandardInput.FlushAsync(); + } + } + catch + { + } + } + + foreach (Process worker in workers) + { + try + { + if (!worker.HasExited) + { + using var timeout = new CancellationTokenSource(ChildExitTimeout); + await worker.WaitForExitAsync(timeout.Token); + } + } + catch + { + if (!worker.HasExited) + { + worker.Kill(entireProcessTree: true); + } + } + finally + { + worker.Dispose(); + } + } + } + + private static Process StartPausedAgent( + string agentPath, + string storeName, + int checkpointId, + string workload, + int spillFirstBucket, + int spillSecondBucket) + { + ProcessStartInfo start = CreateAgentStartInfo(agentPath); + byte[] value = new byte[32]; + BenchmarkProtocol.FillGenerationPayload(value, checkpointId, generation: 1); + Span descriptor = stackalloc byte[MaxDescriptorBytes]; + BenchmarkProtocol.WriteDescriptor(descriptor, checkpointId, generation: 1, value.Length); + int workloadOffset = workload == "distributed-key" ? 0 : 1_000; + foreach (string argument in new[] + { + "checkpoint-pause", + storeName, + SlotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + ParticipantRecordCount.ToString(CultureInfo.InvariantCulture), + checkpointId.ToString(CultureInfo.InvariantCulture), + Convert.ToHexString(BenchmarkProtocol.Key(TokenKeyIndex)), + Convert.ToHexString(BenchmarkProtocol.Key(ExistingKeyIndex)), + Convert.ToHexString(BenchmarkProtocol.Key(OperationKeyBase + workloadOffset + checkpointId)), + Convert.ToHexString(BenchmarkProtocol.Key(RecoveryKeyBase + workloadOffset + checkpointId)), + Convert.ToHexString(value), + Convert.ToHexString(descriptor), + spillFirstBucket.ToString(CultureInfo.InvariantCulture), + spillSecondBucket.ToString(CultureInfo.InvariantCulture), + "v2" + }) + { + start.ArgumentList.Add(argument); + } + + return Process.Start(start) ?? throw new InvalidOperationException("Failed to start checkpoint agent."); + } + + private static async Task ReadCheckpointCatalog(string agentPath) + { + ProcessStartInfo start = CreateAgentStartInfo(agentPath); + start.ArgumentList.Add("checkpoint-catalog"); + using Process process = Process.Start(start) + ?? throw new InvalidOperationException("Failed to start checkpoint catalog agent."); + Task stdout = process.StandardOutput.ReadToEndAsync(); + Task stderr = process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync().WaitAsync(ChildExitTimeout); + string json = await stdout; + string error = await stderr; + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"Checkpoint catalog agent exited {process.ExitCode}: {error.Trim()}"); + } + + return JsonSerializer.Deserialize(json) + ?? throw new InvalidOperationException("Checkpoint catalog agent emitted invalid JSON."); + } + + private static ProcessStartInfo CreateSelfStartInfo() + { + string processPath = Environment.ProcessPath + ?? throw new InvalidOperationException("Cannot determine executable path."); + var start = new ProcessStartInfo(processPath) + { + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + if (string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase)) + { + start.ArgumentList.Add(Assembly.GetExecutingAssembly().Location); + } + + return start; + } + + private static ProcessStartInfo CreateAgentStartInfo(string agentPath) + { + var start = new ProcessStartInfo("dotnet") + { + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + start.ArgumentList.Add("exec"); + start.ArgumentList.Add(agentPath); + return start; + } + + private static string FindAgentAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null + && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + if (directory is null) + { + throw new InvalidOperationException("Cannot locate repository root for LockFreeAgent."); + } + + string path = Path.Combine( + directory.FullName, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + if (!File.Exists(path)) + { + throw new FileNotFoundException( + "LockFreeAgent must be built in the same configuration as SyncProbe.", + path); + } + + return path; + } + + private static SharedMemoryStoreOptions CreateOptions(string name, OpenMode mode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + SlotCount, + MaxValueBytes, + MaxDescriptorBytes, + MaxKeyBytes, + LeaseRecordCount, + ParticipantRecordCount, + mode, + enableLeaseRecovery: true); + + private static DiagnosticsSnapshot GetDiagnostics(Store store) + { + for (var attempt = 0; attempt < 64; attempt++) + { + StoreStatus status = store.TryGetDiagnostics(out DiagnosticsSnapshot snapshot); + if (status == StoreStatus.Success) + { + return snapshot; + } + + if (status != StoreStatus.StoreBusy) + { + throw new InvalidOperationException("Suspension diagnostics failed: " + status); + } + + Thread.SpinWait(32 << Math.Min(attempt, 10)); + } + + throw new InvalidOperationException("Suspension diagnostics exhausted its bounded retry budget."); + } + + private static bool IsStoreFullProofCheckpoint(int checkpointId) => checkpointId is + (int)LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification + or (int)LockFreeCheckpointId.StoreFullAfterExactDoubleCollect; + + private static void RemoveStoreFullFillers(Store store, int checkpointId) + { + for (var index = 0; index < SlotCount; index++) + { + byte[] key = CreateStoreFullFillerKey(checkpointId, index); + for (var attempt = 0; attempt < 4096; attempt++) + { + StoreStatus status = store.TryRemove(key); + if (status is StoreStatus.Success or StoreStatus.NotFound) + { + break; + } + + if (status is not (StoreStatus.RemovePending or StoreStatus.StoreBusy)) + { + throw new InvalidOperationException( + "StoreFull proof filler cleanup failed: " + status); + } + + if (attempt == 4095) + { + throw new InvalidOperationException( + "StoreFull proof filler cleanup exhausted its retry budget."); + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + } + } + } + + private static byte[] CreateStoreFullFillerKey(int checkpointId, int index) => + BitConverter.GetBytes( + 0x6f00_0000_0000_0000UL + | ((ulong)(byte)checkpointId << 48) + | checked((uint)(index + 1))); + + private static bool CapacityPermits(DiagnosticsSnapshot snapshot, int healthyProcesses) => + snapshot.FreeSlotCount >= 32 + && snapshot.FreeLeaseCount >= healthyProcesses + 1 + && snapshot.FreeParticipantCount >= 1 + && snapshot.GetFailureCount(StoreStatus.StoreFull) == 0 + && snapshot.GetFailureCount(StoreStatus.LeaseTableFull) == 0; + + private static SuspensionCapacityEvidence CapacityEvidence(DiagnosticsSnapshot snapshot) => + new( + snapshot.FreeSlotCount, + snapshot.PublishedSlotCount, + snapshot.ActiveLeaseCount, + snapshot.ActiveReservationCount, + snapshot.InitializingSlotCount, + snapshot.ReservedSlotCount, + snapshot.ReclaimingSlotCount, + snapshot.FreeLeaseCount, + snapshot.ClaimingLeaseCount, + snapshot.RecoveringLeaseCount, + snapshot.FreeParticipantCount, + snapshot.ActiveParticipantCount, + snapshot.RegisteringParticipantCount, + snapshot.ClosingParticipantCount, + snapshot.RecoveringParticipantCount, + snapshot.ReclaimingParticipantCount, + snapshot.GetFailureCount(StoreStatus.StoreFull), + snapshot.GetFailureCount(StoreStatus.LeaseTableFull), + snapshot.ContentionBudgetExhaustionCount); + + private static async Task ReadLine(Process process, TimeSpan timeout) => + await process.StandardOutput.ReadLineAsync().WaitAsync(timeout); + + private static int GetAvailableProcessorCount() + { + if (!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + { + return Environment.ProcessorCount; + } + + try + { + using Process process = Process.GetCurrentProcess(); + return BitOperations.PopCount(unchecked((ulong)(nuint)process.ProcessorAffinity)); + } + catch + { + return Environment.ProcessorCount; + } + } + + private static SuspensionEnvironment CaptureEnvironment(int availableProcessors) + { + string assembly = Assembly.GetExecutingAssembly().Location; + return new SuspensionEnvironment( + RuntimeInformation.OSDescription, + RuntimeInformation.OSArchitecture.ToString(), + RuntimeInformation.ProcessArchitecture.ToString(), + RuntimeInformation.FrameworkDescription, + Environment.Version.ToString(), + Environment.ProcessorCount, + availableProcessors, + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(assembly))), + Stopwatch.Frequency); + } + + private static SuspensionCheckpointResult NotQualifiedForProcessors( + CheckpointCatalogEntry checkpoint, + SuspensionWorkload workload, + int baselineSeconds, + int pauseSeconds, + double minimumRatio, + int availableProcessors, + int requiredProcessors) => + EmptyResult( + checkpoint, + workload, + baselineSeconds, + pauseSeconds, + minimumRatio, + availableProcessors, + requiredProcessors, + "not-qualified-insufficient-processors", + [$"available-processors={availableProcessors}; required-processors={requiredProcessors}"]); + + private static SuspensionCheckpointResult HarnessFailure( + CheckpointCatalogEntry checkpoint, + SuspensionWorkload workload, + int baselineSeconds, + int pauseSeconds, + double minimumRatio, + int availableProcessors, + int requiredProcessors, + Exception exception) => + EmptyResult( + checkpoint, + workload, + baselineSeconds, + pauseSeconds, + minimumRatio, + availableProcessors, + requiredProcessors, + pauseSeconds >= 30 ? "qualified-fail" : "smoke-fail", + ["harness-error=" + exception.GetType().Name + ": " + exception.Message]); + + private static SuspensionCheckpointResult EmptyResult( + CheckpointCatalogEntry checkpoint, + SuspensionWorkload workload, + int baselineSeconds, + int pauseSeconds, + double minimumRatio, + int availableProcessors, + int requiredProcessors, + string qualification, + string[] errors) => + new( + checkpoint.Id, + checkpoint.Name, + checkpoint.Family, + checkpoint.Position, + checkpoint.Pause, + checkpoint.Crash, + checkpoint.Race, + checkpoint.IsPublicOrderingPoint, + workload.Name, + workload.ReaderCount, + workload.WriterCount, + workload.HealthyProcessCount, + baselineSeconds, + pauseSeconds, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + minimumRatio, + false, + errors.LongLength, + errors, + 0, + false, + -1, + "not-applied", + -1, + -1, + availableProcessors, + requiredProcessors, + 0, + -1, + qualification, + false, + [], + [], + default, + default, + default, + default, + default); + + private static int ReadPositiveInt(string[] arguments, string name, int fallback) + { + string? text = ReadString(arguments, name); + return text is null + ? fallback + : int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out int value) && value > 0 + ? value + : throw new ArgumentException(name + " must be a positive integer."); + } + + private static int ReadNonNegativeInt(string[] arguments, string name, int fallback) + { + string? text = ReadString(arguments, name); + return text is null + ? fallback + : int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out int value) && value >= 0 + ? value + : throw new ArgumentException(name + " must be a non-negative integer."); + } + + private static double ReadRatio(string[] arguments, string name, double fallback) + { + string? text = ReadString(arguments, name); + return text is null + ? fallback + : double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out double value) + && value > 0 + && value <= 1 + ? value + : throw new ArgumentException(name + " must be in (0,1]."); + } + + private static string[] ParseList(string[] arguments, string name) => + (ReadString(arguments, name) ?? string.Empty) + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + private static string? ReadString(string[] arguments, string name) + { + int index = Array.IndexOf(arguments, name); + if (index < 0) + { + return null; + } + + if (index == arguments.Length - 1) + { + throw new ArgumentException(name + " requires a value."); + } + + return arguments[index + 1]; + } + + private sealed class WorkerState(string workload, string role, int workerId) + { + internal string Workload { get; } = workload; + internal string Role { get; } = role; + internal int WorkerId { get; } = workerId; + internal byte[][] Keys { get; } = CreateWorkloadKeys(workload); + internal long Cycle { get; set; } + } + + private sealed class WindowCounters + { + private readonly StatusCounters _statusCounters = new(); + + internal long AttemptedCycles { get; set; } + internal long CompletedCycles { get; set; } + internal long ApiCalls => _statusCounters.TotalOperations; + internal long Failures { get; set; } + + internal void Record(OperationKind operation, StoreStatus status) => + _statusCounters.Record(operation, status); + + internal void RecordChecksumFailure() => _statusCounters.RecordChecksumFailure(); + + internal SortedDictionary ToHistogram() => _statusCounters.ToHistogram(); + } +} + +internal sealed record SuspensionWorkload(string Name, int ReaderCount, int WriterCount) +{ + internal int HealthyProcessCount => ReaderCount + WriterCount; +} + +internal sealed record CheckpointCatalogEntry( + int Id, + string Name, + string Family, + string Position, + string Pause, + string Crash, + string Race, + bool IsPublicOrderingPoint, + string Description); + +internal sealed record CheckpointSignal( + int Id, + string Name, + string Family, + string Position, + string Crash, + int ProcessId, + ulong StoreId, + ulong ParticipantToken, + ulong SlotBinding, + ulong LeaseToken); + +internal sealed record SuspensionWorkerReady( + int WorkerId, + string Role, + int ProcessId, + bool AffinityApplied, + int AssignedProcessor, + string AffinityStrategy); + +internal sealed record SuspensionWindowResult( + int WorkerId, + string Role, + int ProcessId, + string Window, + long AttemptedCycles, + long CompletedCycles, + long ApiCalls, + double ElapsedSeconds, + double CompletedCyclesPerSecond, + double ApiCallsPerSecond, + long Failures, + bool AffinityApplied, + int AssignedProcessor, + string AffinityStrategy, + SortedDictionary StatusHistogram); + +internal readonly record struct SuspensionCapacityEvidence( + int FreeSlotCount, + int PublishedSlotCount, + int ActiveLeaseCount, + int ActiveReservationCount, + int InitializingSlotCount, + int ReservedSlotCount, + int ReclaimingSlotCount, + int FreeLeaseCount, + int ClaimingLeaseCount, + int RecoveringLeaseCount, + int FreeParticipantCount, + int ActiveParticipantCount, + int RegisteringParticipantCount, + int ClosingParticipantCount, + int RecoveringParticipantCount, + int ReclaimingParticipantCount, + long StoreFullFailures, + long LeaseTableFullFailures, + long ContentionBudgetExhaustionCount); + +internal sealed record SuspensionCheckpointResult( + int CheckpointId, + string CheckpointName, + string CheckpointFamily, + string CheckpointPosition, + string PauseClassification, + string CrashClassification, + string RaceClassification, + bool IsPublicOrderingPoint, + string Workload, + int ReaderProcessCount, + int WriterProcessCount, + int HealthyProcessCount, + int BaselineWindowSeconds, + int SuspendedWindowSeconds, + double BaselineCompletedCyclesPerSecond, + double SuspendedCompletedCyclesPerSecond, + long BaselineAttemptedCycles, + long BaselineCompletedCycles, + long BaselineApiCalls, + long SuspendedAttemptedCycles, + long SuspendedCompletedCycles, + long SuspendedApiCalls, + double BaselineApiCallsPerSecond, + double SuspendedApiCallsPerSecond, + double ThroughputRatio, + double MinimumThroughputRatio, + bool CapacityPermits, + long CorrectnessFailureCount, + string[] CorrectnessErrors, + int HealthyAffinityAppliedCount, + bool PausedParticipantAffinityApplied, + int PausedParticipantProcessor, + string PausedParticipantAffinityStrategy, + int AgentSpillFirstBucket, + int AgentSpillSecondBucket, + int AvailableProcessorCount, + int RequiredProcessorCount, + int PausedParticipantProcessId, + int PausedParticipantExitCode, + string Qualification, + bool GatePassed, + SuspensionWindowResult[] BaselineWorkers, + SuspensionWindowResult[] SuspendedWorkers, + SuspensionCapacityEvidence BeforeBaselineCapacity, + SuspensionCapacityEvidence AfterBaselineCapacity, + SuspensionCapacityEvidence BeforeSuspendedCapacity, + SuspensionCapacityEvidence AfterSuspendedCapacity, + SuspensionCapacityEvidence AfterResumeCapacity); + +internal sealed record SuspensionConfiguration( + int BaselineWindowSeconds, + int SuspendedWindowSeconds, + int WarmupSeconds, + double MinimumThroughputRatio, + bool AffinityRequested, + string AffinityPolicy, + int IncludedCheckpointCount, + string[] Workloads, + int CatalogCheckpointCount, + string[] ExcludedFamilies, + string ComparisonMethod); + +internal sealed record SuspensionEnvironment( + string OperatingSystem, + string OperatingSystemArchitecture, + string ProcessArchitecture, + string Framework, + string RuntimeVersion, + int LogicalProcessorCount, + int AvailableProcessorCount, + string ProbeAssemblySha256, + long StopwatchFrequency); + +internal sealed record SuspensionQualificationReport( + int SchemaVersion, + DateTimeOffset TimestampUtc, + SuspensionEnvironment Environment, + SuspensionConfiguration Configuration, + IReadOnlyList Results, + int RequiredResultCount, + int QualifiedPassCount, + int SmokePassCount, + int FailCount, + int NotQualifiedCount, + bool AllRequiredQualifiedAndPassed); diff --git a/docs/architecture.md b/docs/architecture.md index 0e74444..016f202 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -131,20 +131,39 @@ announce payload length and descriptor bytes before payload writes, expose runtime-appropriate writable views, record exact progress, and publish only after an exact commit. -Pending reservations are invisible to readers but occupy capacity and block -duplicate keys. Abort, dispose, and recovery remove the pending index entry -before reclaiming the slot. Public memory lifetime rules are in the +Successfully ordered explicit reservations are invisible to readers, occupy +capacity, and block duplicate keys. Lock-free v2 records immutable +`PublicationIntent` ordinary metadata for each current lifecycle. `TryReserve` +uses `ExplicitReservation` and orders at `Initializing -> Reserved`; the +`TryPublish` and `TryPublishSegments` convenience workflows use +`AtomicPublication`, remain tentative through internal `Reserved`, and order +only at `Reserved -> Published`. Tentative `Initializing` and +`Reserved(AtomicPublication)` claims are helpable and consume physical capacity +but do not alone block a duplicate key. Abort, dispose, and recovery remove the +pending index entry before reclaiming the slot. Public memory lifetime rules are +in the [reservation memory contract](../specs/005-api-production-readiness/contracts/reservation-memory-contract.md). ## Synchronization and Waits -Public operations synchronize through the platform store lock and the -process-local lifecycle gate. Windows uses named synchronization. Linux uses a -deterministic shared lock resource in the runtime shared-memory location. -`StoreWaitOptions` controls how long an operation waits for shared -synchronization. Busy and canceled waits return `StoreBusy` or +Legacy-profile public operations synchronize through the platform store lock +and the process-local lifecycle gate. Windows uses named synchronization; Linux +uses a deterministic shared lock resource in the runtime shared-memory +location. Lock-free-v2 steady-state operations instead use bounded mapped +atomics, helping, and generation revalidation, and never enter that global +operation lock. `StoreWaitOptions` bounds either legacy lock waiting or v2 local +retry/helping. Busy and canceled waits return `StoreBusy` or `OperationCanceled`. +Lock-free directory observations are cached atomic-reference witnesses, not +ownership of a slot. Before reporting a would-be corrupt binding, v2 rereads the +exact raw reference word around a fresh stable classification of its separately +decoded slot binding. A primary/overflow source equals the binding; a versioned +spill summary is the complete encoded `Present(binding)` source word. Source +movement causes a budgeted lookup or maintenance retry, while only an unchanged +exact source enclosing a repeated malformed slot may fail closed. This +source/slot/source rule adds no shared epoch, multi-word atomic, or global owner. + The native adapters reproduce the same Windows mapping/mutex and Linux region, byte-range lock, owner-sidecar, lifecycle-lock, permission, and cleanup rules. Matching mapped bytes without matching resource participation is not @@ -163,7 +182,11 @@ Recovery is explicit and owner-scoped: active, unsupported, and failed records. Recovery exists to make cleanup policy observable. It must not become automatic -background reclamation. +background reclamation. Normal v2 recovery preserves owner-controlled resources +whose exact participant is live Active. Current-process lease or reservation +overrides are administrative test/controlled-shutdown policies and require the +corresponding process-wide borrowed-view and operation quiescence; racing an +override with current-process activity is outside the supported result contract. ## Diagnostics @@ -185,6 +208,21 @@ benchmark commands or measured validation notes. See [Performance scope](performance.md) and [`benchmarks/SharedMemoryStore.Benchmarks/`](../benchmarks/SharedMemoryStore.Benchmarks/). +Each lock-free open handle eagerly owns a process-local `long[SlotCount]` +scratch snapshot used only to certify rare `StoreFull` candidates. It costs +approximately eight bytes per slot (about 8 MiB at the v2 slot ceiling), is +reused without per-operation allocation, and is protected by a nonblocking +local guard. The guard and buffer are not mapped state and cannot serialize +another process or handle. + +The same handle owns a separate `long[LeaseRecordCount]` snapshot and local +guard for rare `LeaseTableFull` candidates. Both capacity outcomes require two +same-order, structurally valid, entirely non-Free, exactly equal collects; a +malformed control fails closed, while movement, reusable capacity, or local +guard contention follows the caller's wait policy. The lease buffer has the same +eight-bytes-per-record private-memory cost and likewise adds no shared or OS +synchronization. + ## Portability Model The repository now contains managed, C++20, and Python 3.10+ implementations diff --git a/docs/errors.md b/docs/errors.md index 8fe3045..c90e66f 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -25,36 +25,59 @@ exceptions. | `InsufficientCapacity` | `TotalBytes` cannot contain the requested layout. | Recalculate with `CalculateRequiredBytes`. | | `AccessDenied` | The process lacks mapping access. | Review process identity and OS permissions. | | `MappingFailed` | The runtime failed to create or open the memory mapping. | Capture OS, runtime, options, and failure context for support. | -| `StoreBusy` | Shared synchronization was not acquired within the wait policy. | Retry according to caller policy or use a longer wait. | -| `OperationCanceled` | Cancellation was observed before synchronization was acquired. | Honor caller cancellation. | +| `StoreBusy` | The selected wait policy expired before the operation ordered. Legacy uses a shared lock; lock-free v2 exhausts its local retry/revalidation/helping budget. | Retry according to caller policy or use a longer wait. | +| `OperationCanceled` | Cancellation was observed before the operation ordering point. | Honor caller cancellation. | ## Operation Statuses | Status | Meaning | Typical cause | |--------|---------|---------------| | `Success` | Operation completed. | Expected path. | -| `DuplicateKey` | Key maps to a published, pending-removal, or pending-reservation value. | Producer attempted to overwrite without removal. | +| `DuplicateKey` | Key maps to a published or pending-removal value, or to `Reserved(ExplicitReservation)`. | Producer attempted to overwrite a public same-key lifecycle. Tentative `Initializing` and `Reserved(AtomicPublication)` alone do not qualify. | | `NotFound` | Key is absent or no longer published. | Reader looked up a missing, pending, removed, or recovered key. | | `InvalidKey` | Key is empty or otherwise invalid. | Caller supplied empty key bytes. | | `KeyTooLarge` | Key exceeds `MaxKeyBytes`. | Configuration or encoding mismatch. | | `ValueTooLarge` | Payload exceeds `MaxValueBytes`. | Capacity is too small for the payload. | | `DescriptorTooLarge` | Descriptor exceeds `MaxDescriptorBytes`. | Metadata is too large for configured capacity. | -| `StoreFull` | No reusable value slot is available. | Published, pending-removal, or pending-reservation values occupy all slots. | -| `LeaseTableFull` | No reusable lease record is available. | Too many concurrent readers or leaked leases. | +| `StoreFull` | No reusable value slot is available. | Published, pending-removal, explicit-reservation, tentative initialization/atomic-publication, cleanup, or retired lifecycles occupy all physical slots. | +| `LeaseTableFull` | Two exact, structurally valid lease-control collects confirm that no reusable lease record is available. | Too many concurrent readers or leaked leases. | | `InvalidLease` | Lease token does not match an active record. | Default token, stale token, or recovered/reclaimed record. | | `LeaseAlreadyReleased` | Lease was already released. | Repeated release or dispose after release. | -| `RemovePending` | Removal was requested while active readers still hold leases. | Readers need to release before slot reuse. | +| `RemovePending` | The key is logically absent, but an active lease or incomplete bounded post-removal work delays physical reclamation. | Release readers or let later remove, release, or allocation-pressure helping finish reclamation. | | `UnsupportedPlatform` | Operation is unsupported on this platform. | Platform or owner-liveness capability mismatch. | | `StoreDisposed` | Store handle has been disposed. | Operation used a closed handle or raced with disposal. | | `CorruptStore` | Unsafe shared-memory state was detected. | Inconsistent metadata or external mutation. | | `AccessDenied` | Process lacks required access. | OS permissions or mapping access issue. | | `UnknownFailure` | Unexpected runtime failure occurred. | Capture diagnostics and reproduction details. | -| `InvalidReservation` | Reservation token does not match a pending slot generation. | Default, stale, committed, aborted, disposed, or recovered token. | +| `InvalidReservation` | Reservation token does not match a pending slot generation, or the tentative claim was legally canceled before explicit reservation ordered. | Default, stale, committed, aborted, disposed, or recovered token; or exact-generation cancellation before ordering. | | `ReservationIncomplete` | Commit was attempted before exact announced length was advanced. | Producer has not written all bytes. | | `ReservationAlreadyCompleted` | Reservation already committed, aborted, disposed, or recovered. | Repeated completion path. | | `ReservationWriteOutOfRange` | Advance would move outside announced payload length. | Producer wrote too many bytes or used wrong count. | -| `StoreBusy` | Shared synchronization was not acquired within the wait policy. | Contention under `NoWait` or bounded timeout. | -| `OperationCanceled` | Cancellation was observed before synchronization was acquired. | Caller canceled the operation. | +| `StoreBusy` | The operation did not order within the wait policy. Legacy may be waiting for its shared lock; lock-free v2 exhausted bounded local retry/revalidation/helping. | Contention under `NoWait` or bounded timeout. | +| `OperationCanceled` | Cancellation was observed before the operation ordering point. | Caller canceled the operation. | + +In lock-free v2, each lifecycle records a publication intent. `TryReserve` +orders at `Initializing -> Reserved(ExplicitReservation)`, which becomes a +duplicate-key witness. `TryPublish` and `TryPublishSegments` use +`AtomicPublication`; their internal `Initializing` and `Reserved` states are +tentative and the outer operation orders only at `Reserved -> Published`. +Tentative states are helpable and consume physical capacity, but they do not +alone justify `DuplicateKey`; bounded revalidation may instead return +`StoreBusy`. After an initial absent-key lookup, a raced operation may instead +return `StoreFull` at candidate claim before final duplicate arbitration; +duplicate status does not take precedence over genuine physical exhaustion in +that race. A missed allocation scan alone is not enough: v2 returns `StoreFull` +only after two same-order, structurally valid, all-non-Free control snapshots +match exactly. `Initializing`/`Reserved` require a structurally valid configured +participant token; `Free`/`Published`/`RemoveRequested`/`Aborting`/`Reclaiming`/ +`Retired` require participant zero, every generation is nonzero and bounded, and +`Retired` is terminal. An invalid state/generation/owner shape returns +`CorruptStore`, even when two malformed words compare equal. A free or changing +slot, or contention for that handle's private proof buffer, follows the caller's +wait policy and can return `StoreBusy` instead. Normal recovery preserves a +lifecycle owned by an exact live Active participant. The current-process +reservation override is supported only after process-wide publication and +writable-view quiescence. ## Common Symptoms @@ -65,8 +88,11 @@ var first = store.TryPublish(key, [1]); var second = store.TryPublish(key, [2]); ``` -Expected statuses: `Success`, then `DuplicateKey`. Remove the key first or use -a new key. +With candidate capacity available, expected statuses are `Success`, then +`DuplicateKey`. If every physical slot is already occupied, the second +concurrent operation may return `StoreFull` after its initial absent lookup and +before final duplicate arbitration. +Remove the key first, free capacity, or use a new key. Missing key: @@ -93,7 +119,11 @@ var status = store.TryAcquire(key, out var lease); ``` Expected status under lease-record pressure: `LeaseTableFull`. Increase -`LeaseRecordCount` or release leases sooner. +`LeaseRecordCount` or release leases sooner. An exhausted scan alone is not +enough: the lock-free profile confirms two identical, structurally valid, +all-non-Free snapshots. Movement or another operation using the handle-local +proof buffer follows the wait policy and may return `StoreBusy`; malformed lease +controls fail `CorruptStore`. Invalid release: @@ -146,9 +176,15 @@ layout version returns `IncompatibleLayout`. See Corruption signal: -`CorruptStore` means the process detected unsafe shared state. Stop unsafe -access, capture options, operation, status, platform, package version, and -diagnostics, then follow [SUPPORT.md](../SUPPORT.md). +`CorruptStore` means a process proved unsafe persistent shared state. Layout v2 +irreversibly latches that condition in the mapped store control, so subsequent +operations in every attached process fail before a new projection or mutation; +a later open reports `IncompatibleLayout`. Already borrowed spans cannot be +revoked. Stop access, capture options, operation, status, platform, package +version, and any diagnostics captured before the latch, then follow +[SUPPORT.md](../SUPPORT.md). Invalid caller input and ordinary concurrency, +capacity, cancellation, and token-history outcomes do not set the corruption +latch. ## Diagnostics To Inspect diff --git a/docs/examples.md b/docs/examples.md index 13324e3..fc47167 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -173,6 +173,9 @@ immediate writes. The [zero-copy ingest sample](../samples/ZeroCopyIngest/README.md) demonstrates direct chunked writes, a runnable length-prefixed stream adapter, abort cleanup, reader acquire, remove, and segmented publication. +That adapter is application-side input plumbing: the store remains a keyed +value lifecycle and adds no stream position, delivery, dequeue, or +acknowledgement semantics. ## Segmented Payloads diff --git a/docs/getting-started.md b/docs/getting-started.md index e608aaf..5504f88 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -2,7 +2,9 @@ This guide selects one of the .NET, native C++, or Python distributions and gets a clean consumer to a complete create/open, publish, acquire, release, -remove, and close workflow. All three use layout `1.2` and resource naming `1`. +remove, and close workflow. Layout `1.2` and resource naming `1` remain the +cross-runtime boundary. The managed package additionally supports explicit C# +layout `2.0`/resource protocol `2`; native and Python clients reject that layout. ## Prerequisites @@ -15,13 +17,13 @@ remove, and close workflow. All three use layout `1.2` and resource naming `1`. - Docker Engine or Docker Desktop only when validating same-host container sharing. -The managed package version is `1.0.2`; the native and Python distributions are +The managed package version is `2.0.0`; the native and Python distributions are independently versioned `0.1.0`. If an artifact has not been published to its ecosystem feed, build it locally from the repository. | Consumer | Artifact | Public entry point | |----------|----------|--------------------| -| .NET | NuGet `SharedMemoryStore` `1.0.2` | `MemoryStore` | +| .NET | NuGet `SharedMemoryStore` `2.0.0` | `MemoryStore` | | C++ | CMake `SharedMemoryStore` `0.1.0` | `shared_memory_store::memory_store` | | Python | wheel `shared-memory-store` `0.1.0` | `shared_memory_store.MemoryStore` | diff --git a/docs/lifecycle.md b/docs/lifecycle.md index 8ae0b48..2bacf26 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -82,8 +82,13 @@ Producers: `TryRecoverLeases` is owner controlled. When `RecoverCurrentProcessLeases` is `true`, the store may recover current-process leases and stale-owner leases. It -must still skip leases owned by another live process. Reports include scanned, -recovered, active, unsupported, and failed counts. +must still skip leases owned by another live process. Before selecting true, the +caller must quiesce all current-process lease acquisition, projection, +borrowed-span use, and release across every handle attached to the mapping and +keep that activity quiescent until recovery returns. This is an administrative +test/controlled-shutdown precondition; no hot-path gate enforces it. False is the +normal mode and remains safe during concurrent lease activity. Reports include +scanned, recovered, active, unsupported, and failed counts. `TryRecoverReservations` scans pending reservations, evaluates producer liveness where supported, removes pending index entries, and reclaims slots without @@ -129,6 +134,30 @@ handle closes. A zero-length per-name lifecycle lock file may remain so a later opener cannot race a different lock inode; applications should use stable store names rather than generating an unbounded sequence of one-time names. +Each current managed Linux handle also holds a private per-owner `flock` +liveness anchor while its mapped view exists. Lifecycle cleanup treats a locked +anchor as live even when the owner's PID is hidden by a container PID namespace, +and treats an unlocked anchor as stale. Missing anchors use the existing +PID/start-token check so C++, Python, and older managed participants remain +compatible. Close unmaps before releasing the anchor; process termination +releases it automatically. These files and checks are cold lifecycle metadata +and are not entered by publish, acquire, release, or remove. + +After a lifecycle operation atomically commits the replacement owner sidecar, +it may repair an orphan left by a crash between anchor creation and owner-line +publication. The repair considers only the exact store's canonical +`.owners.anchor.<32-lowercase-hex-guid>` names that are not referenced by the +committed sidecar. It opens each candidate separately with symbolic-link +following disabled, verifies a regular file, and deletes it only while holding +a successfully acquired nonblocking exclusive `flock`. Referenced, locked, +ambiguous, non-regular, symbolic-link, directory, malformed, or access-error +artifacts are retained. Final-handle cleanup does not use a broad anchor glob. + +If bounded close cannot commit exact owner-line removal, a finalized release +marker lets it safely release the local anchor after unmapping. The owner line, +anchor pathname, or both may remain until a later lifecycle operation reconciles +the marker and performs the same post-commit conservative sweep. + ## Long-Running Identity Reusable slots carry generation and reuse-epoch identity. Index entries, lease diff --git a/docs/packaging.md b/docs/packaging.md index 958bf29..080beea 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -1,8 +1,11 @@ # Packaging SharedMemoryStore ships independently versioned NuGet, CMake, and Python -artifacts. They share layout `1.2` and resource naming `1`; matching package -versions are not required. The managed runtime package is built from +artifacts. They interoperate through layout `1.2` and resource naming `1`; +matching package versions are not required. Managed `2.0.0` additionally +supports explicit C# layout `2.0` and resource protocol `2`, while the current +native and Python distributions reject it. The managed runtime package is built +from [`src/SharedMemoryStore/SharedMemoryStore.csproj`](../src/SharedMemoryStore/SharedMemoryStore.csproj) and targets `net10.0`. Runtime dependencies are limited to the .NET BCL. @@ -11,14 +14,14 @@ and targets `net10.0`. Runtime dependencies are limited to the .NET BCL. | Field | Value | |-------|-------| | `PackageId` | `SharedMemoryStore` | -| `Version` | `1.0.2` | +| `Version` | `2.0.0` | | `TargetFramework` | `net10.0` | | `Description` | `A bounded named shared-memory key-value store for opaque binary values.` | | `PackageTags` | `shared-memory;memory-mapped-file;zero-copy;linux;windows;docker;library` | | `PackageLicenseExpression` | `MIT` | | `PackageProjectUrl` | `https://github.com/rantri/SharedMemoryStore` | | `PackageReadmeFile` | `README.md` | -| `PackageReleaseNotes` | `NuGet SharedMemoryStore 1.0.2 preserves Linux, Windows, and same-host Docker support, corrects Linux layout-mismatch reporting, and rejects unsupported managed processes early while preserving the public API, status values, BCL-only runtime surface, layout 1.2, and resource naming 1. Repository source adds independently versioned native C++ and Python 0.1.0 sibling distributions using C ABI 1.0, validated with .NET on Windows and Linux; v1.0.2 neither includes them in NuGet nor publishes them to PyPI or a native package registry.` | +| `PackageReleaseNotes` | `NuGet SharedMemoryStore 2.0.0 adds an explicitly selected C# layout 2.0 lock-free key-value profile and resource protocol 2, including zero-copy reservation publication, shared read leases, generation-fenced removal/reuse, participant records, explicit recovery, and additive diagnostics. The legacy profile remains the default and preserves mapped layout 1.2, resource naming 1, existing status numbers, and Linux, Windows, and same-host Docker support. Layout 1.2 and 2.0 mappings are distinct: upgrade and rollback require drain, close, recreate, and application-owned republish; there is no in-place conversion or mixed-layout writer mode. Independently versioned C++ and Python 0.1.0 distributions remain layout 1.2-only and fail closed on layout 2.0.` | | `RepositoryType` | `git` | | `RepositoryUrl` | `https://github.com/rantri/SharedMemoryStore` | | `SymbolPackageFormat` | `snupkg` | @@ -95,7 +98,7 @@ pwsh ./scripts/validate-python.ps1 -Configuration Release | Distribution | Version | ABI requirement | Creates/reads | Resource naming | |--------------|---------|-----------------|---------------|-----------------| -| NuGet `SharedMemoryStore` | `1.0.2` | Not applicable | layout `1.2` | `1` | +| NuGet `SharedMemoryStore` | `2.0.0` | Not applicable | layouts `1.2`, `2.0` | `1`, `2` | | CMake `SharedMemoryStore` | `0.1.0` | provides C ABI `1.0` | layout `1.2` | `1` | | Python `shared-memory-store` | `0.1.0` | requires C ABI `1.0` | layout `1.2` | `1` | diff --git a/docs/performance.md b/docs/performance.md index bd82346..4d96e8d 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -80,6 +80,14 @@ pending removals, and pending reservations. `LeaseRecordCount` must cover concurrent active readers. `MaxKeyBytes`, `MaxDescriptorBytes`, and `MaxValueBytes` must cover encoded keys, descriptors, and payloads. +Every lock-free open handle also allocates private `long[SlotCount]` and +`long[LeaseRecordCount]` capacity-proof buffers at open time. Budget about eight +bytes per configured value slot or lease record per process/handle. The slot +buffer is roughly 8 MiB at 1,048,575 slots; the lease buffer is 1 KiB for 128 +records, 64 KiB for 8,192, and 8 MiB for 1,048,576. The buffers are reused on +rare full-candidate paths; they add no per-operation allocation and no shared or +OS synchronization. + Use [Diagnostics](diagnostics.md) to track `StoreFull`, `LeaseTableFull`, `CapacityPressureCount`, active leases, active reservations, pending removals, and key-index health. diff --git a/docs/portability.md b/docs/portability.md index da15a34..8267b00 100644 --- a/docs/portability.md +++ b/docs/portability.md @@ -5,7 +5,9 @@ ordinary same-host workflows on 64-bit little-endian Linux and Windows. Same-host Linux Docker participation requires deployment configuration that exposes compatible shared-memory, synchronization, owner-liveness, permission, and capacity capabilities. Layout `1.2` and resource naming `1` are the common -interoperability boundary; similar public APIs alone are insufficient. +cross-runtime interoperability boundary; similar public APIs alone are +insufficient. The managed package also implements explicitly selected layout +`2.0` and resource protocol `2`, which current C++ and Python clients reject. Detailed sources: @@ -19,13 +21,14 @@ Detailed sources: ## Current Baseline -- Managed distribution: NuGet `SharedMemoryStore` `1.0.2`, targeting - `net10.0`. +- Managed distribution: NuGet `SharedMemoryStore` `2.0.0`, targeting + `net10.0`; it supports legacy layout `1.2` and explicit lock-free layout `2.0`. - Native distribution: CMake `SharedMemoryStore` `0.1.0`, C++20, C ABI `1.0`. - Python distribution: `shared-memory-store` `0.1.0`, Python 3.10 or newer, using `ctypes` over the bundled native library. -- Shared identities: layout `1.2`, resource naming `1`, little-endian 64-bit - process model. +- Shared cross-runtime identities: layout `1.2`, resource naming `1`, + little-endian 64-bit process model. Managed-only lock-free participation uses + layout `2.0` and resource protocol `2` on qualified x64 hosts. - Implementation targets: Linux and Windows. Release validation for each distribution and ordered runtime pairing must be recorded separately. - Managed supported container profile: Linux-based same-host Docker containers @@ -95,7 +98,7 @@ the public API. Windows uses named operating-system memory mappings and named synchronization. An explicit `Global\` mapping name uses global synchronization in managed -`1.0.2` and native `0.1.0`. All participants must implement compatible +`2.0.0` and native `0.1.0`. All participants must implement compatible resource-naming version `1` behavior. Ordinary unqualified and explicit `Local\` names retain session-local synchronization. Linux uses deterministic files in a shared runtime memory location such as @@ -104,14 +107,23 @@ prevention hash. Docker containers participate in the Linux model only when their IPC and process-liveness configuration lets all participants see the same resources and classify owners safely. +Current managed handles supplement PID/start-token checks with a private locked +per-owner liveness anchor. Consequently, an opener does not delete a live +managed mapping merely because that owner's PID is hidden by a different PID +namespace. A missing anchor retains the PID/start-token fallback needed for +C++, Python, and older managed owners. This improves region-lifetime safety but +does not relax the documented namespace requirements for lease/reservation +recovery or make default-isolated containers a supported topology. + The managed and native implementations must derive the same resources and participate in the same lock. Python inherits the native behavior rather than reimplementing it. The Linux resource directory is owner-only (`0700`), and region, -synchronization, owner, and lifecycle files are owner-only (`0600`). Cooperating -host processes must therefore run as the same Unix identity. Containers must -share a compatible identity as well as IPC and process-liveness namespaces. +synchronization, owner, lifecycle, and managed owner-anchor files are owner-only +(`0600`). Cooperating host processes must therefore run as the same Unix +identity. Containers must share a compatible identity as well as the namespaces +required by the operations and recovery policies they use. ## Unsupported Scenarios diff --git a/docs/releases.md b/docs/releases.md index 76035ee..da204b9 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -2,10 +2,11 @@ This guide is the maintainer checklist for preparing independently versioned managed, native, and Python releases. The current source identities are NuGet -`SharedMemoryStore` `1.0.2`, CMake `SharedMemoryStore` `0.1.0`, Python -`shared-memory-store` `0.1.0`, C ABI `1.0`, mapped layout `1.2`, and resource -naming `1`. Update this guide when any distribution, ABI, protocol, support, -security, documentation, or sample contract changes. +`SharedMemoryStore` `2.0.0`, CMake `SharedMemoryStore` `0.1.0`, Python +`shared-memory-store` `0.1.0`, C ABI `1.0`, managed mapped layouts `1.2`/`2.0`, +and resource protocols `1`/`2`. Native and Python remain layout-1.2/resource-1 +only. Update this guide when any distribution, ABI, protocol, support, security, +documentation, or sample contract changes. ## Automated Release Process @@ -287,8 +288,8 @@ The repository now contains the implementation and packaging inputs for: conformance fixtures, test-only JSON-lines agents, and the ordered 3x3 core exchange harness. -These sibling distributions do not change the managed `1.0.2` public API, -status numbers, NuGet runtime dependencies, or mapped layout. Their `0.1.0` +These sibling distributions remain independently versioned beside managed +`2.0.0`; they do not implement its layout-2.0 lock-free protocol. Their `0.1.0` versions are alpha lines and are not included in the NuGet package. Before publishing native or Python artifacts or describing a platform as diff --git a/docs/samples.md b/docs/samples.md index db99cf8..4531c7d 100644 --- a/docs/samples.md +++ b/docs/samples.md @@ -72,6 +72,9 @@ demonstrates chunked writes into `ValueReservation.GetSpan()`, exact `Advance()` progress, trusted stream reads through `ValueReservation.DangerousGetMemory()`, `Commit()`, `Abort()`, reader acquisition, cleanup, a pipeline adapter, and `TryPublishSegments`. +The socket, stream, and pipeline adapters feed application-owned bytes into a +keyed reservation; they do not turn SharedMemoryStore into a stream, queue, +broker, or delivery coordinator. Focused commands: diff --git a/docs/usage.md b/docs/usage.md index 74b63af..a0e5352 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -158,17 +158,39 @@ when you do not need the status. ## Remove and Reuse -Removal removes the key from the visible index. If no readers hold leases, the -slot is reclaimed immediately. If readers still hold leases, removal returns -`RemovePending`, new acquires for that key fail, and final release reclaims the -slot. +Removal first makes the key logically absent. It returns `RemovePending` when an +active lease protects that generation or when the caller's bounded policy ends +before post-removal classification or reclamation completes. Existing leases +remain readable, new acquires for that key fail, and final release, a later +remove, or allocation-pressure helping can finish physical reclamation. A +`Success` result confirms that the bounded lease classification found no +protecting lease, but callers should still use capacity outcomes rather than +assume that this call alone completed every physical cleanup step. + +`LeaseTableFull` is an exact physical-capacity result, not an exhausted-scan +guess. The lock-free profile confirms two identical, structurally valid, +all-non-Free lease-control snapshots. A released/moving record or contention on +the handle-local proof buffer follows `StoreWaitOptions` and may yield +`StoreBusy`; malformed controls yield `CorruptStore`. ```csharp var remove = store.TryRemove(key); ``` Publishing the same key while the value is still published, pending removal, or -pending reservation returns `DuplicateKey`. +owned by `Reserved(ExplicitReservation)` returns `DuplicateKey`. An internal +`Reserved(AtomicPublication)` used by `TryPublish` or `TryPublishSegments` +remains tentative and does not alone justify that result. Because lock-free-v2 +candidate claim follows an initial lookup but precedes final arbitration, a +raced caller can return genuine `StoreFull` when no candidate is reusable. +The result is certified by two identical, structurally valid, all-non-Free +slot-control snapshots. `Initializing`/`Reserved` require a structurally valid +configured participant token; all other states require participant zero and +`Retired` additionally requires the terminal generation. A malformed state, +generation, or owner shape returns `CorruptStore`, even if both snapshots contain +the same malformed word. Movement or a free slot is contention and is retried +according to the supplied `StoreWaitOptions` rather than reported as false +capacity exhaustion. ## Direct Reservation Ingest @@ -193,6 +215,14 @@ completion returns `ReservationIncomplete`. Advancing beyond the remaining payload length returns `ReservationWriteOutOfRange`. `Abort()` and active reservation disposal remove the pending key without exposing partial bytes. +In the lock-free profile, `TryReserve` records `ExplicitReservation` and orders +when its exact slot changes from `Initializing` to `Reserved`. The earlier state +is a tentative physical claim: other callers may help it, but it does not alone +justify `DuplicateKey`. Normal recovery preserves a lifecycle owned by an exact +live Active participant. A stale owner or one that has published exact +`Closing`/`Recovering` authority can be recovered; the current-process override +is an administrative quiescent-shutdown policy described below. + Writable spans are valid only while the reservation is pending and the store handle remains open. Use `DangerousGetMemory` only for trusted direct-I/O adapters, such as stream or socket reads that require `Memory`. The @@ -209,11 +239,22 @@ ReadOnlySequence payload = GetPayloadSequence(); var publish = store.TryPublishSegments(key, payload, descriptor, out var copiedBytes); ``` -The helper acquires shared synchronization once, reserves a contiguous slot, -copies each segment in order, and publishes only after the copied byte count -matches the sequence length. A copy or validation failure reclaims the internal -slot before synchronization is released, so bounded contention cannot strand a -caller-inaccessible reservation. +The helper reserves a contiguous slot, copies each segment in order, and +publishes only after the copied byte count matches the sequence length. Under +the legacy profile it holds the operation's shared lock through cleanup. Under +the lock-free v2 profile it uses the caller's local retry budget and publishes +only helpable shared transitions; no operation-wide cross-process lock is +acquired. A copy, validation, timeout, or pre-order cancellation failure +relinquishes ownership or hands it to exact-generation cleanup, so bounded +contention cannot strand a caller-inaccessible reservation. + +In lock-free v2, both this helper and `TryPublish` record +`AtomicPublication`. Their internal `Initializing` and `Reserved` states are +tentative; the one public operation orders only when the slot becomes +`Published`. A same-key caller helps/revalidates those states and may return +`DuplicateKey` only after a public duplicate witness appears, or `StoreBusy` +when its bounded budget expires. Tentative states still occupy physical slot +capacity and can therefore contribute to `StoreFull`. ## Diagnostics @@ -251,6 +292,22 @@ Use recovery for controlled owner workflows, tests, or cleanup after abnormal termination. Normal readers and producers should still release leases, commit or abort reservations, and dispose store handles directly. +`RecoverCurrentProcessLeases: false` remains safe while lease activity continues. +Before using the true test/controlled-shutdown override, stop and drain lease +acquisition, `ValueSpan`/`DescriptorSpan` projection, borrowed-span consumption, +and release across every current-process handle attached to that mapping. Keep +the process quiescent until recovery returns; the store does not impose a +hot-path gate to enforce this administrative precondition. + +`RecoverCurrentProcessReservations: false` likewise preserves reservations and +atomic publications whose exact participant remains live Active. Before using +the true reservation override, stop and drain `TryReserve`, `TryPublish`, +`TryPublishSegments`, writable projection and borrowed-memory use, `Advance`, +`Commit`, `Abort`, reservation disposal, and `MemoryStore` handle disposal +across every current-process handle attached to the mapping. Keep that +writer/view quiescence until recovery returns. Racing the override with those +operations is outside the supported result contract. + ## Dispose Dispose every store handle when finished: diff --git a/protocol/README.md b/protocol/README.md index ff30066..a44c820 100644 --- a/protocol/README.md +++ b/protocol/README.md @@ -10,19 +10,29 @@ these contracts. | Contract | Current identity | Canonical definition | |---|---:|---| -| Mapped layout | `1.2` | [layout-v1.2.md](layout-v1.2.md) | -| Platform resource naming | `1` | [resource-naming-v1.md](resource-naming-v1.md) | -| Conformance manifest | `1` | [fixtures/v1.2/manifest.json](fixtures/v1.2/manifest.json) | +| Legacy mapped layout | `1.2` | [layout-v1.2.md](layout-v1.2.md) | +| Lock-free mapped layout | `2.0` | [layout-v2.0.md](layout-v2.0.md) | +| Platform resource naming | `1`, `2` | [resource-naming-v1.md](resource-naming-v1.md), [resource-naming-v2.md](resource-naming-v2.md) | +| Conformance manifests | `1`, `2` | [fixtures/v1.2/manifest.json](fixtures/v1.2/manifest.json), [fixtures/v2.0/manifest.json](fixtures/v2.0/manifest.json) | | Native C ABI | `1.0` | [native-c-api.md](../specs/008-cpp-python-implementations/contracts/native-c-api.md) | Package versions are independent of all four identities. A package release must declare the layout versions it can create and open, the resource-naming version it implements, and its C ABI range when applicable. -## Layout version boundary +## Layout version boundary and client support -New stores created by this repository use layout major `1`, minor `2`. The -advertised read and create boundary for this feature is exactly `1.2`. +C# selects its layout explicitly: the default legacy profile creates/opens +`1.2`, while `StoreProfile.LockFree` creates/opens `2.0`. C++ and Python remain +`1.2`-only in this release and must reject `SMS2` before reading directory, +slot, lease, descriptor, or payload data. Package, layout, resource-naming, and +C ABI versions remain independent. + +Layout 2.0 is a bounded shared-memory key-value protocol, not a queue or stream. +It replaces steady-state named-lock participation with generation-fenced atomic +state machines and explicit record-local helping/recovery. This is lock-free +system-wide progress, not a wait-free guarantee for each caller. The trust +boundary remains cooperating same-host processes with mapped-memory access. The two header numbers are not sufficient evidence of compatibility. An opener must also validate the magic, major version, header and record sizes, configured @@ -61,3 +71,10 @@ not live. Changes to an executable constant or algorithm must update the narrative, manifest, every language's static conformance tests, and the cross-runtime matrix in one review. + +Layout 2.0 currently requires feature bit 0 for its versioned-empty +exact-generation spill summary, bit 1 for per-slot publication intent, and bit +2 for exact store/participant Linux PID-namespace identities. The exact +required mask is `7`. These bits intentionally fence the earlier pre-release +required-features-zero, bit-0-only, and mask-3 v2 shapes in both directions +without changing the 2.0 topology or resource-protocol number. diff --git a/protocol/compatibility.json b/protocol/compatibility.json index 19762c8..9e43e35 100644 --- a/protocol/compatibility.json +++ b/protocol/compatibility.json @@ -1,8 +1,24 @@ { - "schema_version": 1, + "schema_version": 2, "shared_protocol": { - "layout": { "major": 1, "minor": 2 }, - "resource_naming": 1, + "layouts": [ + { + "version": "1.2", + "major": 1, + "minor": 2, + "magic": "SMS1", + "resource_protocol": 1 + }, + { + "version": "2.0", + "major": 2, + "minor": 0, + "magic": "SMS2", + "resource_protocol": 2, + "required_features_mask": 7, + "required_features": ["versioned_empty_spill_summary", "publication_intent", "pid_namespace_identity"] + } + ], "byte_order": "little", "required_pointer_width": 64 }, @@ -10,10 +26,14 @@ { "name": "SharedMemoryStore", "ecosystem": "NuGet", - "version": "1.0.2", - "creates_layouts": ["1.2"], - "reads_layouts": ["1.2"], - "resource_naming": 1, + "version": "2.0.0", + "creates_layouts": ["1.2", "2.0"], + "reads_layouts": ["1.2", "2.0"], + "recognizes_layout_headers": ["1.2", "2.0"], + "layout_resource_protocols": { + "1.2": 1, + "2.0": 2 + }, "validated_targets": ["windows-x86_64", "linux-x86_64"] }, { @@ -23,7 +43,13 @@ "c_abi": "1.0", "creates_layouts": ["1.2"], "reads_layouts": ["1.2"], - "resource_naming": 1, + "recognizes_layout_headers": ["1.2", "2.0"], + "rejects_layouts": ["2.0"], + "rejection_status": "IncompatibleLayout", + "payload_access_before_rejection": false, + "layout_resource_protocols": { + "1.2": 1 + }, "validated_targets": ["windows-x86_64", "linux-x86_64"] }, { @@ -33,7 +59,13 @@ "requires_c_abi": "1.0", "creates_layouts": ["1.2"], "reads_layouts": ["1.2"], - "resource_naming": 1, + "recognizes_layout_headers": ["1.2", "2.0"], + "rejects_layouts": ["2.0"], + "rejection_status": "IncompatibleLayout", + "payload_access_before_rejection": false, + "layout_resource_protocols": { + "1.2": 1 + }, "validated_targets": ["windows-x86_64", "linux-x86_64"] } ], @@ -55,7 +87,10 @@ "reservation-commit-abort", "bounded-lock-contention", "explicit-crash-recovery", - "linux-owner-cleanup" + "linux-owner-cleanup", + "layout-2.0-rejects-required-features-zero-bit-0-only-and-mask-3-drafts", + "older-layout-2.0-drafts-reject-required-features-mask-7", + "v1.2-only-client-rejects-layout-2.0-before-payload-access" ] } } diff --git a/protocol/fixtures/v2.0/manifest.json b/protocol/fixtures/v2.0/manifest.json new file mode 100644 index 0000000..f33b9f6 --- /dev/null +++ b/protocol/fixtures/v2.0/manifest.json @@ -0,0 +1,145 @@ +{ + "format_version": 1, + "protocol": { + "layout_major": 2, + "layout_minor": 0, + "resource_protocol": 2, + "magic_ascii": "SMS2", + "magic_integer_hex": "32534d53", + "little_endian_bytes_hex": "534d5332", + "byte_order": "little", + "required_architecture": "x86_64", + "atomic_width": 8, + "rmw_order": "sequentially-consistent", + "required_features": 7, + "incompatible_draft_required_feature_masks": [0, 1, 3], + "required_feature_bits": { + "versioned_empty_spill_summary": 1, + "publication_intent": 2, + "pid_namespace_identity": 4 + } + }, + "records": { + "store_header": { "size": 512, "alignment": 64, "fields": { "pid_namespace_id": 264, "pid_namespace_mode": 272 } }, + "participant": { + "size": 64, + "fields": { "control": 0, "identity_kind": 8, "reserved": 12, "process_start_value": 16, "open_sequence": 24, "pid_namespace_id": 32 } + }, + "primary_directory_bucket": { + "size": 128, + "fields": { "spill_summary": 0, "mutation": 8, "lanes": 16 }, + "lane_count": 8 + }, + "lease": { + "size": 64, + "fields": { "control": 0, "slot_binding": 8, "acquire_sequence": 16 } + }, + "value_slot": { + "size": 128, + "fields": { + "control": 0, + "directory_binding": 8, + "directory_location": 16, + "directory_operation": 24, + "key_hash": 32, + "key_length": 40, + "descriptor_length": 44, + "value_length": 48, + "publication_intent": 52, + "bytes_advanced": 56, + "commit_sequence": 64, + "key_offset": 72, + "descriptor_offset": 80, + "payload_offset": 88 + } + } + }, + "binding": { + "slot_index_plus_one": { "least_bit": 0, "bit_count": 31 }, + "slot_generation": { "least_bit": 31, "bit_count": 33 }, + "empty": "0000000000000000" + }, + "spill_summary": { + "slot_index_plus_one": { "least_bit": 0, "bit_count": 20 }, + "slot_generation": { "least_bit": 20, "bit_count": 33 }, + "present": { "bit": 53, "meaning": "one_requires_overflow_scan" }, + "reserved_bits": [54, 63], + "initial_empty": "0000000000000000", + "slot_index_constraint": "slot_index < header.slot_count", + "clear_transition": "Present(X) -> Empty(X) by exact full-word CAS", + "witness_repoint_transition": "Present(X) -> Present(Y) only for an exact current same-canonical overflow witness under the revalidated canonical mutation" + }, + "participant_token": { + "bit_count": 28, + "index_bits": "ceil(log2(participant_record_count + 1))", + "generation_bits": "28 - index_bits", + "encoding": "generation << index_bits | record_index + 1" + }, + "pid_namespace_identity": { + "header_id_offset": 264, + "header_mode_offset": 272, + "participant_id_offset": 32, + "windows_id": 0, + "linux_source": "/proc/self/ns/pid numeric token", + "modes": { "recovery_enabled": 1, "mixed_or_unproven": 2 }, + "mixed_transition": "release-store before the differing or unproven opener's first Registering CAS", + "registering_recovery_in_mixed": "unsupported", + "active_recovery_in_mixed": "requires exact per-record namespace proof" + }, + "directory_location": { + "kind_bits": [0, 1], + "index_bits": [2, 23], + "slot_generation_bits": [24, 56], + "reserved_bits": [57, 63], + "kinds": { "none": 0, "primary": 1, "overflow": 2 } + }, + "directory_operation": { + "intent_bits": [0, 1], + "phase_bits": [2, 4], + "target_kind_bits": [5, 6], + "target_index_bits": [7, 28], + "slot_generation_bits": [29, 61], + "reserved_bits": [62, 63], + "intents": { "none": 0, "insert": 1, "unlink": 2 }, + "phases": { "none": 0, "prepared": 1, "target_selected": 2, "binding_changed": 3, "rejected": 4, "complete": 5 } + }, + "states": { + "participant": { "free": 0, "registering": 1, "active": 2, "closing": 3, "recovering": 4, "reclaiming": 5, "retired": 6 }, + "slot": { "free": 0, "initializing": 1, "reserved": 2, "published": 3, "remove_requested": 4, "aborting": 5, "reclaiming": 6, "retired": 7 }, + "lease": { "free": 0, "claiming": 1, "active": 2, "releasing": 3, "recovering": 4, "retired": 5 } + }, + "publication_intent": { + "field_offset": 52, + "values": { "none": 0, "explicit_reservation": 1, "atomic_publication": 2 }, + "metadata_ready_marker": "current-generation directory_operation Insert/Prepared release publication", + "marker_precedes": ["canonical_bucket_mutation", "directory_cell_binding"], + "pre_metadata_initializing": "directory_operation is zero and no current-generation mutation or directory-cell reference exists", + "unreferenced_cleanup_of_pre_metadata_claim_interprets_intent": false, + "current_reference_without_marker": "corrupt_store", + "unknown_discoverable_intent": "corrupt_store", + "ignored_states": ["free", "retired", "initializing_pre_metadata"], + "explicit_reservation_ordering": "Initializing -> Reserved", + "atomic_publication_ordering": "Reserved -> Published", + "duplicate_witnesses": { + "explicit_reservation": ["reserved", "published", "remove_requested"], + "atomic_publication": ["published", "remove_requested"] + } + }, + "capacity_outcomes": { + "store_full_basis": "all_value_slots_non_free", + "tentative_states_consume_capacity": ["initializing", "reserved_atomic_publication"], + "initial_same_key_lookup_precedes_candidate_claim": true, + "candidate_claim_precedes_final_same_key_arbitration": true, + "duplicate_key_precedence_over_store_full": false + }, + "sizing": { + "primary_lane_count": "NextPowerOfTwo(max(32, checked(slot_count * 4)))", + "primary_lanes_per_bucket": 8, + "overflow_count": "slot_count", + "slot_count_min": 1, + "slot_count_max": 1048575, + "participant_record_count_min": 1, + "participant_record_count_max": 1048575, + "byte_storage_alignment": 8 + } +} diff --git a/protocol/layout-v2.0.md b/protocol/layout-v2.0.md new file mode 100644 index 0000000..941516c --- /dev/null +++ b/protocol/layout-v2.0.md @@ -0,0 +1,305 @@ +# Mapped Layout 2.0 + +Layout 2.0 is the C# lock-free profile's language-neutral mapped format. All +multi-byte values are little-endian two's-complement integers. The supported +runtime architecture is x86-64; every shared atomic word is an 8-byte-aligned +signed `int64` accessed with acquire/release `Volatile` operations or +sequentially consistent `Interlocked` read-modify-write operations. + +The magic integer is `0x32534d53`, whose little-endian bytes spell `SMS2`. +The layout version is `2.0` and the shared-resource protocol is `2`. +Required-feature bit 0 (value `0x1`) is +`versioned_empty_spill_summary` and bit 1 (value `0x2`) is +`publication_intent`; bit 2 (value `0x4`) is `pid_namespace_identity`. +This pre-release v2 contract requires the exact mask `7`. +Required-features-zero, bit-0-only, and mask-3 draft v2 binaries reject the current +mapping, and the current binary rejects all older shapes, before payload +projection. + +## Canonical region order + +```text +StoreHeaderV2[512] +ParticipantRecordV2[participant_record_count] stride 64 +PrimaryDirectoryBucketV2[primary_bucket_count] stride 128 +OverflowBinding[slot_count] stride 8 +LeaseRecordV2[lease_record_count] stride 64 +ValueSlotMetadataV2[slot_count] stride 128 +KeyStorage[slot_count] Align8(max(1,max_key_bytes)) +DescriptorStorage[slot_count] Align8(max(1,max_descriptor_bytes)) +PayloadStorage[slot_count] Align8(max(1,max_value_bytes)) +``` + +Every section begins on an 8-byte boundary. The participant, primary bucket, +lease, and slot sections begin on 64-byte boundaries. Arithmetic is checked. + +```text +primary_lane_count = NextPowerOfTwo(max(32, checked(slot_count * 4))) +primary_bucket_count = primary_lane_count / 8 +participant_index_bits = ceil(log2(participant_record_count + 1)) +participant_generation_bits = 28 - participant_index_bits +``` + +Participant count is 1..1,048,575 and generation bits are therefore at least +8. Slot count is also 1..1,048,575, which bounds `primary_lane_count` at +`2^22`. Required bytes is the aligned end of payload storage. + +## Store control + +The aligned atomic header control encodes `Initializing=1`, `Ready=2`, +`Corrupt=3`, and `Unsupported=4`. Initialization release-publishes `Ready`. +After exact-reference/race revalidation proves persistent mapped structural +corruption, any participant may full-word-CAS `Ready` to terminal `Corrupt`. +Every mapped-data operation acquire-loads this word before a new projection or +mutation, and later openers reject a corrupt mapping as incompatible. Invalid +caller-owned inputs and normal capacity, contention, cancellation, disposal, +or concurrent-lifecycle outcomes must not set the latch. No OS synchronization +or process-held lock participates in this mechanism. + +## Fixed records + +### ParticipantRecordV2: 64 bytes + +| Field | Type | Offset | +|---|---:|---:| +| `control` | atomic uint64 | 0 | +| `identity_kind` | int32 | 8 | +| reserved | int32 | 12 | +| `process_start_value` | int64 | 16 | +| `open_sequence` | int64 | 24 | +| `pid_namespace_id` | uint64 | 32 | +| reserved | bytes | 40 | + +Participant control uses state bits 0..2, incarnation bits 3..30, PID bits +31..62, and a zero reserved bit 63. States are Free=0, Registering=1, +Active=2, Closing=3, Recovering=4, Reclaiming=5, Retired=6. The hot token is +`generation << participant_index_bits | record_index + 1`. Generation begins +at one and terminal generations retire instead of wrapping. + +`Closing` is published by a handle only after its local operation gate drains, +and before cleanup begins. `Closing` and `Recovering` are claim-closed handoff +states: exact referenced slot/lease controls may be recovered without PID/start +liveness classification, while participant reuse still requires a fresh exact- +token absence scan followed by full-word `Closing/Recovering -> Reclaiming` CAS. + +The header stores the creator's `pid_namespace_id` at byte offset 264 and an +atomic recovery mode at offset 272 (`Enabled=1`, `Mixed=2`). Linux uses the +positive numeric token parsed from `/proc/self/ns/pid`; Windows stores zero. A +different or unproven Linux opener release-publishes the irreversible Mixed mode +before its first Registering CAS, then continues ordinary KV access. Registering +recovery is unsupported in Mixed. A Registering record writes its own observed +value before Active publication; stable Active snapshots include that value and +may be classified only after an exact current-namespace comparison. Closing and +Recovering remain explicit, namespace-independent handoffs. + +### PrimaryDirectoryBucketV2: 128 bytes + +| Field | Type | Offset | +|---|---:|---:| +| `spill_summary` | atomic uint64 | 0 | +| `mutation` | atomic uint64 | 8 | +| `lane[8]` | atomic uint64[8] | 16 | +| reserved | bytes | 80 | + +`spill_summary` is a versioned negative-cache word. Bits 0..19 contain candidate +slot-index-plus-one, bits 20..52 contain its exact 33-bit slot generation, bit +53 is Present, and bits 54..63 are zero. Raw zero is only initial EmptyNone. +Overflow publication full-word-CASes the exact prior version to +`Present(candidate)` before its cell CAS. After a stable empty full scan under +the exact canonical mutation, cleanup CASes only `Present(X) -> Empty(X)` and +never restores zero. Lookup scans overflow only for Present. Malformed tokens +fail closed. When a full scan instead finds another exact current spill Y, the +same revalidated mutation may full-word-CAS `Present(X) -> Present(Y)` so the +summary continues to name a directly valid witness. This may repeat a Present +word only while Y's nonwrapping binding remains current; it never reintroduces +an Empty identity and cannot validate a prior stable-empty clearer. `mutation` +contains the exact binding of a fully described helpable +insert or unlink operation; it is not a process-owned lock. + +### LeaseRecordV2: 64 bytes + +| Field | Type | Offset | +|---|---:|---:| +| `control` | atomic uint64 | 0 | +| `slot_binding` | uint64 | 8 | +| `acquire_sequence` | int64 | 16 | +| reserved | bytes | 24 | + +Lease control uses state bits 0..2, a 33-bit record generation in bits 3..35, +and the complete participant token in bits 36..63. States are Free=0, +Claiming=1, Active=2, Releasing=3, Recovering=4, Retired=5. + +### ValueSlotMetadataV2: 128 bytes + +| Field | Type | Offset | +|---|---:|---:| +| `control` | atomic uint64 | 0 | +| `directory_binding` | uint64 | 8 | +| `directory_location` | atomic uint64 | 16 | +| `directory_operation` | atomic uint64 | 24 | +| `key_hash` | uint64 | 32 | +| `key_length` | int32 | 40 | +| `descriptor_length` | int32 | 44 | +| `value_length` | int32 | 48 | +| `publication_intent` | int32 | 52 | +| `bytes_advanced` | atomic int64 | 56 | +| `commit_sequence` | int64 | 64 | +| `key_offset` | int64 | 72 | +| `descriptor_offset` | int64 | 80 | +| `payload_offset` | int64 | 88 | +| reserved | bytes | 96 | + +Slot control uses state bits 0..2, a 33-bit generation in bits 3..35, and a +28-bit participant token in bits 36..63 only while Initializing or Reserved. +States are Free=0, Initializing=1, Reserved=2, Published=3, +RemoveRequested=4, Aborting=5, Reclaiming=6, Retired=7. + +`publication_intent` assignments are None=0, ExplicitReservation=1, and +AtomicPublication=2; every other signed 32-bit value is invalid on a current +discoverable lifecycle. The exclusive claimant writes intent before publishing +the current-generation Insert/Prepared directory operation and does not change +it during that generation. The release publication of that exact nonzero +operation is the metadata-ready marker; canonical mutation and directory-cell +binding publication follow it. Pre-metadata Initializing is exactly a current +Initializing state with operation zero and no current-generation mutation/cell +reference. It, Free, and Retired ignore stale/None intent bytes. A current +mutation/cell without the required operation marker is corruption. A marked +or referenced current lifecycle requires a known nonzero intent and otherwise +fails closed as corrupt. Direct unreferenced cleanup of a recovered pre-metadata +claim does not interpret stale ordinary intent bytes. + +ExplicitReservation orders publicly at Initializing-to-Reserved and its +Reserved state is a duplicate-key witness. AtomicPublication covers +`TryPublish` and `TryPublishSegments`; its Reserved state remains an internal +tentative stage and the public operation orders only at Reserved-to-Published. +Lookup must not return DuplicateKey solely from Initializing or from +Reserved(AtomicPublication). A bounded contender may return StoreBusy while it +cannot classify a terminal key owner. StoreFull remains physical: every non-Free +slot, including tentative Initializing/Reserved, is unavailable for reuse. A +raced insertion whose initial same-key lookup returned absent may return +StoreFull at candidate claim before final directory arbitration; duplicate +status does not take precedence over genuine physical exhaustion in that race. +Scan exhaustion is provisional. A public StoreFull result requires two forward +collects of every slot control to be structurally valid, non-Free, and exactly +equal; the confirmed ordering point lies between the collects. Slot controls +never roll back within one generation, so equality cannot hide ABA. The scratch +snapshot and nonblocking guard are private process memory and add no mapped +field, shared counter, named primitive, or OS synchronization. +Every collected control must obey the complete state/generation/owner rules +below. An equal malformed word is corruption, never StoreFull evidence; a free +or moving word or local guard conflict is contention under the caller's wait +policy. + +Lease-record scan exhaustion is provisional for the same reason. A public +LeaseTableFull result requires two forward collects of every lease control to +be structurally valid, non-Free, and exactly equal; its confirmed ordering point +lies between those collects. Claiming/Active require a configured, structurally +valid participant token; Free/Releasing/Recovering/Retired require participant +zero, and Retired requires the terminal incarnation. Invalid state, +incarnation, owner shape, or participant token is corruption. Lease incarnation +advance or retirement prevents control ABA. Each open handle may keep an eager +private `int64[lease_record_count]` snapshot behind a nonblocking local guard; +neither is mapped state or OS/cross-process synchronization. + +## Binding and descriptor words + +Directory bindings use slot-index-plus-one in bits 0..30 and slot generation +in bits 31..63. Zero is empty. Generation zero and index-plus-one zero are +invalid. + +Spill-summary identities use the layout's tighter slot cap rather than the +general binding shape: index-plus-one bits 0..19, generation bits 20..52, +Present bit 53, and ten reserved-zero bits. Present and Empty versions preserve +the same candidate identity, and nonwrapping generation makes every later +candidate version distinct. Every nonzero identity must name a slot below the +header's configured `slot_count`; a mapping-out-of-range identity is corrupt +even when its reserved bits and generation are otherwise codec-valid. + +Directory location is zero for None; otherwise kind Primary=1 or Overflow=2 +occupies bits 0..1, the absolute section cell index occupies bits 2..23, and +the exact nonzero 33-bit slot generation occupies bits 24..56. Bits 57..63 are +zero. + +Directory operation uses intent None=0, Insert=1, Unlink=2 in bits 0..1; +phase None=0, Prepared=1, TargetSelected=2, BindingChanged=3, Rejected=4, +Complete=5 in bits 2..4; target kind in bits 5..6; and target index in bits +7..28. The exact nonzero 33-bit slot generation occupies bits 29..61 and bits +62..63 are zero. Helpers compare and replace the complete word. + +Prepared and Rejected carry no target. TargetSelected and BindingChanged carry +a section-valid target. Insert Complete retains its target; Unlink Complete may +carry no target only when the exact binding was already absent and no directory +cell required clearing. + +Every nonzero operation/location generation must equal both the bucket mutation +binding generation and the current slot-control generation before a helper +acts. Phase changes, location publication/clear, and directory-cell mutation use +exact full-word CAS. A helper paused across reclaim/reuse cannot match a newer +lifecycle; any old binding it installed remains generation-tagged and may be +exactly rolled back or helped clear without touching the newer generation. + +For directory-location publication, the validated tuple is the canonical +mutation word, exact operation word, current location word, slot control, +immutable directory binding, and selected or competing target cells. A helper +MUST NOT report terminal corruption until two acquire collections return the +same tuple, exact no-op compare/exchanges confirm every atomic member, and a +fresh immutable-binding read still agrees. A lost comparison proves concurrent +progress and requires retry. + +Cancellation may hand an Insert to `Unlink/Prepared`. At that handoff, an empty +target or structurally valid different in-range binding is legal target-loss +progress, while a malformed or out-of-range target is corruption. The first +valid `Unlink/Prepared` location publication wins; a losing publisher may +exact-clear only its distinct recovered old binding. `Unlink/TargetSelected` +MUST tolerate a structurally valid same-generation alternate location and +exact-clean its selected and alternate old-binding witnesses plus the alternate +location, preserving empty cells and valid replacements. Source loss after a +location CAS withdraws only the publisher's exact old target and location; a +committed Insert successor or valid replacement is preserved. + +An older location is stale exact-cleanable residue. A future location proves +benign reuse only when another old-tuple member has moved; a future location +inside the confirmed exact old-generation tuple is corruption and remains +untouched for diagnosis. This protocol changes validation and helping only; the +directory-operation and directory-location offsets and encodings are unchanged. + +An atomic directory-reference read is a cached witness. The exact raw source +word and its decoded slot binding are separate: a primary/overflow word equals +the binding, while a spill-summary source is the complete encoded +`Present(binding)` word. If later slot classification would be corrupt, readers +or maintenance helpers must acquire-read that exact raw source word, obtain a +fresh stable snapshot of the separately decoded slot binding, and acquire-read +the same raw word again. A source that no longer equals the exact raw reference +on either side is a stale observation that requires a fresh lookup or +maintenance retry; only an unchanged reference word enclosing a repeated +invalid slot snapshot is corruption. Slot classification validates the complete +control shape: Initializing/Reserved require a configured, structurally valid +participant token; Free/Published/RemoveRequested/Aborting/Reclaiming/Retired +require participant zero; every generation is nonzero and bounded; and Retired +is terminal. A newer generation is stale only when that newer control is itself +structurally valid. + +## Visibility and safety + +- Publication becomes visible at the slot-control CAS from Reserved to + Published, after immutable key/descriptor/payload metadata and bytes exist. +- Acquire activates an exact lease record and then revalidates the directory + and Published slot generation. +- Removal becomes logical at Published to RemoveRequested. It rejects new + leases while existing exact leases continue protecting immutable bytes. +- Reclaim requires a stable no-active-exact-lease scan, exact directory unlink, + exact generation-tagged helper-word cleanup, and release publication of the + next Free generation. Ordinary slot metadata is ignored while Free/Retired; + reclaim helpers do not zero it, and the next Initializing owner overwrites it + before directory publication. +- All terminal generations retire. No index, slot, lease, or participant + incarnation wraps to a previously valid value. +- A completed/canceled insert or unlink does not release its exact canonical + mutation until spill-summary cleanup either revalidates the summary's exact + current overflow witness, finds another current same-bucket witness by bounded + scan, exact-clears a stable empty Present token to its versioned Empty form, + or retains the helpable mutation on budget/uncertainty. + +The executable authority for these offsets and encodings is +`LockFreeLayoutContractTests`; the machine-readable copy is +[`fixtures/v2.0/manifest.json`](fixtures/v2.0/manifest.json). diff --git a/protocol/resource-naming-v1.md b/protocol/resource-naming-v1.md index 4c23a10..6ab1232 100644 --- a/protocol/resource-naming-v1.md +++ b/protocol/resource-naming-v1.md @@ -69,13 +69,32 @@ are forced back to `0600` when touched. ## Linux locking -Both `.lock` and `.lifecycle` use a nonblocking POSIX record lock on byte range -`[0, 1)`, retried according to the caller's wait policy. This is the behavior -exposed by .NET `FileStream.Lock(0, 1)`; using `flock` is not compatible. Each -process must additionally serialize contenders through a process-local mutex -keyed by the absolute lock-file path because POSIX record locks alone do not -provide the required same-process ownership boundary. Release unlocks the byte -range before releasing the local mutex. +Both `.lock` and `.lifecycle` use a nonblocking record lock on byte range +`[0, 1)`, retried according to the caller's wait policy. Current C# and native +implementations issue Linux `F_OFD_SETLK` open-file-description locks and fail +closed as `UnsupportedPlatform` when that command is unavailable. OFD locks +conflict with other descriptors in the same PID, including independently loaded +managed assemblies and native modules, and with traditional `F_SETLK` locks. +They therefore remain mutually exclusive across processes with released v1 +clients while avoiding the traditional rule that closing any sibling descriptor +releases all locks owned by that process. Using `flock` for either interoperable +resource is not compatible. + +One wrapper may still be called by several local threads on the same descriptor, +so it uses a non-reentrant local gate before entering the kernel. Release unlocks +the byte range before releasing that gate. If explicit unlock fails, the wrapper +closes/retires its descriptor before reopening the local gate because close is +the OFD-lock release boundary. Concurrent use of a released implementation that +still uses process-associated `F_SETLK` and a current OFD implementation inside +one OS process is unsupported: closing any descriptor can invalidate the old +implementation's process-associated lock. Cross-process compatibility and +same-process coexistence among current OFD implementations remain supported. + +This prohibition applies to the interoperable `.lock` and `.lifecycle` +resources. A current C# participant may also hold `flock` on its own private +per-owner liveness anchor as described below. That anchor is not a replacement +for either protocol record lock and no foreign participant is required to lock +it. Retries observe cancellation and bounded time using a monotonic clock. The managed baseline retries at intervals no longer than 10 milliseconds or the @@ -96,6 +115,34 @@ plus the process start time in .NET UTC ticks. The unique token is a lowercase 32-hex-digit GUID without separators. Owner readers also tolerate legacy PID-only lines conservatively, but writers emit all three fields. +Current C# packages add one private liveness artifact for each managed owner: + +```text +.owners.anchor.<32-hex-unique-token> +``` + +The artifact is mode `0600`; its suffix is the unchanged third field of the +owner line. The managed process holds an exclusive open-description `flock` for +the lifetime of its mapped view. This is an additive managed safety extension, +not a new owner-line field or a requirement on resource-protocol-1 C++, Python, +or older C# participants. Those participants neither create nor interpret the +artifact. + +The canonical anchor name is the exact `.owners` path plus `.anchor.` and the +owner token rendered as 32 lowercase hexadecimal digits. Anchor reconciliation +is scoped to that one store: names with any other prefix, suffix length, case, +or token syntax are malformed for this purpose and are never selected for +automatic deletion. + +While holding `.lifecycle`, a current C# reader classifies a valid referenced +anchor before consulting PID state: a contended lock is authoritative evidence +that the owner is live, including when its PID is hidden by another PID +namespace; an acquirable lock is stale. A missing anchor preserves the normal +PID/start-token rule for implementations that do not create anchors. Access +failure, a symbolic link, a directory, or any other ambiguous probe is retained +conservatively. A same-process registry and a separately opened probe descriptor +make local probing explicit rather than relying on process-scoped lock behavior. + Owner updates occur while holding `.lifecycle`. A reader trims each line, splits it into at most three colon-separated parts, and requires the first part to be a positive decimal PID. It compares a start token only when all three parts are @@ -106,11 +153,48 @@ an owner when liveness cannot be determined. Updates write the complete set to `.owners.tmp`, force mode `0600`, then atomically replace `.owners`; best-effort cleanup removes a leftover temporary file. -When no live owner remains, stale cleanup removes `.region`, `.lock`, `.owners`, -and `.owners.tmp`. The `.lifecycle` file is deliberately retained because it is -the rendezvous used while cleanup is in progress and by later openers. Closing a -non-final handle removes only its owner record. Closing the final live handle -performs the same stale-resource deletion while holding the lifecycle lock. +An owner-sidecar rewrite that excludes an unlocked owner commits before its +anchor is deleted. On orderly managed close, the mapped view is released first; +the anchor is unlocked only after the exact owner line is committed absent or a +finalized exact-owner release marker has been atomically published. Process +termination closes the anchor descriptor and releases `flock` automatically, so +the next lifecycle operation can classify the stale line and remove a canonical +artifact only when its independent probe proves deletion safe. + +While holding `.lifecycle`, a current C# lifecycle operation performs an +advisory orphan-anchor sweep only after the replacement `.owners` sidecar has +committed. It derives the referenced-token set from canonical three-field lines +in that committed sidecar and considers only canonical anchor names for this +store. Each unreferenced candidate is opened through a separate descriptor with +`O_NOFOLLOW` and verified to be a regular file before a nonblocking exclusive +`flock` is attempted. The candidate is deleted only while that separate lock is +held. Referenced or locked anchors, ambiguous probes, non-regular files, +symbolic links, directories, malformed names, and artifacts that cannot be +enumerated, opened, inspected, locked, or deleted because of an access error +are retained conservatively. The sweep is cold lifecycle repair for a crash +between anchor creation and owner-line publication; it is never a key-value +operation path. + +A finalized release-marker fallback records permission to release the local +anchor; it does not assert that the owner line or anchor pathname has already +been removed. Either artifact can remain until a later lifecycle operation +reconciles the marker, commits the filtered sidecar, and repeats the conservative +anchor sweep. + +When no live owner remains, current stale cleanup removes `.region`, `.owners`, +`.owners.tmp`, and applicable release-marker artifacts. It does not blindly +remove every per-owner anchor. Exact anchors are deleted by orderly owner +release or by the post-commit sweep only when they are canonical, unreferenced, +regular files whose lock is acquirable; every uncertain artifact remains for a +later reconciliation or operator diagnosis. The empty mode-`0600` `.lock` and +`.lifecycle` files are deliberately retained as stable rendezvous inodes; they +contain no store-generation state. Ordinary synchronization is disposed before +mapped-region cleanup can enter `.lifecycle`, so even an older participant that +deletes a no-owner `.lock` cannot strand a current closing descriptor on an +obsolete inode. Closing a non-final handle commits removal of only its exact +owner record and then attempts the same safe anchor cleanup. Closing the final +live handle performs stale data-resource deletion while holding the lifecycle +lock and remains subject to the same conservative anchor rules. The sidecar start token protects resource cleanup from PID reuse. Layout-1.2 lease and reservation records themselves contain only a PID; their explicit diff --git a/protocol/resource-naming-v2.md b/protocol/resource-naming-v2.md new file mode 100644 index 0000000..e5e6c23 --- /dev/null +++ b/protocol/resource-naming-v2.md @@ -0,0 +1,172 @@ +# Shared Resource Protocol 2 + +Resource protocol 2 changes synchronization participation without changing +the physical identity derived from a public store name. This deliberate reuse +makes layout 1.2 and layout 2.0 discover one another and fail closed rather +than creating two unrelated stores under one name. + +## Physical resources + +The mapping and lifecycle names/paths are exactly those specified by +[`resource-naming-v1.md`](resource-naming-v1.md): + +- the same Windows named mapping and named synchronization object; +- the same Linux `.region`, `.owners`, `.lifecycle`, and operation-lock paths. + +An opener maps enough of an existing region at its actual capacity to inspect +the magic and header before projecting caller-requested dimensions. `CreateNew` +reports AlreadyExists for either layout. A different requested layout reports +IncompatibleLayout before any directory, slot, descriptor, or payload access. + +## Cold synchronization + +Layout-v2 create, zero-header initialization, complete header validation, and +participant registration occur while the existing named synchronization +resource is held. This preserves serialization with already released v1 +clients. Participant retirement is an aligned-atomic layout-v2 participant +protocol transition and does not enter that named resource. The mapping is not +Ready during creation, and no handle is returned until all sections have been +initialized and the participant record is Active. + +Physical creation ownership, rather than `OpenMode` or an observed zero magic, +authorizes initialization. A cold-open attempt records whether its physical +mapping call created a new region or opened an existing one. Only the former may +clear and initialize the header. An existing zero header is never written by an +opener: `CreateOrOpen` reports `StoreBusy`, because an older creator may still be +between mapping and synchronization acquisition under resource protocol 1, +while `OpenExisting` reports `IncompatibleLayout`. `CreateNew` reports +`AlreadyExists` without mapping the existing payload. + +On Windows, the named mutex is acquired before the physical mapping is created +or opened and remains held through header work and participant registration. On +Linux, `.lifecycle` is acquired first; release-marker reconciliation and stale +data-resource deletion complete before the persistent `.lock` inode is opened +and acquired. The mapping, +private owner-anchor lock, owner-line commit, header work, and participant +registration then occur while both gates are held. Release order is `.lock` +followed by `.lifecycle`. Failed-open cleanup first releases both gates, then +disposes the ordinary synchronization descriptor, and only then disposes the +mapping/owner registration that may re-enter lifecycle coordination. Current +cleanup retains `.lock` as a stable empty rendezvous file. Together these rules +prevent active and reopening participants from splitting across an unlinked and +replacement inode. + +The caller's original wait and cancellation budget covers the complete cold +transaction, including both gates, mapping, header work, and participant +registration. A deadline or cancellation observed before mapping or owner +publication prevents those side effects. + +Linux owner registration and cleanup remain protected by `.lifecycle`. Every +open layout-v2 handle writes one live v1-compatible owner line: + +```text +decimal-pid:proc-start-or-utc-token:32-hex-guid +``` + +This prevents an older opener from deleting a live SMS2 region as stale. Close +retires the ordinary descriptor before removing only the exact handle's line; +final-owner cleanup follows the existing resource protocol and retains the +stable `.lock`/`.lifecycle` rendezvous files. + +Each current managed Linux owner also creates the private mode-`0600` path + +```text +.owners.anchor.<32-hex-owner-guid> +``` + +before publishing its owner line, then holds an exclusive open-description +`flock` until its mapped view is gone and its owner release is safely recorded. +This lock is deliberately private to the owner and distinct from the POSIX +record-lock protocol on `.lock` and `.lifecycle`; it never appears on a hot data +path. C++/Python and older managed owners remain compatible because the +three-field sidecar format is unchanged and they do not need to create anchors. +The canonical name is the exact store `.owners` path plus `.anchor.` and the +owner GUID rendered as exactly 32 lowercase hexadecimal digits; anchor cleanup +never widens that per-store name pattern. + +Under `.lifecycle`, a current managed reader probes a referenced anchor through +a separately opened descriptor. Lock contention means live even if the recorded +PID is invisible in the reader's PID namespace. Successful lock acquisition +means stale. A missing anchor falls back to the v1 PID/start-token check for +older, C++, and Python owners. Access errors, symbolic links, directories, and +other ambiguous results retain the owner conservatively. A same-process anchor +registry makes local-owner classification explicit. + +Close and failed-open cleanup never wait indefinitely for `.lifecycle`. After +the mapped view is released, a C# resource-protocol-2 participant waits at most +250 milliseconds to remove its exact owner line. If it cannot acquire the lock, +or if cleanup fails before the owner-sidecar replacement commits, it publishes: + +```text +.owners.released.<32-hex-owner-guid>.ready +``` + +The file contains the exact v1-compatible owner line. It is created as a unique +`0600` temporary file beside `.owners`, flushed, and atomically renamed to its +final name. The owner GUID in the filename must equal the third field in the +content. Temporary artifacts have the same prefix but no `.ready` suffix. + +While holding `.lifecycle`, a resource-protocol-2 opener or releaser reconciles +finalized markers before process-liveness filtering. It reads the raw owner +sidecar, applies the existing line-trimming rule, removes only each marker's +ordinal-exact owner record, atomically +rewrites `.owners`, and only then deletes the corresponding marker. Replaying a +marker after a crash between rewrite and deletion is therefore idempotent. A +marker that arrives after the scan is conservative: its still-present owner line +continues to protect the region until a later lifecycle operation. A malformed +finalized marker fails the cold operation closed and is retained for diagnosis. + +When the committed live-owner set is empty, stale-resource deletion also removes +the exact resource's finalized and temporary marker glob. The empty owner set is +atomically committed before this deletion. Resource-protocol-1 C#, C++, and +Python participants do not interpret release markers; they continue to see the +unreconciled owner line and therefore remain conservatively fail closed until a +protocol-2 participant reconciles it or normal PID/start-token liveness proves it +stale. + +The C# 2.0 package uses this bounded cleanup extension for both mapped profiles +because layout 1.2 and layout 2.0 share the same Linux ownership resources. This +does not change layout-1.2 bytes or its required per-operation `.lock` behavior; +older resource-protocol-1 implementations remain compatible and conservative as +described above. + +The owner-sidecar rewrite is the commit point before deleting an unlocked stale +anchor. Orderly close unmaps first, then releases the anchor only after either +that exact owner is absent from the committed sidecar or its finalized release +marker has been atomically published. If both recording paths fail, the managed +process deliberately retains the anchor. Process termination closes the file +descriptor and releases `flock` automatically; later lifecycle cleanup removes +the now-unlocked artifact only when the conservative probe rules below prove it +safe. + +Every current C# orphan-anchor sweep runs under `.lifecycle` and only after the +replacement `.owners` sidecar commits. The sweep builds its referenced-token set +from canonical three-field records in that committed sidecar, enumerates only +canonical anchor names belonging to the same store, and ignores malformed +names. It considers only unreferenced candidates. Each candidate is opened on a +separate descriptor with `O_NOFOLLOW`, verified through that descriptor to be a +regular file, and deleted only after and while a nonblocking exclusive `flock` +succeeds. Referenced or locked anchors, ambiguous probes, non-regular files, +symbolic links, directories, malformed names, and enumeration/open/stat/lock/ +delete access errors are retained conservatively. No final-owner glob removes +these uncertain artifacts. + +Publishing a finalized release marker permits the closing participant to +release its local anchor after unmapping, but the fallback may leave both the +compatible owner line and the anchor pathname in place. A later lifecycle +operation reconciles the exact line, commits the sidecar, and then applies the +same conservative orphan sweep. This repair remains entirely on the cold +lifecycle path. + +## Hot data paths + +After open, layout-v2 publish, reservation, acquire, projection, release, +remove, reclaim, recovery, and diagnostics do not enter any named semaphore, +mutex, or file lock. They use only aligned mapped atomics, immutable bytes, +bounded scans, and helpable descriptors. A retained cold synchronization +object is unreachable from these paths and exists only for compatible close or +recovery lifecycle work. + +Layout-v1.2 handles continue using resource protocol 1 and acquire the ordinary +named synchronization object per data operation. Resource protocol 2 does not +change their bytes or behavior. diff --git a/samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj b/samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj new file mode 100644 index 0000000..947eb59 --- /dev/null +++ b/samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj @@ -0,0 +1,18 @@ + + + + + + + + + + + + Exe + net10.0 + enable + enable + + + diff --git a/samples/LockFreeBrokerKeys/Program.cs b/samples/LockFreeBrokerKeys/Program.cs new file mode 100644 index 0000000..e6ac63d --- /dev/null +++ b/samples/LockFreeBrokerKeys/Program.cs @@ -0,0 +1,227 @@ +using System.Buffers.Binary; +using System.Threading.Channels; +using SharedMemoryStore; + +const int PayloadBytes = 4 * 1024; +int workerCount = ParseOption(args, "--workers", defaultValue: 6); +int frameCount = ParseOption(args, "--frames", defaultValue: 48); +if (workerCount is < 6 or > 12 || frameCount < workerCount) +{ + Console.Error.WriteLine("Use --workers 6..12 and --frames >= workers."); + return 2; +} + +string storeName = $"sms-lock-free-broker-{Guid.NewGuid():N}"; +int slotCount = frameCount; +SharedMemoryStoreOptions createOptions = Options(storeName, slotCount, OpenMode.CreateNew); +StoreOpenStatus open = MemoryStore.TryCreateOrOpen(createOptions, out MemoryStore? producer); +if (open != StoreOpenStatus.Success || producer is null) +{ + Console.Error.WriteLine($"Producer open failed: {open}"); + return 3; +} + +using (producer) +{ + if (producer.Profile != StoreProfile.LockFree + || producer.ProtocolInfo.LayoutMajorVersion != 2 + || producer.ProtocolInfo.LayoutMinorVersion != 0 + || producer.ProtocolInfo.ResourceProtocolVersion != 2) + { + throw new InvalidOperationException( + $"Unexpected profile/protocol: {producer.Profile}, {producer.ProtocolInfo}."); + } + + // These channels stand in for an application-owned broker. The store is + // still only a key-value store: it neither queues nor assigns work. + Channel workKeys = Channel.CreateUnbounded(); + Channel observerKeys = Channel.CreateUnbounded(); + var publishedKeys = new byte[frameCount][]; + long workerChecksum = 0; + long observerChecksum = 0; + int processed = 0; + + Task[] workers = Enumerable.Range(0, workerCount) + .Select(workerId => Task.Run(async () => + { + using MemoryStore workerStore = OpenExisting(storeName, slotCount); + await foreach (byte[] key in workKeys.Reader.ReadAllAsync()) + { + StoreStatus acquire = workerStore.TryAcquire(key, out ValueLease lease); + if (acquire != StoreStatus.Success) + { + throw new InvalidOperationException( + $"Worker {workerId} could not acquire {Convert.ToHexString(key)}: {acquire}"); + } + + using (lease) + { + Interlocked.Add(ref workerChecksum, Checksum(lease.ValueSpan)); + } + + Interlocked.Increment(ref processed); + } + })) + .ToArray(); + + Task observer = Task.Run(async () => + { + using MemoryStore observerStore = OpenExisting(storeName, slotCount); + await foreach (byte[] key in observerKeys.Reader.ReadAllAsync()) + { + if (observerStore.TryAcquire(key, out ValueLease lease) == StoreStatus.Success) + { + using (lease) + { + Interlocked.Add(ref observerChecksum, Checksum(lease.ValueSpan)); + } + } + } + }); + + byte[] descriptor = new byte[sizeof(int)]; + for (var frame = 0; frame < frameCount; frame++) + { + byte[] key = new byte[sizeof(long)]; + BinaryPrimitives.WriteInt64LittleEndian(key, frame); + publishedKeys[frame] = key; + + BinaryPrimitives.WriteInt32LittleEndian(descriptor, frame); + StoreStatus reserve = producer.TryReserve( + key, + PayloadBytes, + descriptor, + out ValueReservation reservation); + if (reserve != StoreStatus.Success) + { + throw new InvalidOperationException($"Reserve {frame} failed: {reserve}"); + } + + using (reservation) + { + Span destination = reservation.GetSpan(PayloadBytes); + destination.Fill(unchecked((byte)frame)); + StoreStatus advance = reservation.Advance(PayloadBytes); + StoreStatus commit = advance == StoreStatus.Success + ? reservation.Commit() + : advance; + if (commit != StoreStatus.Success) + { + throw new InvalidOperationException($"Commit {frame} failed: {commit}"); + } + } + + // Broker messages contain keys, never the 4 KiB payload. + await workKeys.Writer.WriteAsync(key); + await observerKeys.Writer.WriteAsync(key); + } + + workKeys.Writer.Complete(); + observerKeys.Writer.Complete(); + await Task.WhenAll(workers.Append(observer)); + + // Demonstrate that a non-worker reader protects one exact value while its + // logical removal remains local to that key. + using MemoryStore independentReader = OpenExisting(storeName, slotCount); + StoreStatus held = independentReader.TryAcquire(publishedKeys[0], out ValueLease heldLease); + if (held != StoreStatus.Success) + { + throw new InvalidOperationException($"Independent reader failed: {held}"); + } + + StoreStatus pending = producer.TryRemove(publishedKeys[0]); + if (pending != StoreStatus.RemovePending || heldLease.Release() != StoreStatus.Success) + { + throw new InvalidOperationException($"Expected lease-protected RemovePending, got {pending}."); + } + + for (var frame = 1; frame < frameCount; frame++) + { + StoreStatus remove = producer.TryRemove(publishedKeys[frame]); + if (remove != StoreStatus.Success) + { + throw new InvalidOperationException($"Remove {frame} failed: {remove}"); + } + } + + var boundedRead = new StoreWaitOptions(TimeSpan.FromMilliseconds(25), CancellationToken.None); + StoreStatus missing = producer.TryAcquire( + new byte[] { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + boundedRead, + out _); + + _ = producer.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: false), + out LeaseRecoveryReport leaseRecovery); + _ = producer.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: false), + out ReservationRecoveryReport reservationRecovery); + + StoreStatus diagnosticsStatus = producer.TryGetDiagnostics(out DiagnosticsSnapshot diagnostics); + if (diagnosticsStatus != StoreStatus.Success + || diagnostics.Profile != StoreProfile.LockFree + || diagnostics.ProtocolInfo.LayoutMajorVersion != 2) + { + throw new InvalidOperationException($"Diagnostics failed: {diagnosticsStatus}."); + } + + Console.WriteLine( + $"RESULT workers={workerCount} frames={frameCount} processed={processed} " + + $"workerChecksum={workerChecksum} observerChecksum={observerChecksum} " + + $"pendingRemove={pending} missing={missing} diagnostics={diagnosticsStatus} " + + $"profile={producer.Profile} layout={producer.ProtocolInfo.LayoutMajorVersion}." + + $"{producer.ProtocolInfo.LayoutMinorVersion} " + + $"recoveredLeases={leaseRecovery.RecoveredLeaseCount} " + + $"recoveredReservations={reservationRecovery.RecoveredReservationCount}"); +} + +return 0; + +static SharedMemoryStoreOptions Options(string name, int slotCount, OpenMode mode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: PayloadBytes, + maxDescriptorBytes: sizeof(int), + maxKeyBytes: sizeof(long), + leaseRecordCount: 64, + participantRecordCount: 32, + openMode: mode, + enableLeaseRecovery: true); + +static MemoryStore OpenExisting(string name, int slotCount) +{ + // A burst of worker processes may serialize briefly on cold lifecycle + // registration. Give that startup phase an explicit bounded budget. + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, slotCount, OpenMode.OpenExisting), + new StoreWaitOptions(TimeSpan.FromSeconds(10)), + out MemoryStore? store); + return status == StoreOpenStatus.Success && store is not null + ? store + : throw new InvalidOperationException($"OpenExisting failed: {status}"); +} + +static int ParseOption(string[] arguments, string name, int defaultValue) +{ + for (var index = 0; index + 1 < arguments.Length; index++) + { + if (arguments[index] == name && int.TryParse(arguments[index + 1], out int value)) + { + return value; + } + } + + return defaultValue; +} + +static int Checksum(ReadOnlySpan bytes) +{ + var checksum = 17; + foreach (byte value in bytes) + { + checksum = unchecked((checksum * 31) + value); + } + + return checksum; +} diff --git a/scripts/run-lock-free-qualification.ps1 b/scripts/run-lock-free-qualification.ps1 new file mode 100644 index 0000000..f38c9b5 --- /dev/null +++ b/scripts/run-lock-free-qualification.ps1 @@ -0,0 +1,5527 @@ +[CmdletBinding()] +param( + [ValidateSet('pr', 'nightly', 'release')] + [string]$Tier = 'pr', + [string]$Configuration = 'Release', + [string]$OutputDirectory = 'artifacts/lock-free-qualification', + [ValidatePattern('^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$')] + [string]$EvidenceRunId = '', + [switch]$SkipPerformance, + [switch]$SkipOsValidation, + [string[]]$AdditionalOsEvidence = @(), + [switch]$ValidateOnly +) + +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) +$git = (Get-Command git -ErrorAction Stop).Source +if (-not $ValidateOnly) { + $earlyStatus = @(& $git -C $root ` + '-c' 'core.autocrlf=true' ` + '-c' 'core.safecrlf=false' ` + 'status' '--porcelain=v2' '--untracked-files=normal' 2>$null) + if ($LASTEXITCODE -ne 0) { + throw 'Executable qualification could not determine repository cleanliness.' + } + if ($earlyStatus.Count -ne 0) { + throw 'Executable qualification requires a clean working tree.' + } +} +$runStartedUtc = [DateTimeOffset]::UtcNow +$configPath = Join-Path $root 'specs/009-lock-free-publish-read/qualification-config.json' +$config = Get-Content -LiteralPath $configPath -Raw | ConvertFrom-Json +$selected = $config.tiers.$Tier +if ($null -eq $selected) { + throw "Qualification tier '$Tier' is absent from $configPath." +} +$churnTestSourceRelativePath = 'tests/SharedMemoryStore.IntegrationTests/LockFreeChurnIntegrationTests.cs' +$churnTestNamespace = 'SharedMemoryStore.IntegrationTests' +$churnTestClass = 'LockFreeChurnIntegrationTests' +$churnQualificationTestMethod = 'CollisionHeavyMultiProcessRemoveReuseRestoresCapacityAndKeepsLateLatencyBounded' +$churnQualificationTestNameFragment = "$churnTestClass.$churnQualificationTestMethod" + +$outputRoot = if ([IO.Path]::IsPathFullyQualified($OutputDirectory)) { + [IO.Path]::GetFullPath($OutputDirectory) +} +else { + [IO.Path]::GetFullPath((Join-Path $root $OutputDirectory)) +} +$allowedRoot = [IO.Path]::GetFullPath((Join-Path $root 'artifacts')) + [IO.Path]::DirectorySeparatorChar +if (-not ($outputRoot + [IO.Path]::DirectorySeparatorChar).StartsWith($allowedRoot, [StringComparison]::OrdinalIgnoreCase)) { + throw "Qualification output must remain below '$allowedRoot'." +} + +New-Item -ItemType Directory -Path $outputRoot -Force | Out-Null +$runId = if ([string]::IsNullOrWhiteSpace($EvidenceRunId)) { + (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ') + '-' + $Tier + '-' + [Guid]::NewGuid().ToString('N').Substring(0, 8) +} +else { + $EvidenceRunId +} +$runRoot = Join-Path $outputRoot $runId +if (Test-Path -LiteralPath $runRoot) { + throw "Refusing to reuse qualification evidence directory '$runRoot'." +} +New-Item -ItemType Directory -Path $runRoot | Out-Null +$results = [Collections.Generic.List[object]]::new() +$notQualifiedReasons = [Collections.Generic.List[string]]::new() +$acceptedOsEvidence = [Collections.Generic.List[object]]::new() +$overallStatus = 'running' +$failureMessage = $null +$dotnet = (Get-Command dotnet -ErrorAction Stop).Source +$powershell = [Diagnostics.Process]::GetCurrentProcess().MainModule.FileName + +function Get-StringSha256 { + param([AllowEmptyString()][string]$Value) + + $bytes = [Text.Encoding]::UTF8.GetBytes($Value) + return [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($bytes)) +} + +function Get-FileSha256 { + param([Parameter(Mandatory)][string]$Path) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return $null + } + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash +} + +function Invoke-TextCommand { + param( + [Parameter(Mandatory)][string]$FileName, + [Parameter(Mandatory)][string[]]$Arguments) + + try { + $output = & $FileName @Arguments 2>$null + if ($LASTEXITCODE -ne 0) { + return 'unknown' + } + return (($output | ForEach-Object { [string]$_ }) -join "`n").Trim() + } + catch { + return 'unknown' + } +} + +function Get-SourceManifestSha256 { + $discoveredPaths = @(& $git ` + -c core.autocrlf=true ` + -c core.safecrlf=false ` + -C $root ` + ls-files --cached --others --exclude-standard 2>$null) + if ($LASTEXITCODE -ne 0) { + return 'unknown' + } + + # Source provenance must be identical for the same Git content on Windows + # and Linux regardless of either host's ambient core.autocrlf setting. + # Use an ordinal path set/order and an explicit clean-filter policy. + $pathSet = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($discoveredPath in $discoveredPaths) { + if (-not [string]::IsNullOrEmpty([string]$discoveredPath)) { + $null = $pathSet.Add([string]$discoveredPath) + } + } + $orderedPaths = [Collections.Generic.List[string]]::new($pathSet) + $orderedPaths.Sort([StringComparer]::Ordinal) + $paths = @($orderedPaths) + + $existing = @($paths | Where-Object { Test-Path -LiteralPath (Join-Path $root $_) -PathType Leaf }) + $hashes = if ($existing.Count -eq 0) { + @() + } + else { + @($existing | & $git ` + -c core.autocrlf=true ` + -c core.safecrlf=false ` + -C $root ` + hash-object --stdin-paths 2>$null) + } + if ($LASTEXITCODE -ne 0 -or $hashes.Count -ne $existing.Count) { + return 'unknown' + } + $hashByPath = @{} + for ($index = 0; $index -lt $existing.Count; $index++) { + $hashByPath[$existing[$index]] = [string]$hashes[$index] + } + $entries = foreach ($path in $paths) { + "$path`0$(if ($hashByPath.ContainsKey($path)) { $hashByPath[$path] } else { 'missing' })" + } + return Get-StringSha256 ($entries -join "`n") +} + +function Remove-SolutionProjectBuildOutputs { + param([Parameter(Mandatory)][string]$ReportPath) + + $solutionPath = Join-Path $root 'SharedMemoryStore.slnx' + [xml]$solution = Get-Content -LiteralPath $solutionPath -Raw + $projects = @( + $solution.SelectNodes('//Project[@Path]') | + ForEach-Object { [string]$_.Path } | + Where-Object { [IO.Path]::GetExtension($_).Equals('.csproj', [StringComparison]::OrdinalIgnoreCase) } | + Sort-Object -Unique) + if ($projects.Count -eq 0) { + throw 'SharedMemoryStore.slnx does not contain any C# projects.' + } + + $repositoryProjects = @( + & $git -C $root ls-files --cached --others --exclude-standard -- '*.csproj' 2>$null | + Sort-Object -Unique) + if ($LASTEXITCODE -ne 0 -or $repositoryProjects.Count -eq 0) { + throw 'Could not enumerate tracked and nonignored repository project files.' + } + + $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + $pathComparer = if ($IsWindows) { [StringComparer]::OrdinalIgnoreCase } else { [StringComparer]::Ordinal } + $repositoryProjectSet = [Collections.Generic.HashSet[string]]::new($pathComparer) + foreach ($repositoryProject in $repositoryProjects) { + $null = $repositoryProjectSet.Add($repositoryProject.Replace('\', '/')) + } + $normalizedProjects = @($projects | ForEach-Object { $_.Replace('\', '/') } | Sort-Object -Unique) + $solutionProjectSet = [Collections.Generic.HashSet[string]]::new($pathComparer) + foreach ($normalizedProjectPath in $normalizedProjects) { + $null = $solutionProjectSet.Add($normalizedProjectPath) + } + $outsideSolution = @($repositoryProjectSet | Where-Object { -not $solutionProjectSet.Contains($_) }) + $missingFromRepository = @($solutionProjectSet | Where-Object { -not $repositoryProjectSet.Contains($_) }) + if ($outsideSolution.Count -ne 0 -or $missingFromRepository.Count -ne 0) { + throw "SharedMemoryStore.slnx must contain every tracked or nonignored C# project; outsideSolution='$($outsideSolution -join ',')'; missingFromRepository='$($missingFromRepository -join ',')'." + } + + $normalizedRoot = [IO.Path]::GetFullPath($root).TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + $rootPrefix = $normalizedRoot + [IO.Path]::DirectorySeparatorChar + $removed = 0 + $targetRecords = [Collections.Generic.List[object]]::new() + foreach ($project in $projects) { + $normalizedProject = $project.Replace('\', '/') + if ([IO.Path]::IsPathRooted($project) -or -not $repositoryProjectSet.Contains($normalizedProject)) { + throw "Solution project is not a tracked or nonignored repository project: '$project'." + } + + $projectPath = [IO.Path]::GetFullPath((Join-Path $normalizedRoot $project)) + if (-not $projectPath.StartsWith($rootPrefix, $comparison) ` + -or -not (Test-Path -LiteralPath $projectPath -PathType Leaf)) { + throw "Solution project path is missing or outside the repository: '$projectPath'." + } + + $projectItem = Get-Item -LiteralPath $projectPath -Force + if (($projectItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to clean outputs for linked solution project '$projectPath'." + } + + $projectDirectory = [IO.Path]::GetDirectoryName($projectPath) + $ancestor = $projectDirectory + while ($true) { + $ancestorItem = Get-Item -LiteralPath $ancestor -Force + if (-not $ancestorItem.PSIsContainer ` + -or ($ancestorItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to clean through non-directory or linked ancestor '$ancestor'." + } + if ($ancestor.Equals($normalizedRoot, $comparison)) { + break + } + if (-not $ancestor.StartsWith($rootPrefix, $comparison)) { + throw "Project ancestor escaped the repository while checking '$projectPath'." + } + $parent = [IO.Directory]::GetParent($ancestor) + if ($null -eq $parent) { + throw "Repository root was not reached while checking '$projectPath'." + } + $ancestor = $parent.FullName + } + + foreach ($directoryName in @('bin', 'obj')) { + $target = [IO.Path]::GetFullPath((Join-Path $projectDirectory $directoryName)) + if (-not $target.StartsWith($rootPrefix, $comparison)) { + throw "Refusing to clean project output outside the repository: '$target'." + } + + $relativeTarget = [IO.Path]::GetRelativePath($normalizedRoot, $target).Replace('\', '/') + $protectedEntries = @( + & $git -C $normalizedRoot --literal-pathspecs ls-files --cached --others --exclude-standard -- $relativeTarget 2>$null) + if ($LASTEXITCODE -ne 0) { + throw "Could not prove that '$relativeTarget' contains only ignored build output." + } + if ($protectedEntries.Count -ne 0) { + throw "Refusing to clean '$relativeTarget' because it contains tracked or nonignored entry '$($protectedEntries[0])'." + } + $existed = Test-Path -LiteralPath $target + if (-not $existed) { + $targetRecords.Add([pscustomobject][ordered]@{ + path = $relativeTarget + existed = $false + removed = $false + verifiedAbsent = $false + trackedOrNonignoredFiles = 0 + }) + continue + } + + $item = Get-Item -LiteralPath $target -Force + if (-not $item.PSIsContainer ` + -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to recursively clean non-directory or linked output path '$target'." + } + $linkedDescendants = @(Get-ChildItem -LiteralPath $target -Recurse -Force -ErrorAction Stop | + Where-Object { ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 } | + Sort-Object { $_.FullName.Length } -Descending) + foreach ($linkedDescendant in $linkedDescendants) { + Remove-Item -LiteralPath $linkedDescendant.FullName -Force + if (Test-Path -LiteralPath $linkedDescendant.FullName) { + throw "Could not unlink project output entry '$($linkedDescendant.FullName)'." + } + } + Remove-Item -LiteralPath $target -Recurse -Force + if (Test-Path -LiteralPath $target) { + throw "Project output remained after recursive cleanup: '$target'." + } + $targetRecords.Add([pscustomobject][ordered]@{ + path = $relativeTarget + existed = $true + removed = $true + verifiedAbsent = $false + trackedOrNonignoredFiles = 0 + }) + $removed++ + } + } + + $stabilized = $false + for ($stabilizationPass = 1; $stabilizationPass -le 5; $stabilizationPass++) { + $recleaned = $false + foreach ($targetRecord in $targetRecords) { + $absoluteTarget = [IO.Path]::GetFullPath((Join-Path $normalizedRoot $targetRecord.path)) + $remainingItem = Get-Item -LiteralPath $absoluteTarget -Force -ErrorAction SilentlyContinue + if ($null -eq $remainingItem -and -not (Test-Path -LiteralPath $absoluteTarget)) { + continue + } + + $protectedEntries = @( + & $git -C $normalizedRoot --literal-pathspecs ls-files --cached --others --exclude-standard -- $targetRecord.path 2>$null) + if ($LASTEXITCODE -ne 0 -or $protectedEntries.Count -ne 0) { + throw "Refusing to re-clean recreated output '$($targetRecord.path)' without a zero-protected-file proof." + } + if ($null -eq $remainingItem ` + -or -not $remainingItem.PSIsContainer ` + -or ($remainingItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to re-clean non-directory or linked output '$absoluteTarget'." + } + $linkedDescendants = @(Get-ChildItem -LiteralPath $absoluteTarget -Recurse -Force -ErrorAction Stop | + Where-Object { ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 } | + Sort-Object { $_.FullName.Length } -Descending) + foreach ($linkedDescendant in $linkedDescendants) { + Remove-Item -LiteralPath $linkedDescendant.FullName -Force + } + Remove-Item -LiteralPath $absoluteTarget -Recurse -Force + $targetRecord.existed = $true + if (-not $targetRecord.removed) { + $targetRecord.removed = $true + $removed++ + } + $recleaned = $true + } + + Start-Sleep -Milliseconds 100 + $remainingTargets = @($targetRecords | Where-Object { + $candidate = [IO.Path]::GetFullPath((Join-Path $normalizedRoot $_.path)) + $null -ne (Get-Item -LiteralPath $candidate -Force -ErrorAction SilentlyContinue) ` + -or (Test-Path -LiteralPath $candidate) + }) + if ($remainingTargets.Count -eq 0) { + foreach ($targetRecord in $targetRecords) { + $targetRecord.verifiedAbsent = $true + } + $stabilized = $true + break + } + } + if (-not $stabilized) { + throw 'Project outputs did not converge to an absent state after five guarded cleanup passes.' + } + + $orderedTargets = @($targetRecords | Sort-Object path) + $expectedTargetCount = $normalizedProjects.Count * 2 + if ($orderedTargets.Count -ne $expectedTargetCount ` + -or @($orderedTargets | ForEach-Object path | Sort-Object -Unique).Count -ne $expectedTargetCount ` + -or @($orderedTargets | Where-Object { -not $_.verifiedAbsent }).Count -ne 0) { + throw 'Cross-platform pre-clean did not verify every unique solution project output target as absent.' + } + + if (Test-Path -LiteralPath $ReportPath) { + throw "Refusing to overwrite pre-clean evidence '$ReportPath'." + } + $report = [pscustomobject][ordered]@{ + schemaVersion = 1 + completedUtc = [DateTimeOffset]::UtcNow + solution = 'SharedMemoryStore.slnx' + solutionProjects = $normalizedProjects + solutionProjectCount = $normalizedProjects.Count + solutionProjectSetSha256 = Get-StringSha256 ($normalizedProjects -join "`n") + targets = $orderedTargets + targetCount = $orderedTargets.Count + targetSetSha256 = Get-StringSha256 (@($orderedTargets | ForEach-Object path) -join "`n") + existedCount = @($orderedTargets | Where-Object existed).Count + removedCount = @($orderedTargets | Where-Object removed).Count + verifiedAbsentCount = @($orderedTargets | Where-Object verifiedAbsent).Count + trackedOrNonignoredFileCount = [int64](@($orderedTargets | Measure-Object trackedOrNonignoredFiles -Sum).Sum) + } + New-Item -ItemType Directory -Path (Split-Path -Parent $ReportPath) -Force | Out-Null + [IO.File]::WriteAllText($ReportPath, ($report | ConvertTo-Json -Depth 6), [Text.UTF8Encoding]::new($false)) + $reportRelativePath = [IO.Path]::GetRelativePath($normalizedRoot, $ReportPath).Replace('\', '/') + $reportSha256 = Get-FileSha256 $ReportPath + $summary = [pscustomobject][ordered]@{ + schemaVersion = 1 + solutionProjectCount = $normalizedProjects.Count + uniqueSolutionProjectCount = $solutionProjectSet.Count + solutionProjectSetSha256 = $report.solutionProjectSetSha256 + targetCount = $orderedTargets.Count + uniqueTargetCount = @($orderedTargets | ForEach-Object path | Sort-Object -Unique).Count + targetSetSha256 = $report.targetSetSha256 + existedBeforeCount = $report.existedCount + removedCount = $report.removedCount + verifiedAbsentCount = $report.verifiedAbsentCount + protectedFileCount = $report.trackedOrNonignoredFileCount + reportPath = $reportRelativePath + reportSha256 = $reportSha256 + } + + return [pscustomobject]@{ + SolutionProjectCount = $normalizedProjects.Count + TargetCount = $orderedTargets.Count + RemovedDirectoryCount = $removed + VerifiedAbsentCount = $orderedTargets.Count + ReportPath = $ReportPath + ReportSha256 = $reportSha256 + Summary = $summary + } +} + +function Get-RepositoryProvenance { + $status = Invoke-TextCommand $git @( + '-c', 'core.autocrlf=true', + '-c', 'core.safecrlf=false', + '-C', $root, + 'status', '--porcelain=v2', '--untracked-files=normal') + return [ordered]@{ + commit = Invoke-TextCommand $git @('-C', $root, 'rev-parse', 'HEAD') + headTree = Invoke-TextCommand $git @('-C', $root, 'rev-parse', 'HEAD^{tree}') + workingTreeState = if ([string]::IsNullOrWhiteSpace($status)) { 'clean' } elseif ($status -eq 'unknown') { 'unknown' } else { 'dirty' } + statusSha256 = Get-StringSha256 $status + sourceManifestSha256 = Get-SourceManifestSha256 + } +} + +$repositoryProvenance = Get-RepositoryProvenance +$completionProvenance = $null +$testedAssemblyManifest = @() +$completionAssemblyManifest = @() + +function Assert-KnownProvenance { + param( + [Parameter(Mandatory)]$Provenance, + [Parameter(Mandatory)][string]$Context) + + foreach ($property in @('commit', 'headTree', 'workingTreeState', 'statusSha256', 'sourceManifestSha256')) { + $value = [string]$Provenance[$property] + if ([string]::IsNullOrWhiteSpace($value) -or $value -eq 'unknown') { + throw "$Context provenance property '$property' is unknown." + } + } +} + +function Assert-ProvenanceStable { + param( + [Parameter(Mandatory)]$Start, + [Parameter(Mandatory)]$End) + + Assert-KnownProvenance $Start 'start' + Assert-KnownProvenance $End 'completion' + foreach ($property in @('commit', 'headTree', 'workingTreeState', 'statusSha256', 'sourceManifestSha256')) { + if ([string]$Start[$property] -ne [string]$End[$property]) { + throw "Repository provenance changed during qualification: '$property' start='$($Start[$property])' completion='$($End[$property])'." + } + } +} + +function Test-IsIntegerNumber { + param($Value) + + return $Value -is [byte] -or $Value -is [sbyte] ` + -or $Value -is [int16] -or $Value -is [uint16] ` + -or $Value -is [int32] -or $Value -is [uint32] ` + -or $Value -is [int64] -or $Value -is [uint64] +} + +function Test-IsNumericValue { + param($Value) + + return (Test-IsIntegerNumber $Value) -or $Value -is [single] ` + -or $Value -is [double] -or $Value -is [decimal] +} + +function Get-RequiredPropertyValue { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context) + + $member = $Object.PSObject.Properties[$Property] + if ($null -eq $member -or $null -eq $member.Value) { + throw "$Context is missing required property '$Property'." + } + return $member.Value +} + +function Get-StrictInt64 { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context, + $Minimum = [int64]::MinValue, + $Maximum = [int64]::MaxValue) + + # Command-mode arguments can preserve a static-member token as text. Only + # these exact internal bound tokens are accepted; evidence values remain + # required to be real JSON numbers below. + $minimumBound = if (Test-IsIntegerNumber $Minimum) { + [Convert]::ToInt64($Minimum, [Globalization.CultureInfo]::InvariantCulture) + } + elseif ([string]$Minimum -eq '[int64]::MinValue') { + [int64]::MinValue + } + else { + throw "$Context.$Property received an invalid internal minimum bound '$Minimum'." + } + $maximumBound = if (Test-IsIntegerNumber $Maximum) { + [Convert]::ToInt64($Maximum, [Globalization.CultureInfo]::InvariantCulture) + } + elseif ([string]$Maximum -eq '[int32]::MaxValue') { + [int32]::MaxValue + } + elseif ([string]$Maximum -eq '[int64]::MaxValue') { + [int64]::MaxValue + } + else { + throw "$Context.$Property received an invalid internal maximum bound '$Maximum'." + } + + $value = Get-RequiredPropertyValue $Object $Property $Context + if (-not (Test-IsIntegerNumber $value)) { + throw "$Context.$Property must be an integer JSON number, actual type '$($value.GetType().FullName)'." + } + try { + $converted = [Convert]::ToInt64($value, [Globalization.CultureInfo]::InvariantCulture) + } + catch { + throw "$Context.$Property is outside signed 64-bit range." + } + if ($converted -lt $minimumBound -or $converted -gt $maximumBound) { + throw "$Context.$Property=$converted is outside [$minimumBound,$maximumBound]." + } + return $converted +} + +function Get-StrictDouble { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context, + $Minimum = -[double]::MaxValue, + $Maximum = [double]::MaxValue, + [switch]$Positive) + + $minimumBound = if (Test-IsNumericValue $Minimum) { + [Convert]::ToDouble($Minimum, [Globalization.CultureInfo]::InvariantCulture) + } + elseif ([string]$Minimum -eq '-[double]::MaxValue') { + -[double]::MaxValue + } + else { + throw "$Context.$Property received an invalid internal minimum bound '$Minimum'." + } + $maximumBound = if (Test-IsNumericValue $Maximum) { + [Convert]::ToDouble($Maximum, [Globalization.CultureInfo]::InvariantCulture) + } + elseif ([string]$Maximum -eq '[double]::MaxValue') { + [double]::MaxValue + } + else { + throw "$Context.$Property received an invalid internal maximum bound '$Maximum'." + } + + $value = Get-RequiredPropertyValue $Object $Property $Context + if (-not (Test-IsNumericValue $value)) { + throw "$Context.$Property must be a numeric JSON value, actual type '$($value.GetType().FullName)'." + } + $converted = [Convert]::ToDouble($value, [Globalization.CultureInfo]::InvariantCulture) + if (-not [double]::IsFinite($converted) -or $converted -lt $minimumBound -or $converted -gt $maximumBound ` + -or ($Positive -and $converted -le 0)) { + throw "$Context.$Property=$converted is not a valid finite value." + } + return $converted +} + +function Get-StrictBoolean { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context) + + $value = Get-RequiredPropertyValue $Object $Property $Context + if ($value -isnot [bool]) { + throw "$Context.$Property must be a JSON Boolean." + } + return [bool]$value +} + +function Get-StrictString { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context, + [switch]$AllowEmpty) + + $value = Get-RequiredPropertyValue $Object $Property $Context + if ($value -isnot [string] -or (-not $AllowEmpty -and [string]::IsNullOrWhiteSpace($value))) { + throw "$Context.$Property must be a JSON string$(if ($AllowEmpty) { '' } else { ' with content' })." + } + return [string]$value +} + +function Get-CanonicalCheckpointCatalog { + $catalogPath = Join-Path $root 'src/SharedMemoryStore/LockFree/LockFreeCheckpoint.cs' + $source = Get-Content -LiteralPath $catalogPath -Raw + $enumMatch = [regex]::Match( + $source, + '(?s)internal\s+enum\s+LockFreeCheckpointId\s*\{(?.*?)\}') + if (-not $enumMatch.Success) { + throw "Cannot parse LockFreeCheckpointId from '$catalogPath'." + } + + $idByName = @{} + foreach ($match in [regex]::Matches( + $enumMatch.Groups['body'].Value, + '(?m)^\s*(?[A-Za-z][A-Za-z0-9_]*)\s*=\s*(?[0-9]+)\s*,?\s*$')) { + $name = $match.Groups['name'].Value + $id = [int]$match.Groups['id'].Value + if ($idByName.ContainsKey($name)) { + throw "Checkpoint enum contains duplicate name '$name'." + } + $idByName[$name] = $id + } + + $catalogMatches = [regex]::Matches( + $source, + '(?s)(?Before|After)\(\s*' + + 'LockFreeCheckpointId\.(?[A-Za-z][A-Za-z0-9_]*)\s*,\s*' + + 'LockFreeCheckpointFamily\.(?[A-Za-z][A-Za-z0-9_]*)\s*,\s*' + + 'LockFreePauseClassification\.(?[A-Za-z][A-Za-z0-9_]*)\s*,\s*' + + 'LockFreeCrashClassification\.(?[A-Za-z][A-Za-z0-9_]*)\s*,\s*' + + 'LockFreeRaceClassification\.(?[A-Za-z][A-Za-z0-9_]*)\s*,\s*' + + '(?:orderingPoint:\s*(?true|false)\s*,\s*)?"') + if ($idByName.Count -eq 0 -or $catalogMatches.Count -ne $idByName.Count) { + throw "Checkpoint catalog must classify every enum member exactly once; enum=$($idByName.Count), catalog=$($catalogMatches.Count)." + } + + $entries = foreach ($match in $catalogMatches) { + $name = $match.Groups['name'].Value + if (-not $idByName.ContainsKey($name)) { + throw "Checkpoint catalog references unknown enum member '$name'." + } + [pscustomobject][ordered]@{ + id = [int]$idByName[$name] + name = $name + family = $match.Groups['family'].Value + position = $match.Groups['position'].Value + pause = $match.Groups['pause'].Value + crash = $match.Groups['crash'].Value + race = $match.Groups['race'].Value + isPublicOrderingPoint = $match.Groups['ordering'].Success -and + $match.Groups['ordering'].Value -ceq 'true' + } + } + $ordered = @($entries | Sort-Object id) + if (@($ordered | Group-Object id | Where-Object Count -ne 1).Count -ne 0 ` + -or @($ordered | Group-Object name | Where-Object Count -ne 1).Count -ne 0) { + throw 'Checkpoint catalog contains a duplicate or omits a canonical identifier.' + } + for ($index = 0; $index -lt $ordered.Count; $index++) { + if ([int]$ordered[$index].id -ne ($index + 1)) { + throw "Checkpoint identifiers must remain append-only and contiguous; index=$index id=$($ordered[$index].id)." + } + } + return $ordered +} + +$checkpointCatalog = @(Get-CanonicalCheckpointCatalog) +$canonicalSuspensionCheckpointIds = @($checkpointCatalog | + Where-Object { $_.family -notin @('Participant', 'Disposal') } | + ForEach-Object { [int]$_.id }) + +function Get-TestedAssemblyManifest { + [xml]$solution = Get-Content -LiteralPath (Join-Path $root 'SharedMemoryStore.slnx') -Raw + $projectPaths = @($solution.SelectNodes("//*[local-name()='Project']") | ForEach-Object { [string]$_.Path }) + if ($projectPaths.Count -eq 0) { + throw 'Solution does not expose any project paths for assembly provenance.' + } + + $files = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($projectPath in $projectPaths) { + $projectDirectory = Split-Path -Parent (Join-Path $root $projectPath) + $assemblyName = [IO.Path]::GetFileNameWithoutExtension($projectPath) + '.dll' + $outputDirectory = Join-Path $projectDirectory "bin/$Configuration/net10.0" + foreach ($fileName in @($assemblyName, 'SharedMemoryStore.dll') | Sort-Object -Unique) { + $fullPath = Join-Path $outputDirectory $fileName + if (Test-Path -LiteralPath $fullPath -PathType Leaf) { + [void]$files.Add([IO.Path]::GetFullPath($fullPath)) + } + elseif ($fileName -eq $assemblyName) { + throw "Expected freshly built assembly '$fullPath' is missing." + } + } + } + + return @($files | Sort-Object | ForEach-Object { + [pscustomobject][ordered]@{ + path = [IO.Path]::GetRelativePath($root, $_) + length = (Get-Item -LiteralPath $_).Length + sha256 = Get-FileSha256 $_ + } + }) +} + +function Assert-AssemblyManifestStable { + param( + [Parameter(Mandatory)][object[]]$Start, + [Parameter(Mandatory)][object[]]$End) + + $startCanonical = @($Start | ForEach-Object { "$($_.path)|$($_.length)|$($_.sha256)" }) -join "`n" + $endCanonical = @($End | ForEach-Object { "$($_.path)|$($_.length)|$($_.sha256)" }) -join "`n" + if ([string]::IsNullOrWhiteSpace($startCanonical) -or $startCanonical -ne $endCanonical) { + throw 'Tested assembly manifest changed after the clean build or while qualification was running.' + } +} + +function Get-TestedAssemblyHash { + param([Parameter(Mandatory)][string]$RelativePath) + + $normalized = $RelativePath.Replace('\', '/') + $assemblyRows = @($testedAssemblyManifest | Where-Object { + ([string]$_.path).Replace('\', '/') -ceq $normalized + }) + if ($assemblyRows.Count -ne 1) { + throw "Fresh-build assembly manifest lacks unique path '$RelativePath'." + } + $hash = [string]$assemblyRows[0].sha256 + if ($hash -notmatch '^[0-9A-F]{64}$') { + throw "Fresh-build assembly manifest lacks unique path '$RelativePath'." + } + return $hash +} + +function Invoke-BoundedStep { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$FileName, + [Parameter(Mandatory)][string[]]$Arguments, + [hashtable]$Environment = @{}, + [int[]]$AllowedExitCodes = @(0) + ) + + $stdoutPath = Join-Path $runRoot ($Name + '.stdout.log') + $stderrPath = Join-Path $runRoot ($Name + '.stderr.log') + $start = [Diagnostics.ProcessStartInfo]::new() + $start.FileName = $FileName + $start.WorkingDirectory = $root + $start.UseShellExecute = $false + $start.CreateNoWindow = $true + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + foreach ($argument in $Arguments) { + $start.ArgumentList.Add($argument) + } + foreach ($entry in $Environment.GetEnumerator()) { + $start.Environment[$entry.Key] = [string]$entry.Value + } + + $process = [Diagnostics.Process]::new() + $process.StartInfo = $start + $startedUtc = [DateTimeOffset]::UtcNow + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + if (-not $process.Start()) { + throw "Could not start qualification step '$Name'." + } + + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + $timeoutMilliseconds = [int64]$selected.stepTimeoutSeconds * 1000 + $completed = $process.WaitForExit([int][Math]::Min([int]::MaxValue, $timeoutMilliseconds)) + $terminationSucceeded = $true + $terminationDetail = $null + if (-not $completed) { + try { + $process.Kill($true) + } + catch { + $terminationSucceeded = $false + $terminationDetail = "process-tree kill failed: $($_.Exception.Message)" + } + if ($terminationSucceeded -and -not $process.WaitForExit(30000)) { + $terminationSucceeded = $false + $terminationDetail = 'process tree did not terminate within 30 seconds of Kill' + } + } + + $streamsDrained = $false + $streamDrainDetail = $null + try { + $streamsDrained = [Threading.Tasks.Task]::WaitAll( + [Threading.Tasks.Task[]]@($stdoutTask, $stderrTask), + 30000) + } + catch { + $streamDrainDetail = "redirected-stream drain failed: $($_.Exception.GetBaseException().Message)" + } + if (-not $streamsDrained -and [string]::IsNullOrWhiteSpace($streamDrainDetail)) { + $streamDrainDetail = 'redirected streams did not close within 30 seconds' + } + $stdout = if ($stdoutTask.IsCompletedSuccessfully) { + $stdoutTask.GetAwaiter().GetResult() + } + else { + "[qualification-runner] stdout unavailable: $streamDrainDetail" + } + $stderr = if ($stderrTask.IsCompletedSuccessfully) { + $stderrTask.GetAwaiter().GetResult() + } + else { + "[qualification-runner] stderr unavailable: $streamDrainDetail" + } + if (-not $terminationSucceeded) { + $stderr += [Environment]::NewLine + "[qualification-runner] $terminationDetail" + } + [IO.File]::WriteAllText($stdoutPath, $stdout) + [IO.File]::WriteAllText($stderrPath, $stderr) + $stopwatch.Stop() + + $hasExited = $false + try { + $hasExited = $process.HasExited + } + catch { + $hasExited = $false + } + $exitCode = if ($hasExited -and $completed) { $process.ExitCode } else { -1 } + $executionFailed = -not $completed -or -not $terminationSucceeded -or -not $streamsDrained ` + -or $exitCode -notin $AllowedExitCodes + + $result = [ordered]@{ + name = $Name + command = $FileName + ' ' + ($Arguments -join ' ') + startedUtc = $startedUtc + elapsedSeconds = $stopwatch.Elapsed.TotalSeconds + timeoutSeconds = [int]$selected.stepTimeoutSeconds + timedOut = -not $completed + exitCode = $exitCode + environment = [ordered]@{} + $Environment + stdout = [IO.Path]::GetRelativePath($root, $stdoutPath) + stderr = [IO.Path]::GetRelativePath($root, $stderrPath) + stdoutSha256 = Get-FileSha256 $stdoutPath + stderrSha256 = Get-FileSha256 $stderrPath + status = if ($executionFailed) { + 'failed' + } + elseif ($exitCode -eq 0) { + 'passed' + } + else { + 'not-qualified' + } + qualification = 'execution-only' + validation = @() + } + $results.Add([pscustomobject]$result) + $process.Dispose() + if ($executionFailed) { + throw "Qualification step '$Name' failed; see '$stdoutPath' and '$stderrPath'." + } +} + +function Add-EvidenceResult { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][ValidateSet('passed', 'failed', 'not-qualified', 'smoke-only')][string]$Status, + [Parameter(Mandatory)][string]$Qualification, + [Parameter(Mandatory)][string[]]$Evidence, + [string[]]$Artifacts = @()) + + $results.Add([pscustomobject][ordered]@{ + name = $Name + command = $null + startedUtc = [DateTimeOffset]::UtcNow + elapsedSeconds = 0 + timeoutSeconds = 0 + timedOut = $false + exitCode = $null + environment = [ordered]@{} + stdout = $null + stderr = $null + stdoutSha256 = $null + stderrSha256 = $null + status = $Status + qualification = $Qualification + validation = $Evidence + artifacts = $Artifacts + }) +} + +function Get-StepResult { + param([Parameter(Mandatory)][string]$Name) + + $matches = @($results | Where-Object { $_.name -eq $Name }) + if ($matches.Count -ne 1) { + throw "Expected exactly one result for qualification step '$Name'." + } + + return $matches[0] +} + +function Set-StepValidation { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][ValidateSet('passed', 'failed', 'not-qualified', 'smoke-only')][string]$Status, + [Parameter(Mandatory)][string]$Qualification, + [Parameter(Mandatory)][string[]]$Evidence + ) + + $result = Get-StepResult $Name + $result.status = $Status + $result.qualification = $Qualification + $result.validation = $Evidence +} + +function Fail-StepValidation { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$Message + ) + + Set-StepValidation $Name 'failed' 'validation-failed' @($Message) + throw "Qualification evidence '$Name' failed validation: $Message" +} + +function Mark-StepNotQualified { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$Reason, + [string[]]$Evidence = @() + ) + + Set-StepValidation $Name 'not-qualified' 'environment-not-qualified' (@($Reason) + $Evidence) + $notQualifiedReasons.Add("$Name`: $Reason") +} + +function Assert-ExactStringSet { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][object[]]$Actual, + [Parameter(Mandatory)][string[]]$Expected) + + $actualSet = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($value in $Actual) { + if ($value -isnot [string] -or -not $actualSet.Add([string]$value)) { + throw "Qualification config '$Name' contains a nonstring or duplicate value." + } + } + $expectedSet = [Collections.Generic.HashSet[string]]::new($Expected, [StringComparer]::Ordinal) + if ($Actual.Count -ne $Expected.Count -or -not $actualSet.SetEquals($expectedSet)) { + throw "Qualification config '$Name' must be exactly [$($Expected -join ', ')], actual [$($Actual -join ', ')]." + } +} + +function Assert-LinuxTinySyncTopology { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Context) + + if ((Get-StrictInt64 $Object 'syncKeysPerWorker' $Context 2 2) -ne 2 ` + -or (Get-StrictInt64 $Object 'syncMaximumWorkerCount' $Context 12 12) -ne 12 ` + -or (Get-StrictInt64 $Object 'syncCanonicalBucketCount' $Context 16 16) -ne 16) { + throw "$Context does not describe the exact two-key/12-worker/16-bucket synchronization topology." + } + + $digest = Get-StrictString $Object 'syncKeyCatalogSha256' $Context + if ($digest -cne '9A7E93EB1382F2665155971C64C10D4C29039916CD9E314DB72B9906549656D2' ` + -or $digest -cnotmatch '^[0-9A-F]{64}$') { + throw "$Context has the wrong deterministic synchronization-key catalog digest." + } + + $assignments = @(Get-RequiredPropertyValue $Object 'syncKeyCanonicalBucketAssignments' $Context) + if ($assignments.Count -ne 24) { + throw "$Context must contain exactly 24 synchronization-key bucket assignments." + } + for ($index = 0; $index -lt $assignments.Count; $index++) { + [int64]$expected = [Math]::Floor($index / 2) + if (-not (Test-IsIntegerNumber $assignments[$index]) ` + -or [int64]$assignments[$index] -ne $expected) { + throw "$Context synchronization-key bucket assignment $index must be $expected." + } + } +} + +function Assert-RequiredBenchmarkHardwareMetadata { + param( + [Parameter(Mandatory)]$Environment, + [Parameter(Mandatory)][string]$Context) + + $logicalCount = Get-StrictInt64 $Environment 'logicalProcessorCount' $Context 1 ([int32]::MaxValue) + [void](Get-StrictInt64 $Environment 'physicalCoreCount' $Context 1 $logicalCount) + [void](Get-StrictInt64 $Environment 'totalMemoryBytes' $Context 1048576 ([int64]::MaxValue)) + $processorModel = Get-StrictString $Environment 'processorModel' $Context + $processorIdentifier = Get-StrictString $Environment 'processorIdentifier' $Context + foreach ($value in @($processorModel, $processorIdentifier)) { + if ($value.Trim() -match '^(?i:(?:unknown|unavailable|not[- ]available|n/?a)(?:\s+(?:cpu|processor|model))?)$') { + throw "$Context contains unknown processor-model evidence '$value'." + } + } +} + +function Assert-ExactBenchmarkStoreDimensions { + param( + [Parameter(Mandatory)]$Dimensions, + [Parameter(Mandatory)][string]$Context, + [Parameter(Mandatory)][int64]$SlotCount, + [Parameter(Mandatory)][int64]$MaxValueBytes, + [Parameter(Mandatory)][int64]$MaxDescriptorBytes, + [Parameter(Mandatory)][int64]$MaxKeyBytes, + [Parameter(Mandatory)][int64]$LeaseRecordCount, + [Parameter(Mandatory)][int64]$LockFreeParticipantRecordCount) + + $expected = [ordered]@{ + slotCount = $SlotCount + maxValueBytes = $MaxValueBytes + maxDescriptorBytes = $MaxDescriptorBytes + maxKeyBytes = $MaxKeyBytes + leaseRecordCount = $LeaseRecordCount + lockFreeParticipantRecordCount = $LockFreeParticipantRecordCount + } + foreach ($entry in $expected.GetEnumerator()) { + if ((Get-StrictInt64 $Dimensions $entry.Key $Context 0 ([int32]::MaxValue)) -ne $entry.Value) { + throw "$Context.$($entry.Key) does not match the exact benchmark store topology." + } + } +} + +function Assert-BenchmarkScenarioStoreDimensions { + param( + [Parameter(Mandatory)]$Configuration, + [Parameter(Mandatory)][string[]]$ExpectedScenarios, + [Parameter(Mandatory)][string]$Context) + + $allDimensions = Get-RequiredPropertyValue $Configuration 'scenarioStoreDimensions' $Context + Assert-ExactStringSet "$Context scenarioStoreDimensions" ` + @($allDimensions.PSObject.Properties.Name) $ExpectedScenarios + foreach ($scenario in $ExpectedScenarios) { + $expected = switch ($scenario) { + { $_ -cin @('acquire-release', 'publish-remove') } { @(32, 8, 0, 8, 64, 64); break } + { $_ -cin @('same-key-read', 'distributed-key-read') } { @(256, 256, 0, 8, 64, 64); break } + { $_ -cin @('broker-directed', 'large-ingest') } { + @(256, (Get-StrictInt64 $Configuration 'largeFrameBytes' $Context 1 ([int32]::MaxValue)), 16, 8, 64, 64) + break + } + 'mixed-churn' { @(768, 256, 16, 8, 128, 64); break } + 'sticky-overflow-miss' { + @((Get-StrictInt64 $Configuration 'stickyOverflowSlotCount' $Context 1 ([int32]::MaxValue)), 1, 0, 8, 64, 64) + break + } + default { throw "$Context has no store-dimension contract for scenario '$scenario'." } + } + Assert-ExactBenchmarkStoreDimensions ` + $allDimensions.$scenario ` + "$Context scenarioStoreDimensions.$scenario" ` + $expected[0] $expected[1] $expected[2] $expected[3] $expected[4] $expected[5] + } +} + +function Get-Sc017SourceTransitionCount { + $sourcePath = Join-Path $root 'tests/SharedMemoryStore.UnitTests/LockFreeDirectoryGenerationStressTests.cs' + $source = Get-Content -LiteralPath $sourcePath -Raw + $matches = [regex]::Matches( + $source, + '(?m)^\s*private const int QualificationTransitionCount = (?[1-9][0-9]*);\s*$') + if ($matches.Count -ne 1) { + throw 'The SC-017 test must declare exactly one machine-readable QualificationTransitionCount.' + } + + return [Convert]::ToInt64( + $matches[0].Groups['count'].Value, + [Globalization.CultureInfo]::InvariantCulture) +} + +function Assert-Sc017TierConfiguration { + param( + [Parameter(Mandatory)]$QualificationConfig, + [Parameter(Mandatory)][int64]$TransitionCount) + + $tiers = Get-RequiredPropertyValue $QualificationConfig 'tiers' 'qualification config' + foreach ($tierName in @('pr', 'nightly', 'release')) { + $tierConfig = Get-RequiredPropertyValue $tiers $tierName 'qualification config tiers' + $configured = Get-StrictInt64 $tierConfig 'directoryGenerationStressRepetitions' ` + "qualification tier '$tierName'" 1 [int64]::MaxValue + if ($configured -lt $TransitionCount) { + throw "Qualification tier '$tierName' directoryGenerationStressRepetitions must be at least the source-owned SC-017 transition count $TransitionCount; actual=$configured." + } + } +} + +function Invoke-Sc017ConfigurationVerifierSelfTest { + $transitionCount = Get-Sc017SourceTransitionCount + Assert-Sc017TierConfiguration $config $transitionCount + + $tampered = $config | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $tampered.tiers.pr.directoryGenerationStressRepetitions = $transitionCount - 1 + $rejected = $false + try { + Assert-Sc017TierConfiguration $tampered $transitionCount + } + catch { + $rejected = $_.Exception.Message -like '*must be at least the source-owned SC-017 transition count*' + } + if (-not $rejected) { + throw "SC-017 configuration verifier self-test accepted $($transitionCount - 1) repetitions for $transitionCount transitions." + } + + return 2 +} + +function Get-ChurnQualificationTestContract { + param( + [Parameter(Mandatory)]$QualificationConfig, + [AllowEmptyString()][string]$SourceOverride) + + $assertions = @(Get-RequiredPropertyValue $QualificationConfig ` + 'requiredLeakAssertions' 'qualification config') + if ($assertions.Count -ne 3) { + throw 'Qualification config must contain exactly three required leak assertions.' + } + $expectedEvidenceSteps = [ordered]@{ + 'slot-owner-count=0' = 'churn' + 'lease-owner-count=0' = 'churn' + 'unreferenced-stale-participant-count=0' = 'recovery' + } + foreach ($expected in $expectedEvidenceSteps.GetEnumerator()) { + $matches = @($assertions | Where-Object { + [string]$_.id -ceq $expected.Key + }) + if ($matches.Count -ne 1 ` + -or [string]$matches[0].evidenceStep -cne [string]$expected.Value) { + throw "Leak assertion '$($expected.Key)' must map exactly once to evidence step '$($expected.Value)'." + } + } + + $churnAssertions = @($assertions | Where-Object { + [string]$_.evidenceStep -ceq 'churn' + }) + if ($churnAssertions.Count -ne 2) { + throw "Qualification config must map exactly two owner/leak assertions to the churn evidence step; actual=$($churnAssertions.Count)." + } + + $distinctMappings = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($assertion in $churnAssertions) { + $mapping = Get-StrictString $assertion 'testNameContains' ` + "qualification leak assertion '$($assertion.id)'" + [void]$distinctMappings.Add($mapping) + } + if ($distinctMappings.Count -ne 1) { + throw 'Every churn owner/leak assertion must map to one identical test method.' + } + + $mapping = @($distinctMappings)[0] + $match = [regex]::Match( + $mapping, + '^(?[A-Za-z_][A-Za-z0-9_]*)\.(?[A-Za-z_][A-Za-z0-9_]*)$', + [Text.RegularExpressions.RegexOptions]::CultureInvariant) + if (-not $match.Success -or $match.Groups['class'].Value -cne $churnTestClass) { + throw "Configured churn evidence test '$mapping' must name one method on LockFreeChurnIntegrationTests." + } + if ($mapping -cne $churnQualificationTestNameFragment) { + throw "Configured churn evidence test must remain the SC-016 collision workload '$churnQualificationTestNameFragment'; actual='$mapping'." + } + + $sourcePath = Join-Path $root $churnTestSourceRelativePath + $source = if ($PSBoundParameters.ContainsKey('SourceOverride')) { + $SourceOverride + } + else { + Get-Content -LiteralPath $sourcePath -Raw + } + $method = $match.Groups['method'].Value + $namespacePattern = '(?m)^[ \t]*namespace[ \t]+' + + [regex]::Escape($churnTestNamespace) + ';[ \t]*$' + $classPattern = '(?m)^[ \t]*public[ \t]+sealed[ \t]+class[ \t]+' + + [regex]::Escape($churnTestClass) + '[ \t]*$' + $methodDeclarationPattern = '^[ \t]*\[Fact\][ \t]*\r?\n' + + '(?:[ \t]*\[[^\r\n]+\][ \t]*\r?\n)*' + + '[ \t]*public[ \t]+(?:async[ \t]+)?(?:void|Task(?:<[^>\r\n]+>)?)[ \t]+' + + [regex]::Escape($method) + '[ \t]*\(' + $methodPattern = '(?ms)' + $methodDeclarationPattern + $sourceContractPattern = '(?ms)' + + '^[ \t]*namespace[ \t]+' + [regex]::Escape($churnTestNamespace) + ';[ \t]*\r?\n' + + '[ \t\r\n]*' + + '^[ \t]*public[ \t]+sealed[ \t]+class[ \t]+' + + [regex]::Escape($churnTestClass) + '[ \t]*\r?\n[ \t]*\{[ \t]*\r?\n' + + '(?:(?!^}[ \t]*$)(?!^[^\r\n]*\b(?:class|struct|record|interface|enum)\b).)*?' + + $methodDeclarationPattern + if (([regex]::Matches($source, $namespacePattern).Count -ne 1) ` + -or ([regex]::Matches($source, $classPattern).Count -ne 1) ` + -or ([regex]::Matches($source, $methodPattern).Count -ne 1) ` + -or ([regex]::Matches($source, $sourceContractPattern).Count -ne 1)) { + throw "Configured churn evidence FQN '$churnTestNamespace.$mapping' must identify the exact namespace, class, and one [Fact] in $churnTestSourceRelativePath." + } + + return [pscustomobject]@{ + testNameFragment = $mapping + fullyQualifiedName = "$churnTestNamespace.$mapping" + sourcePath = $churnTestSourceRelativePath + } +} + +function Assert-QualificationConfiguration { + Assert-KnownProvenance $repositoryProvenance 'start' + if (-not $ValidateOnly -and $repositoryProvenance.workingTreeState -ne 'clean') { + throw 'Executable qualification requires a clean working tree.' + } + if ((Get-StrictInt64 $config 'schemaVersion' 'qualification config' 5 5) -ne 5) { + throw "Qualification config schemaVersion must be 5." + } + if ($Configuration -ne 'Release' -and -not $ValidateOnly) { + throw "Qualification evidence must be built in Release; '$Configuration' is diagnostic-only." + } + + $positiveProperties = @( + 'stepTimeoutSeconds', + 'checkerHistoryRepetitionsPerFamily', + 'productionHistoryCountPerFamily', + 'productionRaceRepetitionsPerFamily', + 'churnCycles', + 'recoveryCases', + 'performanceWarmupSeconds', + 'performanceDurationSeconds', + 'performanceDurationBoundGraceSeconds', + 'performanceTrials', + 'mixedOperations', + 'largeFrames', + 'largeFrameBytes', + 'suspensionBaselineSeconds', + 'suspensionPauseSeconds', + 'suspensionWarmupSeconds', + 'directoryGenerationStressRepetitions') + foreach ($property in $positiveProperties) { + [void](Get-StrictInt64 $selected $property "qualification tier '$Tier'" 1 [int64]::MaxValue) + } + $sc017TransitionCount = Get-Sc017SourceTransitionCount + Assert-Sc017TierConfiguration $config $sc017TransitionCount + [void](Get-StrictInt64 $config 'seed' 'qualification config' 0 [int32]::MaxValue) + [void](Get-StrictDouble $config 'suspensionMinimumHealthyThroughputRatio' 'qualification config' 0 1 -Positive) + $expectedMode = if ($Tier -eq 'release') { 'full' } else { 'all' } + $modeValue = Get-RequiredPropertyValue $selected 'performanceMode' "qualification tier '$Tier'" + if ($modeValue -isnot [string] -or $modeValue -ne $expectedMode) { + throw "Qualification tier '$Tier' must use performanceMode '$expectedMode'." + } + if ((Get-StrictInt64 $selected 'performanceDurationBoundGraceSeconds' ` + "qualification tier '$Tier'" 60 60) -ne 60) { + throw "Qualification tier '$Tier' must use exactly 60 seconds of duration-bound watchdog grace." + } + + if ((Get-StrictInt64 $config 'boundedOperationSlackMilliseconds' 'qualification config' 250 250) -ne 250) { + throw 'boundedOperationSlackMilliseconds must remain the contracted 250 ms.' + } + $waitPolicySource = Get-Content -LiteralPath (Join-Path $root 'tests/SharedMemoryStore.IntegrationTests/LockFreeWaitPolicyMatrixIntegrationTests.cs') -Raw + $slackPattern = 'CompletionAllowance\s*=\s*TimeSpan\.FromMilliseconds\(' + + [regex]::Escape([string][int]$config.boundedOperationSlackMilliseconds) + '\)' + if ($waitPolicySource -notmatch $slackPattern) { + throw 'The bounded-operation slack setting is not enforced by LockFreeWaitPolicyMatrixIntegrationTests.' + } + + Assert-ExactStringSet 'platforms' @($config.platforms) @('windows-x64', 'linux-x64') + Assert-ExactStringSet 'requiredLeakAssertions' @($config.requiredLeakAssertions | ForEach-Object id) @( + 'slot-owner-count=0', + 'lease-owner-count=0', + 'unreferenced-stale-participant-count=0') + foreach ($assertion in @($config.requiredLeakAssertions)) { + if ([string]::IsNullOrWhiteSpace([string]$assertion.testNameContains) ` + -or [string]::IsNullOrWhiteSpace([string]$assertion.assertedState) ` + -or [string]$assertion.evidenceStep -notin @('churn', 'recovery')) { + throw "Leak assertion '$($assertion.id)' lacks an executable test mapping." + } + } + [void](Get-ChurnQualificationTestContract $config) + + Assert-ExactStringSet 'referenceModelFamilies' @($config.completionEvidence.referenceModelFamilies) @( + 'publish-publish', 'publish-reserve', 'reserve-reserve', 'commit-acquire', + 'acquire-remove', 'release-reclaim', + 'recovery-live-action', 'disposal-operation', 'participant-capacity', + 'value-capacity', 'lease-capacity', 'cancellation', 'stale-token') + Assert-ExactStringSet 'productionHistoryFamilies' @($config.completionEvidence.productionHistoryFamilies) @( + 'publish-publish', 'publish-reserve', 'reserve-reserve', 'commit-acquire', + 'acquire-remove', 'release-reclaim', 'recovery-live-lease', 'disposal-operation') + Assert-ExactStringSet 'productionRaceFamilies' @($config.completionEvidence.productionRaceFamilies) @( + 'publish-publish', 'publish-reserve', 'reserve-reserve', 'commit-acquire', + 'acquire-remove', 'release-reclaim', 'recovery-live-lease', 'disposal-operation') + Assert-ExactStringSet 'performance profiles' @($config.performanceMatrix.profiles) @('Legacy', 'LockFree') + Assert-ExactStringSet 'count-bound performance profiles' @($config.performanceMatrix.countBoundProfiles) @('LockFree') + Assert-ExactStringSet 'lock-free-only performance scenarios' @($config.performanceMatrix.lockFreeOnlyScenarios) @('sticky-overflow-miss') + $linuxTiny = Get-RequiredPropertyValue $config 'linuxTinyPerformance' 'qualification config' + $expectedLinuxTinyProperties = @( + 'mode', 'profiles', 'scenarios', 'processCounts', 'syncKeysPerWorker', + 'syncMaximumWorkerCount', 'syncCanonicalBucketCount', 'syncKeyCatalogSha256', + 'syncKeyCanonicalBucketAssignments', 'minimumThroughputRatio', + 'maximumUncontendedP99Ratio', 'maximumScaleP99Ratio', + 'maximumP99Microseconds', 'maximumStallMicroseconds') + if ((@($linuxTiny.PSObject.Properties.Name) -join ',') -cne ($expectedLinuxTinyProperties -join ',')) { + throw "linuxTinyPerformance properties must be exactly [$($expectedLinuxTinyProperties -join ', ')]." + } + if ((Get-StrictString $linuxTiny 'mode' 'qualification config linuxTinyPerformance') -cne 'sync') { + throw 'linuxTinyPerformance.mode must be sync.' + } + Assert-ExactStringSet 'linuxTinyPerformance profiles' @($linuxTiny.profiles) @('Legacy', 'LockFree') + Assert-ExactStringSet 'linuxTinyPerformance scenarios' @($linuxTiny.scenarios) @('acquire-release', 'publish-remove') + [void](Assert-LinuxTinySyncTopology $linuxTiny 'qualification config linuxTinyPerformance') + $linuxProcessCounts = @($linuxTiny.processCounts) + if ($linuxProcessCounts.Count -ne 2 ` + -or -not (Test-IsIntegerNumber $linuxProcessCounts[0]) -or [int64]$linuxProcessCounts[0] -ne 1 ` + -or -not (Test-IsIntegerNumber $linuxProcessCounts[1]) -or [int64]$linuxProcessCounts[1] -ne 8 ` + -or (Get-StrictDouble $linuxTiny 'minimumThroughputRatio' 'qualification config linuxTinyPerformance' 1 1) -ne 1 ` + -or (Get-StrictDouble $linuxTiny 'maximumUncontendedP99Ratio' 'qualification config linuxTinyPerformance' 1 1) -ne 1 ` + -or (Get-StrictDouble $linuxTiny 'maximumScaleP99Ratio' 'qualification config linuxTinyPerformance' 3 3) -ne 3 ` + -or (Get-StrictDouble $linuxTiny 'maximumP99Microseconds' 'qualification config linuxTinyPerformance' 10 10) -ne 10 ` + -or (Get-StrictDouble $linuxTiny 'maximumStallMicroseconds' 'qualification config linuxTinyPerformance' 10000 10000) -ne 10000) { + throw 'linuxTinyPerformance must require process counts [1,8], LF1/Legacy1 p99<=1, LF8/Legacy8 throughput>=1, LF8/LF1 p99<=3, LF8 p99<=10us, and every raw lock-free stall<=10000us.' + } + if ((Get-StrictInt64 $config.tiers.release 'performanceWarmupSeconds' 'qualification config release tier' 10 10) -ne 10 ` + -or (Get-StrictInt64 $config.tiers.release 'performanceDurationSeconds' 'qualification config release tier' 60 60) -ne 60 ` + -or (Get-StrictInt64 $config.tiers.release 'performanceDurationBoundGraceSeconds' 'qualification config release tier' 60 60) -ne 60 ` + -or (Get-StrictInt64 $config.tiers.release 'performanceTrials' 'qualification config release tier' 3 3) -ne 3) { + throw 'The Linux tiny release gate requires exactly 10s warmup, 60s measurement, 60s watchdog grace, and three trials.' + } + + $contractCounts = [ordered]@{ + 'acquire-release' = @(1, 2, 4, 8, 12) + 'publish-remove' = @(1, 2, 4, 8, 12) + 'same-key-read' = @(1, 2, 4, 6, 8, 12) + 'distributed-key-read' = @(1, 2, 4, 6, 8, 12) + 'broker-directed' = @(1, 12) + 'mixed-churn' = @(12) + 'large-ingest' = @(1, 12) + 'sticky-overflow-miss' = @(1) + } + foreach ($entry in $contractCounts.GetEnumerator()) { + $source = if ($config.performanceMatrix.shortScenarios.PSObject.Properties.Name -contains $entry.Key) { + $config.performanceMatrix.shortScenarios.$($entry.Key) + } + else { + $config.performanceMatrix.releaseOnlyScenarios.$($entry.Key) + } + $actual = @($source | ForEach-Object { + if (-not (Test-IsIntegerNumber $_)) { + throw "Performance matrix '$($entry.Key)' contains a non-integer process count." + } + [Convert]::ToInt32($_, [Globalization.CultureInfo]::InvariantCulture) + }) + if (($actual -join ',') -ne (@($entry.Value) -join ',')) { + throw "Performance matrix '$($entry.Key)' does not match the contracted process counts." + } + } + + $configuredCheckpointIds = @($config.suspensionCheckpointIds | ForEach-Object { + if (-not (Test-IsIntegerNumber $_)) { + throw 'suspensionCheckpointIds contains a non-integer value.' + } + [Convert]::ToInt32($_, [Globalization.CultureInfo]::InvariantCulture) + }) + if (($configuredCheckpointIds -join ',') -cne ($canonicalSuspensionCheckpointIds -join ',') ` + -or @($configuredCheckpointIds | Sort-Object -Unique).Count -ne $canonicalSuspensionCheckpointIds.Count) { + throw 'suspensionCheckpointIds must exactly match the source-derived non-Participant/non-Disposal checkpoint catalog.' + } + if ((Get-StrictInt64 $selected 'recoveryCases' "qualification tier '$Tier'" 1 [int32]::MaxValue) ` + -lt $checkpointCatalog.Count) { + throw "Qualification tier '$Tier' must execute at least one recovery case for each of the $($checkpointCatalog.Count) canonical checkpoints." + } + + if ($Tier -eq 'release') { + $releaseMinimums = [ordered]@{ + checkerHistoryRepetitionsPerFamily = 10000 + productionHistoryCountPerFamily = 16 + productionRaceRepetitionsPerFamily = 1000000 + churnCycles = 100000000 + recoveryCases = 10000 + performanceWarmupSeconds = 10 + performanceDurationSeconds = 60 + performanceDurationBoundGraceSeconds = 60 + performanceTrials = 3 + mixedOperations = 100000000 + largeFrames = 100000 + largeFrameBytes = 1363148 + suspensionBaselineSeconds = 10 + suspensionPauseSeconds = 30 + suspensionWarmupSeconds = 10 + directoryGenerationStressRepetitions = 1000000 + } + foreach ($minimum in $releaseMinimums.GetEnumerator()) { + $actualMinimum = Get-StrictInt64 $selected $minimum.Key "release qualification tier" 1 [int64]::MaxValue + if ($actualMinimum -lt [int64]$minimum.Value) { + throw "Release '$($minimum.Key)' must be at least $($minimum.Value)." + } + } + } +} + +function Get-TrxResults { + param([Parameter(Mandatory)][string]$Directory) + + $files = @(Get-ChildItem -LiteralPath $Directory -Recurse -Filter '*.trx' -File -ErrorAction SilentlyContinue) + if ($files.Count -eq 0) { + throw "No TRX files were produced below '$Directory'." + } + $rows = [Collections.Generic.List[object]]::new() + foreach ($file in $files) { + [xml]$document = Get-Content -LiteralPath $file.FullName -Raw + foreach ($node in @($document.SelectNodes("//*[local-name()='UnitTestResult']"))) { + $rows.Add([pscustomobject]@{ + testName = [string]$node.testName + outcome = [string]$node.outcome + file = [IO.Path]::GetRelativePath($root, $file.FullName) + }) + } + } + return @($rows) +} + +function Assert-TrxStepEvidence { + param( + [Parameter(Mandatory)][string]$Step, + [Parameter(Mandatory)][string]$Directory, + [int]$ExpectedPassedCount = -1, + [string[]]$RequiredTestNameContains = @()) + + $rows = @(Get-TrxResults $Directory) + $passed = @($rows | Where-Object { $_.outcome -ceq 'Passed' }) + $nonPassed = @($rows | Where-Object { $_.outcome -cne 'Passed' }) + if ($nonPassed.Count -ne 0 -or $passed.Count -eq 0) { + $outcomes = @($rows | Group-Object outcome | Sort-Object Name | ForEach-Object { + "$($_.Name)=$($_.Count)" + }) -join ', ' + Fail-StepValidation $Step "TRX evidence must contain only Passed rows; passed=$($passed.Count), nonPassed=$($nonPassed.Count), outcomes=[$outcomes]." + } + if ($ExpectedPassedCount -ge 0 -and $passed.Count -ne $ExpectedPassedCount) { + Fail-StepValidation $Step "TRX evidence has $($passed.Count) passed rows; expected exactly $ExpectedPassedCount." + } + foreach ($fragment in $RequiredTestNameContains) { + if (@($passed | Where-Object { $_.testName -like "*$fragment*" }).Count -eq 0) { + Fail-StepValidation $Step "TRX evidence lacks a passed test containing '$fragment'." + } + } + Set-StepValidation $Step 'passed' 'trx-test-execution-proven' @( + "passed=$($passed.Count)", + 'nonPassed=0', + "trxFiles=$(@($rows.file | Sort-Object -Unique).Count)") + return $passed +} + +function Assert-ExactPassedTrxRows { + param( + [Parameter(Mandatory)][object[]]$Rows, + [Parameter(Mandatory)][string[]]$ExpectedTestNames) + + if ($ExpectedTestNames.Count -eq 0) { + throw 'Exact TRX verification requires at least one expected test name.' + } + $expected = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($testName in $ExpectedTestNames) { + if ([string]::IsNullOrWhiteSpace($testName) -or -not $expected.Add($testName)) { + throw 'Exact TRX verification contains an empty or duplicate expected test name.' + } + } + + $passed = @($Rows | Where-Object { [string]$_.outcome -ceq 'Passed' }) + $nonPassed = @($Rows | Where-Object { [string]$_.outcome -cne 'Passed' }) + if ($nonPassed.Count -ne 0) { + $outcomes = @($Rows | Group-Object outcome | Sort-Object Name | ForEach-Object { + "$($_.Name)=$($_.Count)" + }) -join ', ' + throw "TRX evidence must contain only Passed rows; outcomes=[$outcomes]." + } + if ($passed.Count -ne $ExpectedTestNames.Count) { + throw "TRX evidence has $($passed.Count) passed rows; expected exactly $($ExpectedTestNames.Count)." + } + + $actual = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($row in $passed) { + $testName = [string]$row.testName + if ([string]::IsNullOrWhiteSpace($testName) -or -not $actual.Add($testName)) { + throw "TRX evidence contains an empty or duplicate passed test name '$testName'." + } + } + if (-not $actual.SetEquals($expected)) { + throw "TRX evidence test names must be exactly [$($ExpectedTestNames -join ', ')]; actual [$(@($passed.testName) -join ', ')]." + } + + return $passed +} + +function Assert-ExactTrxStepEvidence { + param( + [Parameter(Mandatory)][string]$Step, + [Parameter(Mandatory)][string]$Directory, + [Parameter(Mandatory)][string[]]$ExpectedTestNames) + + $rows = @(Get-TrxResults $Directory) + try { + $passed = @(Assert-ExactPassedTrxRows $rows $ExpectedTestNames) + } + catch { + Fail-StepValidation $Step $_.Exception.Message + } + Set-StepValidation $Step 'passed' 'exact-trx-test-execution-proven' @( + "passed=$($passed.Count)", + 'nonPassed=0', + "trxFiles=$(@($rows.file | Sort-Object -Unique).Count)", + "exactTestNames=$($ExpectedTestNames -join ',')") + return $passed +} + +function Invoke-ChurnQualificationVerifierSelfTest { + $contract = Get-ChurnQualificationTestContract $config + $validRow = [pscustomobject]@{ + testName = $contract.fullyQualifiedName + outcome = 'Passed' + file = 'synthetic.trx' + } + [void](Assert-ExactPassedTrxRows @($validRow) @($contract.fullyQualifiedName)) + $assertions = 1 + + $differentMappings = $config | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $leaseMappings = @($differentMappings.requiredLeakAssertions | Where-Object { + [string]$_.id -ceq 'lease-owner-count=0' -and [string]$_.evidenceStep -ceq 'churn' + }) + if ($leaseMappings.Count -ne 1) { + throw 'Churn qualification verifier self-test could not locate the configured lease-owner mapping.' + } + $leaseMappings[0].testNameContains = + 'LockFreeChurnIntegrationTests.BenchmarkFixedKeysSurviveRepeatedEightProcessPublishRemoveChurn' + $mappingRejected = $false + try { + [void](Get-ChurnQualificationTestContract $differentMappings) + } + catch { + $mappingRejected = $_.Exception.Message -like '*must map to one identical test method*' + } + if (-not $mappingRejected) { + throw 'Churn qualification verifier self-test accepted two distinct churn leak-evidence mappings.' + } + $assertions++ + + $missingMapping = $config | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $missingMapping.requiredLeakAssertions = @( + $missingMapping.requiredLeakAssertions | Where-Object { + [string]$_.id -cne 'lease-owner-count=0' + }) + $missingMappingRejected = $false + try { + [void](Get-ChurnQualificationTestContract $missingMapping) + } + catch { + $missingMappingRejected = $_.Exception.Message -like '*must contain exactly three required leak assertions*' + } + if (-not $missingMappingRejected) { + throw 'Churn qualification verifier self-test accepted a missing churn leak-evidence mapping.' + } + $assertions++ + + $swappedEvidenceSteps = $config | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $leaseMapping = @($swappedEvidenceSteps.requiredLeakAssertions | Where-Object { + [string]$_.id -ceq 'lease-owner-count=0' + }) + $participantMapping = @($swappedEvidenceSteps.requiredLeakAssertions | Where-Object { + [string]$_.id -ceq 'unreferenced-stale-participant-count=0' + }) + if ($leaseMapping.Count -ne 1 -or $participantMapping.Count -ne 1) { + throw 'Churn qualification verifier self-test could not locate exact role-swap mappings.' + } + $leaseMapping[0].evidenceStep = 'recovery' + $participantMapping[0].evidenceStep = 'churn' + $roleSwapRejected = $false + try { + [void](Get-ChurnQualificationTestContract $swappedEvidenceSteps) + } + catch { + $roleSwapRejected = $_.Exception.Message -like '*must map exactly once to evidence step*' + } + if (-not $roleSwapRejected) { + throw 'Churn qualification verifier self-test accepted swapped leak-evidence roles.' + } + $assertions++ + + $wrongExistingTarget = $config | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($assertion in @($wrongExistingTarget.requiredLeakAssertions | Where-Object { + [string]$_.evidenceStep -ceq 'churn' + })) { + $assertion.testNameContains = + 'LockFreeChurnIntegrationTests.BenchmarkFixedKeysSurviveRepeatedEightProcessPublishRemoveChurn' + } + $semanticTargetRejected = $false + try { + [void](Get-ChurnQualificationTestContract $wrongExistingTarget) + } + catch { + $semanticTargetRejected = $_.Exception.Message -like '*must remain the SC-016 collision workload*' + } + if (-not $semanticTargetRejected) { + throw 'Churn qualification verifier self-test accepted the fixed-key sibling as configured SC-016 evidence.' + } + $assertions++ + + $source = Get-Content -LiteralPath (Join-Path $root $churnTestSourceRelativePath) -Raw + $sourceWithoutDirectMethod = $source.Replace( + $churnQualificationTestMethod, + 'MissingConfiguredChurnEvidenceMethod', + [StringComparison]::Ordinal) + $outerClassClose = $sourceWithoutDirectMethod.LastIndexOf('}') + if ($outerClassClose -lt 0) { + throw 'Churn qualification verifier self-test source has no outer class terminator.' + } + $nestedMethodSource = $sourceWithoutDirectMethod.Insert( + $outerClassClose, + " private sealed class NestedChurnEvidence`r`n {`r`n" + + " [Fact]`r`n public void $churnQualificationTestMethod()`r`n" + + " {`r`n }`r`n }`r`n") + $sourceNegativeCases = @( + [pscustomobject]@{ + name = 'missing-method' + source = $source.Replace( + $churnQualificationTestMethod, + 'MissingConfiguredChurnEvidenceMethod', + [StringComparison]::Ordinal) + }, + [pscustomobject]@{ + name = 'wrong-namespace' + source = $source.Replace( + "namespace $churnTestNamespace;", + "namespace $churnTestNamespace.Wrong;", + [StringComparison]::Ordinal) + }, + [pscustomobject]@{ + name = 'wrong-class' + source = $source.Replace( + "public sealed class $churnTestClass", + "public sealed class WrongChurnIntegrationTests", + [StringComparison]::Ordinal) + }, + [pscustomobject]@{ + name = 'method-outside-configured-class' + source = $source.Replace( + $churnQualificationTestMethod, + 'MissingConfiguredChurnEvidenceMethod', + [StringComparison]::Ordinal) + + "`r`n`r`npublic sealed class WrongChurnEvidenceContainer`r`n{`r`n" + + " [Fact]`r`n public void $churnQualificationTestMethod()`r`n {`r`n }`r`n}`r`n" + }, + [pscustomobject]@{ + name = 'configured-class-nested-under-another-type' + source = $source.Replace( + "public sealed class $churnTestClass", + "public sealed class OuterChurnContainer`r`n{`r`n public sealed class $churnTestClass", + [StringComparison]::Ordinal) + + "`r`n}`r`n" + }, + [pscustomobject]@{ + name = 'configured-method-nested-under-another-type' + source = $nestedMethodSource + }) + foreach ($case in $sourceNegativeCases) { + $sourceRejected = $false + try { + [void](Get-ChurnQualificationTestContract $config -SourceOverride $case.source) + } + catch { + $sourceRejected = $_.Exception.Message -match 'must identify the exact namespace, class, and one \[Fact\]' + } + if (-not $sourceRejected) { + throw "Churn qualification verifier self-test accepted invalid source case '$($case.name)'." + } + $assertions++ + } + + $siblingRow = [pscustomobject]@{ + testName = "$churnTestNamespace.LockFreeChurnIntegrationTests.BenchmarkFixedKeysSurviveRepeatedEightProcessPublishRemoveChurn" + outcome = 'Passed' + file = 'synthetic.trx' + } + $wrongRow = [pscustomobject]@{ + testName = "$churnTestNamespace.LockFreeChurnIntegrationTests.WrongChurnEvidence" + outcome = 'Passed' + file = 'synthetic.trx' + } + $nonPassedRow = [pscustomobject]@{ + testName = $contract.fullyQualifiedName + outcome = 'Failed' + file = 'synthetic.trx' + } + $negativeCases = @( + [pscustomobject]@{ name = 'missing'; rows = @() }, + [pscustomobject]@{ name = 'extra-sibling'; rows = @($validRow, $siblingRow) }, + [pscustomobject]@{ name = 'wrong-only'; rows = @($wrongRow) }, + [pscustomobject]@{ name = 'duplicate'; rows = @($validRow, $validRow) }, + [pscustomobject]@{ name = 'non-passed'; rows = @($nonPassedRow) }) + foreach ($case in $negativeCases) { + $rejected = $false + try { + [void](Assert-ExactPassedTrxRows @($case.rows) @($contract.fullyQualifiedName)) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw "Churn exact-TRX verifier self-test accepted invalid case '$($case.name)'." + } + $assertions++ + } + + $trxSelfTestRoot = Join-Path $runRoot 'churn-trx-verifier-self-test' + New-Item -ItemType Directory -Path $trxSelfTestRoot | Out-Null + $validTrxPath = Join-Path $trxSelfTestRoot 'valid.trx' + $extraTrxPath = Join-Path $trxSelfTestRoot 'extra.trx' + [IO.File]::WriteAllText( + $validTrxPath, + ('' ` + -f $contract.fullyQualifiedName)) + $parsedRows = @(Get-TrxResults $trxSelfTestRoot) + [void](Assert-ExactPassedTrxRows $parsedRows @($contract.fullyQualifiedName)) + $assertions++ + + [IO.File]::WriteAllText( + $extraTrxPath, + ('' ` + -f $siblingRow.testName)) + $temporaryStepName = 'self-test-churn-exact-trx-wrapper-negative' + Add-EvidenceResult $temporaryStepName 'passed' 'self-test-setup' @('pending negative assertion') + $temporaryStep = Get-StepResult $temporaryStepName + $wrapperRejected = $false + try { + [void](Assert-ExactTrxStepEvidence ` + $temporaryStepName $trxSelfTestRoot @($contract.fullyQualifiedName)) + } + catch { + $wrapperRejected = $_.Exception.Message -like '*TRX evidence has 2 passed rows; expected exactly 1*' ` + -and $temporaryStep.status -ceq 'failed' ` + -and $temporaryStep.qualification -ceq 'validation-failed' + } + finally { + [void]$results.Remove($temporaryStep) + } + if (-not $wrapperRejected) { + throw 'Churn exact-TRX verifier self-test did not reject and record an XML-parsed extra sibling row.' + } + $assertions++ + + return $assertions +} + +function Get-FamilyCompletionSeed { + param( + [Parameter(Mandatory)][string]$Step, + [Parameter(Mandatory)][string]$Family) + + if ($Step -eq 'reference-model-histories') { + return [int64](Get-StrictInt64 $config 'seed' 'qualification config' 0 [int32]::MaxValue) + } + + $ordinal = switch ($Family) { + 'publish-publish' { 1 } + 'publish-reserve' { 2 } + 'reserve-reserve' { 3 } + 'commit-acquire' { 4 } + 'acquire-remove' { 5 } + 'release-reclaim' { 6 } + 'recovery-live-lease' { 7 } + 'disposal-operation' { 8 } + default { throw "No derived-seed ordinal is defined for '$Family'." } + } + $rootSeed = Get-StrictInt64 $config 'seed' 'qualification config' 0 [int32]::MaxValue + $unsigned = (([uint64][uint32][int32]$rootSeed) + + ([uint64]$ordinal * [uint64]2654435769)) -band [uint64]4294967295 + if ($unsigned -ge [uint64]2147483648) { + return [int64]$unsigned - [int64]4294967296 + } + + return [int64]$unsigned +} + +function Get-UniqueFamilyMarkerLine { + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$Text, + [Parameter(Mandatory)][string]$Family) + + $pattern = '^[\t ]*family=' + [regex]::Escape($Family) + '(?:[\t ]|$)' + $lines = @($Text -split "\r?\n" | Where-Object { $_ -match $pattern }) + if ($lines.Count -ne 1) { + throw "Expected exactly one marker line for family '$Family'; found $($lines.Count)." + } + + return [string]$lines[0].Trim() +} + +function Assert-ExactFamilyMarkerSet { + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$Text, + [Parameter(Mandatory)][string[]]$Families) + + $expected = [Collections.Generic.HashSet[string]]::new($Families, [StringComparer]::Ordinal) + $markerLines = @($Text -split "\r?\n" | Where-Object { $_ -match '^[\t ]*family=(?[a-z0-9-]+)(?:[\t ]|$)' }) + if ($markerLines.Count -ne $Families.Count) { + throw "Expected exactly $($Families.Count) family marker lines; found $($markerLines.Count)." + } + + foreach ($line in $markerLines) { + $match = [regex]::Match($line, '^[\t ]*family=(?[a-z0-9-]+)(?:[\t ]|$)') + if (-not $match.Success -or -not $expected.Contains($match.Groups['family'].Value)) { + throw "Unexpected family marker line: '$($line.Trim())'." + } + } +} + +function Assert-FamilyCompletionMarkerText { + param( + [Parameter(Mandatory)][string]$Step, + [Parameter(Mandatory)][AllowEmptyString()][string]$Text, + [Parameter(Mandatory)][string]$Field, + [Parameter(Mandatory)][int64]$ExpectedCount, + [Parameter(Mandatory)][string[]]$Families) + + Assert-ExactFamilyMarkerSet $Text $Families + foreach ($family in $Families) { + $expectedSeed = Get-FamilyCompletionSeed $Step $family + $line = Get-UniqueFamilyMarkerLine $Text $family + $pattern = '^family=' + [regex]::Escape($family) + + '\s+seed=' + [regex]::Escape([string]$expectedSeed) + '\s+' + + [regex]::Escape($Field) + '=' + [regex]::Escape([string]$ExpectedCount) + + '(?:\s+\S.*)?$' + if ($line -notmatch $pattern) { + throw "Family '$family' does not have the exact seed=$expectedSeed $Field=$ExpectedCount marker prefix." + } + } +} + +function ConvertTo-MarkerInt64 { + param( + [Parameter(Mandatory)][string]$Value, + [Parameter(Mandatory)][string]$Context) + + [int64]$parsed = 0 + if (-not [int64]::TryParse( + $Value, + [Globalization.NumberStyles]::None, + [Globalization.CultureInfo]::InvariantCulture, + [ref]$parsed)) { + throw "Marker field '$Context' is not a nonnegative Int64: '$Value'." + } + + return $parsed +} + +function Assert-ProductionRaceMarkerText { + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$Text, + [Parameter(Mandatory)][int64]$ExpectedCount, + [Parameter(Mandatory)][string[]]$Families) + + Assert-ExactFamilyMarkerSet $Text $Families + $recoverySuccesses = $null + $recoveryBusy = $null + $liveActiveWitnesses = $null + $operationWins = $null + $disposalWins = $null + + foreach ($family in $Families) { + $expectedSeed = Get-FamilyCompletionSeed 'production-race-stress' $family + $line = Get-UniqueFamilyMarkerLine $Text $family + $common = '^family=' + [regex]::Escape($family) + + '\s+seed=' + [regex]::Escape([string]$expectedSeed) + + '\s+completed=' + [regex]::Escape([string]$ExpectedCount) + + '\s+productionOperationRaces=' + [regex]::Escape([string]$ExpectedCount) + + if ($family -eq 'recovery-live-lease') { + $pattern = $common + + '\s+control=persistent-two-phase-barrier' + + '\s+recoverySuccesses=(?[0-9]+)' + + '\s+recoveryBusy=(?[0-9]+)' + + '\s+liveActiveWitnesses=(?[0-9]+)$' + $match = [regex]::Match($line, $pattern) + if (-not $match.Success) { + throw "Family '$family' lacks its exact production race and recovery witness marker." + } + + $recoverySuccesses = ConvertTo-MarkerInt64 $match.Groups['successes'].Value "$family.recoverySuccesses" + $recoveryBusy = ConvertTo-MarkerInt64 $match.Groups['busy'].Value "$family.recoveryBusy" + $liveActiveWitnesses = ConvertTo-MarkerInt64 $match.Groups['witnesses'].Value "$family.liveActiveWitnesses" + if ($recoverySuccesses -le 0 ` + -or $recoveryBusy -gt $ExpectedCount ` + -or $recoverySuccesses -ne ($ExpectedCount - $recoveryBusy) ` + -or $liveActiveWitnesses -ne $recoverySuccesses) { + throw "Family '$family' has invalid recovery witnesses: successes=$recoverySuccesses, busy=$recoveryBusy, liveActiveWitnesses=$liveActiveWitnesses, expected=$ExpectedCount." + } + } + elseif ($family -eq 'disposal-operation') { + $pattern = $common + + '\s+disposeCalls=' + [regex]::Escape([string]$ExpectedCount) + + '\s+freshHandles=' + [regex]::Escape([string]$ExpectedCount) + + '\s+operationWins=(?[0-9]+)' + + '\s+disposalWins=(?[0-9]+)' + + '\s+control=persistent-two-phase-barrier$' + $match = [regex]::Match($line, $pattern) + if (-not $match.Success) { + throw "Family '$family' lacks its exact per-repetition disposal race marker." + } + + $operationWins = ConvertTo-MarkerInt64 $match.Groups['operationWins'].Value "$family.operationWins" + $disposalWins = ConvertTo-MarkerInt64 $match.Groups['disposalWins'].Value "$family.disposalWins" + if ($operationWins -le 0 ` + -or $disposalWins -le 0 ` + -or $disposalWins -gt $ExpectedCount ` + -or $operationWins -ne ($ExpectedCount - $disposalWins)) { + throw "Family '$family' has invalid disposal order witnesses: operationWins=$operationWins, disposalWins=$disposalWins, expected=$ExpectedCount." + } + } + else { + $pattern = $common + '\s+control=persistent-two-phase-barrier$' + if ($line -notmatch $pattern) { + throw "Family '$family' lacks its exact productionOperationRaces=$ExpectedCount marker." + } + } + } + + return [pscustomobject][ordered]@{ + recoverySuccesses = $recoverySuccesses + recoveryBusy = $recoveryBusy + liveActiveWitnesses = $liveActiveWitnesses + operationWins = $operationWins + disposalWins = $disposalWins + } +} + +function Invoke-ProductionRaceMarkerParserSelfTest { + $families = @($config.completionEvidence.productionRaceFamilies) + [int64]$count = 17 + $lines = [Collections.Generic.List[string]]::new() + foreach ($family in $families) { + $seed = Get-FamilyCompletionSeed 'production-race-stress' $family + $common = "family=$family seed=$seed completed=$count productionOperationRaces=$count" + if ($family -eq 'recovery-live-lease') { + $lines.Add("$common control=persistent-two-phase-barrier recoverySuccesses=9 recoveryBusy=8 liveActiveWitnesses=9") + } + elseif ($family -eq 'disposal-operation') { + $lines.Add("$common disposeCalls=$count freshHandles=$count operationWins=7 disposalWins=10 control=persistent-two-phase-barrier") + } + else { + $lines.Add("$common control=persistent-two-phase-barrier") + } + } + + $validText = $lines -join "`n" + $valid = Assert-ProductionRaceMarkerText $validText $count $families + if ($valid.recoverySuccesses -ne 9 ` + -or $valid.liveActiveWitnesses -ne 9 ` + -or $valid.operationWins -ne 7 ` + -or $valid.disposalWins -ne 10) { + throw 'Production race marker parser self-test did not preserve the validated witness counts.' + } + + $invalidCases = [Collections.Generic.List[object]]::new() + $invalidCases.Add([pscustomobject]@{ + name = 'duplicate-family' + text = $validText + "`n" + $lines[0] + }) + + $wrongRaceCount = @($lines | ForEach-Object { $_ }) + $wrongRaceCount[0] = $wrongRaceCount[0].Replace('productionOperationRaces=17', 'productionOperationRaces=16') + $invalidCases.Add([pscustomobject]@{ name = 'wrong-production-count'; text = $wrongRaceCount -join "`n" }) + + $wrongRecoveryWitness = @($lines | ForEach-Object { $_ }) + $recoveryIndex = [Array]::IndexOf($families, 'recovery-live-lease') + $wrongRecoveryWitness[$recoveryIndex] = $wrongRecoveryWitness[$recoveryIndex].Replace( + 'liveActiveWitnesses=9', + 'liveActiveWitnesses=8') + $invalidCases.Add([pscustomobject]@{ name = 'wrong-recovery-witness'; text = $wrongRecoveryWitness -join "`n" }) + + $wrongRecoveryTotal = @($lines | ForEach-Object { $_ }) + $wrongRecoveryTotal[$recoveryIndex] = $wrongRecoveryTotal[$recoveryIndex].Replace('recoveryBusy=8', 'recoveryBusy=7') + $invalidCases.Add([pscustomobject]@{ name = 'wrong-recovery-total'; text = $wrongRecoveryTotal -join "`n" }) + + $wrongDisposeCalls = @($lines | ForEach-Object { $_ }) + $disposalIndex = [Array]::IndexOf($families, 'disposal-operation') + $wrongDisposeCalls[$disposalIndex] = $wrongDisposeCalls[$disposalIndex].Replace('disposeCalls=17', 'disposeCalls=16') + $invalidCases.Add([pscustomobject]@{ name = 'wrong-dispose-calls'; text = $wrongDisposeCalls -join "`n" }) + + $missingDisposalOrdering = @($lines | ForEach-Object { $_ }) + $missingDisposalOrdering[$disposalIndex] = $missingDisposalOrdering[$disposalIndex].Replace( + 'operationWins=7 disposalWins=10', + 'operationWins=0 disposalWins=17') + $invalidCases.Add([pscustomobject]@{ name = 'missing-disposal-ordering'; text = $missingDisposalOrdering -join "`n" }) + + $wrongSeed = @($lines | ForEach-Object { $_ }) + $wrongSeed[0] = $wrongSeed[0] -replace 'seed=-?[0-9]+', 'seed=0' + $invalidCases.Add([pscustomobject]@{ name = 'wrong-derived-seed'; text = $wrongSeed -join "`n" }) + + foreach ($case in $invalidCases) { + $rejected = $false + try { + $null = Assert-ProductionRaceMarkerText $case.text $count $families + } + catch { + $rejected = $true + } + + if (-not $rejected) { + throw "Production race marker parser self-test accepted invalid case '$($case.name)'." + } + } + + return 1 + $invalidCases.Count +} + +function Assert-FamilyCompletionMarkers { + param( + [Parameter(Mandatory)][string]$Step, + [Parameter(Mandatory)][string]$Field, + [Parameter(Mandatory)][int64]$ExpectedCount, + [Parameter(Mandatory)][string[]]$Families, + [Parameter(Mandatory)][string]$Qualification) + + $stepResult = Get-StepResult $Step + $stdout = Get-Content -LiteralPath (Join-Path $root $stepResult.stdout) -Raw + try { + Assert-FamilyCompletionMarkerText $Step $stdout $Field $ExpectedCount $Families + } + catch { + Fail-StepValidation $Step $_.Exception.Message + } + Set-StepValidation $Step 'passed' $Qualification @( + "families=$($Families.Count)", + "$Field=$ExpectedCount", + "total=$([int64]$Families.Count * $ExpectedCount)", + "rootSeed=$([int]$config.seed)") +} + +function Assert-ProductionRaceEvidence { + param( + [Parameter(Mandatory)][int64]$ExpectedCount, + [Parameter(Mandatory)][string[]]$Families) + + $step = 'production-race-stress' + $stepResult = Get-StepResult $step + $stdout = Get-Content -LiteralPath (Join-Path $root $stepResult.stdout) -Raw + try { + $witnesses = Assert-ProductionRaceMarkerText $stdout $ExpectedCount $Families + } + catch { + Fail-StepValidation $step $_.Exception.Message + } + + Set-StepValidation $step 'passed' 'sc011-production-race-count-and-witnesses-proven' @( + "families=$($Families.Count)", + "completedPerFamily=$ExpectedCount", + "productionOperationRacesPerFamily=$ExpectedCount", + "total=$([int64]$Families.Count * $ExpectedCount)", + 'familyMarkers=exactly-one-per-configured-family', + "recoverySuccesses=$($witnesses.recoverySuccesses)", + "recoveryBusy=$($witnesses.recoveryBusy)", + "liveActiveWitnesses=$($witnesses.liveActiveWitnesses)", + "disposalOperationWins=$($witnesses.operationWins)", + "disposalWins=$($witnesses.disposalWins)", + "rootSeed=$([int]$config.seed)") +} + +function Assert-FullSuiteEvidence { + param([Parameter(Mandatory)][string]$TrxDirectory) + + Assert-TrxStepEvidence 'full-test-suite' $TrxDirectory | Out-Null + $result = Get-StepResult 'full-test-suite' + $result.qualification = 'full-solution-trx-passed' +} + +function Assert-OwnerLeakEvidence { + param([Parameter(Mandatory)][hashtable]$TrxDirectories) + + $evidence = [Collections.Generic.List[string]]::new() + foreach ($assertion in @($config.requiredLeakAssertions)) { + $step = [string]$assertion.evidenceStep + if (-not $TrxDirectories.ContainsKey($step)) { + throw "Leak assertion '$($assertion.id)' names unavailable evidence step '$step'." + } + $passed = @(Get-TrxResults $TrxDirectories[$step] | Where-Object outcome -eq 'Passed') + $matching = @($passed | Where-Object { $_.testName -like "*$($assertion.testNameContains)*" }) + if ($matching.Count -eq 0) { + throw "Leak assertion '$($assertion.id)' has no passing executable test in '$step'." + } + $evidence.Add("$($assertion.id); step=$step; passedRows=$($matching.Count); test=$($assertion.testNameContains); state=$($assertion.assertedState)") + } + Add-EvidenceResult 'owner-leak-assertions' 'passed' 'configured-owner-leak-claims-executed' @($evidence) @( + $TrxDirectories.Values | ForEach-Object { [IO.Path]::GetRelativePath($root, [string]$_) }) +} + +function Assert-RecoveryCheckpointEvidence { + param( + [Parameter(Mandatory)][object[]]$PassedRows, + [Parameter(Mandatory)][int64]$ExpectedCases) + + $testPrefix = 'SharedMemoryStore.IntegrationTests.LockFreeCrashRecoveryIntegrationTests.' + + 'EveryCanonicalCheckpointCanBeKilledRecoveredAndFilledToCapacity' + $parsed = [Collections.Generic.List[object]]::new() + foreach ($row in $PassedRows) { + $match = [regex]::Match( + [string]$row.testName, + '^' + [regex]::Escape($testPrefix) + + '\(caseIndex:\s*(?[0-9]+),\s*checkpointValue:\s*(?[0-9]+)\)$') + if (-not $match.Success) { + Fail-StepValidation 'recovery' "Recovery TRX row has an unparseable identity: '$($row.testName)'." + } + $caseIndex = [int64]::Parse($match.Groups['case'].Value, [Globalization.CultureInfo]::InvariantCulture) + $checkpointId = [int]::Parse($match.Groups['checkpoint'].Value, [Globalization.CultureInfo]::InvariantCulture) + if ($caseIndex -lt 0 -or $caseIndex -ge $ExpectedCases) { + Fail-StepValidation 'recovery' "Recovery TRX case index $caseIndex is outside [0,$($ExpectedCases - 1)]." + } + $expectedCheckpointId = [int]$checkpointCatalog[[int]($caseIndex % $checkpointCatalog.Count)].id + if ($checkpointId -ne $expectedCheckpointId) { + Fail-StepValidation 'recovery' "Recovery case $caseIndex executed checkpoint $checkpointId; expected canonical checkpoint $expectedCheckpointId." + } + $parsed.Add([pscustomobject]@{ caseIndex = $caseIndex; checkpointId = $checkpointId }) + } + + $caseIndexes = @($parsed | ForEach-Object caseIndex | Sort-Object) + if ($parsed.Count -ne $ExpectedCases ` + -or @($caseIndexes | Sort-Object -Unique).Count -ne $ExpectedCases) { + Fail-StepValidation 'recovery' 'Recovery TRX evidence omitted or duplicated a configured case index.' + } + for ($index = 0; $index -lt $caseIndexes.Count; $index++) { + if ([int64]$caseIndexes[$index] -ne [int64]$index) { + Fail-StepValidation 'recovery' "Recovery TRX case-index set is not contiguous at index $index." + } + } + $actualCheckpointSet = @($parsed | ForEach-Object checkpointId | Sort-Object -Unique) + $expectedCheckpointSet = @($checkpointCatalog | ForEach-Object { [int]$_.id }) + if (($actualCheckpointSet -join ',') -cne ($expectedCheckpointSet -join ',')) { + Fail-StepValidation 'recovery' 'Recovery TRX evidence does not cover the exact source-derived checkpoint catalog.' + } + + $checkpointSetDigest = Get-StringSha256 ( + ($expectedCheckpointSet | ForEach-Object { '{0:D3}' -f $_ }) -join "`n") + $result = Get-StepResult 'recovery' + $result.validation = @($result.validation) + @( + "configuredCases=$ExpectedCases", + "catalogCheckpointCount=$($checkpointCatalog.Count)", + "checkpointSetDigest=$checkpointSetDigest") +} + +function Get-ExpectedReleaseOsRows { + param([Parameter(Mandatory)][string]$PlatformId) + + $allNames = @( + 'self-test-architecture', 'self-test-atomic', 'self-test-raw', + 'self-test-no-lock', 'self-test-crash', 'self-test-release-tests', + 'self-test-interop', 'self-test-samples', 'self-test-pack', + 'dotnet-info', 'clean', 'restore', 'build', + 'architecture', 'atomic', 'raw', 'no-lock-held', 'no-lock-linux-strace', + 'linux-tiny-performance', + 'crash-checkpoint-kill', 'crash-linux-sigstop', 'crash-linux-docker-pause', + 'release-tests', 'native', 'python', 'docker', 'sample-6', 'sample-12', 'pack') + $requirements = [ordered]@{} + foreach ($name in $allNames) { + $requirements[$name] = $true + } + $requirements['crash-linux-docker-pause'] = $false + if ($PlatformId -eq 'windows-x64') { + $requirements['no-lock-linux-strace'] = $false + $requirements['crash-linux-sigstop'] = $false + $requirements['linux-tiny-performance'] = $false + } + elseif ($PlatformId -ne 'linux-x64') { + throw "Release OS row contract does not support '$PlatformId'." + } + return $requirements +} + +function Get-OsAssemblyHash { + param( + [Parameter(Mandatory)]$Report, + [Parameter(Mandatory)][string]$RelativePath, + [Parameter(Mandatory)][string]$EvidencePath) + + $assemblyMatches = @($Report.testedAssemblies | Where-Object { + ([string]$_.path).Replace('\', '/') -ceq $RelativePath.Replace('\', '/') + }) + if ($assemblyMatches.Count -ne 1 -or [string]$assemblyMatches[0].sha256 -notmatch '^[0-9A-F]{64}$') { + throw "OS evidence '$EvidencePath' does not uniquely bind tested assembly '$RelativePath'." + } + return [string]($assemblyMatches[0].sha256) +} + +function Assert-OsDerivedDouble { + param( + [Parameter(Mandatory)][double]$Actual, + [Parameter(Mandatory)][double]$Expected, + [Parameter(Mandatory)][string]$Context) + + $tolerance = [Math]::Max(0.000000001, [Math]::Abs($Expected) * 0.000000000001) + if (-not [double]::IsFinite($Actual) -or -not [double]::IsFinite($Expected) ` + -or [Math]::Abs($Actual - $Expected) -gt $tolerance) { + throw "$Context is not reproducible from the raw OS performance evidence." + } +} + +function Assert-LinuxTinyOsPerformanceEvidence { + param( + [Parameter(Mandatory)]$Report, + [Parameter(Mandatory)][string]$EvidencePath) + + $row = @($Report.results | Where-Object { [string]$_.name -ceq 'linux-tiny-performance' }) + if ($row.Count -ne 1 -or -not (Get-StrictBoolean $row[0] 'required' 'Linux tiny performance OS row') ` + -or [string]$row[0].status -cne 'pass') { + throw "Linux OS evidence '$EvidencePath' lacks its required passing linux-tiny-performance row." + } + $expectedCommandTokens = @( + 'SharedMemoryStore.SyncProbe.csproj', '--mode sync', '--profile both', + '--scenario acquire-release,publish-remove', '--process-counts 1,8', + '--warmup 10', '--duration 60', '--trials 3') + foreach ($token in $expectedCommandTokens) { + if ([string]$row[0].command -cnotlike "*$token*") { + throw "Linux tiny performance command is missing exact token '$token'." + } + } + + $performance = Get-RequiredPropertyValue $row[0] 'performanceEvidence' 'Linux tiny performance OS row' + if ((Get-StrictInt64 $performance 'schemaVersion' 'Linux tiny performance row evidence' 1 1) -ne 1) { + throw 'Linux tiny performance row evidence schema must be 1.' + } + $expectedRawPath = [IO.Path]::GetFullPath((Join-Path ` + (Join-Path (Split-Path -Parent $EvidencePath) ([IO.Path]::GetFileNameWithoutExtension($EvidencePath) + '.evidence')) ` + 'linux-tiny-performance.json')) + $rawPathValue = Get-StrictString $performance 'reportPath' 'Linux tiny performance row evidence' + $actualRawPath = if ([IO.Path]::IsPathFullyQualified($rawPathValue)) { + [IO.Path]::GetFullPath($rawPathValue) + } + else { + [IO.Path]::GetFullPath((Join-Path $root $rawPathValue)) + } + $pathComparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + if (-not $actualRawPath.Equals($expectedRawPath, $pathComparison) ` + -or -not (Test-Path -LiteralPath $actualRawPath -PathType Leaf) ` + -or (Get-StrictString $performance 'reportSha256' 'Linux tiny performance row evidence') -cne + (Get-FileSha256 $actualRawPath)) { + throw "Linux tiny performance row does not bind the exact sibling raw report '$expectedRawPath'." + } + $manifestMatches = @($Report.evidenceManifest | Where-Object { + $manifestPath = if ([IO.Path]::IsPathFullyQualified([string]$_.path)) { + [IO.Path]::GetFullPath([string]$_.path) + } + else { + [IO.Path]::GetFullPath((Join-Path $root ([string]$_.path))) + } + $manifestPath.Equals($actualRawPath, $pathComparison) ` + -and [string]$_.sha256 -ceq (Get-FileSha256 $actualRawPath) + }) + if ($manifestMatches.Count -ne 1) { + throw 'Linux tiny performance raw JSON is not uniquely bound by the OS evidence manifest.' + } + + $raw = Get-Content -LiteralPath $actualRawPath -Raw | ConvertFrom-Json -Depth 30 + if ((Get-StrictInt64 $raw 'schemaVersion' 'Linux tiny performance raw report' 8 8) -ne 8 ` + -or (Get-StrictInt64 $raw 'minimumCompatibleSchemaVersion' 'Linux tiny performance raw report' 8 8) -ne 8) { + throw 'Linux tiny performance raw report must be exact schema 8/minimum-compatible 8.' + } + [void](Get-StrictString $raw 'schemaCompatibility' 'Linux tiny performance raw report') + $environment = Get-RequiredPropertyValue $raw 'environment' 'Linux tiny performance raw report' + $osHost = Get-RequiredPropertyValue $Report 'host' 'Linux OS evidence report' + if ((Get-StrictString $environment 'repositoryCommit' 'Linux tiny performance environment') -cne + [string]$Report.provenance.repositoryCommit ` + -or (Get-StrictString $environment 'repositoryWorkingTreeState' 'Linux tiny performance environment') -cne 'clean' ` + -or (Get-StrictString $Report 'platform' 'Linux OS evidence report') -cne 'linux' ` + -or (Get-StrictString $Report 'architecture' 'Linux OS evidence report') -cne 'x64' ` + -or (Get-StrictString $environment 'operatingSystem' 'Linux tiny performance environment') -cne + (Get-StrictString $osHost 'operatingSystem' 'Linux OS host evidence') ` + -or (Get-StrictString $environment 'operatingSystemArchitecture' 'Linux tiny performance environment') -cne + (Get-StrictString $osHost 'operatingSystemArchitecture' 'Linux OS host evidence') ` + -or (Get-StrictString $environment 'operatingSystemArchitecture' 'Linux tiny performance environment') -cne 'X64' ` + -or (Get-StrictString $environment 'processArchitecture' 'Linux tiny performance environment') -cne + (Get-StrictString $osHost 'processArchitecture' 'Linux OS host evidence') ` + -or (Get-StrictString $environment 'processArchitecture' 'Linux tiny performance environment') -cne 'X64' ` + -or (Get-StrictInt64 $environment 'logicalProcessorCount' 'Linux tiny performance environment' 8 [int32]::MaxValue) -ne + (Get-StrictInt64 $osHost 'logicalProcessorCount' 'Linux OS host evidence' 8 [int32]::MaxValue)) { + throw 'Linux tiny performance raw environment does not match its clean Linux OS evidence report.' + } + foreach ($property in @('framework', 'runtimeVersion', 'processorIdentifier')) { + [void](Get-StrictString $environment $property 'Linux tiny performance environment') + } + Assert-RequiredBenchmarkHardwareMetadata $environment 'Linux tiny performance environment' + [void](Get-StrictBoolean $environment 'serverGarbageCollection' 'Linux tiny performance environment') + [void](Get-StrictInt64 $environment 'stopwatchFrequency' 'Linux tiny performance environment' 1 [int64]::MaxValue) + $probeAssembly = "benchmarks/SharedMemoryStore.SyncProbe/bin/$($Report.configuration)/net10.0/SharedMemoryStore.SyncProbe.dll" + $storeAssembly = "benchmarks/SharedMemoryStore.SyncProbe/bin/$($Report.configuration)/net10.0/SharedMemoryStore.dll" + $actualProbeHash = Get-StrictString $environment 'probeAssemblySha256' 'Linux tiny performance environment' + $actualStoreHash = Get-StrictString $environment 'sharedMemoryStoreAssemblySha256' 'Linux tiny performance environment' + $expectedProbeHash = $(Get-OsAssemblyHash -Report $Report -RelativePath $probeAssembly -EvidencePath $EvidencePath) + $expectedStoreHash = $(Get-OsAssemblyHash -Report $Report -RelativePath $storeAssembly -EvidencePath $EvidencePath) + if ($actualProbeHash -cne $expectedProbeHash -or $actualStoreHash -cne $expectedStoreHash) { + throw "Linux tiny performance raw assembly hashes do not match the OS tested-assembly manifest (probe=$actualProbeHash/$expectedProbeHash; store=$actualStoreHash/$expectedStoreHash)." + } + + $configuration = Get-RequiredPropertyValue $raw 'configuration' 'Linux tiny performance raw report' + if ((Get-StrictString $configuration 'mode' 'Linux tiny performance configuration') -cne 'sync' ` + -or (Get-StrictInt64 $configuration 'durationSeconds' 'Linux tiny performance configuration' 60 60) -ne 60 ` + -or (Get-StrictInt64 $configuration 'durationBoundGraceSeconds' 'Linux tiny performance configuration' 60 60) -ne 60 ` + -or (Get-StrictInt64 $configuration 'warmupSeconds' 'Linux tiny performance configuration' 10 10) -ne 10 ` + -or (Get-StrictInt64 $configuration 'warmupCycles' 'Linux tiny performance configuration' 0 0) -ne 0 ` + -or (Get-StrictInt64 $configuration 'samplingInterval' 'Linux tiny performance configuration' 64 64) -ne 64 ` + -or (Get-StrictInt64 $configuration 'maxLatencySamplesPerWorker' 'Linux tiny performance configuration' 65536 65536) -ne 65536 ` + -or (Get-StrictInt64 $configuration 'trials' 'Linux tiny performance configuration' 3 3) -ne 3 ` + -or -not (Get-StrictBoolean $configuration 'affinityRequested' 'Linux tiny performance configuration')) { + throw 'Linux tiny performance raw configuration is not the exact 10s/60s/3-trial affinity workload.' + } + [void](Assert-LinuxTinySyncTopology $configuration 'Linux tiny performance configuration') + Assert-BenchmarkScenarioStoreDimensions ` + $configuration @('acquire-release', 'publish-remove') 'Linux tiny performance configuration' + if ((@($configuration.profiles) -join ',') -cne 'Legacy,LockFree' ` + -or (@($configuration.countBoundProfiles) -join ',') -cne 'LockFree' ` + -or (@($configuration.scenarios) -join ',') -cne 'acquire-release,publish-remove' ` + -or (@($configuration.scenarioProcessCounts.PSObject.Properties.Name) -join ',') -cne + 'acquire-release,publish-remove') { + throw 'Linux tiny performance raw profile/scenario matrix is not exact.' + } + foreach ($scenario in @('acquire-release', 'publish-remove')) { + $counts = @($configuration.scenarioProcessCounts.$scenario) + if ($counts.Count -ne 2 ` + -or -not (Test-IsIntegerNumber $counts[0]) -or [int64]$counts[0] -ne 1 ` + -or -not (Test-IsIntegerNumber $counts[1]) -or [int64]$counts[1] -ne 8) { + throw "Linux tiny performance '$scenario' process-count matrix must be exactly [1,8]." + } + } + + $runs = @($raw.runs) + $summaries = @($raw.summary) + if ($runs.Count -ne 24 -or $summaries.Count -ne 8) { + throw "Linux tiny performance raw matrix must contain 24 runs and 8 summaries, actual=$($runs.Count)/$($summaries.Count)." + } + $expectedRunKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + $expectedSummaryKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($profile in @('Legacy', 'LockFree')) { + foreach ($scenario in @('acquire-release', 'publish-remove')) { + foreach ($processCount in @(1, 8)) { + [void]$expectedSummaryKeys.Add("$profile|$scenario|$processCount") + foreach ($trial in 1..3) { + [void]$expectedRunKeys.Add("$profile|$scenario|$processCount|$trial") + } + } + } + } + $actualRunKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($run in $runs) { + $context = "Linux tiny run $($run.profile)/$($run.scenario)/$($run.processCount)/trial-$($run.trial)" + $profile = Get-StrictString $run 'profile' $context + $scenario = Get-StrictString $run 'scenario' $context + $processCount = Get-StrictInt64 $run 'processCount' $context 1 8 + if ($processCount -notin @(1, 8)) { + throw "$context has an unsupported process count." + } + $trial = Get-StrictInt64 $run 'trial' $context 1 3 + $key = "$profile|$scenario|$processCount|$trial" + if (-not $expectedRunKeys.Contains($key) -or -not $actualRunKeys.Add($key)) { + throw "$context is unexpected or duplicated." + } + if ((Get-StrictString $run 'qualification' $context) -cne 'qualification-measurement' ` + -or (Get-StrictInt64 $run 'failures' $context 0 0) -ne 0 ` + -or (Get-StrictInt64 $run 'operationTarget' $context 0 0) -ne 0 ` + -or (Get-StrictInt64 $run 'frameTarget' $context 0 0) -ne 0 ` + -or (Get-StrictBoolean $run 'oversubscribed' $context)) { + throw "$context is not a correctness-clean qualification measurement." + } + $readerCount = Get-StrictInt64 $run 'readerProcessCount' $context 0 $processCount + $publisherCount = Get-StrictInt64 $run 'publisherProcessCount' $context 0 $processCount + $observerCount = Get-StrictInt64 $run 'observerProcessCount' $context 0 0 + if (($scenario -ceq 'acquire-release' -and ($readerCount -ne $processCount -or $publisherCount -ne 0)) ` + -or ($scenario -ceq 'publish-remove' -and ($readerCount -ne 0 -or $publisherCount -ne $processCount)) ` + -or $observerCount -ne 0) { + throw "$context has the wrong process-role topology." + } + $cycles = Get-StrictInt64 $run 'cycles' $context 1 [int64]::MaxValue + $operations = Get-StrictInt64 $run 'operations' $context 1 [int64]::MaxValue + if ([decimal]$operations -lt ([decimal]2 * [decimal]$cycles)) { + throw "$context has fewer than the two recorded store operations required per completed cycle." + } + $measuredSeconds = Get-StrictDouble $run 'measuredSeconds' $context 60 [double]::MaxValue + $wallSeconds = Get-StrictDouble $run 'wallSeconds' $context $measuredSeconds [double]::MaxValue + [void]$wallSeconds + [int64]$minimumWindowSamples = [int64]$processCount * 1024 + [int64]$maximumWindowSamples = [int64]$processCount * 32768 + [int64]$minimumTotalSamples = $minimumWindowSamples * 2 + [int64]$maximumTotalSamples = $maximumWindowSamples * 2 + $earlySampleCount = Get-StrictInt64 $run 'earlySampleCount' $context $minimumWindowSamples $maximumWindowSamples + $lateSampleCount = Get-StrictInt64 $run 'lateSampleCount' $context $minimumWindowSamples $maximumWindowSamples + $sampleCount = Get-StrictInt64 $run 'sampleCount' $context $minimumTotalSamples $maximumTotalSamples + if ($sampleCount -ne ($earlySampleCount + $lateSampleCount) -or $sampleCount -gt $cycles) { + throw "$context sampleCount must equal its early/late windows and cannot exceed completed cycles." + } + Assert-OsDerivedDouble ` + (Get-StrictDouble $run 'apiCallsPerSecond' $context 0 [double]::MaxValue -Positive) ` + ([double]$operations / $measuredSeconds) "$context.apiCallsPerSecond" + $p50 = Get-StrictDouble $run 'p50Microseconds' $context 0 [double]::MaxValue + $p95 = Get-StrictDouble $run 'p95Microseconds' $context 0 [double]::MaxValue + $p99 = Get-StrictDouble $run 'p99Microseconds' $context 0 [double]::MaxValue + $maximum = Get-StrictDouble $run 'maxMicroseconds' $context 0 [double]::MaxValue + [void](Get-StrictDouble $run 'earlyP99Microseconds' $context 0 [double]::MaxValue -Positive) + [void](Get-StrictDouble $run 'lateP99Microseconds' $context 0 [double]::MaxValue -Positive) + if ($p50 -gt $p95 -or $p95 -gt $p99 -or $p99 -gt $maximum ` + -or ($profile -ceq 'LockFree' -and $maximum -gt + (Get-StrictDouble $config.linuxTinyPerformance 'maximumStallMicroseconds' ` + 'qualification config linuxTinyPerformance' 10000 10000))) { + throw "$context violates p99/maximum ordering or the every-run 10000us lock-free stall gate." + } + $assigned = @($run.assignedProcessors) + if ($assigned.Count -ne $processCount ` + -or @($assigned | Sort-Object -Unique).Count -ne $processCount ` + -or (Get-StrictInt64 $run 'affinityAppliedCount' $context $processCount $processCount) -ne $processCount) { + throw "$context lacks complete unique $processCount-process affinity evidence." + } + foreach ($processor in $assigned) { + if (-not (Test-IsIntegerNumber $processor) ` + -or [int64]$processor -lt 0 ` + -or [int64]$processor -gt 63) { + throw "$context has a processor assignment outside the probe's 64-bit affinity mask [0,63]." + } + } + $workerCycles = @($run.workerCycles) + [decimal]$workerCycleTotal = 0 + if ($workerCycles.Count -ne $processCount) { + throw "$context must contain exactly $processCount worker-cycle rows." + } + foreach ($workerCycle in $workerCycles) { + if (-not (Test-IsIntegerNumber $workerCycle) -or [int64]$workerCycle -lt 0) { + throw "$context has an invalid worker-cycle row." + } + $workerCycleTotal += [decimal]$workerCycle + } + if ($workerCycleTotal -ne [decimal]$cycles) { throw "$context worker cycles do not sum to Cycles." } + [decimal]$statusTotal = 0 + $histogram = Get-RequiredPropertyValue $run 'statusHistogram' $context + if (@($histogram.PSObject.Properties).Count -eq 0) { throw "$context has an empty status histogram." } + foreach ($entry in $histogram.PSObject.Properties) { + if (-not (Test-IsIntegerNumber $entry.Value) -or [int64]$entry.Value -lt 0) { + throw "$context status '$($entry.Name)' is invalid." + } + if ($entry.Name -ceq 'Validation.ChecksumMismatch' ` + -or $entry.Name -clike 'CorruptReason.*') { + throw "$context contains forbidden checksum/corruption evidence '$($entry.Name)'." + } + if ($entry.Name -match '^(Acquire|Release|Publish|Remove)\.') { $statusTotal += [decimal]$entry.Value } + } + if ($statusTotal -ne [decimal]$operations) { throw "$context status histogram does not sum to Operations." } + $requiredSuccesses = if ($scenario -ceq 'acquire-release') { + @('Acquire.Success', 'Release.Success') + } + else { + @('Publish.Success', 'Remove.Success') + } + foreach ($successName in $requiredSuccesses) { + $successRows = @($histogram.PSObject.Properties | Where-Object { $_.Name -ceq $successName }) + if ($successRows.Count -ne 1 ` + -or -not (Test-IsIntegerNumber $successRows[0].Value) ` + -or [int64]$successRows[0].Value -ne $cycles) { + throw "$context must contain exactly $cycles '$successName' operations, one per completed cycle." + } + } + } + if (-not $actualRunKeys.SetEquals($expectedRunKeys)) { + throw 'Linux tiny performance raw run tuple set is incomplete.' + } + + $actualSummaryKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($summary in $summaries) { + $context = "Linux tiny summary $($summary.profile)/$($summary.scenario)/$($summary.processCount)" + $profile = Get-StrictString $summary 'profile' $context + $scenario = Get-StrictString $summary 'scenario' $context + $processCount = Get-StrictInt64 $summary 'processCount' $context 1 8 + if ($processCount -notin @(1, 8)) { + throw "$context has an unsupported process count." + } + $key = "$profile|$scenario|$processCount" + if (-not $expectedSummaryKeys.Contains($key) -or -not $actualSummaryKeys.Add($key)) { + throw "$context is unexpected or duplicated." + } + $matching = @($runs | Where-Object { + [string]$_.profile -ceq $profile -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq $processCount + }) + if ($matching.Count -ne 3 -or (Get-StrictInt64 $summary 'totalFailures' $context 0 0) -ne 0) { + throw "$context does not summarize exactly three correctness-clean trials." + } + foreach ($pair in @( + @('medianApiCallsPerSecond', 'apiCallsPerSecond'), + @('medianP99Microseconds', 'p99Microseconds'), + @('medianMaxMicroseconds', 'maxMicroseconds'))) { + $values = [double[]]@($matching | ForEach-Object { + Get-StrictDouble $_ $pair[1] $context 0 [double]::MaxValue + }) + Assert-OsDerivedDouble ` + (Get-StrictDouble $summary $pair[0] $context 0 [double]::MaxValue) ` + (Get-MedianValue $values) "$context.$($pair[0])" + } + $merged = [ordered]@{} + foreach ($run in $matching) { + foreach ($entry in $run.statusHistogram.PSObject.Properties) { + if (-not $merged.Contains($entry.Name)) { $merged[$entry.Name] = [int64]0 } + $merged[$entry.Name] = [int64]$merged[$entry.Name] + [int64]$entry.Value + } + } + $summaryHistogram = Get-RequiredPropertyValue $summary 'statusHistogram' $context + if ((@($summaryHistogram.PSObject.Properties.Name) -join ',') -cne (@($merged.Keys) -join ',')) { + throw "$context status histogram keys do not match its raw trials." + } + foreach ($entry in $summaryHistogram.PSObject.Properties) { + if (-not (Test-IsIntegerNumber $entry.Value) -or [int64]$entry.Value -ne [int64]$merged[$entry.Name]) { + throw "$context status '$($entry.Name)' is not the raw-trial total." + } + } + } + if (-not $actualSummaryKeys.SetEquals($expectedSummaryKeys)) { + throw 'Linux tiny performance summary tuple set is incomplete.' + } + foreach ($scenario in @('acquire-release', 'publish-remove')) { + $legacyOne = @($summaries | Where-Object { + [string]$_.profile -ceq 'Legacy' -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq 1 + })[0] + $lockFreeOne = @($summaries | Where-Object { + [string]$_.profile -ceq 'LockFree' -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq 1 + })[0] + $legacyEight = @($summaries | Where-Object { + [string]$_.profile -ceq 'Legacy' -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq 8 + })[0] + $lockFreeEight = @($summaries | Where-Object { + [string]$_.profile -ceq 'LockFree' -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq 8 + })[0] + $legacyOneP99 = Get-StrictDouble $legacyOne 'medianP99Microseconds' "$scenario legacy/1p summary" 0 [double]::MaxValue -Positive + $lockFreeOneP99 = Get-StrictDouble $lockFreeOne 'medianP99Microseconds' "$scenario lock-free/1p summary" 0 [double]::MaxValue -Positive + $legacyEightRate = Get-StrictDouble $legacyEight 'medianApiCallsPerSecond' "$scenario legacy/8p summary" 0 [double]::MaxValue -Positive + $lockFreeEightRate = Get-StrictDouble $lockFreeEight 'medianApiCallsPerSecond' "$scenario lock-free/8p summary" 0 [double]::MaxValue -Positive + $lockFreeEightP99 = Get-StrictDouble $lockFreeEight 'medianP99Microseconds' "$scenario lock-free/8p summary" 0 [double]::MaxValue -Positive + $uncontendedP99Ratio = $lockFreeOneP99 / $legacyOneP99 + $throughputRatio = $lockFreeEightRate / $legacyEightRate + $scaleP99Ratio = $lockFreeEightP99 / $lockFreeOneP99 + if (-not [double]::IsFinite($uncontendedP99Ratio) ` + -or $uncontendedP99Ratio -gt [double]$config.linuxTinyPerformance.maximumUncontendedP99Ratio ` + -or -not [double]::IsFinite($throughputRatio) ` + -or $throughputRatio -lt [double]$config.linuxTinyPerformance.minimumThroughputRatio ` + -or -not [double]::IsFinite($scaleP99Ratio) ` + -or $scaleP99Ratio -gt [double]$config.linuxTinyPerformance.maximumScaleP99Ratio ` + -or $lockFreeEightP99 -gt [double]$config.linuxTinyPerformance.maximumP99Microseconds) { + throw "Linux tiny performance '$scenario' gate failed: uncontendedP99Ratio=$uncontendedP99Ratio throughputRatio=$throughputRatio scaleP99Ratio=$scaleP99Ratio lockFreeEightP99Microseconds=$lockFreeEightP99." + } + } + $declared = Get-RequiredPropertyValue $performance 'validation' 'Linux tiny performance row evidence' + $expectedDeclaredProperties = @( + 'schemaVersion', 'runCount', 'summaryCount', 'warmupSeconds', 'durationSeconds', + 'trials', 'processCounts', 'minimumThroughputRatio', 'maximumUncontendedP99Ratio', + 'maximumScaleP99Ratio', 'maximumP99Microseconds', 'maximumStallMicroseconds', 'metrics') + if ((@($declared.PSObject.Properties.Name) -join ',') -cne ($expectedDeclaredProperties -join ',')) { + throw 'Linux tiny performance row declared validation has unexpected properties or property order.' + } + $declaredProcessCounts = @(Get-RequiredPropertyValue $declared 'processCounts' 'Linux tiny declared validation') + if ($declaredProcessCounts.Count -ne 2 ` + -or -not (Test-IsIntegerNumber $declaredProcessCounts[0]) -or [int64]$declaredProcessCounts[0] -ne 1 ` + -or -not (Test-IsIntegerNumber $declaredProcessCounts[1]) -or [int64]$declaredProcessCounts[1] -ne 8 ` + -or (Get-StrictInt64 $declared 'schemaVersion' 'Linux tiny declared validation' 2 2) -ne 2 ` + -or (Get-StrictInt64 $declared 'runCount' 'Linux tiny declared validation' 24 24) -ne 24 ` + -or (Get-StrictInt64 $declared 'summaryCount' 'Linux tiny declared validation' 8 8) -ne 8 ` + -or (Get-StrictInt64 $declared 'warmupSeconds' 'Linux tiny declared validation' 10 10) -ne 10 ` + -or (Get-StrictInt64 $declared 'durationSeconds' 'Linux tiny declared validation' 60 60) -ne 60 ` + -or (Get-StrictInt64 $declared 'trials' 'Linux tiny declared validation' 3 3) -ne 3 ` + -or (Get-StrictDouble $declared 'minimumThroughputRatio' 'Linux tiny declared validation' 1 1) -ne 1 ` + -or (Get-StrictDouble $declared 'maximumUncontendedP99Ratio' 'Linux tiny declared validation' 1 1) -ne 1 ` + -or (Get-StrictDouble $declared 'maximumScaleP99Ratio' 'Linux tiny declared validation' 3 3) -ne 3 ` + -or (Get-StrictDouble $declared 'maximumP99Microseconds' 'Linux tiny declared validation' 10 10) -ne 10 ` + -or (Get-StrictDouble $declared 'maximumStallMicroseconds' 'Linux tiny declared validation' 10000 10000) -ne 10000) { + throw 'Linux tiny performance row declared validation does not match the recomputed gate.' + } + $declaredMetrics = @($declared.metrics) + if ($declaredMetrics.Count -ne 8) { + throw 'Linux tiny performance row must declare exactly eight recomputable metric rows.' + } + $actualDeclaredMetricKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($metric in $declaredMetrics) { + $metricContext = "Linux tiny declared metric $($metric.profile)/$($metric.scenario)/$($metric.processCount)" + $profile = Get-StrictString $metric 'profile' $metricContext + $scenario = Get-StrictString $metric 'scenario' $metricContext + $metricProcessCount = Get-StrictInt64 $metric 'processCount' $metricContext 1 8 + if ($metricProcessCount -notin @(1, 8)) { + throw "$metricContext has an unsupported process count." + } + $metricKey = "$profile|$scenario|$metricProcessCount" + if (-not $expectedSummaryKeys.Contains($metricKey) -or -not $actualDeclaredMetricKeys.Add($metricKey)) { + throw "$metricContext is unexpected or duplicated." + } + $matchingSummary = @($summaries | Where-Object { + [string]$_.profile -ceq $profile -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq $metricProcessCount + }) + $matchingRuns = @($runs | Where-Object { + [string]$_.profile -ceq $profile -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq $metricProcessCount + }) + if ($matchingSummary.Count -ne 1 -or $matchingRuns.Count -ne 3) { + throw "$metricContext does not identify one exact recomputed summary tuple." + } + Assert-OsDerivedDouble ` + (Get-StrictDouble $metric 'medianApiCallsPerSecond' $metricContext 0 [double]::MaxValue) ` + ([double]$matchingSummary[0].medianApiCallsPerSecond) "$metricContext.medianApiCallsPerSecond" + Assert-OsDerivedDouble ` + (Get-StrictDouble $metric 'medianP99Microseconds' $metricContext 0 [double]::MaxValue) ` + ([double]$matchingSummary[0].medianP99Microseconds) "$metricContext.medianP99Microseconds" + Assert-OsDerivedDouble ` + (Get-StrictDouble $metric 'maximumRawStallMicroseconds' $metricContext 0 [double]::MaxValue) ` + ([double](($matchingRuns | Measure-Object maxMicroseconds -Maximum).Maximum)) ` + "$metricContext.maximumRawStallMicroseconds" + } + if (-not $actualDeclaredMetricKeys.SetEquals($expectedSummaryKeys)) { + throw 'Linux tiny performance declared metric tuple set is incomplete.' + } + return [IO.Path]::GetRelativePath($root, $actualRawPath) +} + +function Invoke-LinuxTinyOsPerformanceVerifierSelfTest { + $reportPath = Join-Path $runRoot 'linux-tiny-os-performance-self-test.json' + $treeRoot = Join-Path $runRoot 'linux-tiny-os-performance-self-test.evidence' + New-Item -ItemType Directory -Path $treeRoot | Out-Null + $rawPath = Join-Path $treeRoot 'linux-tiny-performance.json' + $runs = [Collections.Generic.List[object]]::new() + $summaries = [Collections.Generic.List[object]]::new() + foreach ($profile in @('Legacy', 'LockFree')) { + foreach ($scenario in @('acquire-release', 'publish-remove')) { + foreach ($processCount in @(1, 8)) { + $api = if ($profile -ceq 'Legacy') { 1000.0 } else { 1100.0 } + $p99 = if ($processCount -eq 1) { + if ($profile -ceq 'Legacy') { 5.0 } else { 4.0 } + } + else { + if ($profile -ceq 'Legacy') { 3.0 } else { 8.0 } + } + $maximum = if ($profile -ceq 'Legacy') { 500.0 } else { 9000.0 } + [int64]$cycles = if ($profile -ceq 'Legacy') { 30000 } else { 33000 } + [int64]$operations = $cycles * 2 + [int64]$workerCycle = $cycles / $processCount + [int64]$windowSamples = [int64]$processCount * 1024 + $workerCycles = @() + for ($worker = 0; $worker -lt $processCount; $worker++) { + $workerCycles += $workerCycle + } + $histogram = if ($scenario -ceq 'acquire-release') { + [pscustomobject][ordered]@{ 'Acquire.Success' = $cycles; 'Release.Success' = $cycles } + } + else { + [pscustomobject][ordered]@{ 'Publish.Success' = $cycles; 'Remove.Success' = $cycles } + } + foreach ($trial in 1..3) { + $runs.Add([pscustomobject][ordered]@{ + Profile = $profile; Scenario = $scenario; ProcessCount = $processCount; Trial = $trial + ReaderProcessCount = $(if ($scenario -ceq 'acquire-release') { $processCount } else { 0 }) + PublisherProcessCount = $(if ($scenario -ceq 'publish-remove') { $processCount } else { 0 }) + ObserverProcessCount = 0; Cycles = $cycles; Operations = $operations + ApiCallsPerSecond = $api; P50Microseconds = 1.0; P95Microseconds = 2.0 + P99Microseconds = $p99; MaxMicroseconds = $maximum + EarlyP99Microseconds = $p99; LateP99Microseconds = $p99 + Failures = 0; MeasuredSeconds = 60.0; WallSeconds = 70.0 + EarlySampleCount = $windowSamples; LateSampleCount = $windowSamples + SampleCount = ($windowSamples * 2); AffinityAppliedCount = $processCount + AssignedProcessors = @(0..($processCount - 1)); Oversubscribed = $false + Qualification = 'qualification-measurement'; StatusHistogram = $histogram + WorkerCycles = @($workerCycles); OperationTarget = 0; FrameTarget = 0 + }) + } + $summaryHistogram = if ($scenario -ceq 'acquire-release') { + [pscustomobject][ordered]@{ + 'Acquire.Success' = ($cycles * 3); 'Release.Success' = ($cycles * 3) + } + } + else { + [pscustomobject][ordered]@{ + 'Publish.Success' = ($cycles * 3); 'Remove.Success' = ($cycles * 3) + } + } + $summaries.Add([pscustomobject][ordered]@{ + Profile = $profile; Scenario = $scenario; ProcessCount = $processCount + MedianApiCallsPerSecond = $api; MedianP99Microseconds = $p99 + MedianMaxMicroseconds = $maximum; TotalFailures = 0; StatusHistogram = $summaryHistogram + }) + } + } + } + $raw = [pscustomobject][ordered]@{ + SchemaVersion = 8 + Environment = [pscustomobject][ordered]@{ + RepositoryCommit = 'synthetic'; RepositoryWorkingTreeState = 'clean' + SharedMemoryStoreAssemblySha256 = ('A' * 64); ProbeAssemblySha256 = ('B' * 64) + OperatingSystem = 'Ubuntu 24.04 synthetic'; OperatingSystemArchitecture = 'X64'; ProcessArchitecture = 'X64' + Framework = '.NET synthetic'; RuntimeVersion = 'synthetic'; LogicalProcessorCount = 8 + PhysicalCoreCount = 4; TotalMemoryBytes = 17179869184; ProcessorModel = 'Synthetic CPU' + ProcessorIdentifier = 'Synthetic CPU'; ServerGarbageCollection = $false; StopwatchFrequency = 10000000 + } + Configuration = [pscustomobject][ordered]@{ + Mode = 'sync'; DurationSeconds = 60; DurationBoundGraceSeconds = 60 + Trials = 3; Profiles = @('Legacy', 'LockFree') + CountBoundProfiles = @('LockFree') + Scenarios = @('acquire-release', 'publish-remove') + ScenarioProcessCounts = [pscustomobject][ordered]@{ + 'acquire-release' = @(1, 8); 'publish-remove' = @(1, 8) + } + ScenarioStoreDimensions = [pscustomobject][ordered]@{ + 'acquire-release' = [pscustomobject][ordered]@{ + SlotCount = 32; MaxValueBytes = 8; MaxDescriptorBytes = 0; MaxKeyBytes = 8 + LeaseRecordCount = 64; LockFreeParticipantRecordCount = 64 + } + 'publish-remove' = [pscustomobject][ordered]@{ + SlotCount = 32; MaxValueBytes = 8; MaxDescriptorBytes = 0; MaxKeyBytes = 8 + LeaseRecordCount = 64; LockFreeParticipantRecordCount = 64 + } + } + WarmupCycles = 0; WarmupSeconds = 10; AffinityRequested = $true + SamplingInterval = 64; MaxLatencySamplesPerWorker = 65536 + SyncKeysPerWorker = [int]$config.linuxTinyPerformance.syncKeysPerWorker + SyncMaximumWorkerCount = [int]$config.linuxTinyPerformance.syncMaximumWorkerCount + SyncCanonicalBucketCount = [int]$config.linuxTinyPerformance.syncCanonicalBucketCount + SyncKeyCatalogSha256 = [string]$config.linuxTinyPerformance.syncKeyCatalogSha256 + SyncKeyCanonicalBucketAssignments = @($config.linuxTinyPerformance.syncKeyCanonicalBucketAssignments) + } + Runs = @($runs); Summary = @($summaries); MinimumCompatibleSchemaVersion = 8 + SchemaCompatibility = 'synthetic Schema v8 release-runner verifier self-test' + } + $raw | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $rawRelativePath = [IO.Path]::GetRelativePath($root, $rawPath) + $rawManifest = [pscustomobject][ordered]@{ + path = $rawRelativePath + length = (Get-Item -LiteralPath $rawPath).Length + sha256 = Get-FileSha256 $rawPath + } + $declaredMetrics = @($summaries | ForEach-Object { + $summary = $_ + $matchingRuns = @($runs | Where-Object { + [string]$_.Profile -ceq [string]$summary.Profile ` + -and [string]$_.Scenario -ceq [string]$summary.Scenario ` + -and [int64]$_.ProcessCount -eq [int64]$summary.ProcessCount + }) + [pscustomobject][ordered]@{ + profile = [string]$summary.Profile + scenario = [string]$summary.Scenario + processCount = [int64]$summary.ProcessCount + medianApiCallsPerSecond = [double]$summary.MedianApiCallsPerSecond + medianP99Microseconds = [double]$summary.MedianP99Microseconds + maximumRawStallMicroseconds = [double](($matchingRuns | Measure-Object MaxMicroseconds -Maximum).Maximum) + } + }) + $osReport = [pscustomobject][ordered]@{ + platform = 'linux' + architecture = 'x64' + configuration = 'Release' + provenance = [pscustomobject][ordered]@{ repositoryCommit = 'synthetic' } + host = [pscustomobject][ordered]@{ + operatingSystem = 'Ubuntu 24.04 synthetic' + operatingSystemArchitecture = 'X64' + processArchitecture = 'X64' + logicalProcessorCount = 8 + } + testedAssemblies = @( + [pscustomobject][ordered]@{ + path = 'benchmarks/SharedMemoryStore.SyncProbe/bin/Release/net10.0/SharedMemoryStore.dll' + sha256 = ('A' * 64) + }, + [pscustomobject][ordered]@{ + path = 'benchmarks/SharedMemoryStore.SyncProbe/bin/Release/net10.0/SharedMemoryStore.SyncProbe.dll' + sha256 = ('B' * 64) + }) + results = @([pscustomobject][ordered]@{ + name = 'linux-tiny-performance'; required = $true; status = 'pass' + command = 'dotnet SharedMemoryStore.SyncProbe.csproj --mode sync --profile both --scenario acquire-release,publish-remove --process-counts 1,8 --warmup 10 --duration 60 --trials 3' + performanceEvidence = [pscustomobject][ordered]@{ + schemaVersion = 1; reportPath = $rawRelativePath; reportSha256 = $rawManifest.sha256 + validation = [pscustomobject][ordered]@{ + schemaVersion = 2; runCount = 24; summaryCount = 8 + warmupSeconds = 10; durationSeconds = 60; trials = 3; processCounts = @(1, 8) + minimumThroughputRatio = 1.0; maximumUncontendedP99Ratio = 1.0 + maximumScaleP99Ratio = 3.0; maximumP99Microseconds = 10.0 + maximumStallMicroseconds = 10000.0; metrics = $declaredMetrics + } + } + }) + evidenceManifest = @($rawManifest) + } + [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) + $hostMismatch = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $hostMismatch.host.operatingSystem = 'Debian synthetic' + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $hostMismatch $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted a raw report from a different host OS description.' + } + [int]$assertions = 2 + + $oddMedian = Get-MedianValue ([double[]]@(30.0, 10.0, 20.0)) + $evenMedian = Get-MedianValue ([double[]]@(40.0, 10.0, 30.0, 20.0)) + if ($oddMedian -ne 20.0 -or $evenMedian -ne 25.0) { + throw 'Linux OS performance verifier self-test did not compute canonical odd/even medians.' + } + $assertions++ + + $tampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $tamperedLockFreeRun = @($tampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq 'acquire-release' ` + -and [int64]$_.ProcessCount -eq 1 + })[0] + $tamperedLockFreeRun.MaxMicroseconds = 10001.0 + $tampered | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $osReport.evidenceManifest[0].sha256 = $tamperedHash + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted an over-limit raw lock-free stall.' + } + $assertions++ + + $sampleTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $sampleTamperedOneProcessRun = @($sampleTampered.Runs | Where-Object { + [int64]$_.ProcessCount -eq 1 + })[0] + $sampleTamperedOneProcessRun.LateSampleCount = 1023 + $sampleTamperedOneProcessRun.SampleCount = 2047 + $sampleTampered | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $osReport.evidenceManifest[0].sha256 = $tamperedHash + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted incoherent early/late sample counts.' + } + $assertions++ + + $assertRawTamperRejected = { + param( + [Parameter(Mandatory)]$TamperedRaw, + [Parameter(Mandatory)]$TamperedOsReport, + [Parameter(Mandatory)][string]$FailureMessage) + + $TamperedRaw | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $TamperedOsReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $TamperedOsReport.evidenceManifest[0].sha256 = $tamperedHash + $TamperedOsReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $wasRejected = $false + try { + [void](Assert-LinuxTinyOsPerformanceEvidence $TamperedOsReport $reportPath) + } + catch { + $wasRejected = $true + } + if (-not $wasRejected) { + throw $FailureMessage + } + } + + $unknownProcessorTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $unknownProcessorTampered.Environment.ProcessorModel = ' Unknown CPU ' + $unknownProcessorOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + & $assertRawTamperRejected $unknownProcessorTampered $unknownProcessorOsReport ` + 'Linux OS performance verifier self-test accepted an unknown processor model.' + $assertions++ + + $missingMemoryTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $missingMemoryTampered.Environment.PSObject.Properties.Remove('TotalMemoryBytes') + $missingMemoryOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + & $assertRawTamperRejected $missingMemoryTampered $missingMemoryOsReport ` + 'Linux OS performance verifier self-test accepted missing memory metadata.' + $assertions++ + + $storeDimensionTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $storeDimensionTampered.Configuration.ScenarioStoreDimensions.'acquire-release'.SlotCount = 31 + $storeDimensionOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + & $assertRawTamperRejected $storeDimensionTampered $storeDimensionOsReport ` + 'Linux OS performance verifier self-test accepted incorrect store dimensions.' + $assertions++ + + $countPolicyTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $countPolicyTampered.Configuration.CountBoundProfiles = @('Legacy') + $countPolicyOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + & $assertRawTamperRejected $countPolicyTampered $countPolicyOsReport ` + 'Linux OS performance verifier self-test accepted a swapped count-bound profile policy.' + $assertions++ + + $targetTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $targetTampered.Runs[0].OperationTarget = 1 + $targetOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + & $assertRawTamperRejected $targetTampered $targetOsReport ` + 'Linux OS performance verifier self-test accepted a count target on a duration-only tiny row.' + $assertions++ + + $durationTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $durationTampered.Runs[0].MeasuredSeconds = 59.999 + $durationOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + & $assertRawTamperRejected $durationTampered $durationOsReport ` + 'Linux OS performance verifier self-test accepted a short duration-bound row.' + $assertions++ + + foreach ($scenario in @('acquire-release', 'publish-remove')) { + $uncontendedTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($run in @($uncontendedTampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 1 + })) { + $run.P99Microseconds = 6.0 + $run.EarlyP99Microseconds = 6.0 + $run.LateP99Microseconds = 6.0 + } + @($uncontendedTampered.Summary | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 1 + })[0].MedianP99Microseconds = 6.0 + $uncontendedOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + @($uncontendedOsReport.results[0].performanceEvidence.validation.metrics | Where-Object { + [string]$_.profile -ceq 'LockFree' ` + -and [string]$_.scenario -ceq $scenario ` + -and [int64]$_.processCount -eq 1 + })[0].medianP99Microseconds = 6.0 + & $assertRawTamperRejected $uncontendedTampered $uncontendedOsReport ` + "Linux OS performance verifier self-test accepted an over-limit '$scenario' uncontended p99 ratio." + $assertions++ + + $scaleTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($run in @($scaleTampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 1 + })) { + $run.P99Microseconds = 2.0 + $run.EarlyP99Microseconds = 2.0 + $run.LateP99Microseconds = 2.0 + } + @($scaleTampered.Summary | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 1 + })[0].MedianP99Microseconds = 2.0 + $scaleOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + @($scaleOsReport.results[0].performanceEvidence.validation.metrics | Where-Object { + [string]$_.profile -ceq 'LockFree' ` + -and [string]$_.scenario -ceq $scenario ` + -and [int64]$_.processCount -eq 1 + })[0].medianP99Microseconds = 2.0 + & $assertRawTamperRejected $scaleTampered $scaleOsReport ` + "Linux OS performance verifier self-test accepted an over-limit '$scenario' scale p99 ratio." + $assertions++ + + $absoluteTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($run in @($absoluteTampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 8 + })) { + $run.P99Microseconds = 11.0 + $run.EarlyP99Microseconds = 11.0 + $run.LateP99Microseconds = 11.0 + } + @($absoluteTampered.Summary | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 8 + })[0].MedianP99Microseconds = 11.0 + $absoluteOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + @($absoluteOsReport.results[0].performanceEvidence.validation.metrics | Where-Object { + [string]$_.profile -ceq 'LockFree' ` + -and [string]$_.scenario -ceq $scenario ` + -and [int64]$_.processCount -eq 8 + })[0].medianP99Microseconds = 11.0 + & $assertRawTamperRejected $absoluteTampered $absoluteOsReport ` + "Linux OS performance verifier self-test accepted an over-limit '$scenario' absolute p99." + $assertions++ + + $throughputTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($run in @($throughputTampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 8 + })) { + $run.MeasuredSeconds = 120.0 + $run.WallSeconds = 130.0 + $run.ApiCallsPerSecond = [double]$run.Operations / 120.0 + } + @($throughputTampered.Summary | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 8 + })[0].MedianApiCallsPerSecond = 550.0 + $throughputOsReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + @($throughputOsReport.results[0].performanceEvidence.validation.metrics | Where-Object { + [string]$_.profile -ceq 'LockFree' ` + -and [string]$_.scenario -ceq $scenario ` + -and [int64]$_.processCount -eq 8 + })[0].medianApiCallsPerSecond = 550.0 + & $assertRawTamperRejected $throughputTampered $throughputOsReport ` + "Linux OS performance verifier self-test accepted an under-limit '$scenario' 8-process throughput ratio." + $assertions++ + } + + $topologyTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $topologyTampered.Configuration.SyncKeyCanonicalBucketAssignments[1] = 1 + $topologyTampered | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $osReport.evidenceManifest[0].sha256 = $tamperedHash + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted a colliding synchronization-key topology.' + } + $assertions++ + + $raw | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = Get-FileSha256 $rawPath + $osReport.evidenceManifest[0].sha256 = Get-FileSha256 $rawPath + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $duplicateMetricReport = $osReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $duplicateMetricReport.results[0].performanceEvidence.validation.metrics[1] = + $duplicateMetricReport.results[0].performanceEvidence.validation.metrics[0] + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $duplicateMetricReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted duplicated declared metric tuples.' + } + $assertions++ + + $affinityTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $affinityTamperedEightProcessRun = @($affinityTampered.Runs | Where-Object { + [int64]$_.ProcessCount -eq 8 + })[0] + $affinityTamperedEightProcessRun.AssignedProcessors = @(64, 65, 66, 67, 68, 69, 70, 71) + $affinityTampered | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $osReport.evidenceManifest[0].sha256 = $tamperedHash + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted processor IDs outside its 64-bit affinity mask.' + } + $assertions++ + + $operationTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $operationTampered.Runs[0].Operations = [int64]$operationTampered.Runs[0].Cycles + $operationTampered.Runs[0].ApiCallsPerSecond = + [double]$operationTampered.Runs[0].Operations / [double]$operationTampered.Runs[0].MeasuredSeconds + $operationTampered | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $osReport.evidenceManifest[0].sha256 = $tamperedHash + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted fewer than two operations per completed cycle.' + } + $assertions++ + + $successTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $successTampered.Runs[0].StatusHistogram = [pscustomobject][ordered]@{ + 'Acquire.NotFound' = [int64]$successTampered.Runs[0].Operations + } + $successTampered | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $osReport.evidenceManifest[0].sha256 = $tamperedHash + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted a cycle set without exact paired success counts.' + } + $assertions++ + + $corruptionTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $corruptionTampered.Runs[0].StatusHistogram | Add-Member ` + -NotePropertyName 'Validation.ChecksumMismatch' -NotePropertyValue 1 + $corruptionTampered | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $osReport.evidenceManifest[0].sha256 = $tamperedHash + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted checksum/corruption evidence.' + } + $assertions++ + + $corruptReasonTampered = $raw | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $corruptReasonTampered.Runs[0].StatusHistogram | Add-Member ` + -NotePropertyName 'CorruptReason.Forged' -NotePropertyValue 1 + $corruptReasonTampered | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $tamperedHash = Get-FileSha256 $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = $tamperedHash + $osReport.evidenceManifest[0].sha256 = $tamperedHash + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $rejected = $false + try { [void](Assert-LinuxTinyOsPerformanceEvidence $osReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux OS performance verifier self-test accepted a corruption-reason row.' + } + $assertions++ + + $raw | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $rawPath + $osReport.results[0].performanceEvidence.reportSha256 = Get-FileSha256 $rawPath + $osReport.evidenceManifest[0].sha256 = Get-FileSha256 $rawPath + $osReport.evidenceManifest[0].length = (Get-Item -LiteralPath $rawPath).Length + $osReport | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $reportPath + return $assertions +} + +function Assert-OsResultCommandEvidence { + param( + [Parameter(Mandatory)]$Row, + [Parameter(Mandatory)][string]$Context) + + $members = @{} + foreach ($property in @('command', 'stdout', 'stderr', 'stdoutSha256', 'stderrSha256')) { + $member = $Row.PSObject.Properties[$property] + if ($null -eq $member) { + throw "$Context is missing nullable execution-evidence property '$property'." + } + $members[$property] = $member.Value + } + + if ($null -eq $members.command) { + foreach ($property in @('stdout', 'stderr', 'stdoutSha256', 'stderrSha256')) { + if ($null -ne $members[$property]) { + throw "$Context has '$property' evidence without a command." + } + } + return $false + } + + if ($members.command -isnot [string] -or [string]::IsNullOrWhiteSpace($members.command)) { + throw "$Context has an empty or non-string command." + } + foreach ($stream in @('stdout', 'stderr')) { + if ($members[$stream] -isnot [string] -or [string]::IsNullOrWhiteSpace($members[$stream])) { + throw "$Context has an empty or non-string $stream path." + } + $digest = $members[$stream + 'Sha256'] + if ($digest -isnot [string] -or $digest -notmatch '^[0-9A-F]{64}$') { + throw "$Context has no SHA-256-bound $stream evidence." + } + } + return $true +} + +function Assert-ExactReleaseOsRows { + param( + [Parameter(Mandatory)]$Report, + [Parameter(Mandatory)][string]$PlatformId, + [Parameter(Mandatory)][string]$EvidencePath) + + if ([string]$Report.command -ne 'all') { + throw "Release OS evidence '$EvidencePath' must have command=all." + } + $expected = Get-ExpectedReleaseOsRows $PlatformId + $rows = @($Report.results) + if ($rows.Count -ne $expected.Count) { + throw "Release OS evidence '$EvidencePath' has $($rows.Count) rows; expected exactly $($expected.Count)." + } + $duplicates = @($rows | Group-Object name | Where-Object Count -ne 1) + if ($duplicates.Count -ne 0) { + throw "Release OS evidence '$EvidencePath' has missing/duplicate result names: $($duplicates.Name -join ',')." + } + foreach ($entry in $expected.GetEnumerator()) { + $rowMatches = @($rows | Where-Object { [string]$_.name -ceq [string]$entry.Key }) + if ($rowMatches.Count -ne 1) { + throw "Release OS evidence '$EvidencePath' is missing exact row '$($entry.Key)'." + } + $required = Get-StrictBoolean $rowMatches[0] 'required' "OS row '$($entry.Key)'" + if ($required -ne [bool]$entry.Value) { + throw "Release OS row '$($entry.Key)' has required=$required; expected $($entry.Value)." + } + $status = Get-RequiredPropertyValue $rowMatches[0] 'status' "OS row '$($entry.Key)'" + if ($status -isnot [string] -or ($required -and $status -ne 'pass') ` + -or (-not $required -and $status -notin @('pass', 'not-qualified'))) { + throw "Release OS row '$($entry.Key)' has invalid status '$status'." + } + [void](Get-StrictDouble $rowMatches[0] 'elapsedSeconds' "OS row '$($entry.Key)'" 0 [double]::MaxValue) + $timeoutSeconds = Get-StrictInt64 $rowMatches[0] 'timeoutSeconds' "OS row '$($entry.Key)'" 0 [int32]::MaxValue + $timedOut = Get-StrictBoolean $rowMatches[0] 'timedOut' "OS row '$($entry.Key)'" + $hasCommandEvidence = Assert-OsResultCommandEvidence $rowMatches[0] "OS row '$($entry.Key)'" + $requiresCommand = $status -ceq 'pass' ` + -and [string]$entry.Key -cnotlike 'self-test-*' + if ($timedOut) { + throw "Release OS row '$($entry.Key)' timed out." + } + if ($requiresCommand -and -not $hasCommandEvidence) { + throw "Release OS passing executable row '$($entry.Key)' cannot omit its bounded command/log evidence." + } + if ($hasCommandEvidence) { + $exitCode = Get-StrictInt64 $rowMatches[0] 'exitCode' "OS row '$($entry.Key)'" ` + ([int64]-1) [int32]::MaxValue + if ($timeoutSeconds -le 0 -or $exitCode -ne 0) { + throw "Release OS executable row '$($entry.Key)' lacks bounded successful command/log evidence." + } + } + } + $unexpected = @($rows | Where-Object { -not $expected.Contains([string]$_.name) }) + if ($unexpected.Count -ne 0) { + throw "Release OS evidence '$EvidencePath' has unexpected rows: $($unexpected.name -join ',')." + } + + $cleanRow = @($rows | Where-Object { [string]$_.name -ceq 'clean' })[0] + $preclean = Get-RequiredPropertyValue $cleanRow 'preclean' "OS evidence '$EvidencePath' clean row" + if ((Get-StrictInt64 $preclean 'schemaVersion' "OS evidence '$EvidencePath' preclean" 1 1) -ne 1) { + throw "Release OS evidence '$EvidencePath' has an unsupported pre-clean schema." + } + $projectCount = Get-StrictInt64 $preclean 'solutionProjectCount' "OS evidence '$EvidencePath' preclean" 1 [int32]::MaxValue + $uniqueProjectCount = Get-StrictInt64 $preclean 'uniqueSolutionProjectCount' "OS evidence '$EvidencePath' preclean" 1 [int32]::MaxValue + $targetCount = Get-StrictInt64 $preclean 'targetCount' "OS evidence '$EvidencePath' preclean" 2 [int32]::MaxValue + $uniqueTargetCount = Get-StrictInt64 $preclean 'uniqueTargetCount' "OS evidence '$EvidencePath' preclean" 2 [int32]::MaxValue + $existedBeforeCount = Get-StrictInt64 $preclean 'existedBeforeCount' "OS evidence '$EvidencePath' preclean" 0 $targetCount + $removedCount = Get-StrictInt64 $preclean 'removedCount' "OS evidence '$EvidencePath' preclean" 0 $targetCount + $verifiedAbsentCount = Get-StrictInt64 $preclean 'verifiedAbsentCount' "OS evidence '$EvidencePath' preclean" 0 $targetCount + $protectedFileCount = Get-StrictInt64 $preclean 'protectedFileCount' "OS evidence '$EvidencePath' preclean" 0 [int32]::MaxValue + if ($uniqueProjectCount -ne $projectCount ` + -or $targetCount -ne (2 * $projectCount) ` + -or $uniqueTargetCount -ne $targetCount ` + -or $existedBeforeCount -ne $removedCount ` + -or $verifiedAbsentCount -ne $targetCount ` + -or $protectedFileCount -ne 0) { + throw "Release OS evidence '$EvidencePath' lacks exact, unique, protected-file-free pre-clean coverage." + } + foreach ($hashProperty in @('solutionProjectSetSha256', 'targetSetSha256', 'reportSha256')) { + $hash = Get-RequiredPropertyValue $preclean $hashProperty "OS evidence '$EvidencePath' preclean" + if ($hash -isnot [string] -or $hash -notmatch '^[0-9A-F]{64}$') { + throw "Release OS evidence '$EvidencePath' preclean.$hashProperty is not a SHA-256 digest." + } + } + $precleanReportPath = Get-RequiredPropertyValue $preclean 'reportPath' "OS evidence '$EvidencePath' preclean" + if ($precleanReportPath -isnot [string] ` + -or [string]::IsNullOrWhiteSpace($precleanReportPath) ` + -or $precleanReportPath.Replace('\', '/') -notmatch '^artifacts/.+\.evidence/preclean\.json$') { + throw "Release OS evidence '$EvidencePath' has an invalid pre-clean sidecar path." + } + $manifestMatches = @($Report.evidenceManifest | Where-Object { + ([string]$_.path).Replace('\', '/') -ceq $precleanReportPath.Replace('\', '/') ` + -and [string]$_.sha256 -ceq [string]$preclean.reportSha256 + }) + if ($manifestMatches.Count -ne 1) { + throw "Release OS evidence '$EvidencePath' does not uniquely manifest its pre-clean sidecar and digest." + } + if ($PlatformId -eq 'linux-x64') { + return Assert-LinuxTinyOsPerformanceEvidence $Report $EvidencePath + } + $windowsPerformanceRow = @($rows | Where-Object { [string]$_.name -ceq 'linux-tiny-performance' }) + if ($windowsPerformanceRow.Count -ne 1 ` + -or (Get-StrictBoolean $windowsPerformanceRow[0] 'required' 'Windows optional Linux performance row') ` + -or [string]$windowsPerformanceRow[0].status -cne 'not-qualified' ` + -or $null -ne $windowsPerformanceRow[0].command) { + throw "Windows OS evidence '$EvidencePath' must retain one optional non-executed Linux performance row." + } + return $null +} + +function Assert-OsReportProvenance { + param( + [Parameter(Mandatory)]$Report, + [Parameter(Mandatory)][string]$EvidencePath) + + foreach ($sectionName in @('provenance', 'completionProvenance')) { + $section = Get-RequiredPropertyValue $Report $sectionName "OS evidence '$EvidencePath'" + foreach ($property in @('repositoryCommit', 'headTree', 'workingTreeState', 'statusSha256', 'sourceManifestSha256')) { + $value = Get-RequiredPropertyValue $section $property "OS evidence '$EvidencePath'.$sectionName" + if ($value -isnot [string] -or [string]::IsNullOrWhiteSpace($value) -or $value -eq 'unknown') { + throw "OS evidence '$EvidencePath'.$sectionName.$property is unknown or invalid." + } + } + } + foreach ($property in @('repositoryCommit', 'headTree', 'workingTreeState', 'statusSha256', 'sourceManifestSha256')) { + if ([string]$Report.provenance.$property -ne [string]$Report.completionProvenance.$property) { + throw "OS evidence '$EvidencePath' changed provenance property '$property' during execution." + } + } + $runnerToOsProperty = [ordered]@{ + commit = 'repositoryCommit' + headTree = 'headTree' + workingTreeState = 'workingTreeState' + statusSha256 = 'statusSha256' + sourceManifestSha256 = 'sourceManifestSha256' + } + foreach ($entry in $runnerToOsProperty.GetEnumerator()) { + if ([string]$Report.provenance.($entry.Value) -ne [string]$repositoryProvenance[$entry.Key]) { + throw "OS evidence '$EvidencePath' does not match qualification provenance '$($entry.Key)'." + } + } + $assemblies = @($Report.testedAssemblies) + $completionAssemblies = @($Report.completionTestedAssemblies) + if ($assemblies.Count -eq 0 -or $completionAssemblies.Count -ne $assemblies.Count ` + -or @($assemblies | Group-Object path | Where-Object Count -ne 1).Count -ne 0 ` + -or @($completionAssemblies | Group-Object path | Where-Object Count -ne 1).Count -ne 0) { + throw "OS evidence '$EvidencePath' lacks stable unique tested-assembly manifests." + } + foreach ($manifest in @($assemblies, $completionAssemblies)) { + foreach ($assembly in @($manifest)) { + if ([string]::IsNullOrWhiteSpace([string]$assembly.path) ` + -or [string]$assembly.sha256 -notmatch '^[0-9A-F]{64}$' ` + -or (Get-StrictInt64 $assembly 'length' "OS tested assembly '$($assembly.path)'" 1 [int64]::MaxValue) -le 0) { + throw "OS evidence '$EvidencePath' has an invalid tested assembly row." + } + } + } + $startCanonical = @($assemblies | Sort-Object path | ForEach-Object { + "$($_.path)|$($_.length)|$($_.sha256)" + }) -join "`n" + $completionCanonical = @($completionAssemblies | Sort-Object path | ForEach-Object { + "$($_.path)|$($_.length)|$($_.sha256)" + }) -join "`n" + if ($startCanonical -cne $completionCanonical) { + throw "OS evidence '$EvidencePath' tested assembly manifest changed during execution." + } +} + +function Assert-NoReparsePath { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][string]$Context) + + $fullPath = [IO.Path]::GetFullPath($Path) + $artifactBoundary = [IO.Path]::GetFullPath((Join-Path $root 'artifacts')) + $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + if (-not $fullPath.Equals($artifactBoundary, $comparison) ` + -and -not $fullPath.StartsWith($artifactBoundary + [IO.Path]::DirectorySeparatorChar, $comparison)) { + throw "$Context resolves outside the repository artifacts root." + } + $current = $fullPath + while ($true) { + if (Test-Path -LiteralPath $current) { + $item = Get-Item -LiteralPath $current -Force + if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "$Context traverses reparse point '$current'." + } + } + if ($current.Equals($artifactBoundary, $comparison)) { + break + } + $parent = [IO.Path]::GetDirectoryName($current) + if ([string]::IsNullOrWhiteSpace($parent) -or $parent.Equals($current, $comparison)) { + throw "$Context cannot be proven below the artifacts root." + } + $current = $parent + } +} + +function Assert-OsEvidenceTree { + param( + [Parameter(Mandatory)]$Report, + [Parameter(Mandatory)][string]$EvidencePath) + + $reportPath = [IO.Path]::GetFullPath($EvidencePath) + if (-not (Test-Path -LiteralPath $reportPath -PathType Leaf)) { + throw "OS evidence report '$reportPath' is missing." + } + Assert-NoReparsePath $reportPath "OS evidence report '$reportPath'" + $evidenceRoot = [IO.Path]::GetFullPath((Join-Path ` + (Split-Path -Parent $reportPath) ` + ([IO.Path]::GetFileNameWithoutExtension($reportPath) + '.evidence'))) + if (-not (Test-Path -LiteralPath $evidenceRoot -PathType Container)) { + throw "OS evidence '$reportPath' is missing exact sibling evidence root '$evidenceRoot'." + } + Assert-NoReparsePath $evidenceRoot "OS evidence root '$evidenceRoot'" + + $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + $pathComparer = if ($IsWindows) { [StringComparer]::OrdinalIgnoreCase } else { [StringComparer]::Ordinal } + $evidencePrefix = $evidenceRoot + [IO.Path]::DirectorySeparatorChar + $manifestRows = @(Get-RequiredPropertyValue $Report 'evidenceManifest' "OS evidence '$reportPath'") + if ($manifestRows.Count -eq 0) { + throw "OS evidence '$reportPath' has an empty evidence manifest." + } + $manifestByPath = [Collections.Generic.Dictionary[string, object]]::new($pathComparer) + foreach ($entry in $manifestRows) { + $manifestPath = Get-StrictString $entry 'path' "OS evidence '$reportPath' manifest row" + if ([IO.Path]::IsPathFullyQualified($manifestPath)) { + throw "OS evidence manifest path '$manifestPath' must be repository relative." + } + $fullPath = [IO.Path]::GetFullPath((Join-Path $root $manifestPath)) + if (-not $fullPath.StartsWith($evidencePrefix, $comparison)) { + throw "OS evidence manifest path '$manifestPath' escapes exact sibling root '$evidenceRoot'." + } + $canonicalRepositoryPath = [IO.Path]::GetRelativePath($root, $fullPath).Replace('\', '/') + if ($manifestPath.Replace('\', '/') -cne $canonicalRepositoryPath) { + throw "OS evidence manifest path '$manifestPath' is not normalized as '$canonicalRepositoryPath'." + } + if (-not $manifestByPath.TryAdd($fullPath, $entry)) { + throw "OS evidence manifest duplicates normalized path '$manifestPath'." + } + Assert-NoReparsePath $fullPath "OS evidence manifest path '$manifestPath'" + if (-not (Test-Path -LiteralPath $fullPath -PathType Leaf)) { + throw "OS evidence manifest file '$manifestPath' is missing." + } + $length = Get-StrictInt64 $entry 'length' "OS evidence manifest '$manifestPath'" 0 [int64]::MaxValue + $hash = Get-StrictString $entry 'sha256' "OS evidence manifest '$manifestPath'" + $actual = Get-Item -LiteralPath $fullPath -Force + if ([int64]$actual.Length -ne $length -or $hash -notmatch '^[0-9A-F]{64}$' ` + -or $hash -cne (Get-FileSha256 $fullPath)) { + throw "OS evidence manifest length/hash mismatch for '$manifestPath'." + } + } + + $allItems = @(Get-ChildItem -LiteralPath $evidenceRoot -Recurse -Force) + $reparseItems = @($allItems | Where-Object { + ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 + }) + if ($reparseItems.Count -ne 0) { + throw "OS evidence tree contains reparse entries: $($reparseItems.FullName -join ', ')." + } + $actualFiles = @($allItems | Where-Object { -not $_.PSIsContainer }) + $actualPaths = [Collections.Generic.HashSet[string]]::new($pathComparer) + foreach ($file in $actualFiles) { + if (-not $actualPaths.Add([IO.Path]::GetFullPath($file.FullName))) { + throw "OS evidence tree enumerated duplicate path '$($file.FullName)'." + } + } + if ($actualPaths.Count -ne $manifestByPath.Count ` + -or @($actualPaths | Where-Object { -not $manifestByPath.ContainsKey($_) }).Count -ne 0 ` + -or @($manifestByPath.Keys | Where-Object { -not $actualPaths.Contains($_) }).Count -ne 0) { + throw "OS evidence '$reportPath' manifest is not the exact actual file set (manifest=$($manifestByPath.Count), actual=$($actualPaths.Count))." + } + + foreach ($row in @($Report.results)) { + $statusMember = $row.PSObject.Properties['status'] + $nameMember = $row.PSObject.Properties['name'] + $requiresCommand = $null -ne $statusMember ` + -and [string]$statusMember.Value -ceq 'pass' ` + -and $null -ne $nameMember ` + -and [string]$nameMember.Value -cnotlike 'self-test-*' + $hasCommandEvidence = Assert-OsResultCommandEvidence $row "OS result row '$($row.name)'" + if ($requiresCommand -and -not $hasCommandEvidence) { + throw "OS passing executable row '$($row.name)' cannot omit its command and manifested logs." + } + if (-not $hasCommandEvidence) { + continue + } + foreach ($stream in @('stdout', 'stderr')) { + $streamPath = Get-StrictString $row $stream "OS executable row '$($row.name)'" + if ([IO.Path]::IsPathFullyQualified($streamPath)) { + throw "OS executable row '$($row.name)' $stream path must be repository relative." + } + $fullStreamPath = [IO.Path]::GetFullPath((Join-Path $root $streamPath)) + if (-not $manifestByPath.ContainsKey($fullStreamPath)) { + throw "OS executable row '$($row.name)' $stream log is not uniquely present in the exact evidence manifest." + } + $declaredHash = Get-StrictString $row ($stream + 'Sha256') "OS executable row '$($row.name)'" + if ($declaredHash -cne [string]$manifestByPath[$fullStreamPath].sha256 ` + -or $declaredHash -cne (Get-FileSha256 $fullStreamPath)) { + throw "OS executable row '$($row.name)' $stream hash is not bound to its manifested log." + } + } + } + + $canonicalRows = [Collections.Generic.List[string]]::new() + foreach ($path in $manifestByPath.Keys) { + $entry = $manifestByPath[$path] + $canonicalRows.Add( + ([IO.Path]::GetRelativePath($evidenceRoot, $path).Replace('\', '/') + + "|$($entry.length)|$($entry.sha256)")) + } + $canonicalRows.Sort([StringComparer]::Ordinal) + return [pscustomobject][ordered]@{ + reportPath = [IO.Path]::GetRelativePath($root, $reportPath) + reportSha256 = Get-FileSha256 $reportPath + evidenceRoot = [IO.Path]::GetRelativePath($root, $evidenceRoot) + evidenceFileCount = $manifestByPath.Count + evidenceTreeSha256 = Get-StringSha256 (@($canonicalRows) -join "`n") + } +} + +function Assert-AcceptedOsEvidenceStable { + [int]$validated = 0 + foreach ($accepted in $acceptedOsEvidence) { + $reportPath = if ([IO.Path]::IsPathFullyQualified([string]$accepted.reportPath)) { + [IO.Path]::GetFullPath([string]$accepted.reportPath) + } + else { + [IO.Path]::GetFullPath((Join-Path $root ([string]$accepted.reportPath))) + } + $report = Get-Content -LiteralPath $reportPath -Raw | ConvertFrom-Json -Depth 30 + $current = Assert-OsEvidenceTree $report $reportPath + if ([string]$current.reportSha256 -cne [string]$accepted.reportSha256 ` + -or [string]$current.evidenceTreeSha256 -cne [string]$accepted.evidenceTreeSha256 ` + -or [int64]$current.evidenceFileCount -ne [int64]$accepted.evidenceFileCount) { + throw "Accepted OS evidence '$reportPath' changed before qualification completion." + } + $validated++ + } + return $validated +} + +function Invoke-OsEvidenceManifestVerifierSelfTest { + $reportPath = Join-Path $runRoot 'os-evidence-manifest-self-test.json' + $treeRoot = Join-Path $runRoot 'os-evidence-manifest-self-test.evidence' + New-Item -ItemType Directory -Path $treeRoot | Out-Null + $stdout = Join-Path $treeRoot 'synthetic.stdout.log' + $stderr = Join-Path $treeRoot 'synthetic.stderr.log' + $structural = Join-Path $treeRoot 'structural.json' + [IO.File]::WriteAllText($stdout, "synthetic stdout`n") + [IO.File]::WriteAllText($stderr, "synthetic stderr`n") + [IO.File]::WriteAllText($structural, "{}`n") + [IO.File]::WriteAllText($reportPath, "{}`n") + $manifest = @(@($stdout, $stderr, $structural) | ForEach-Object { + $file = Get-Item -LiteralPath $_ + [pscustomobject][ordered]@{ + path = [IO.Path]::GetRelativePath($root, $file.FullName) + length = $file.Length + sha256 = Get-FileSha256 $file.FullName + } + }) + $report = [pscustomobject][ordered]@{ + results = @( + [pscustomobject][ordered]@{ + name = 'synthetic'; status = 'pass'; required = $true; command = 'synthetic command' + stdout = [IO.Path]::GetRelativePath($root, $stdout) + stderr = [IO.Path]::GetRelativePath($root, $stderr) + stdoutSha256 = Get-FileSha256 $stdout + stderrSha256 = Get-FileSha256 $stderr + }, + [pscustomobject][ordered]@{ + name = 'self-test-synthetic'; status = 'pass'; required = $true; command = $null + stdout = $null; stderr = $null; stdoutSha256 = $null; stderrSha256 = $null + }, + [pscustomobject][ordered]@{ + name = 'optional-synthetic'; status = 'not-qualified'; required = $false; command = $null + stdout = $null; stderr = $null; stdoutSha256 = $null; stderrSha256 = $null + }) + evidenceManifest = $manifest + } + [void](Assert-OsEvidenceTree $report $reportPath) + [int]$assertions = 3 + + $invalidFields = @( + @{ Name = 'empty structural command'; Row = 1; Property = 'command'; Value = '' }, + @{ Name = 'whitespace optional command'; Row = 2; Property = 'command'; Value = ' ' }, + @{ Name = 'empty executable stdout'; Row = 0; Property = 'stdout'; Value = '' }, + @{ Name = 'whitespace executable stderr'; Row = 0; Property = 'stderr'; Value = "`t" }) + foreach ($case in $invalidFields) { + $invalidReport = $report | ConvertTo-Json -Depth 10 | ConvertFrom-Json + $invalidReport.results[$case.Row].($case.Property) = $case.Value + $rejected = $false + try { [void](Assert-OsEvidenceTree $invalidReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { + throw "OS evidence verifier self-test accepted $($case.Name) pseudo-evidence." + } + $assertions++ + } + + $extra = Join-Path $treeRoot 'unexpected.log' + [IO.File]::WriteAllText($extra, 'unexpected') + $rejected = $false + try { [void](Assert-OsEvidenceTree $report $reportPath) } catch { $rejected = $true } + Remove-Item -LiteralPath $extra + if (-not $rejected) { throw 'OS evidence verifier self-test accepted an unmanifested extra file.' } + $assertions++ + + [IO.File]::WriteAllText($stdout, 'tampered') + $rejected = $false + try { [void](Assert-OsEvidenceTree $report $reportPath) } catch { $rejected = $true } + [IO.File]::WriteAllText($stdout, "synthetic stdout`n") + if (-not $rejected) { throw 'OS evidence verifier self-test accepted a content/hash mismatch.' } + $assertions++ + + $tamperedReport = $report | ConvertTo-Json -Depth 10 | ConvertFrom-Json + $tamperedReport.results[0].stdoutSha256 = ('A' * 64) + $rejected = $false + try { [void](Assert-OsEvidenceTree $tamperedReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { throw 'OS evidence verifier self-test accepted an unbound command log digest.' } + $assertions++ + + $commandlessReport = $report | ConvertTo-Json -Depth 10 | ConvertFrom-Json + $commandlessReport.results[0].command = $null + $commandlessReport.results[0].name = 'clean' + $commandlessReport.results[0].stdout = $null + $commandlessReport.results[0].stderr = $null + $commandlessReport.results[0].stdoutSha256 = $null + $commandlessReport.results[0].stderrSha256 = $null + $commandlessReport.evidenceManifest = @($commandlessReport.evidenceManifest | Where-Object { + [string]$_.path -notin @( + [IO.Path]::GetRelativePath($root, $stdout), + [IO.Path]::GetRelativePath($root, $stderr)) + }) + Remove-Item -LiteralPath $stdout, $stderr + $rejected = $false + try { [void](Assert-OsEvidenceTree $commandlessReport $reportPath) } catch { $rejected = $true } + [IO.File]::WriteAllText($stdout, "synthetic stdout`n") + [IO.File]::WriteAllText($stderr, "synthetic stderr`n") + if (-not $rejected) { throw 'OS evidence verifier self-test accepted a passing clean row with command/logs removed consistently.' } + $assertions++ + + $escapedReport = $report | ConvertTo-Json -Depth 10 | ConvertFrom-Json + $escapedReport.evidenceManifest[0].path = 'specs/009-lock-free-publish-read/qualification-config.json' + $rejected = $false + try { [void](Assert-OsEvidenceTree $escapedReport $reportPath) } catch { $rejected = $true } + if (-not $rejected) { throw 'OS evidence verifier self-test accepted an out-of-root manifest path.' } + $assertions++ + return $assertions +} + +function Read-OsEvidence { + param([Parameter(Mandatory)][string]$Path) + + $fullPath = if ([IO.Path]::IsPathFullyQualified($Path)) { + [IO.Path]::GetFullPath($Path) + } + else { + [IO.Path]::GetFullPath((Join-Path $root $Path)) + } + $pathComparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + if (-not $fullPath.StartsWith($allowedRoot, $pathComparison)) { + throw "OS evidence '$fullPath' must remain below '$allowedRoot'." + } + if (-not (Test-Path -LiteralPath $fullPath -PathType Leaf)) { + throw "OS evidence '$fullPath' does not exist." + } + $report = Get-Content -LiteralPath $fullPath -Raw | ConvertFrom-Json + if ((Get-StrictInt64 $report 'schemaVersion' "OS evidence '$fullPath'" 3 3) -ne 3 ` + -or (Get-StrictBoolean $report 'validationOnly' "OS evidence '$fullPath'")) { + throw "OS evidence '$fullPath' is not an executable exact schema-v3 result." + } + $platformValue = Get-RequiredPropertyValue $report 'platform' "OS evidence '$fullPath'" + $architectureValue = Get-RequiredPropertyValue $report 'architecture' "OS evidence '$fullPath'" + if ($platformValue -isnot [string] -or $architectureValue -isnot [string]) { + throw "OS evidence '$fullPath' platform and architecture must be strings." + } + $platformId = ($platformValue + '-' + $architectureValue).ToLowerInvariant() + return [pscustomobject]@{ + path = $fullPath + relativePath = [IO.Path]::GetRelativePath($root, $fullPath) + sha256 = Get-FileSha256 $fullPath + report = $report + platformId = $platformId + } +} + +function Assert-OsEvidenceSet { + param( + [Parameter(Mandatory)][string]$CurrentPath, + [string[]]$AdditionalPaths = @()) + + $paths = @($CurrentPath) + @($AdditionalPaths) + $evidence = [Collections.Generic.List[object]]::new() + foreach ($path in $paths) { + try { + $item = Read-OsEvidence $path + } + catch { + Add-EvidenceResult 'dual-platform-os-evidence' 'not-qualified' 'invalid-os-evidence' @($_.Exception.Message) + $notQualifiedReasons.Add("dual-platform-os-evidence: $($_.Exception.Message)") + return + } + + $report = $item.report + try { + $treeSnapshot = Assert-OsEvidenceTree $report $item.path + if ([string]$treeSnapshot.reportSha256 -cne [string]$item.sha256) { + throw "OS evidence '$($item.path)' changed while its exact evidence tree was being validated." + } + } + catch { + Add-EvidenceResult "os-evidence-$($item.platformId)" 'not-qualified' 'invalid-evidence-tree' @( + $_.Exception.Message) @($item.relativePath) + $notQualifiedReasons.Add("os-evidence-$($item.platformId): invalid exact evidence tree") + continue + } + try { + Assert-OsReportProvenance $report $item.path + } + catch { + Add-EvidenceResult "os-evidence-$($item.platformId)" 'not-qualified' 'source-provenance-mismatch' @( + $_.Exception.Message) @($item.relativePath) + $notQualifiedReasons.Add("os-evidence-$($item.platformId): mismatched source or binary provenance") + continue + } + if ([string]$report.overallStatus -eq 'fail') { + Add-EvidenceResult "os-evidence-$($item.platformId)" 'failed' 'os-validation-failed' @( + "sha256=$($item.sha256)") @($item.relativePath) + throw "OS validation failed for $($item.platformId); see '$($item.path)'." + } + $performanceArtifact = $null + if ($Tier -eq 'release') { + try { + $performanceArtifact = Assert-ExactReleaseOsRows $report $item.platformId $item.path + } + catch { + Add-EvidenceResult "os-evidence-$($item.platformId)" 'not-qualified' 'invalid-or-partial-os-evidence' @( + $_.Exception.Message) @($item.relativePath) + $notQualifiedReasons.Add("os-evidence-$($item.platformId): invalid or partial command=all evidence") + continue + } + } + if ([string]$report.configuration -ne 'Release') { + Add-EvidenceResult "os-evidence-$($item.platformId)" 'not-qualified' 'non-release-os-evidence' @( + "configuration=$($report.configuration)") @($item.relativePath) + $notQualifiedReasons.Add("os-evidence-$($item.platformId): configuration is not Release") + continue + } + $qualifiedArchitecture = Get-StrictBoolean $report 'qualifiedArchitecture' "OS evidence '$($item.path)'" + if ([string]$report.overallStatus -ne 'pass' -or -not $qualifiedArchitecture) { + Add-EvidenceResult "os-evidence-$($item.platformId)" 'not-qualified' 'os-validation-not-qualified' @( + "overallStatus=$($report.overallStatus)", + "qualifiedArchitecture=$($report.qualifiedArchitecture)", + "sha256=$($item.sha256)") @($item.relativePath) + $notQualifiedReasons.Add("os-evidence-$($item.platformId): validation is not qualified") + continue + } + if (@($report.results | Where-Object { $_.required -eq $true -and $_.status -ne 'pass' }).Count -ne 0) { + Add-EvidenceResult "os-evidence-$($item.platformId)" 'failed' 'required-os-row-not-passed' @( + "sha256=$($item.sha256)") @($item.relativePath) + throw "OS evidence '$($item.path)' claims pass while a required row is not pass." + } + if (@($evidence | Where-Object platformId -eq $item.platformId).Count -ne 0) { + Add-EvidenceResult "os-evidence-$($item.platformId)-duplicate" 'not-qualified' 'duplicate-platform-evidence' @( + "path=$($item.relativePath)") @($item.relativePath) + $notQualifiedReasons.Add("os-evidence-$($item.platformId): duplicate platform evidence") + continue + } + if ((Get-FileSha256 $item.path) -cne [string]$treeSnapshot.reportSha256) { + Add-EvidenceResult "os-evidence-$($item.platformId)" 'not-qualified' 'os-report-changed-during-validation' @( + "path=$($item.relativePath)") @($item.relativePath) + $notQualifiedReasons.Add("os-evidence-$($item.platformId): report changed during validation") + continue + } + + $evidence.Add($item) + $acceptedOsEvidence.Add([pscustomobject][ordered]@{ + platformId = $item.platformId + reportPath = $treeSnapshot.reportPath + reportSha256 = $treeSnapshot.reportSha256 + evidenceRoot = $treeSnapshot.evidenceRoot + evidenceFileCount = $treeSnapshot.evidenceFileCount + evidenceTreeSha256 = $treeSnapshot.evidenceTreeSha256 + }) + $acceptedArtifacts = @($item.relativePath) + if (-not [string]::IsNullOrWhiteSpace([string]$performanceArtifact)) { + $acceptedArtifacts += [string]$performanceArtifact + } + Add-EvidenceResult "os-evidence-$($item.platformId)" 'passed' 'same-source-release-os-validation-pass' @( + "sha256=$($item.sha256)", + "commit=$($repositoryProvenance.commit)", + "sourceManifestSha256=$($repositoryProvenance.sourceManifestSha256)", + "evidenceFiles=$($treeSnapshot.evidenceFileCount)", + "evidenceTreeSha256=$($treeSnapshot.evidenceTreeSha256)") $acceptedArtifacts + } + + $requiredPlatforms = if ($Tier -eq 'release') { + @($config.platforms | ForEach-Object { [string]$_ }) + } + else { + @($(if ($IsWindows) { 'windows' } elseif ($IsLinux) { 'linux' } else { 'unsupported' }) + '-' + + [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()) + } + $passedPlatforms = @($evidence | ForEach-Object platformId | Sort-Object -Unique) + $missing = @($requiredPlatforms | Where-Object { $_ -notin $passedPlatforms }) + if ($missing.Count -ne 0) { + Add-EvidenceResult 'dual-platform-os-evidence' 'not-qualified' 'required-platform-evidence-missing' @( + "required=$($requiredPlatforms -join ',')", + "passed=$($passedPlatforms -join ',')", + "missing=$($missing -join ',')") @($evidence.relativePath) + $notQualifiedReasons.Add("dual-platform-os-evidence: missing $($missing -join ',')") + return + } + Add-EvidenceResult 'dual-platform-os-evidence' 'passed' $(if ($Tier -eq 'release') { 'windows-and-linux-release-qualified' } else { 'current-platform-smoke-qualified' }) @( + "platforms=$($passedPlatforms -join ',')", + "commit=$($repositoryProvenance.commit)", + "sourceManifestSha256=$($repositoryProvenance.sourceManifestSha256)") @($evidence.relativePath) +} + +function Get-EvidenceManifest { + $summaryPath = Join-Path $runRoot 'summary.json' + return @(Get-ChildItem -LiteralPath $runRoot -Recurse -File | Where-Object { + $_.FullName -ne $summaryPath + } | Sort-Object FullName | ForEach-Object { + [pscustomobject][ordered]@{ + path = [IO.Path]::GetRelativePath($root, $_.FullName) + length = $_.Length + sha256 = Get-FileSha256 $_.FullName + } + }) +} + +function Assert-Sc017Evidence { + param([int64]$ExpectedRepetitions) + + $step = Get-StepResult 'directory-generation-stress' + $stdout = Get-Content -LiteralPath (Join-Path $root $step.stdout) -Raw + $expectedSeedHex = ([uint64](Get-StrictInt64 $config 'seed' 'qualification config' 0 [int32]::MaxValue)).ToString('X16') + $transitionCount = Get-Sc017SourceTransitionCount + $startPattern = 'SC017 start: seed=0x' + [regex]::Escape($expectedSeedHex) + + '; configuredRepetitions=' + [regex]::Escape([string]$ExpectedRepetitions) + + '; transitionCount=' + [regex]::Escape([string]$transitionCount) + + '; distribution=quotient-plus-remainder\.' + if ($stdout -notmatch $startPattern) { + Fail-StepValidation 'directory-generation-stress' ` + "Missing exact SC017 start evidence for $ExpectedRepetitions repetitions across $transitionCount transitions." + } + + $transitionMarkers = [regex]::Matches( + $stdout, + 'SC017 transition=(?[a-z0-9-]+); seed=0x[0-9A-F]{16}; repetitions=(?[1-9][0-9]*); result=pass\.') + $uniqueNames = @($transitionMarkers | ForEach-Object { $_.Groups['name'].Value } | Sort-Object -Unique) + [int64]$markerRepetitions = 0 + foreach ($marker in $transitionMarkers) { + $markerRepetitions += [Convert]::ToInt64( + $marker.Groups['repetitions'].Value, + [Globalization.CultureInfo]::InvariantCulture) + } + if ($transitionMarkers.Count -ne $transitionCount ` + -or $uniqueNames.Count -ne $transitionCount ` + -or $markerRepetitions -ne $ExpectedRepetitions) { + Fail-StepValidation 'directory-generation-stress' ` + "SC017 transition evidence must contain exactly $transitionCount unique passing markers whose repetitions sum to $ExpectedRepetitions; markers=$($transitionMarkers.Count), unique=$($uniqueNames.Count), sum=$markerRepetitions." + } + + $pattern = 'SC017 complete: seed=0x' + [regex]::Escape($expectedSeedHex) + '; executedRepetitions=' + + [regex]::Escape([string]$ExpectedRepetitions) + + '; wrongGenerationMutations=0; corruption=0; falseMisses=0; leakedCapacity=0\.' + if ($stdout -notmatch $pattern) { + Fail-StepValidation 'directory-generation-stress' ` + "Missing exact SC017 completion evidence for $ExpectedRepetitions repetitions and zero failures/leaks." + } + + Set-StepValidation 'directory-generation-stress' 'passed' 'sc017-qualified-count-and-correctness' @( + "executedRepetitions=$ExpectedRepetitions", + "transitionCount=$transitionCount", + "uniqueTransitionMarkers=$($uniqueNames.Count)", + "seed=0x$expectedSeedHex", + 'wrongGenerationMutations=0', + 'corruption=0', + 'falseMisses=0', + 'leakedCapacity=0') +} + +function Assert-SuspensionEvidence { + param([Parameter(Mandatory)][string]$Path) + + $report = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json + $context = 'suspension report' + $environment = Get-RequiredPropertyValue $report 'environment' $context + foreach ($property in @( + 'operatingSystem', 'operatingSystemArchitecture', 'processArchitecture', + 'framework', 'runtimeVersion')) { + [void](Get-StrictString $environment $property "$context.environment") + } + if ((Get-StrictString $environment 'processArchitecture' "$context.environment") -ne + [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString() ` + -or (Get-StrictString $environment 'operatingSystemArchitecture' "$context.environment") -ne + [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() ` + -or (Get-StrictString $environment 'probeAssemblySha256' "$context.environment") -cne + (Get-TestedAssemblyHash "benchmarks/SharedMemoryStore.SyncProbe/bin/$Configuration/net10.0/SharedMemoryStore.SyncProbe.dll")) { + Fail-StepValidation 'participant-suspension' 'Suspension report architecture or probe assembly hash does not match the qualification build.' + } + [void](Get-StrictInt64 $environment 'logicalProcessorCount' "$context.environment" 1 [int32]::MaxValue) + [void](Get-StrictInt64 $environment 'availableProcessorCount' "$context.environment" 1 [int32]::MaxValue) + [void](Get-StrictInt64 $environment 'stopwatchFrequency' "$context.environment" 1 [int64]::MaxValue) + $configuration = Get-RequiredPropertyValue $report 'configuration' $context + $baselineSeconds = Get-StrictInt64 $configuration 'baselineWindowSeconds' "$context.configuration" 1 [int32]::MaxValue + $pauseSeconds = Get-StrictInt64 $configuration 'suspendedWindowSeconds' "$context.configuration" 1 [int32]::MaxValue + $warmupSeconds = Get-StrictInt64 $configuration 'warmupSeconds' "$context.configuration" 1 [int32]::MaxValue + $minimumRatio = Get-StrictDouble $configuration 'minimumThroughputRatio' "$context.configuration" 0 1 -Positive + $affinityRequested = Get-StrictBoolean $configuration 'affinityRequested' "$context.configuration" + $affinityPolicy = Get-StrictString $configuration 'affinityPolicy' "$context.configuration" + $comparisonMethod = Get-StrictString $configuration 'comparisonMethod' "$context.configuration" + if ($baselineSeconds -ne (Get-StrictInt64 $selected 'suspensionBaselineSeconds' "tier '$Tier'" 1 [int32]::MaxValue) ` + -or $pauseSeconds -ne (Get-StrictInt64 $selected 'suspensionPauseSeconds' "tier '$Tier'" 1 [int32]::MaxValue) ` + -or $warmupSeconds -ne (Get-StrictInt64 $selected 'suspensionWarmupSeconds' "tier '$Tier'" 1 [int32]::MaxValue) ` + -or $minimumRatio -ne (Get-StrictDouble $config 'suspensionMinimumHealthyThroughputRatio' 'qualification config' 0 1 -Positive) ` + -or -not $affinityRequested ` + -or $affinityPolicy -ne 'physical-core-first-then-siblings' ` + -or $comparisonMethod -ne 'Same persistent healthy process set is measured immediately before and while one external participant is blocked inside a production checkpoint.') { + Fail-StepValidation 'participant-suspension' 'Suspension report configuration does not exactly match the selected tier/configuration.' + } + if ((Get-StrictInt64 $report 'schemaVersion' $context 1 1) -ne 1) { + Fail-StepValidation 'participant-suspension' 'Suspension report schema must be exactly 1.' + } + $includedCheckpointCount = Get-StrictInt64 $configuration 'includedCheckpointCount' "$context.configuration" 1 [int32]::MaxValue + $catalogCheckpointCount = Get-StrictInt64 $configuration 'catalogCheckpointCount' "$context.configuration" 1 [int32]::MaxValue + $requiredWorkloads = @('distributed-key', 'mixed-churn') + Assert-ExactStringSet 'suspension workloads' @($configuration.workloads) $requiredWorkloads + Assert-ExactStringSet 'suspension excluded families' @($configuration.excludedFamilies) @('Participant', 'Disposal') + $expectedCheckpointIds = @($canonicalSuspensionCheckpointIds) + $expectedResultCount = $expectedCheckpointIds.Count * $requiredWorkloads.Count + $reportedRequiredCount = Get-StrictInt64 $report 'requiredResultCount' $context 1 [int32]::MaxValue + if ($includedCheckpointCount -ne $expectedCheckpointIds.Count ` + -or $catalogCheckpointCount -ne $checkpointCatalog.Count ` + -or $reportedRequiredCount -ne $expectedResultCount) { + Fail-StepValidation 'participant-suspension' 'Suspension report schema/count is invalid.' + } + $resultsRows = @($report.results) + if ($resultsRows.Count -ne $expectedResultCount) { + Fail-StepValidation 'participant-suspension' 'Suspension report omitted required checkpoint/workload rows.' + } + $expectedPairs = foreach ($checkpointId in $expectedCheckpointIds) { + foreach ($workload in $requiredWorkloads) { + '{0:D3}|{1}' -f $checkpointId, $workload + } + } + $actualPairs = foreach ($row in $resultsRows) { + $checkpointId = Get-StrictInt64 $row 'checkpointId' 'suspension result row' 1 ([int]$checkpointCatalog[-1].id) + $workload = Get-RequiredPropertyValue $row 'workload' "suspension checkpoint $checkpointId" + if ($workload -isnot [string] -or $workload -notin $requiredWorkloads) { + Fail-StepValidation 'participant-suspension' "Checkpoint $checkpointId has invalid workload '$workload'." + } + '{0:D3}|{1}' -f $checkpointId, $workload + } + $expectedCanonicalPairs = @($expectedPairs | Sort-Object) + $actualCanonicalPairs = @($actualPairs | Sort-Object) + $pairDigest = Get-StringSha256 ($actualCanonicalPairs -join "`n") + $expectedPairDigest = Get-StringSha256 ($expectedCanonicalPairs -join "`n") + if (($actualCanonicalPairs -join "`n") -cne ($expectedCanonicalPairs -join "`n") ` + -or $pairDigest -ne $expectedPairDigest ` + -or @($actualPairs | Sort-Object -Unique).Count -ne $expectedResultCount) { + Fail-StepValidation 'participant-suspension' "Suspension checkpoint/workload set differs from the exact canonical set; expectedDigest=$expectedPairDigest actualDigest=$pairDigest." + } + $checkpointGroups = @($resultsRows | Group-Object checkpointId) + if ($checkpointGroups.Count -ne $includedCheckpointCount ` + -or @($checkpointGroups | Where-Object { $_.Count -ne $requiredWorkloads.Count }).Count -ne 0) { + Fail-StepValidation 'participant-suspension' 'Every included steady-state checkpoint must have exactly one row for each required workload.' + } + foreach ($checkpointGroup in $checkpointGroups) { + $workloads = @($checkpointGroup.Group.workload | Sort-Object -Unique) + if (($workloads -join ',') -ne (($requiredWorkloads | Sort-Object) -join ',')) { + Fail-StepValidation 'participant-suspension' "Checkpoint $($checkpointGroup.Name) is missing a required workload." + } + } + foreach ($row in $resultsRows) { + $rowContext = "checkpoint $($row.checkpointId)/$($row.workload)" + $qualification = Get-StrictString $row 'qualification' $rowContext + if ($qualification -cnotin @( + 'qualified-pass', 'smoke-pass', 'qualified-fail', 'smoke-fail', + 'not-qualified-capacity-pressure', 'not-qualified-affinity', + 'not-qualified-insufficient-processors')) { + Fail-StepValidation 'participant-suspension' "$rowContext has unknown qualification '$qualification'." + } + $checkpointId = Get-StrictInt64 $row 'checkpointId' $rowContext 1 ([int]$checkpointCatalog[-1].id) + $catalogEntry = $checkpointCatalog[[int]$checkpointId - 1] + if ((Get-StrictString $row 'checkpointName' $rowContext) -cne [string]$catalogEntry.name ` + -or (Get-StrictString $row 'checkpointFamily' $rowContext) -cne [string]$catalogEntry.family ` + -or (Get-StrictString $row 'checkpointPosition' $rowContext) -cne [string]$catalogEntry.position ` + -or (Get-StrictString $row 'pauseClassification' $rowContext) -cne [string]$catalogEntry.pause ` + -or (Get-StrictString $row 'crashClassification' $rowContext) -cne [string]$catalogEntry.crash ` + -or (Get-StrictString $row 'raceClassification' $rowContext) -cne [string]$catalogEntry.race ` + -or (Get-StrictBoolean $row 'isPublicOrderingPoint' $rowContext) -ne [bool]$catalogEntry.isPublicOrderingPoint) { + Fail-StepValidation 'participant-suspension' "$rowContext metadata does not match the source-derived checkpoint catalog." + } + [void](Get-StrictBoolean $row 'capacityPermits' $rowContext) + [void](Get-StrictBoolean $row 'gatePassed' $rowContext) + [void](Get-StrictBoolean $row 'pausedParticipantAffinityApplied' $rowContext) + $expectedReaders = if ($row.workload -ceq 'distributed-key') { 6 } else { 12 } + $expectedWriters = if ($row.workload -ceq 'distributed-key') { 0 } else { 2 } + $expectedHealthy = $expectedReaders + $expectedWriters + if ((Get-StrictInt64 $row 'readerProcessCount' $rowContext 0 [int32]::MaxValue) -ne $expectedReaders ` + -or (Get-StrictInt64 $row 'writerProcessCount' $rowContext 0 [int32]::MaxValue) -ne $expectedWriters ` + -or (Get-StrictInt64 $row 'healthyProcessCount' $rowContext 0 [int32]::MaxValue) -ne $expectedHealthy ` + -or (Get-StrictInt64 $row 'requiredProcessorCount' $rowContext 1 [int32]::MaxValue) -ne ($expectedHealthy + 1)) { + Fail-StepValidation 'participant-suspension' "$rowContext process topology is not the contracted healthy set." + } + foreach ($property in @( + 'healthyAffinityAppliedCount', 'availableProcessorCount', 'agentSpillFirstBucket', + 'agentSpillSecondBucket', 'pausedParticipantProcessor', 'pausedParticipantProcessId', + 'pausedParticipantExitCode')) { + [void](Get-StrictInt64 $row $property $rowContext ([int64]-1) [int32]::MaxValue) + } + [void](Get-StrictString $row 'pausedParticipantAffinityStrategy' $rowContext) + $correctnessErrors = @(Get-RequiredPropertyValue $row 'correctnessErrors' $rowContext) + foreach ($errorValue in $correctnessErrors) { + if ($errorValue -isnot [string] -or [string]::IsNullOrWhiteSpace([string]$errorValue)) { + Fail-StepValidation 'participant-suspension' "$rowContext has an invalid correctness error entry." + } + } + $rowBaselineSeconds = Get-StrictInt64 $row 'baselineWindowSeconds' $rowContext 1 [int32]::MaxValue + $rowPauseSeconds = Get-StrictInt64 $row 'suspendedWindowSeconds' $rowContext 1 [int32]::MaxValue + $rowMinimumRatio = Get-StrictDouble $row 'minimumThroughputRatio' $rowContext 0 1 -Positive + $failureCount = Get-StrictInt64 $row 'correctnessFailureCount' $rowContext 0 [int64]::MaxValue + if ($correctnessErrors.Count -ne $failureCount) { + Fail-StepValidation 'participant-suspension' "$rowContext correctnessErrors count does not equal correctnessFailureCount." + } + foreach ($rateProperty in @( + 'baselineCompletedCyclesPerSecond', 'suspendedCompletedCyclesPerSecond', + 'baselineApiCallsPerSecond', 'suspendedApiCallsPerSecond', 'throughputRatio')) { + [void](Get-StrictDouble $row $rateProperty $rowContext 0 [double]::MaxValue) + } + foreach ($countProperty in @( + 'baselineAttemptedCycles', 'baselineCompletedCycles', 'baselineApiCalls', + 'suspendedAttemptedCycles', 'suspendedCompletedCycles', 'suspendedApiCalls')) { + [void](Get-StrictInt64 $row $countProperty $rowContext 0 [int64]::MaxValue) + } + if ($rowBaselineSeconds -ne $baselineSeconds ` + -or $rowPauseSeconds -ne $pauseSeconds ` + -or $rowMinimumRatio -ne $minimumRatio ` + -or $failureCount -ne 0) { + Fail-StepValidation 'participant-suspension' "$rowContext used a different duration/gate or reported correctness failures." + } + foreach ($capacityName in @( + 'beforeBaselineCapacity', 'afterBaselineCapacity', 'beforeSuspendedCapacity', + 'afterSuspendedCapacity', 'afterResumeCapacity')) { + $capacity = Get-RequiredPropertyValue $row $capacityName $rowContext + foreach ($property in @( + 'freeSlotCount', 'publishedSlotCount', 'activeLeaseCount', 'activeReservationCount', + 'initializingSlotCount', 'reservedSlotCount', 'reclaimingSlotCount', 'freeLeaseCount', + 'claimingLeaseCount', 'recoveringLeaseCount', 'freeParticipantCount', + 'activeParticipantCount', 'registeringParticipantCount', 'closingParticipantCount', + 'recoveringParticipantCount', 'reclaimingParticipantCount', 'storeFullFailures', + 'leaseTableFullFailures', 'contentionBudgetExhaustionCount')) { + [void](Get-StrictInt64 $capacity $property "$rowContext.$capacityName" 0 [int64]::MaxValue) + } + } + } + if (@($resultsRows | Where-Object { $_.qualification -like '*-fail' }).Count -ne 0) { + Fail-StepValidation 'participant-suspension' 'At least one checkpoint had a correctness failure or fell below the 90% smoke/qualification gate.' + } + + foreach ($row in @($resultsRows | Where-Object { $_.qualification -like '*-pass' })) { + $rowContext = "checkpoint $($row.checkpointId)/$($row.workload)" + $capacityPermits = Get-StrictBoolean $row 'capacityPermits' $rowContext + $gatePassed = Get-StrictBoolean $row 'gatePassed' $rowContext + if (-not $capacityPermits -or -not $gatePassed) { + Fail-StepValidation 'participant-suspension' "Checkpoint $($row.checkpointId)/$($row.workload) claims pass without capacity and gate evidence." + } + $baselineRate = Get-StrictDouble $row 'baselineCompletedCyclesPerSecond' $rowContext 0 [double]::MaxValue -Positive + $suspendedRate = Get-StrictDouble $row 'suspendedCompletedCyclesPerSecond' $rowContext 0 [double]::MaxValue -Positive + $throughputRatio = Get-StrictDouble $row 'throughputRatio' $rowContext 0 [double]::MaxValue -Positive + $rowMinimumRatio = Get-StrictDouble $row 'minimumThroughputRatio' $rowContext 0 1 -Positive + $calculatedRatio = $suspendedRate / $baselineRate + if (-not [double]::IsFinite($calculatedRatio) -or [Math]::Abs($calculatedRatio - $throughputRatio) -gt 0.000001) { + Fail-StepValidation 'participant-suspension' "$rowContext has an invalid or inconsistent throughput denominator/ratio." + } + if ($row.qualification -eq 'qualified-pass' ` + -and $throughputRatio -lt $rowMinimumRatio) { + Fail-StepValidation 'participant-suspension' "Checkpoint $($row.checkpointId)/$($row.workload) is below its declared ratio threshold." + } + $expectedHealthy = if ($row.workload -eq 'distributed-key') { 6 } elseif ($row.workload -eq 'mixed-churn') { 14 } else { -1 } + if ((Get-StrictInt64 $row 'healthyProcessCount' $rowContext 0 [int32]::MaxValue) -ne $expectedHealthy ` + -or @($row.baselineWorkers).Count -ne $expectedHealthy ` + -or @($row.suspendedWorkers).Count -ne $expectedHealthy) { + Fail-StepValidation 'participant-suspension' "Checkpoint $($row.checkpointId)/$($row.workload) did not measure the required healthy set." + } + foreach ($worker in @($row.baselineWorkers) + @($row.suspendedWorkers)) { + $workerContext = "$rowContext worker $($worker.workerId)/$($worker.window)" + $workerRole = Get-StrictString $worker 'role' $workerContext + $workerWindow = Get-StrictString $worker 'window' $workerContext + if ($workerRole -cnotin @('reader', 'writer') ` + -or $workerWindow -cnotin @('baseline', 'suspended')) { + Fail-StepValidation 'participant-suspension' "$workerContext has an invalid role or measurement window." + } + [void](Get-StrictInt64 $worker 'workerId' $workerContext 0 [int32]::MaxValue) + [void](Get-StrictInt64 $worker 'processId' $workerContext 1 [int32]::MaxValue) + [void](Get-StrictInt64 $worker 'assignedProcessor' $workerContext 0 [int32]::MaxValue) + [void](Get-StrictInt64 $worker 'attemptedCycles' $workerContext 0 [int64]::MaxValue) + [void](Get-StrictInt64 $worker 'completedCycles' $workerContext 0 [int64]::MaxValue) + [void](Get-StrictInt64 $worker 'apiCalls' $workerContext 0 [int64]::MaxValue) + if ((Get-StrictInt64 $worker 'failures' $workerContext 0 [int64]::MaxValue) -ne 0 ` + -or -not (Get-StrictBoolean $worker 'affinityApplied' $workerContext)) { + Fail-StepValidation 'participant-suspension' "$workerContext has failures or lacks affinity." + } + [void](Get-StrictDouble $worker 'elapsedSeconds' $workerContext 0 [double]::MaxValue -Positive) + [void](Get-StrictDouble $worker 'completedCyclesPerSecond' $workerContext 0 [double]::MaxValue) + [void](Get-StrictDouble $worker 'apiCallsPerSecond' $workerContext 0 [double]::MaxValue) + [void](Get-StrictString $worker 'affinityStrategy' $workerContext) + foreach ($entry in (Get-RequiredPropertyValue $worker 'statusHistogram' $workerContext).PSObject.Properties) { + if (-not (Test-IsIntegerNumber $entry.Value) -or [int64]$entry.Value -lt 0) { + Fail-StepValidation 'participant-suspension' "$workerContext status '$($entry.Name)' is not a nonnegative integer." + } + } + } + foreach ($capacityName in @('afterBaselineCapacity', 'beforeSuspendedCapacity', 'afterSuspendedCapacity')) { + $capacity = Get-RequiredPropertyValue $row $capacityName $rowContext + $freeSlots = Get-StrictInt64 $capacity 'freeSlotCount' "$rowContext.$capacityName" 0 [int32]::MaxValue + $freeLeases = Get-StrictInt64 $capacity 'freeLeaseCount' "$rowContext.$capacityName" 0 [int32]::MaxValue + $freeParticipants = Get-StrictInt64 $capacity 'freeParticipantCount' "$rowContext.$capacityName" 0 [int32]::MaxValue + $storeFullFailures = Get-StrictInt64 $capacity 'storeFullFailures' "$rowContext.$capacityName" 0 [int64]::MaxValue + $leaseFullFailures = Get-StrictInt64 $capacity 'leaseTableFullFailures' "$rowContext.$capacityName" 0 [int64]::MaxValue + if ($freeSlots -lt 32 -or $freeLeases -lt ($expectedHealthy + 1) ` + -or $freeParticipants -lt 1 -or $storeFullFailures -ne 0 -or $leaseFullFailures -ne 0) { + Fail-StepValidation 'participant-suspension' "$rowContext fails the ordinary capacity gate at $capacityName." + } + } + $baselineSet = @($row.baselineWorkers | ForEach-Object { "$($_.role):$($_.workerId):$($_.processId)" } | Sort-Object) + $suspendedSet = @($row.suspendedWorkers | ForEach-Object { "$($_.role):$($_.workerId):$($_.processId)" } | Sort-Object) + if (($baselineSet -join ',') -ne ($suspendedSet -join ',')) { + Fail-StepValidation 'participant-suspension' "Checkpoint $($row.checkpointId)/$($row.workload) changed the healthy process set between windows." + } + $pausedPid = Get-StrictInt64 $row 'pausedParticipantProcessId' $rowContext 1 [int32]::MaxValue + if (@($row.baselineWorkers | Where-Object { $_.processId -eq $pausedPid }).Count -ne 0 ` + -or @($row.suspendedWorkers | Where-Object { $_.processId -eq $pausedPid }).Count -ne 0) { + Fail-StepValidation 'participant-suspension' "Checkpoint $($row.checkpointId)/$($row.workload) included the paused actor in healthy throughput." + } + if ((Get-StrictInt64 $row 'healthyAffinityAppliedCount' $rowContext 0 [int32]::MaxValue) -ne $expectedHealthy ` + -or -not (Get-StrictBoolean $row 'pausedParticipantAffinityApplied' $rowContext)) { + Fail-StepValidation 'participant-suspension' "Checkpoint $($row.checkpointId)/$($row.workload) claims pass without complete affinity evidence." + } + } + + $qualifiedPassCount = Get-StrictInt64 $report 'qualifiedPassCount' $context 0 $expectedResultCount + $smokePassCount = Get-StrictInt64 $report 'smokePassCount' $context 0 $expectedResultCount + $failCount = Get-StrictInt64 $report 'failCount' $context 0 $expectedResultCount + $notQualifiedCount = Get-StrictInt64 $report 'notQualifiedCount' $context 0 $expectedResultCount + $allRequiredQualifiedAndPassed = Get-StrictBoolean $report 'allRequiredQualifiedAndPassed' $context + $actualQualifiedPassCount = @($resultsRows | Where-Object { $_.qualification -ceq 'qualified-pass' }).Count + $actualSmokePassCount = @($resultsRows | Where-Object { $_.qualification -ceq 'smoke-pass' }).Count + $actualFailCount = @($resultsRows | Where-Object { $_.qualification -cin @('qualified-fail', 'smoke-fail') }).Count + $actualNotQualifiedCount = @($resultsRows | Where-Object { $_.qualification -clike 'not-qualified-*' }).Count + if (($qualifiedPassCount + $smokePassCount + $failCount + $notQualifiedCount) -ne $expectedResultCount ` + -or $qualifiedPassCount -ne $actualQualifiedPassCount ` + -or $smokePassCount -ne $actualSmokePassCount ` + -or $failCount -ne $actualFailCount ` + -or $notQualifiedCount -ne $actualNotQualifiedCount ` + -or $allRequiredQualifiedAndPassed -ne ($actualQualifiedPassCount -eq $expectedResultCount)) { + Fail-StepValidation 'participant-suspension' 'Suspension aggregate counts/flag do not match the exact result rows.' + } + $notQualified = @($resultsRows | Where-Object { $_.qualification -like 'not-qualified-*' }) + if ($notQualified.Count -ne 0) { + $reasons = @($notQualified | Group-Object qualification | ForEach-Object { "$($_.Name)=$($_.Count)" }) + Mark-StepNotQualified 'participant-suspension' ($reasons -join '; ') @( + "qualifiedPassCount=$($report.qualifiedPassCount)", + "requiredResultCount=$($report.requiredResultCount)") + return + } + + if ($Tier -eq 'release') { + if ($pauseSeconds -lt 30 ` + -or $qualifiedPassCount -ne $expectedResultCount ` + -or -not $allRequiredQualifiedAndPassed) { + Fail-StepValidation 'participant-suspension' 'Release SC005 requires every row to use a 30-second-or-longer pause and be qualified-pass.' + } + Set-StepValidation 'participant-suspension' 'passed' 'sc005-qualified' @( + "results=$($report.requiredResultCount)", + "minimumRatio=$($report.configuration.minimumThroughputRatio)", + "baselineSeconds=$($report.configuration.baselineWindowSeconds)", + "pauseSeconds=$($report.configuration.suspendedWindowSeconds)", + "checkpointCount=$($expectedCheckpointIds.Count)", + "pairDigest=$pairDigest") + return + } + + if ($pauseSeconds -ge 30 ` + -or $smokePassCount -ne $expectedResultCount) { + Fail-StepValidation 'participant-suspension' 'PR/nightly SC005 smoke must cover every row as smoke-pass without claiming release qualification.' + } + Set-StepValidation 'participant-suspension' 'passed' 'sc005-short-duration-correctness-and-coverage-smoke' @( + "results=$($report.requiredResultCount)", + "minimumRatio=$($report.configuration.minimumThroughputRatio)", + "baselineSeconds=$($report.configuration.baselineWindowSeconds)", + "pauseSeconds=$($report.configuration.suspendedWindowSeconds)", + "checkpointCount=$($expectedCheckpointIds.Count)", + "pairDigest=$pairDigest") +} + +function Get-ProbeSummaryRow { + param($Report, [string]$Profile, [string]$Scenario, [int]$ProcessCount) + + $rows = @($Report.summary | Where-Object { + $_.profile -ceq $Profile -and $_.scenario -ceq $Scenario -and [int]$_.processCount -eq $ProcessCount + }) + if ($rows.Count -ne 1) { + Fail-StepValidation 'sync-probe' "Missing unique probe summary row $Profile/$Scenario/$ProcessCount." + } + return $rows[0] +} + +function Get-ExpectedProbeTuples { + $scenarios = [ordered]@{} + foreach ($property in $config.performanceMatrix.shortScenarios.PSObject.Properties) { + $scenarios[$property.Name] = @($property.Value | ForEach-Object { [int]$_ }) + } + if ($Tier -eq 'release') { + foreach ($property in $config.performanceMatrix.releaseOnlyScenarios.PSObject.Properties) { + $scenarios[$property.Name] = @($property.Value | ForEach-Object { [int]$_ }) + } + } + + $tuples = [Collections.Generic.List[object]]::new() + foreach ($scenario in $scenarios.GetEnumerator()) { + $profiles = if ($scenario.Key -in @($config.performanceMatrix.lockFreeOnlyScenarios)) { + @('LockFree') + } + else { + @($config.performanceMatrix.profiles | ForEach-Object { [string]$_ }) + } + foreach ($profile in $profiles) { + foreach ($processCount in @($scenario.Value)) { + $tuples.Add([pscustomobject]@{ + profile = $profile + scenario = $scenario.Key + processCount = [int]$processCount + }) + } + } + } + return @($tuples) +} + +function Assert-ExactProbeMatrix { + param([Parameter(Mandatory)]$Report) + + $expectedTuples = @(Get-ExpectedProbeTuples) + $expectedRunCount = $expectedTuples.Count * [int]$selected.performanceTrials + if (@($Report.runs).Count -ne $expectedRunCount) { + Fail-StepValidation 'sync-probe' "Performance report has $(@($Report.runs).Count) rows; expected exactly $expectedRunCount." + } + if (@($Report.summary).Count -ne $expectedTuples.Count) { + Fail-StepValidation 'sync-probe' "Performance summary has $(@($Report.summary).Count) rows; expected exactly $($expectedTuples.Count)." + } + + foreach ($tuple in $expectedTuples) { + $summaryRows = @($Report.summary | Where-Object { + $_.profile -ceq $tuple.profile ` + -and $_.scenario -ceq $tuple.scenario ` + -and [int]$_.processCount -eq $tuple.processCount + }) + if ($summaryRows.Count -ne 1) { + Fail-StepValidation 'sync-probe' "Expected one summary row for $($tuple.profile)/$($tuple.scenario)/$($tuple.processCount)." + } + foreach ($trial in (1..([int]$selected.performanceTrials))) { + $runRows = @($Report.runs | Where-Object { + $_.profile -ceq $tuple.profile ` + -and $_.scenario -ceq $tuple.scenario ` + -and [int]$_.processCount -eq $tuple.processCount ` + -and [int]$_.trial -eq $trial + }) + if ($runRows.Count -ne 1) { + Fail-StepValidation 'sync-probe' "Expected one run for $($tuple.profile)/$($tuple.scenario)/$($tuple.processCount)/trial-$trial." + } + } + } + + foreach ($run in @($Report.runs)) { + if (@($expectedTuples | Where-Object { + $_.profile -ceq $run.profile ` + -and $_.scenario -ceq $run.scenario ` + -and [int]$_.processCount -eq [int]$run.processCount + }).Count -ne 1) { + Fail-StepValidation 'sync-probe' "Unexpected performance row $($run.profile)/$($run.scenario)/$($run.processCount)/trial-$($run.trial)." + } + } + return $expectedRunCount +} + +function Assert-AtLeast { + param([string]$Step, [string]$Gate, [double]$Actual, [double]$Minimum) + if ($Actual -lt $Minimum) { + Fail-StepValidation $Step "$Gate failed: actual=$Actual minimum=$Minimum." + } +} + +function Assert-AtMost { + param([string]$Step, [string]$Gate, [double]$Actual, [double]$Maximum) + if ($Actual -gt $Maximum) { + Fail-StepValidation $Step "$Gate failed: actual=$Actual maximum=$Maximum." + } +} + +function Get-NumericArray { + param( + [Parameter(Mandatory)]$Values, + [Parameter(Mandatory)][string]$Context, + [int]$ExpectedCount = -1) + + $items = @($Values) + if ($ExpectedCount -ge 0 -and $items.Count -ne $ExpectedCount) { + throw "$Context contains $($items.Count) values; expected exactly $ExpectedCount." + } + $converted = foreach ($value in $items) { + if (-not (Test-IsNumericValue $value)) { + throw "$Context contains a nonnumeric value." + } + $number = [Convert]::ToDouble($value, [Globalization.CultureInfo]::InvariantCulture) + if (-not [double]::IsFinite($number) -or $number -lt 0) { + throw "$Context contains a negative or non-finite value." + } + $number + } + return @($converted) +} + +function Get-Percentile99 { + param([Parameter(Mandatory)][double[]]$Values) + + if ($Values.Count -eq 0) { + return 0.0 + } + $sorted = @($Values | Sort-Object) + $index = [Math]::Clamp([int][Math]::Ceiling($sorted.Count * 0.99) - 1, 0, $sorted.Count - 1) + return [double]$sorted[$index] +} + +function Assert-StickyOverflowEvidence { + param([Parameter(Mandatory)]$Run) + + $context = "SC018 trial $($Run.trial)" + $sticky = Get-RequiredPropertyValue $Run 'stickyOverflow' $context + $churnCycles = Get-StrictInt64 $sticky 'churnCycles' $context 10000 [int32]::MaxValue + if ((Get-StrictInt64 $sticky 'slotCount' $context 4096 4096) -ne 4096 ` + -or (Get-StrictInt64 $sticky 'primaryBucketCount' $context 2048 2048) -ne 2048 ` + -or (Get-StrictInt64 $sticky 'exactBucketPairCollisionKeyCount' $context 17 17) -ne 17 ` + -or (Get-StrictInt64 $sticky 'collisionCandidatesExamined' $context 17 [int64]::MaxValue) -lt 17 ` + -or $churnCycles -lt 10000 ` + -or (Get-StrictInt64 $sticky 'missingSamplesPerWindow' $context 16384 [int32]::MaxValue) -lt 16384) { + Fail-StepValidation 'sync-probe' "$context does not use the exact minimum 4,096-slot/10,000-cycle/16,384-sample contract." + } + $samplesPerWindow = [int](Get-StrictInt64 $sticky 'missingSamplesPerWindow' $context 16384 [int32]::MaxValue) + if ((Get-StrictInt64 $Run 'cycles' $context 0 [int64]::MaxValue) -ne $churnCycles ` + -or (Get-StrictInt64 $Run 'sampleCount' $context 0 [int32]::MaxValue) -ne (2 * $samplesPerWindow)) { + Fail-StepValidation 'sync-probe' "$context run totals do not match its raw churn/sample windows." + } + $early = @(Get-NumericArray $sticky.earlyMissingSamplesMicroseconds "$context early samples" $samplesPerWindow) + $late = @(Get-NumericArray $sticky.lateMissingSamplesMicroseconds "$context late samples" $samplesPerWindow) + $earlyP99 = Get-StrictDouble $Run 'earlyP99Microseconds' $context 0 [double]::MaxValue -Positive + $lateP99 = Get-StrictDouble $Run 'lateP99Microseconds' $context 0 [double]::MaxValue -Positive + $ratio = Get-StrictDouble $Run 'lateToEarlyP99Ratio' $context 0 2 -Positive + $reportedGate = Get-StrictDouble $sticky 'lateToEarlyP99Gate' $context 2 2 -Positive + $calculatedEarly = Get-Percentile99 $early + $calculatedLate = Get-Percentile99 $late + $calculatedRatio = $calculatedLate / $calculatedEarly + $tolerance = 0.000000001 + if (-not [double]::IsFinite($calculatedRatio) ` + -or [Math]::Abs($calculatedEarly - $earlyP99) -gt $tolerance ` + -or [Math]::Abs($calculatedLate - $lateP99) -gt $tolerance ` + -or [Math]::Abs($calculatedRatio - $ratio) -gt $tolerance ` + -or $reportedGate -ne 2.0) { + Fail-StepValidation 'sync-probe' "$context raw latency samples do not reproduce the reported p99 fields." + } + + $beforeSpill = Get-StrictInt64 $sticky 'spilledBucketCountBeforeChurn' $context 0 [int32]::MaxValue + $duringSpill = Get-StrictInt64 $sticky 'spilledBucketCountDuringChurn' $context 0 [int32]::MaxValue + $duringOccupancy = Get-StrictInt64 $sticky 'overflowDirectoryOccupancyDuringChurn' $context 0 [int32]::MaxValue + $afterFirstSpill = Get-StrictInt64 $sticky 'spilledBucketCountAfterFirstCleanup' $context 0 [int32]::MaxValue + $afterFirstOccupancy = Get-StrictInt64 $sticky 'overflowDirectoryOccupancyAfterFirstCleanup' $context 0 [int32]::MaxValue + $afterChurnSpill = Get-StrictInt64 $sticky 'spilledBucketCountAfterChurn' $context 0 [int32]::MaxValue + $afterChurnOccupancy = Get-StrictInt64 $sticky 'overflowDirectoryOccupancyAfterChurn' $context 0 [int32]::MaxValue + $scanBeforeCleanup = Get-StrictInt64 $sticky 'overflowScanCountBeforeFirstCleanup' $context 0 [int64]::MaxValue + $scanAfterCleanup = Get-StrictInt64 $sticky 'overflowScanCountAfterFirstCleanup' $context 0 [int64]::MaxValue + $maxAfterCleanup = Get-StrictInt64 $sticky 'maxObservedOverflowScanLengthAfterFirstCleanup' $context 0 [int32]::MaxValue + $scanBeforeLate = Get-StrictInt64 $sticky 'overflowScanCountBeforeLateWindow' $context 0 [int64]::MaxValue + $scanAfterLate = Get-StrictInt64 $sticky 'overflowScanCountAfterLateWindow' $context 0 [int64]::MaxValue + $maxScan = Get-StrictInt64 $sticky 'maxObservedOverflowScanLength' $context 0 [int32]::MaxValue + if ($beforeSpill -ne 0 -or $duringSpill -le 0 -or $duringOccupancy -le 0 ` + -or $afterFirstSpill -ne 0 -or $afterFirstOccupancy -ne 0 ` + -or $afterChurnSpill -ne 0 -or $afterChurnOccupancy -ne 0 ` + -or $scanAfterCleanup -le $scanBeforeCleanup -or $maxAfterCleanup -lt 4096 ` + -or $scanBeforeLate -lt $scanAfterCleanup -or $scanAfterLate -ne $scanBeforeLate ` + -or $maxScan -lt 4096 ` + -or -not (Get-StrictBoolean $sticky 'diagnosticsGatePassed' $context) ` + -or -not (Get-StrictBoolean $sticky 'latencyGatePassed' $context)) { + Fail-StepValidation 'sync-probe' "$context lacks exact collision, spill, cleanup, scan, or latency-gate evidence." + } +} + +function Assert-ProbeEnvironmentEvidence { + param([Parameter(Mandatory)]$Report) + + $environment = Get-RequiredPropertyValue $Report 'environment' 'sync probe report' + $commit = Get-StrictString $environment 'repositoryCommit' 'sync probe environment' + $workingTreeState = Get-StrictString $environment 'repositoryWorkingTreeState' 'sync probe environment' + $storeAssemblyHash = Get-StrictString $environment 'sharedMemoryStoreAssemblySha256' 'sync probe environment' + $probeAssemblyHash = Get-StrictString $environment 'probeAssemblySha256' 'sync probe environment' + if ($commit -eq 'unknown' -or $commit -ne [string]$repositoryProvenance.commit ` + -or $workingTreeState -eq 'unknown' -or $workingTreeState -ne [string]$repositoryProvenance.workingTreeState) { + Fail-StepValidation 'sync-probe' 'Performance report source provenance does not match the qualification run.' + } + $probePath = "benchmarks/SharedMemoryStore.SyncProbe/bin/$Configuration/net10.0/SharedMemoryStore.SyncProbe.dll" + $storePath = "benchmarks/SharedMemoryStore.SyncProbe/bin/$Configuration/net10.0/SharedMemoryStore.dll" + if ($probeAssemblyHash -cne (Get-TestedAssemblyHash $probePath) ` + -or $storeAssemblyHash -cne (Get-TestedAssemblyHash $storePath)) { + Fail-StepValidation 'sync-probe' 'Performance report assembly hashes do not match the fresh tested-assembly manifest.' + } + + foreach ($property in @( + 'operatingSystem', 'operatingSystemArchitecture', 'processArchitecture', + 'framework', 'runtimeVersion')) { + [void](Get-StrictString $environment $property 'sync probe environment') + } + try { + Assert-RequiredBenchmarkHardwareMetadata $environment 'sync probe environment' + } + catch { + Fail-StepValidation 'sync-probe' $_.Exception.Message + } + if ((Get-StrictString $environment 'processArchitecture' 'sync probe environment') -ne + [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString() ` + -or (Get-StrictString $environment 'operatingSystemArchitecture' 'sync probe environment') -ne + [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString()) { + Fail-StepValidation 'sync-probe' 'Performance report architecture does not match the qualification host.' + } + $logicalProcessorCount = Get-StrictInt64 $environment 'logicalProcessorCount' ` + 'sync probe environment' 1 [int32]::MaxValue + if ($logicalProcessorCount -ne [Environment]::ProcessorCount) { + Fail-StepValidation 'sync-probe' 'Performance report logical processor count does not match the qualification host.' + } + [void](Get-StrictInt64 $environment 'stopwatchFrequency' 'sync probe environment' 1 [int64]::MaxValue) + [void](Get-StrictBoolean $environment 'serverGarbageCollection' 'sync probe environment') +} + +function Assert-ProbeConfigurationEvidence { + param([Parameter(Mandatory)]$Report) + + $configuration = Get-RequiredPropertyValue $Report 'configuration' 'sync probe report' + if ((Get-StrictString $configuration 'mode' 'sync probe configuration') -ne [string]$selected.performanceMode ` + -or (Get-StrictInt64 $configuration 'durationSeconds' 'sync probe configuration' 1 [int32]::MaxValue) -ne + (Get-StrictInt64 $selected 'performanceDurationSeconds' "tier '$Tier'" 1 [int32]::MaxValue) ` + -or (Get-StrictInt64 $configuration 'durationBoundGraceSeconds' 'sync probe configuration' 1 [int32]::MaxValue) -ne + (Get-StrictInt64 $selected 'performanceDurationBoundGraceSeconds' "tier '$Tier'" 1 [int32]::MaxValue) ` + -or (Get-StrictInt64 $configuration 'warmupSeconds' 'sync probe configuration' 0 [int32]::MaxValue) -ne + (Get-StrictInt64 $selected 'performanceWarmupSeconds' "tier '$Tier'" 1 [int32]::MaxValue) ` + -or (Get-StrictInt64 $configuration 'trials' 'sync probe configuration' 1 [int32]::MaxValue) -ne + (Get-StrictInt64 $selected 'performanceTrials' "tier '$Tier'" 1 [int32]::MaxValue) ` + -or (Get-StrictInt64 $configuration 'largeFrameBytes' 'sync probe configuration' 1 [int32]::MaxValue) -ne + (Get-StrictInt64 $selected 'largeFrameBytes' "tier '$Tier'" 1 [int32]::MaxValue) ` + -or (Get-StrictInt64 $configuration 'largeFrames' 'sync probe configuration' 1 [int64]::MaxValue) -ne + (Get-StrictInt64 $selected 'largeFrames' "tier '$Tier'" 1 [int64]::MaxValue) ` + -or (Get-StrictInt64 $configuration 'mixedOperationTarget' 'sync probe configuration' 0 [int64]::MaxValue) -ne + (Get-StrictInt64 $selected 'mixedOperations' "tier '$Tier'" 1 [int64]::MaxValue) ` + -or -not (Get-StrictBoolean $configuration 'affinityRequested' 'sync probe configuration')) { + Fail-StepValidation 'sync-probe' 'Performance report configuration does not exactly match the selected tier.' + } + + foreach ($property in @( + 'readerKeyCount', 'readerPayloadBytes', 'brokerRotatingKeyCount', + 'mixedCollisionKeyCount', 'mixedPrimaryBucketCount', 'samplingInterval', + 'maxLatencySamplesPerWorker', 'brokerObserverSamplingInterval')) { + [void](Get-StrictInt64 $configuration $property 'sync probe configuration' 1 [int32]::MaxValue) + } + [void](Assert-LinuxTinySyncTopology $configuration 'sync probe configuration') + if ((Get-StrictInt64 $configuration 'warmupCycles' 'sync probe configuration' 0 0) -ne 0) { + Fail-StepValidation 'sync-probe' 'Performance report must identify time-based warmup with warmupCycles=0.' + } + [void](Get-StrictString $configuration 'affinityPolicy' 'sync probe configuration') + $legacySemantics = Get-StrictString $configuration 'legacyFullPayloadCopiesFieldSemantics' 'sync probe configuration' + if ($legacySemantics -ne ('Retained for v3-v5 readers. Consult FullPayloadCopyCountIsInstrumented and ' + + 'FullPayloadCopyEvidenceKind before interpreting the value as a measured event count.')) { + Fail-StepValidation 'sync-probe' 'Performance report does not declare the legacy copy field as non-authoritative.' + } + + Assert-ExactStringSet 'performance report profiles' @($configuration.profiles) @($config.performanceMatrix.profiles) + Assert-ExactStringSet 'performance report count-bound profiles' ` + @($configuration.countBoundProfiles) @($config.performanceMatrix.countBoundProfiles) + $expectedScenarioCounts = [ordered]@{} + foreach ($property in $config.performanceMatrix.shortScenarios.PSObject.Properties) { + $expectedScenarioCounts[$property.Name] = @($property.Value | ForEach-Object { [int]$_ }) + } + if ($Tier -eq 'release') { + foreach ($property in $config.performanceMatrix.releaseOnlyScenarios.PSObject.Properties) { + $expectedScenarioCounts[$property.Name] = @($property.Value | ForEach-Object { [int]$_ }) + } + } + Assert-ExactStringSet 'performance report scenarios' @($configuration.scenarios) @($expectedScenarioCounts.Keys) + $actualScenarioCounts = Get-RequiredPropertyValue $configuration 'scenarioProcessCounts' 'sync probe configuration' + Assert-ExactStringSet 'performance report scenario-count keys' @($actualScenarioCounts.PSObject.Properties.Name) @($expectedScenarioCounts.Keys) + try { + Assert-BenchmarkScenarioStoreDimensions ` + $configuration @($expectedScenarioCounts.Keys) 'sync probe configuration' + } + catch { + Fail-StepValidation 'sync-probe' $_.Exception.Message + } + foreach ($entry in $expectedScenarioCounts.GetEnumerator()) { + $actual = @($actualScenarioCounts.($entry.Key) | ForEach-Object { + if (-not (Test-IsIntegerNumber $_)) { + Fail-StepValidation 'sync-probe' "Scenario '$($entry.Key)' has a noninteger process count." + } + [Convert]::ToInt32($_, [Globalization.CultureInfo]::InvariantCulture) + }) + if (($actual -join ',') -cne (@($entry.Value) -join ',')) { + Fail-StepValidation 'sync-probe' "Scenario '$($entry.Key)' process-count matrix is not exact." + } + } + + if ($Tier -eq 'release') { + if ((Get-StrictInt64 $configuration 'stickyOverflowSlotCount' 'sync probe configuration' 4096 4096) -ne 4096 ` + -or (Get-StrictInt64 $configuration 'stickyOverflowChurnCycles' 'sync probe configuration' 10000 [int32]::MaxValue) -lt 10000 ` + -or (Get-StrictInt64 $configuration 'stickyOverflowMissingSamplesPerWindow' 'sync probe configuration' 16384 [int32]::MaxValue) -lt 16384) { + Fail-StepValidation 'sync-probe' 'Release SC018 configuration is below its exact minimum workload.' + } + } +} + +function Assert-ProbeDerivedValue { + param( + [Parameter(Mandatory)][string]$Context, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][double]$Actual, + [Parameter(Mandatory)][double]$Expected) + + $tolerance = [Math]::Max(0.000000001, [Math]::Abs($Expected) * 0.000000000001) + if (-not [double]::IsFinite($Actual) -or -not [double]::IsFinite($Expected) ` + -or [Math]::Abs($Actual - $Expected) -gt $tolerance) { + Fail-StepValidation 'sync-probe' "$Context.$Property is not reproducible from its raw counters." + } +} + +function Assert-ProbeRunCompletionEvidence { + param( + [Parameter(Mandatory)]$Run, + [Parameter(Mandatory)][string]$Context, + [Parameter(Mandatory)][int64]$DurationSeconds, + [Parameter(Mandatory)][int64]$MixedOperationTarget, + [Parameter(Mandatory)][int64]$LargeFrameTarget, + [Parameter(Mandatory)][string[]]$CountBoundProfiles) + + $profile = Get-StrictString $Run 'profile' $Context + $scenario = Get-StrictString $Run 'scenario' $Context + if ($profile -cnotin @('Legacy', 'LockFree')) { + throw "$Context has noncanonical profile identity '$profile'." + } + if ($scenario -cnotin @( + 'acquire-release', 'publish-remove', 'same-key-read', 'distributed-key-read', + 'broker-directed', 'mixed-churn', 'large-ingest', 'sticky-overflow-miss')) { + throw "$Context has noncanonical scenario identity '$scenario'." + } + $operationTarget = Get-StrictInt64 $Run 'operationTarget' $Context 0 [int64]::MaxValue + $frameTarget = Get-StrictInt64 $Run 'frameTarget' $Context 0 [int64]::MaxValue + if ($operationTarget -gt 0 -and $frameTarget -gt 0) { + throw "$Context cannot be both operation-bound and frame-bound." + } + + $isCountBoundProfile = $profile -cin $CountBoundProfiles + [int64]$expectedOperationTarget = if ($scenario -ceq 'mixed-churn' -and $isCountBoundProfile) { + $MixedOperationTarget + } + else { 0 } + [int64]$expectedFrameTarget = if ($scenario -ceq 'large-ingest' -and $isCountBoundProfile) { + $LargeFrameTarget + } + else { 0 } + if ($operationTarget -ne $expectedOperationTarget -or $frameTarget -ne $expectedFrameTarget) { + throw "$Context does not match the configured profile-aware count-bound policy." + } + + if ($scenario -ceq 'sticky-overflow-miss') { + return + } + + $operations = Get-StrictInt64 $Run 'operations' $Context 1 [int64]::MaxValue + $frames = Get-StrictInt64 $Run 'frames' $Context 0 [int64]::MaxValue + $measuredSeconds = Get-StrictDouble $Run 'measuredSeconds' $Context 0 [double]::MaxValue -Positive + $earlySamples = Get-StrictInt64 $Run 'earlySampleCount' $Context 1 [int64]::MaxValue + $lateSamples = Get-StrictInt64 $Run 'lateSampleCount' $Context 1 [int64]::MaxValue + [void]$earlySamples + [void]$lateSamples + if ($operationTarget -gt 0) { + if ($operations -lt $operationTarget) { + throw "$Context completed $operations operations below its target $operationTarget." + } + return + } + if ($frameTarget -gt 0) { + if ($frames -lt $frameTarget) { + throw "$Context completed $frames frames below its target $frameTarget." + } + return + } + if ($measuredSeconds -lt $DurationSeconds) { + throw "$Context duration-bound row measured $measuredSeconds seconds below $DurationSeconds." + } +} + +function Invoke-ProbeCompletionVerifierSelfTest { + [int64]$duration = Get-StrictInt64 $selected 'performanceDurationSeconds' "tier '$Tier'" 1 [int32]::MaxValue + [int64]$mixedTarget = Get-StrictInt64 $selected 'mixedOperations' "tier '$Tier'" 1 [int64]::MaxValue + [int64]$frameTarget = Get-StrictInt64 $selected 'largeFrames' "tier '$Tier'" 1 [int64]::MaxValue + [string[]]$countBoundProfiles = @($config.performanceMatrix.countBoundProfiles | ForEach-Object { [string]$_ }) + $legacy = [pscustomobject][ordered]@{ + profile = 'Legacy'; scenario = 'mixed-churn'; operationTarget = 0; frameTarget = 0 + operations = 1; frames = 0; measuredSeconds = [double]$duration + earlySampleCount = 1; lateSampleCount = 1 + } + $lockFreeMixed = [pscustomobject][ordered]@{ + profile = 'LockFree'; scenario = 'mixed-churn'; operationTarget = $mixedTarget; frameTarget = 0 + operations = $mixedTarget; frames = 0; measuredSeconds = 1.0 + earlySampleCount = 1; lateSampleCount = 1 + } + $lockFreeLarge = [pscustomobject][ordered]@{ + profile = 'LockFree'; scenario = 'large-ingest'; operationTarget = 0; frameTarget = $frameTarget + operations = 1; frames = $frameTarget; measuredSeconds = 1.0 + earlySampleCount = 1; lateSampleCount = 1 + } + foreach ($row in @($legacy, $lockFreeMixed, $lockFreeLarge)) { + Assert-ProbeRunCompletionEvidence $row 'completion verifier self-test' ` + $duration $mixedTarget $frameTarget $countBoundProfiles + } + [int]$assertions = 3 + + $mutations = @( + @{ message = 'one-below configured mixed target'; apply = { + param($row) $row.operationTarget = $mixedTarget - 1 + }; source = $lockFreeMixed }, + @{ message = 'one-below completed mixed operations'; apply = { + param($row) $row.operations = $mixedTarget - 1 + }; source = $lockFreeMixed }, + @{ message = 'Legacy count target inheritance'; apply = { + param($row) $row.operationTarget = $mixedTarget + }; source = $legacy }, + @{ message = 'profile target swap'; apply = { + param($row) $row.operationTarget = 0; $row.frameTarget = $frameTarget + }; source = $lockFreeMixed }, + @{ message = 'simultaneous operation and frame targets'; apply = { + param($row) $row.frameTarget = $frameTarget + }; source = $lockFreeMixed }, + @{ message = 'short duration row'; apply = { + param($row) $row.measuredSeconds = [double]$duration - 0.001 + }; source = $legacy }, + @{ message = 'missing operation target metadata'; apply = { + param($row) $row.PSObject.Properties.Remove('operationTarget') + }; source = $lockFreeMixed }, + @{ message = 'one-below completed frame target'; apply = { + param($row) $row.frames = $frameTarget - 1 + }; source = $lockFreeLarge }, + @{ message = 'noncanonical profile casing'; apply = { + param($row) $row.profile = 'lockfree' + }; source = $lockFreeMixed }, + @{ message = 'noncanonical scenario casing'; apply = { + param($row) $row.scenario = 'Mixed-Churn' + }; source = $lockFreeMixed } + ) + foreach ($mutation in $mutations) { + $row = $mutation.source | ConvertTo-Json -Depth 5 | ConvertFrom-Json + & $mutation.apply $row + $rejected = $false + try { + Assert-ProbeRunCompletionEvidence $row 'completion verifier negative self-test' ` + $duration $mixedTarget $frameTarget $countBoundProfiles + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw "Probe completion verifier accepted $($mutation.message)." + } + $assertions++ + } + return $assertions +} + +function Assert-ProbeRowNumericEvidence { + param([Parameter(Mandatory)]$Report) + + $logicalProcessorCount = Get-StrictInt64 $Report.environment 'logicalProcessorCount' ` + 'sync probe environment' 1 [int32]::MaxValue + foreach ($run in @($Report.runs)) { + $context = "probe run $($run.profile)/$($run.scenario)/$($run.processCount)/trial-$($run.trial)" + foreach ($property in @('profile', 'scenario', 'qualification', 'fullPayloadCopyEvidenceKind', 'allocationMeasurementScope')) { + [void](Get-StrictString $run $property $context) + } + $allowedQualifications = if ($run.scenario -ceq 'sticky-overflow-miss') { + @( + 'qualification-passed-versioned-overflow-cleanup', + 'qualification-failed-overflow-diagnostics', + 'qualification-failed-versioned-overflow-latency') + } + else { + @( + 'qualification-measurement', 'smoke-only', + 'smoke-only-insufficient-warmup', 'not-qualified-oversubscribed') + } + if ([string]$run.qualification -cnotin $allowedQualifications) { + Fail-StepValidation 'sync-probe' "$context has unknown qualification '$($run.qualification)'." + } + foreach ($property in @('processCount', 'trial')) { + [void](Get-StrictInt64 $run $property $context 1 [int32]::MaxValue) + } + foreach ($property in @( + 'readerProcessCount', 'publisherProcessCount', 'observerProcessCount', 'cycles', + 'operations', 'frames', 'bytesWritten', 'bytesRead', 'fullPayloadCopies', + 'measuredThreadAllocatedBytes', 'producerStoreOperationAllocatedBytes', 'failures', + 'sampleCount', 'earlySampleCount', 'lateSampleCount', 'affinityAppliedCount', + 'operationTarget', 'frameTarget')) { + [void](Get-StrictInt64 $run $property $context 0 [int64]::MaxValue) + } + try { + Assert-ProbeRunCompletionEvidence $run $context ` + (Get-StrictInt64 $selected 'performanceDurationSeconds' "tier '$Tier'" 1 [int32]::MaxValue) ` + (Get-StrictInt64 $selected 'mixedOperations' "tier '$Tier'" 1 [int64]::MaxValue) ` + (Get-StrictInt64 $selected 'largeFrames' "tier '$Tier'" 1 [int64]::MaxValue) ` + ([string[]]@($config.performanceMatrix.countBoundProfiles | ForEach-Object { [string]$_ })) + } + catch { + Fail-StepValidation 'sync-probe' $_.Exception.Message + } + foreach ($property in @( + 'apiCallsPerSecond', 'p50Microseconds', 'p95Microseconds', 'p99Microseconds', + 'maxMicroseconds', 'earlyP99Microseconds', 'lateP99Microseconds', + 'lateToEarlyP99Ratio', 'framesPerSecond', 'bytesPerSecond', 'fairnessIndex', + 'minWorkerApiCallsPerSecond', 'maxWorkerApiCallsPerSecond', + 'worstWorkerP99Microseconds')) { + [void](Get-StrictDouble $run $property $context 0 [double]::MaxValue) + } + [void](Get-StrictDouble $run 'measuredSeconds' $context 0 [double]::MaxValue -Positive) + [void](Get-StrictDouble $run 'wallSeconds' $context 0 [double]::MaxValue -Positive) + $oversubscribed = Get-StrictBoolean $run 'oversubscribed' $context + [void](Get-StrictBoolean $run 'fullPayloadCopyCountIsInstrumented' $context) + $processCount = Get-StrictInt64 $run 'processCount' $context 1 [int32]::MaxValue + $readerCount = Get-StrictInt64 $run 'readerProcessCount' $context 0 [int32]::MaxValue + $publisherCount = Get-StrictInt64 $run 'publisherProcessCount' $context 0 [int32]::MaxValue + $observerCount = Get-StrictInt64 $run 'observerProcessCount' $context 0 [int32]::MaxValue + $expectedTopology = switch ([string]$run.scenario) { + { $_ -in @('acquire-release', 'same-key-read', 'distributed-key-read') } { + @($processCount, 0, 0, @('reader')); break + } + 'publish-remove' { @(0, $processCount, 0, @('publisher')); break } + 'broker-directed' { @($processCount, 1, 1, @('broker-end-to-end')); break } + 'mixed-churn' { @($processCount, 2, 0, @('publisher', 'reader')); break } + 'large-ingest' { @($processCount, 1, 0, @('broker-end-to-end')); break } + 'sticky-overflow-miss' { @(0, 1, 0, @('missing-key')); break } + default { + Fail-StepValidation 'sync-probe' "$context has no contracted process topology." + } + } + if ($readerCount -ne [int64]$expectedTopology[0] ` + -or $publisherCount -ne [int64]$expectedTopology[1] ` + -or $observerCount -ne [int64]$expectedTopology[2]) { + Fail-StepValidation 'sync-probe' "$context process roles do not match the contracted scenario topology." + } + $participants = $readerCount + $publisherCount + $observerCount + if ($participants -le 0 -or $oversubscribed -ne ($participants -gt $logicalProcessorCount)) { + Fail-StepValidation 'sync-probe' "$context oversubscription flag does not match its raw participant topology." + } + + $assignedProcessors = @(Get-RequiredPropertyValue $run 'assignedProcessors' $context) + if ($assignedProcessors.Count -ne $participants) { + Fail-StepValidation 'sync-probe' "$context has $($assignedProcessors.Count) processor assignments; expected exactly $participants." + } + foreach ($processor in $assignedProcessors) { + if (-not (Test-IsIntegerNumber $processor) -or [int64]$processor -lt -1) { + Fail-StepValidation 'sync-probe' "$context has a noninteger processor assignment." + } + } + $affinityAppliedCount = Get-StrictInt64 $run 'affinityAppliedCount' $context 0 [int32]::MaxValue + $assignedCount = @($assignedProcessors | Where-Object { [int64]$_ -ge 0 }).Count + if ($affinityAppliedCount -ne $assignedCount) { + Fail-StepValidation 'sync-probe' "$context affinityAppliedCount does not match raw assigned processors." + } + if (-not $oversubscribed -and $affinityAppliedCount -eq $participants ` + -and @($assignedProcessors | Sort-Object -Unique).Count -ne $participants) { + Fail-StepValidation 'sync-probe' "$context affinity-qualified processor assignments are not unique." + } + $affinityStrategies = @(Get-RequiredPropertyValue $run 'affinityStrategies' $context) + if ($affinityStrategies.Count -eq 0 ` + -or @($affinityStrategies | Sort-Object -Unique).Count -ne $affinityStrategies.Count) { + Fail-StepValidation 'sync-probe' "$context affinity strategies are empty or duplicated." + } + foreach ($strategy in $affinityStrategies) { + if ($strategy -isnot [string] -or [string]::IsNullOrWhiteSpace([string]$strategy)) { + Fail-StepValidation 'sync-probe' "$context has an invalid affinity strategy." + } + } + + $workerCycles = @(Get-RequiredPropertyValue $run 'workerCycles' $context) + $expectedWorkerCount = if ($run.scenario -cin @('broker-directed', 'large-ingest')) { + $readerCount + } + else { + $participants + } + if ($workerCycles.Count -ne $expectedWorkerCount) { + Fail-StepValidation 'sync-probe' "$context has $($workerCycles.Count) worker-cycle rows; expected exactly $expectedWorkerCount." + } + [decimal]$workerCycleTotal = 0 + foreach ($cycleCount in $workerCycles) { + if (-not (Test-IsIntegerNumber $cycleCount) -or [int64]$cycleCount -lt 0) { + Fail-StepValidation 'sync-probe' "$context has an invalid worker-cycle count." + } + $workerCycleTotal += [decimal]$cycleCount + } + if ($workerCycleTotal -ne [decimal](Get-StrictInt64 $run 'cycles' $context 0 [int64]::MaxValue)) { + Fail-StepValidation 'sync-probe' "$context worker-cycle rows do not sum to the aggregate cycle count." + } + [decimal]$operationStatusTotal = 0 + foreach ($entry in (Get-RequiredPropertyValue $run 'statusHistogram' $context).PSObject.Properties) { + if (-not (Test-IsIntegerNumber $entry.Value) -or [int64]$entry.Value -lt 0) { + Fail-StepValidation 'sync-probe' "$context status '$($entry.Name)' is not a nonnegative integer." + } + if ($entry.Name -match '^(Acquire|Release|Publish|Remove|Reserve|Advance|Commit|RecoverLeases|RecoverReservations)\.') { + $operationStatusTotal += [decimal]$entry.Value + } + } + $operations = Get-StrictInt64 $run 'operations' $context 0 [int64]::MaxValue + if ($operationStatusTotal -ne [decimal]$operations) { + Fail-StepValidation 'sync-probe' "$context operation-status histogram does not sum to operations." + } + $sampleCount = Get-StrictInt64 $run 'sampleCount' $context 0 [int64]::MaxValue + $earlySampleCount = Get-StrictInt64 $run 'earlySampleCount' $context 0 [int64]::MaxValue + $lateSampleCount = Get-StrictInt64 $run 'lateSampleCount' $context 0 [int64]::MaxValue + if ($sampleCount -ne ($earlySampleCount + $lateSampleCount)) { + Fail-StepValidation 'sync-probe' "$context sampleCount does not equal its early/late sample windows." + } + $roleLatency = @(Get-RequiredPropertyValue $run 'roleLatency' $context) + $actualRoles = [Collections.Generic.List[string]]::new() + [int64]$roleSampleTotal = 0 + foreach ($role in $roleLatency) { + $roleContext = "$context role $($role.role)" + $actualRoles.Add((Get-StrictString $role 'role' $roleContext)) + $roleSampleTotal += Get-StrictInt64 $role 'sampleCount' $roleContext 0 [int32]::MaxValue + foreach ($property in @('earlyP99Microseconds', 'lateP99Microseconds', 'lateToEarlyP99Ratio')) { + [void](Get-StrictDouble $role $property $roleContext 0 [double]::MaxValue) + } + } + Assert-ExactStringSet "$context role-latency identities" @($actualRoles) @($expectedTopology[3]) + if ($roleSampleTotal -ne (Get-StrictInt64 $run 'sampleCount' $context 0 [int64]::MaxValue)) { + Fail-StepValidation 'sync-probe' "$context role-latency samples do not sum to sampleCount." + } + + $measuredSeconds = Get-StrictDouble $run 'measuredSeconds' $context 0 [double]::MaxValue -Positive + $frames = Get-StrictInt64 $run 'frames' $context 0 [int64]::MaxValue + $bytesWritten = Get-StrictInt64 $run 'bytesWritten' $context 0 [int64]::MaxValue + $bytesRead = Get-StrictInt64 $run 'bytesRead' $context 0 [int64]::MaxValue + Assert-ProbeDerivedValue $context 'apiCallsPerSecond' ` + (Get-StrictDouble $run 'apiCallsPerSecond' $context 0 [double]::MaxValue) ` + ([double]$operations / $measuredSeconds) + Assert-ProbeDerivedValue $context 'framesPerSecond' ` + (Get-StrictDouble $run 'framesPerSecond' $context 0 [double]::MaxValue) ` + ([double]$frames / $measuredSeconds) + Assert-ProbeDerivedValue $context 'bytesPerSecond' ` + (Get-StrictDouble $run 'bytesPerSecond' $context 0 [double]::MaxValue) ` + ([double]([decimal]$bytesWritten + [decimal]$bytesRead) / $measuredSeconds) + $earlyP99 = Get-StrictDouble $run 'earlyP99Microseconds' $context 0 [double]::MaxValue + $lateP99 = Get-StrictDouble $run 'lateP99Microseconds' $context 0 [double]::MaxValue + Assert-ProbeDerivedValue $context 'lateToEarlyP99Ratio' ` + (Get-StrictDouble $run 'lateToEarlyP99Ratio' $context 0 [double]::MaxValue) ` + $(if ($earlyP99 -eq 0) { 0.0 } else { $lateP99 / $earlyP99 }) + foreach ($role in $roleLatency) { + $roleContext = "$context role $($role.role)" + $roleEarly = Get-StrictDouble $role 'earlyP99Microseconds' $roleContext 0 [double]::MaxValue + $roleLate = Get-StrictDouble $role 'lateP99Microseconds' $roleContext 0 [double]::MaxValue + Assert-ProbeDerivedValue $roleContext 'lateToEarlyP99Ratio' ` + (Get-StrictDouble $role 'lateToEarlyP99Ratio' $roleContext 0 [double]::MaxValue) ` + $(if ($roleEarly -eq 0) { 0.0 } else { $roleLate / $roleEarly }) + } + $fairness = Get-StrictDouble $run 'fairnessIndex' $context 0 1 + $minimumWorkerRate = Get-StrictDouble $run 'minWorkerApiCallsPerSecond' $context 0 [double]::MaxValue + $maximumWorkerRate = Get-StrictDouble $run 'maxWorkerApiCallsPerSecond' $context 0 [double]::MaxValue + if ($fairness -gt 1 -or $minimumWorkerRate -gt $maximumWorkerRate) { + Fail-StepValidation 'sync-probe' "$context fairness or worker-rate bounds are internally inconsistent." + } + + $stickyProperty = $run.PSObject.Properties['stickyOverflow'] + $hasSticky = $null -ne $stickyProperty -and $null -ne $stickyProperty.Value + if (($run.scenario -ceq 'sticky-overflow-miss') -ne $hasSticky) { + Fail-StepValidation 'sync-probe' "$context has inconsistent SC018 raw evidence presence." + } + if ($hasSticky) { + Assert-StickyOverflowEvidence $run + } + } + + foreach ($summary in @($Report.summary)) { + $context = "probe summary $($summary.profile)/$($summary.scenario)/$($summary.processCount)" + foreach ($property in @('profile', 'scenario')) { + [void](Get-StrictString $summary $property $context) + } + [void](Get-StrictInt64 $summary 'processCount' $context 1 [int32]::MaxValue) + foreach ($property in @( + 'totalFrames', 'totalBytesWritten', 'totalFullPayloadCopies', + 'totalMeasuredThreadAllocatedBytes', 'totalFailures', + 'totalProducerStoreOperationAllocatedBytes')) { + [void](Get-StrictInt64 $summary $property $context 0 [int64]::MaxValue) + } + foreach ($property in @( + 'medianApiCallsPerSecond', 'medianP50Microseconds', 'medianP95Microseconds', + 'medianP99Microseconds', 'medianMaxMicroseconds', 'medianEarlyP99Microseconds', + 'medianLateP99Microseconds', 'medianLateToEarlyP99Ratio', 'medianFramesPerSecond', + 'medianBytesPerSecond', 'medianFairnessIndex', 'medianWorstWorkerP99Microseconds')) { + [void](Get-StrictDouble $summary $property $context 0 [double]::MaxValue) + } + [void](Get-StrictBoolean $summary 'fullPayloadCopyCountsAreInstrumented' $context) + foreach ($value in @($summary.fullPayloadCopyEvidenceKinds) + @($summary.allocationMeasurementScopes)) { + if ($value -isnot [string] -or [string]::IsNullOrWhiteSpace([string]$value)) { + Fail-StepValidation 'sync-probe' "$context has an invalid evidence-kind or allocation-scope value." + } + } + foreach ($entry in (Get-RequiredPropertyValue $summary 'statusHistogram' $context).PSObject.Properties) { + if (-not (Test-IsIntegerNumber $entry.Value) -or [int64]$entry.Value -lt 0) { + Fail-StepValidation 'sync-probe' "$context status '$($entry.Name)' is not a nonnegative integer." + } + } + } +} + +function Get-MedianValue { + param([Parameter(Mandatory)][double[]]$Values) + + if ($Values.Count -eq 0) { + throw 'Cannot compute a median for an empty evidence set.' + } + $sorted = @($Values | Sort-Object) + $middleIndex = [int][Math]::Floor($sorted.Count / 2.0) + if (($sorted.Count % 2) -eq 0) { + return ([double]$sorted[$middleIndex - 1] + [double]$sorted[$middleIndex]) / 2.0 + } + return [double]$sorted[$middleIndex] +} + +function Assert-ProbeSummaryConsistency { + param([Parameter(Mandatory)]$Report) + + $medianProperties = [ordered]@{ + medianApiCallsPerSecond = 'apiCallsPerSecond' + medianP50Microseconds = 'p50Microseconds' + medianP95Microseconds = 'p95Microseconds' + medianP99Microseconds = 'p99Microseconds' + medianMaxMicroseconds = 'maxMicroseconds' + medianEarlyP99Microseconds = 'earlyP99Microseconds' + medianLateP99Microseconds = 'lateP99Microseconds' + medianLateToEarlyP99Ratio = 'lateToEarlyP99Ratio' + medianFramesPerSecond = 'framesPerSecond' + medianBytesPerSecond = 'bytesPerSecond' + medianFairnessIndex = 'fairnessIndex' + medianWorstWorkerP99Microseconds = 'worstWorkerP99Microseconds' + } + $totalProperties = [ordered]@{ + totalFrames = 'frames' + totalBytesWritten = 'bytesWritten' + totalFullPayloadCopies = 'fullPayloadCopies' + totalMeasuredThreadAllocatedBytes = 'measuredThreadAllocatedBytes' + totalFailures = 'failures' + totalProducerStoreOperationAllocatedBytes = 'producerStoreOperationAllocatedBytes' + } + + foreach ($summary in @($Report.summary)) { + $context = "probe summary $($summary.profile)/$($summary.scenario)/$($summary.processCount)" + $runs = @($Report.runs | Where-Object { + $_.profile -ceq $summary.profile -and $_.scenario -ceq $summary.scenario ` + -and $_.processCount -eq $summary.processCount + }) + if ($runs.Count -ne [int]$selected.performanceTrials) { + Fail-StepValidation 'sync-probe' "$context cannot be reproduced from the configured number of trials." + } + foreach ($entry in $medianProperties.GetEnumerator()) { + $values = [double[]]@($runs | ForEach-Object { + [double]($_.PSObject.Properties[[string]$entry.Value].Value) + }) + $expected = Get-MedianValue $values + $actual = [double]($summary.PSObject.Properties[[string]$entry.Key].Value) + $tolerance = [Math]::Max(0.000000001, [Math]::Abs($expected) * 0.000000000001) + if ([Math]::Abs($actual - $expected) -gt $tolerance) { + Fail-StepValidation 'sync-probe' "$context.$($entry.Key) does not equal the raw-trial median." + } + } + foreach ($entry in $totalProperties.GetEnumerator()) { + [decimal]$expected = 0 + foreach ($run in $runs) { + $expected += [decimal]($run.PSObject.Properties[[string]$entry.Value].Value) + } + if ([decimal]($summary.PSObject.Properties[[string]$entry.Key].Value) -ne $expected) { + Fail-StepValidation 'sync-probe' "$context.$($entry.Key) does not equal the raw-trial total." + } + } + $allCopyCountersInstrumented = @($runs | Where-Object { -not $_.fullPayloadCopyCountIsInstrumented }).Count -eq 0 + if ([bool]$summary.fullPayloadCopyCountsAreInstrumented -ne $allCopyCountersInstrumented) { + Fail-StepValidation 'sync-probe' "$context copy-instrumentation aggregate is inconsistent." + } + Assert-ExactStringSet "$context copy evidence kinds" @($summary.fullPayloadCopyEvidenceKinds) ` + @($runs.fullPayloadCopyEvidenceKind | Sort-Object -Unique) + Assert-ExactStringSet "$context allocation scopes" @($summary.allocationMeasurementScopes) ` + @($runs.allocationMeasurementScope | Sort-Object -Unique) + + $expectedHistogram = @{} + foreach ($run in $runs) { + foreach ($property in $run.statusHistogram.PSObject.Properties) { + if (-not $expectedHistogram.ContainsKey($property.Name)) { + $expectedHistogram[$property.Name] = [int64]0 + } + $expectedHistogram[$property.Name] = [int64]$expectedHistogram[$property.Name] + [int64]$property.Value + } + } + $actualHistogram = $summary.statusHistogram + Assert-ExactStringSet "$context status histogram keys" @($actualHistogram.PSObject.Properties.Name) @($expectedHistogram.Keys) + foreach ($name in $expectedHistogram.Keys) { + if ([int64]($actualHistogram.PSObject.Properties[$name].Value) -ne [int64]$expectedHistogram[$name]) { + Fail-StepValidation 'sync-probe' "$context status histogram '$name' does not equal the raw-trial total." + } + } + } +} + +function Assert-SyncProbeEvidence { + param([Parameter(Mandatory)][string]$Path) + + $report = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json + if ((Get-StrictInt64 $report 'schemaVersion' 'sync probe report' 8 8) -ne 8 ` + -or (Get-StrictInt64 $report 'minimumCompatibleSchemaVersion' 'sync probe report' 8 8) -ne 8 ` + -or (Get-StrictString $report 'schemaCompatibility' 'sync probe report') -notmatch 'Schema v8' ` + -or @($report.runs).Count -eq 0) { + Fail-StepValidation 'sync-probe' 'Sync probe report must be exact executable schema v8 with nonempty runs.' + } + Assert-ProbeEnvironmentEvidence $report + Assert-ProbeConfigurationEvidence $report + Assert-ProbeRowNumericEvidence $report + Assert-ProbeSummaryConsistency $report + $expectedRunCount = Assert-ExactProbeMatrix $report + if (@($report.runs | Where-Object { $_.failures -ne 0 }).Count -ne 0 ` + -or @($report.summary | Where-Object { $_.totalFailures -ne 0 }).Count -ne 0) { + Fail-StepValidation 'sync-probe' 'A performance workload reported correctness failures.' + } + if (@($report.runs | Where-Object { + $null -ne $_.stickyOverflow -and (-not $_.stickyOverflow.diagnosticsGatePassed -or -not $_.stickyOverflow.latencyGatePassed) + }).Count -ne 0) { + Fail-StepValidation 'sync-probe' 'SC018 diagnostics or latency gate failed.' + } + $notQualified = @($report.runs | Where-Object { $_.qualification -like 'not-qualified-*' }) + if ($notQualified.Count -ne 0) { + Mark-StepNotQualified 'sync-probe' (($notQualified | Group-Object qualification | ForEach-Object { "$($_.Name)=$($_.Count)" }) -join '; ') + return + } + foreach ($run in @($report.runs)) { + $participants = [int64]$run.readerProcessCount + [int64]$run.publisherProcessCount + [int64]$run.observerProcessCount + if ([int64]$run.affinityAppliedCount -ne $participants -or $run.oversubscribed) { + Mark-StepNotQualified 'sync-probe' "Incomplete affinity or oversubscription in $($run.profile)/$($run.scenario)/$($run.processCount)." + return + } + } + + if ($Tier -ne 'release') { + Set-StepValidation 'sync-probe' 'passed' 'short-performance-smoke-not-release-qualified' @( + "durationSeconds=$($selected.performanceDurationSeconds)", + "trials=$($selected.performanceTrials)", + "exactRunRows=$expectedRunCount", + 'correctnessFailures=0') + return + } + + $ordinary = @($report.runs | Where-Object { $_.scenario -cne 'sticky-overflow-miss' }) + if (@($ordinary | Where-Object { $_.qualification -ne 'qualification-measurement' }).Count -ne 0) { + Fail-StepValidation 'sync-probe' 'Release rows were smoke-only or lacked the release warmup/duration/frame target.' + } + foreach ($scenario in @('same-key-read', 'distributed-key-read')) { + $one = Get-ProbeSummaryRow $report 'LockFree' $scenario 1 + $six = Get-ProbeSummaryRow $report 'LockFree' $scenario 6 + $twelve = Get-ProbeSummaryRow $report 'LockFree' $scenario 12 + $minimumSix = if ($scenario -eq 'same-key-read') { 4.0 } else { 4.5 } + $minimumTwelve = if ($scenario -eq 'same-key-read') { 7.0 } else { 8.0 } + $oneRate = Get-StrictDouble $one 'medianApiCallsPerSecond' "$scenario one-reader summary" 0 [double]::MaxValue -Positive + $sixRate = Get-StrictDouble $six 'medianApiCallsPerSecond' "$scenario six-reader summary" 0 [double]::MaxValue -Positive + $twelveRate = Get-StrictDouble $twelve 'medianApiCallsPerSecond' "$scenario twelve-reader summary" 0 [double]::MaxValue -Positive + Assert-AtLeast 'sync-probe' "${scenario}-6-reader-scaling" ` + ($sixRate / $oneRate) $minimumSix + Assert-AtLeast 'sync-probe' "${scenario}-12-reader-scaling" ` + ($twelveRate / $oneRate) $minimumTwelve + } + + $brokerOne = Get-ProbeSummaryRow $report 'LockFree' 'broker-directed' 1 + $brokerTwelve = Get-ProbeSummaryRow $report 'LockFree' 'broker-directed' 12 + $brokerOneRate = Get-StrictDouble $brokerOne 'medianFramesPerSecond' 'broker one-reader summary' 0 [double]::MaxValue -Positive + $brokerTwelveRate = Get-StrictDouble $brokerTwelve 'medianFramesPerSecond' 'broker twelve-reader summary' 0 [double]::MaxValue -Positive + Assert-AtLeast 'sync-probe' 'broker-12-reader-publication-rate' ` + ($brokerTwelveRate / $brokerOneRate) 0.8 + + foreach ($scenario in @('acquire-release', 'publish-remove')) { + $legacyEight = Get-ProbeSummaryRow $report 'Legacy' $scenario 8 + $lockFreeEight = Get-ProbeSummaryRow $report 'LockFree' $scenario 8 + $legacyEightRate = Get-StrictDouble $legacyEight 'medianApiCallsPerSecond' "$scenario legacy/8p summary" 0 [double]::MaxValue -Positive + $lockFreeEightRate = Get-StrictDouble $lockFreeEight 'medianApiCallsPerSecond' "$scenario lock-free/8p summary" 0 [double]::MaxValue -Positive + $legacyEightP99 = Get-StrictDouble $legacyEight 'medianP99Microseconds' "$scenario legacy/8p summary" 0 [double]::MaxValue -Positive + $lockFreeEightP99 = Get-StrictDouble $lockFreeEight 'medianP99Microseconds' "$scenario lock-free/8p summary" 0 [double]::MaxValue -Positive + if ($IsWindows) { + Assert-AtLeast 'sync-probe' "${scenario}-windows-throughput" ` + ($lockFreeEightRate / $legacyEightRate) 4.0 + Assert-AtMost 'sync-probe' "${scenario}-windows-p99" ` + ($lockFreeEightP99 / $legacyEightP99) 0.2 + } + elseif ($IsLinux) { + $legacyOne = Get-ProbeSummaryRow $report 'Legacy' $scenario 1 + $lockFreeOne = Get-ProbeSummaryRow $report 'LockFree' $scenario 1 + $legacyOneP99 = Get-StrictDouble $legacyOne 'medianP99Microseconds' "$scenario legacy/1p summary" 0 [double]::MaxValue -Positive + $lockFreeOneP99 = Get-StrictDouble $lockFreeOne 'medianP99Microseconds' "$scenario lock-free/1p summary" 0 [double]::MaxValue -Positive + Assert-AtMost 'sync-probe' "${scenario}-linux-uncontended-p99-ratio" ` + ($lockFreeOneP99 / $legacyOneP99) ` + (Get-StrictDouble $config.linuxTinyPerformance 'maximumUncontendedP99Ratio' ` + 'qualification config linuxTinyPerformance' 1 1) + Assert-AtLeast 'sync-probe' "${scenario}-linux-8p-throughput-ratio" ` + ($lockFreeEightRate / $legacyEightRate) ` + (Get-StrictDouble $config.linuxTinyPerformance 'minimumThroughputRatio' ` + 'qualification config linuxTinyPerformance' 1 1) + Assert-AtMost 'sync-probe' "${scenario}-linux-scale-p99-ratio" ` + ($lockFreeEightP99 / $lockFreeOneP99) ` + (Get-StrictDouble $config.linuxTinyPerformance 'maximumScaleP99Ratio' ` + 'qualification config linuxTinyPerformance' 3 3) + Assert-AtMost 'sync-probe' "${scenario}-linux-8p-p99-us" ` + $lockFreeEightP99 ` + (Get-StrictDouble $config.linuxTinyPerformance 'maximumP99Microseconds' ` + 'qualification config linuxTinyPerformance' 10 10) + foreach ($run in @($report.runs | Where-Object { + [string]$_.profile -ceq 'LockFree' ` + -and [string]$_.scenario -ceq $scenario ` + -and [int64]$_.processCount -in @(1, 8) + })) { + Assert-AtMost 'sync-probe' "${scenario}-linux-max-stall-us-$($run.processCount)p-trial-$($run.trial)" ` + (Get-StrictDouble $run 'maxMicroseconds' "$scenario lock-free/$($run.processCount)p trial-$($run.trial)" 0 [double]::MaxValue) ` + (Get-StrictDouble $config.linuxTinyPerformance 'maximumStallMicroseconds' ` + 'qualification config linuxTinyPerformance' 10000 10000) + } + } + else { + Mark-StepNotQualified 'sync-probe' 'SC006 supports only Windows-x64 and Linux-x64.' + return + } + } + + $zeroCopyRuns = @($report.runs | Where-Object { + $_.profile -ceq 'LockFree' -and $_.scenario -cin @('broker-directed', 'large-ingest') + }) + foreach ($run in $zeroCopyRuns) { + $context = "$($run.scenario)/$($run.processCount)/trial-$($run.trial)" + if ((Get-StrictBoolean $run 'fullPayloadCopyCountIsInstrumented' $context) ` + -or (Get-StrictInt64 $run 'producerStoreOperationAllocatedBytes' $context 0 0) -ne 0 ` + -or (Get-StrictString $run 'fullPayloadCopyEvidenceKind' $context) -ne + 'structural-direct-reservation-write-and-borrowed-lease-read' ` + -or (Get-StrictString $run 'allocationMeasurementScope' $context) -ne + 'dedicated-producer-and-broker-coordinator-thread-entire-measured-interval') { + Fail-StepValidation 'sync-probe' "Producer allocation/structural-copy gate failed for $context." + } + } + foreach ($run in @($zeroCopyRuns | Where-Object { $_.scenario -ceq 'large-ingest' })) { + $frames = Get-StrictInt64 $run 'frames' "large-ingest trial $($run.trial)" 1 [int64]::MaxValue + Assert-AtLeast 'sync-probe' "large-ingest-frames-$($run.processCount)-trial-$($run.trial)" $frames ` + (Get-StrictInt64 $selected 'largeFrames' "tier '$Tier'" 1 [int64]::MaxValue) + } + + foreach ($run in @($report.runs | Where-Object { $_.profile -ceq 'LockFree' -and $_.scenario -ceq 'mixed-churn' })) { + $context = "mixed-churn trial $($run.trial)" + $operations = Get-StrictInt64 $run 'operations' $context 1 [int64]::MaxValue + $earlyP99 = Get-StrictDouble $run 'earlyP99Microseconds' $context 0 [double]::MaxValue -Positive + $lateP99 = Get-StrictDouble $run 'lateP99Microseconds' $context 0 [double]::MaxValue -Positive + $ratio = Get-StrictDouble $run 'lateToEarlyP99Ratio' $context 0 [double]::MaxValue -Positive + if ([Math]::Abs(($lateP99 / $earlyP99) - $ratio) -gt 0.000001) { + Fail-StepValidation 'sync-probe' "$context has an inconsistent late/early p99 ratio." + } + Assert-AtLeast 'sync-probe' "mixed-churn-operations-trial-$($run.trial)" ` + $operations (Get-StrictInt64 $selected 'mixedOperations' "tier '$Tier'" 1 [int64]::MaxValue) + Assert-AtMost 'sync-probe' "mixed-churn-late-early-p99-trial-$($run.trial)" ` + $ratio 2.0 + } + + Set-StepValidation 'sync-probe' 'passed' 'sc002-sc003-sc004-sc006-sc008-sc009-sc016-sc018-qualified' @( + 'correctnessFailures=0', + 'affinity=complete', + 'producerStoreOperationAllocatedBytes=0', + 'copyEvidence=structural-direct-reservation-write-and-borrowed-lease-read', + 'allQualifiedPerformanceThresholds=passed') +} + +$commonTest = @('-c', $Configuration, '--nologo', '--no-build', '--no-restore') + +try { + Assert-QualificationConfiguration + $churnQualificationContract = Get-ChurnQualificationTestContract $config + Add-EvidenceResult 'configuration-contract' 'passed' 'schema-and-contract-values-validated' @( + "schemaVersion=$($config.schemaVersion)", + "tier=$Tier", + "performanceMode=$($selected.performanceMode)", + "sc017TransitionCount=$(Get-Sc017SourceTransitionCount)", + "churnQualificationTest=$($churnQualificationContract.fullyQualifiedName)", + "boundedOperationSlackMilliseconds=$($config.boundedOperationSlackMilliseconds)", + "leakAssertions=$(@($config.requiredLeakAssertions).Count)") @([IO.Path]::GetRelativePath($root, $configPath)) + + if ($ValidateOnly) { + $sc017ConfigurationAssertions = Invoke-Sc017ConfigurationVerifierSelfTest + Add-EvidenceResult 'sc017-configuration-verifier-self-test' 'passed' ` + 'source-owned-transition-count-positive-and-negative-cases-passed' @( + "assertions=$sc017ConfigurationAssertions", + "sourceTransitionCount=$(Get-Sc017SourceTransitionCount)", + 'all PR/nightly/release tier counts accepted', + 'one-below-source transition count rejected') + $churnVerifierAssertions = Invoke-ChurnQualificationVerifierSelfTest + Add-EvidenceResult 'churn-qualification-verifier-self-test' 'passed' ` + 'configured-exact-test-and-trx-cardinality-positive-and-negative-cases-passed' @( + "assertions=$churnVerifierAssertions", + "exactTest=$($churnQualificationContract.fullyQualifiedName)", + 'two identical semantically pinned mappings and one exact Passed row accepted', + 'distinct/missing/alternate-existing mappings rejected', + 'wrong namespace/class/method source bindings rejected', + 'missing/extra/wrong/duplicate/non-passed rows rejected', + 'XML parsing, wrapper failure recording, and result cleanup exercised') @( + [IO.Path]::GetRelativePath($root, (Join-Path $runRoot 'churn-trx-verifier-self-test/valid.trx')), + [IO.Path]::GetRelativePath($root, (Join-Path $runRoot 'churn-trx-verifier-self-test/extra.trx'))) + $markerParserAssertions = Invoke-ProductionRaceMarkerParserSelfTest + Add-EvidenceResult 'production-race-marker-parser-self-test' 'passed' ` + 'closed-marker-grammar-positive-and-negative-cases-passed' @( + "assertions=$markerParserAssertions", + 'valid-eight-family-marker-set=accepted', + 'duplicate/wrong-count/wrong-seed markers=rejected', + 'invalid-recovery/disposal witnesses=rejected') + $probeCompletionAssertions = Invoke-ProbeCompletionVerifierSelfTest + Add-EvidenceResult 'sync-probe-completion-verifier-self-test' 'passed' ` + 'profile-aware-duration-operation-frame-positive-and-negative-cases-passed' @( + "assertions=$probeCompletionAssertions", + 'duration-bound Legacy plus count-bound LockFree mixed/large rows accepted', + 'below-target/config-swap/dual-target/short-duration/missing-target cases rejected') + $osManifestAssertions = Invoke-OsEvidenceManifestVerifierSelfTest + Add-EvidenceResult 'os-evidence-manifest-verifier-self-test' 'passed' ` + 'exact-tree-positive-and-tamper-negative-cases-passed' @( + "assertions=$osManifestAssertions", + 'exact manifest/file-set/log binding plus null structural/optional rows=accepted', + 'empty/whitespace pseudo-fields plus extra file/content hash/log hash/commandless passing clean row/out-of-root path=rejected') @( + [IO.Path]::GetRelativePath($root, (Join-Path $runRoot 'os-evidence-manifest-self-test.json')), + [IO.Path]::GetRelativePath($root, (Join-Path $runRoot 'os-evidence-manifest-self-test.evidence'))) + $linuxPerformanceAssertions = Invoke-LinuxTinyOsPerformanceVerifierSelfTest + Add-EvidenceResult 'linux-tiny-os-performance-verifier-self-test' 'passed' ` + 'exact-raw-matrix-positive-and-integrity-negative-cases-passed' @( + "assertions=$linuxPerformanceAssertions", + 'exact 24-run/8-summary schema/config/tuple/correctness/affinity/median/schema2-metric evidence=accepted', + 'uncontended/throughput/scale/absolute p99 gate breaches plus over-limit stall/duplicate metric/impossible affinity/incoherent cycle/corruption evidence=rejected') @( + [IO.Path]::GetRelativePath($root, (Join-Path $runRoot 'linux-tiny-os-performance-self-test.json')), + [IO.Path]::GetRelativePath($root, (Join-Path $runRoot 'linux-tiny-os-performance-self-test.evidence/linux-tiny-performance.json'))) + $requiredInputs = @( + 'SharedMemoryStore.slnx', + 'scripts/validate-lock-free-os.ps1', + 'benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj', + 'tests/SharedMemoryStore.LinearizabilityTests/SharedMemoryStore.LinearizabilityTests.csproj', + 'tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj', + $churnTestSourceRelativePath) + $missing = @($requiredInputs | Where-Object { -not (Test-Path -LiteralPath (Join-Path $root $_)) }) + if ($missing.Count -ne 0) { + throw "Qualification dry-run inputs are missing: $($missing -join ', ')." + } + Add-EvidenceResult 'validation-plan' 'passed' 'configuration-and-structure-only-not-qualification' @( + 'no build, tests, benchmarks, or OS validation executed', + 'exit code 0 validates orchestration structure only', + "requiredInputs=$($requiredInputs.Count)") @($requiredInputs) + $completionProvenance = Get-RepositoryProvenance + Assert-ProvenanceStable $repositoryProvenance $completionProvenance + $overallStatus = 'validation-only' + } + else { + Invoke-BoundedStep 'dotnet-info' $dotnet @('--info') + Set-StepValidation 'dotnet-info' 'passed' 'dotnet-toolchain-provisioned' @( + "dotnet=$dotnet", + "version=$(Invoke-TextCommand $dotnet @('--version'))") + $preclean = Remove-SolutionProjectBuildOutputs (Join-Path $runRoot 'preclean.json') + Invoke-BoundedStep 'clean' $dotnet @( + 'clean', 'SharedMemoryStore.slnx', '-c', $Configuration, '--nologo', '--disable-build-servers') + (Get-StepResult 'clean') | Add-Member -NotePropertyName preclean -NotePropertyValue $preclean.Summary + Set-StepValidation 'clean' 'passed' 'solution-output-cleaned-before-qualification-build' @( + 'SharedMemoryStore.slnx', + "configuration=$Configuration", + "solutionProjects=$($preclean.SolutionProjectCount)", + "outputTargets=$($preclean.TargetCount)", + "precleanedOutputDirectories=$($preclean.RemovedDirectoryCount)", + "verifiedAbsent=$($preclean.VerifiedAbsentCount)", + "precleanReport=$([IO.Path]::GetRelativePath($root, $preclean.ReportPath))", + "precleanReportSha256=$($preclean.ReportSha256)") + Invoke-BoundedStep 'restore' $dotnet @('restore', 'SharedMemoryStore.slnx', '--nologo', '--disable-build-servers') + Set-StepValidation 'restore' 'passed' 'explicit-solution-restore-passed' @('SharedMemoryStore.slnx') + Invoke-BoundedStep 'build' $dotnet @( + 'build', 'SharedMemoryStore.slnx', '-c', $Configuration, '--nologo', '--no-restore', '--disable-build-servers') + $testedAssemblyManifest = @(Get-TestedAssemblyManifest) + $testedAssemblyDigest = Get-StringSha256 (@($testedAssemblyManifest | ForEach-Object { + "$($_.path)|$($_.length)|$($_.sha256)" + }) -join "`n") + Set-StepValidation 'build' 'passed' 'fresh-clean-solution-build-and-assembly-manifest' @( + "assemblies=$($testedAssemblyManifest.Count)", + "manifestSha256=$testedAssemblyDigest") + + $fullSuiteTrx = Join-Path $runRoot 'trx/full-test-suite' + New-Item -ItemType Directory -Path $fullSuiteTrx | Out-Null + Invoke-BoundedStep 'full-test-suite' $dotnet @( + 'test', 'SharedMemoryStore.slnx', '-c', $Configuration, '--nologo', '--no-build', '--no-restore', + '--logger', 'trx', '--results-directory', $fullSuiteTrx) + Assert-FullSuiteEvidence $fullSuiteTrx + + Invoke-BoundedStep 'unit-contract' $dotnet (@( + 'test', 'tests/SharedMemoryStore.UnitTests/SharedMemoryStore.UnitTests.csproj') + $commonTest) + Invoke-BoundedStep 'directory-generation-stress' $dotnet (@( + 'test', 'tests/SharedMemoryStore.UnitTests/SharedMemoryStore.UnitTests.csproj') + $commonTest + @( + '--filter', 'FullyQualifiedName~SharedMemoryStore.UnitTests.LockFreeDirectoryGenerationStressTests.ConfiguredProductionStressFencesEveryDirectoryMutationTransitionAcrossSlotReuse', + '--logger', 'console;verbosity=detailed')) @{ + SMS_DIRECTORY_GENERATION_STRESS_REPETITIONS = [int64]$selected.directoryGenerationStressRepetitions + SMS_DIRECTORY_GENERATION_STRESS_SEED = [int]$config.seed + } + Assert-Sc017Evidence ([int64]$selected.directoryGenerationStressRepetitions) + Invoke-BoundedStep 'contract' $dotnet (@( + 'test', 'tests/SharedMemoryStore.ContractTests/SharedMemoryStore.ContractTests.csproj') + $commonTest) + + Invoke-BoundedStep 'reference-model-histories' $dotnet (@( + 'test', 'tests/SharedMemoryStore.LinearizabilityTests/SharedMemoryStore.LinearizabilityTests.csproj') + $commonTest + @( + '--filter', 'FullyQualifiedName~LockFreeHistoryTests', + '--logger', 'console;verbosity=detailed')) @{ + SMS_CHECKER_HISTORY_REPETITIONS = [int]$selected.checkerHistoryRepetitionsPerFamily + SMS_LINEARIZABILITY_SEED = [int]$config.seed + } + Assert-FamilyCompletionMarkers ` + 'reference-model-histories' ` + 'completedCheckerInvocations' ` + ([int64]$selected.checkerHistoryRepetitionsPerFamily) ` + @($config.completionEvidence.referenceModelFamilies) ` + 'configured-reference-model-count-proven' + + Invoke-BoundedStep 'production-generated-histories' $dotnet (@( + 'test', 'tests/SharedMemoryStore.LinearizabilityTests/SharedMemoryStore.LinearizabilityTests.csproj') + $commonTest + @( + '--filter', 'FullyQualifiedName~ProductionGeneratedHistoryTests', + '--logger', 'console;verbosity=detailed')) @{ + SMS_PRODUCTION_HISTORY_COUNT = [int]$selected.productionHistoryCountPerFamily + SMS_PRODUCTION_HISTORY_SEED = [int]$config.seed + } + Assert-FamilyCompletionMarkers ` + 'production-generated-histories' ` + 'completedHistories' ` + ([int64]$selected.productionHistoryCountPerFamily) ` + @($config.completionEvidence.productionHistoryFamilies) ` + 'configured-production-history-count-proven' + + Invoke-BoundedStep 'production-race-stress' $dotnet (@( + 'test', 'tests/SharedMemoryStore.LinearizabilityTests/SharedMemoryStore.LinearizabilityTests.csproj') + $commonTest + @( + '--filter', 'FullyQualifiedName~ProductionRaceStressTests', + '--logger', 'console;verbosity=detailed')) @{ + SMS_PRODUCTION_RACE_REPETITIONS = [int]$selected.productionRaceRepetitionsPerFamily + SMS_PRODUCTION_RACE_SEED = [int]$config.seed + } + Assert-ProductionRaceEvidence ` + ([int64]$selected.productionRaceRepetitionsPerFamily) ` + @($config.completionEvidence.productionRaceFamilies) + + $waitTrx = Join-Path $runRoot 'trx/wait-policy' + New-Item -ItemType Directory -Path $waitTrx | Out-Null + Invoke-BoundedStep 'wait-policy' $dotnet (@( + 'test', 'tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj') + $commonTest + @( + '--filter', 'FullyQualifiedName~LockFreeWaitPolicyMatrixIntegrationTests|FullyQualifiedName~LockFreeNoOperationLockIntegrationTests', + '--logger', 'trx', '--results-directory', $waitTrx)) + Assert-TrxStepEvidence 'wait-policy' $waitTrx -RequiredTestNameContains @( + 'LockFreeWaitPolicyMatrixIntegrationTests', + 'LockFreeNoOperationLockIntegrationTests') | Out-Null + $waitResult = Get-StepResult 'wait-policy' + $waitResult.qualification = 'bounded-wait-plus-250ms-and-no-operation-lock-proven' + $waitResult.validation = @($waitResult.validation) + "completionAllowanceMilliseconds=$($config.boundedOperationSlackMilliseconds)" + + $churnTrx = Join-Path $runRoot 'trx/churn' + New-Item -ItemType Directory -Path $churnTrx | Out-Null + Invoke-BoundedStep 'churn' $dotnet (@( + 'test', 'tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj') + $commonTest + @( + '--filter', "FullyQualifiedName=$($churnQualificationContract.fullyQualifiedName)", + '--logger', 'trx', '--results-directory', $churnTrx)) @{ + SMS_LOCK_FREE_CHURN_CYCLES = [int64]$selected.churnCycles + } + Assert-ExactTrxStepEvidence 'churn' $churnTrx @( + $churnQualificationContract.fullyQualifiedName) | Out-Null + $churnResult = Get-StepResult 'churn' + $churnResult.qualification = 'configured-churn-and-final-capacity-proof-passed' + $churnResult.validation = @($churnResult.validation) + @( + "configuredTotalCycles=$([int64]$selected.churnCycles)", + "configuredTest=$($churnQualificationContract.fullyQualifiedName)") + + $recoveryTrx = Join-Path $runRoot 'trx/recovery' + New-Item -ItemType Directory -Path $recoveryTrx | Out-Null + Invoke-BoundedStep 'recovery' $dotnet (@( + 'test', 'tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj') + $commonTest + @( + '--filter', 'FullyQualifiedName~LockFreeCrashRecoveryIntegrationTests', + '--logger', 'trx', '--results-directory', $recoveryTrx)) @{ + SMS_LOCK_FREE_RECOVERY_CASES = [int]$selected.recoveryCases + } + $recoveryPassed = @(Assert-TrxStepEvidence 'recovery' $recoveryTrx ([int]$selected.recoveryCases) @( + 'LockFreeCrashRecoveryIntegrationTests.EveryCanonicalCheckpointCanBeKilledRecoveredAndFilledToCapacity')) + Assert-RecoveryCheckpointEvidence $recoveryPassed ([int64]$selected.recoveryCases) + $recoveryResult = Get-StepResult 'recovery' + $recoveryResult.qualification = 'configured-recovery-case-count-and-capacity-proof-passed' + $recoveryResult.validation = @($recoveryResult.validation) + "recoveryCases=$([int]$selected.recoveryCases)" + Assert-OwnerLeakEvidence @{ + churn = $churnTrx + recovery = $recoveryTrx + } + + Invoke-BoundedStep 'raw-visibility' $dotnet (@( + 'test', 'tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj') + $commonTest + @( + '--filter', 'FullyQualifiedName~LockFreeRawVisibilityIntegrationTests')) + Invoke-BoundedStep 'package-consumption' $powershell @( + '-NoProfile', '-File', 'scripts/validate-package-consumption.ps1', + '-Configuration', $Configuration) + Set-StepValidation 'package-consumption' 'passed' 'isolated-cache-package-consumption-pass' @( + 'pack=passed', + 'legacy-consumer=passed', + 'lock-free-consumer=passed', + 'nuget-cache=isolated-per-run') + + if ($SkipOsValidation) { + Add-EvidenceResult 'dual-platform-os-evidence' 'not-qualified' 'os-validation-skipped' @( + 'explicitly skipped by -SkipOsValidation') + $notQualifiedReasons.Add('dual-platform-os-evidence: skipped by -SkipOsValidation') + } + else { + $osOutput = Join-Path $runRoot 'os-validation.json' + $osRelativeOutput = [IO.Path]::GetRelativePath($root, $osOutput) + $osCommand = if ($Tier -eq 'release') { 'all' } else { 'self-test' } + Invoke-BoundedStep ` + -Name 'os-validation-current' ` + -FileName $powershell ` + -Arguments @( + '-NoProfile', '-File', 'scripts/validate-lock-free-os.ps1', + '-Command', $osCommand, + '-Configuration', $Configuration, + '-StepTimeoutSeconds', [string]$selected.stepTimeoutSeconds, + '-OutputPath', $osRelativeOutput) ` + -AllowedExitCodes @(0, 2) + $osStep = Get-StepResult 'os-validation-current' + if ($osStep.exitCode -eq 2) { + $osStep.status = 'not-qualified' + $osStep.qualification = 'os-validator-returned-not-qualified' + } + else { + $osStep.status = 'passed' + $osStep.qualification = 'os-validator-executed' + } + $osStep.validation = @( + "command=$osCommand", + "report=$osRelativeOutput", + "reportSha256=$(Get-FileSha256 $osOutput)") + Assert-OsEvidenceSet $osOutput $AdditionalOsEvidence + } + + if (-not $SkipPerformance) { + $benchmarkOutput = Join-Path $runRoot 'sync-probe.json' + Invoke-BoundedStep 'sync-probe' $dotnet @( + 'run', '-c', $Configuration, '--no-build', '--no-restore', + '--project', 'benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj', '--', + '--mode', [string]$selected.performanceMode, + '--profile', 'both', + '--count-bound-profiles', 'v2', + '--warmup', [string]$selected.performanceWarmupSeconds, + '--duration', [string]$selected.performanceDurationSeconds, + '--duration-bound-grace', [string]$selected.performanceDurationBoundGraceSeconds, + '--trials', [string]$selected.performanceTrials, + '--mixed-operations', [string]$selected.mixedOperations, + '--large-frames', [string]$selected.largeFrames, + '--large-frame-bytes', [string]$selected.largeFrameBytes, + '--repository-commit', [string]$repositoryProvenance.commit, + '--repository-working-tree-state', [string]$repositoryProvenance.workingTreeState, + '--output', $benchmarkOutput) + Assert-SyncProbeEvidence $benchmarkOutput + + $suspensionOutput = Join-Path $runRoot 'participant-suspension.json' + Invoke-BoundedStep 'participant-suspension' $dotnet @( + 'run', '-c', $Configuration, '--no-build', '--no-restore', + '--project', 'benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj', '--', + '--mode', 'suspension', + '--profile', 'v2', + '--warmup', [string]$selected.suspensionWarmupSeconds, + '--suspension-baseline-seconds', [string]$selected.suspensionBaselineSeconds, + '--suspension-pause-seconds', [string]$selected.suspensionPauseSeconds, + '--suspension-minimum-ratio', [string]$config.suspensionMinimumHealthyThroughputRatio, + '--output', $suspensionOutput) + Assert-SuspensionEvidence $suspensionOutput + } + else { + Add-EvidenceResult 'performance' 'not-qualified' 'performance-skipped' @( + 'explicitly skipped by -SkipPerformance') + $notQualifiedReasons.Add('performance: skipped by -SkipPerformance') + } + + $revalidatedOsEvidenceCount = Assert-AcceptedOsEvidenceStable + $completionAssemblyManifest = @(Get-TestedAssemblyManifest) + Assert-AssemblyManifestStable $testedAssemblyManifest $completionAssemblyManifest + $completionProvenance = Get-RepositoryProvenance + Assert-ProvenanceStable $repositoryProvenance $completionProvenance + Add-EvidenceResult 'completion-integrity' 'passed' 'source-and-tested-assemblies-stable' @( + "commit=$($completionProvenance.commit)", + "sourceManifestSha256=$($completionProvenance.sourceManifestSha256)", + "testedAssemblyManifestSha256=$testedAssemblyDigest", + "acceptedOsEvidenceRevalidated=$revalidatedOsEvidenceCount") + $overallStatus = if ($notQualifiedReasons.Count -eq 0) { 'passed' } else { 'not-qualified' } + } +} +catch { + $overallStatus = 'failed' + $failureMessage = $_.Exception.Message + throw +} +finally { + if ($null -eq $completionProvenance) { + $completionProvenance = Get-RepositoryProvenance + } + if ($testedAssemblyManifest.Count -gt 0 -and $completionAssemblyManifest.Count -eq 0) { + try { + $completionAssemblyManifest = @(Get-TestedAssemblyManifest) + } + catch { + $completionAssemblyManifest = @() + } + } + $dotnetInfo = @($results | Where-Object name -eq 'dotnet-info' | Select-Object -First 1) + $summary = [ordered]@{ + schemaVersion = 4 + tier = $Tier + runId = $runId + validationOnly = [bool]$ValidateOnly + configuration = $Configuration + platform = if ($IsWindows) { 'windows-' + [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant() } elseif ($IsLinux) { 'linux-' + [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant() } else { 'unsupported' } + overallStatus = $overallStatus + failure = $failureMessage + notQualifiedReasons = $notQualifiedReasons + startedUtc = $runStartedUtc + completedUtc = [DateTimeOffset]::UtcNow + provenance = $repositoryProvenance + completionProvenance = $completionProvenance + testedAssemblies = $testedAssemblyManifest + completionTestedAssemblies = $completionAssemblyManifest + host = [ordered]@{ + operatingSystem = [Runtime.InteropServices.RuntimeInformation]::OSDescription + operatingSystemArchitecture = [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() + processArchitecture = [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString() + framework = [Runtime.InteropServices.RuntimeInformation]::FrameworkDescription + runtimeVersion = [Environment]::Version.ToString() + logicalProcessorCount = [Environment]::ProcessorCount + powershellVersion = $PSVersionTable.PSVersion.ToString() + } + toolchain = [ordered]@{ + dotnetPath = $dotnet + dotnetVersion = Invoke-TextCommand $dotnet @('--version') + dotnetInfoSha256 = if ($dotnetInfo.Count -eq 1) { $dotnetInfo[0].stdoutSha256 } else { $null } + powershellPath = $powershell + gitPath = $git + gitVersion = Invoke-TextCommand $git @('--version') + } + inputs = [ordered]@{ + script = [IO.Path]::GetRelativePath($root, $PSCommandPath) + scriptSha256 = Get-FileSha256 $PSCommandPath + configuration = [IO.Path]::GetRelativePath($root, $configPath) + configurationSha256 = Get-FileSha256 $configPath + solutionSha256 = Get-FileSha256 (Join-Path $root 'SharedMemoryStore.slnx') + } + seed = [int]$config.seed + boundedOperationSlackMilliseconds = [int]$config.boundedOperationSlackMilliseconds + requiredLeakAssertions = $config.requiredLeakAssertions + settings = $selected + acceptedOsEvidence = $acceptedOsEvidence + results = $results + evidenceManifest = Get-EvidenceManifest + } + $summary | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath (Join-Path $runRoot 'summary.json') +} + +if ($overallStatus -eq 'validation-only') { + Write-Host "Qualification orchestration validated without executing workloads. Evidence: $runRoot" +} +elseif ($overallStatus -eq 'not-qualified') { + Write-Warning "Qualification '$Tier' completed but is NOT QUALIFIED. Evidence: $runRoot" + exit 2 +} +elseif ($Tier -eq 'release') { + Write-Host "Release qualification gates passed. Evidence: $runRoot" +} +else { + Write-Host "Qualification tier '$Tier' gates passed; short performance rows are smoke-only, not release qualification. Evidence: $runRoot" +} diff --git a/scripts/validate-docker-shared-memory.ps1 b/scripts/validate-docker-shared-memory.ps1 index 636d605..642690c 100644 --- a/scripts/validate-docker-shared-memory.ps1 +++ b/scripts/validate-docker-shared-memory.ps1 @@ -110,7 +110,7 @@ function Invoke-DockerCleanConsumerValidation { enable - + '@ diff --git a/scripts/validate-docs.ps1 b/scripts/validate-docs.ps1 index ded221d..7c63bb4 100644 --- a/scripts/validate-docs.ps1 +++ b/scripts/validate-docs.ps1 @@ -264,7 +264,7 @@ function Assert-PackageMetadata { $expected = @{ TargetFramework = "net10.0" PackageId = "SharedMemoryStore" - Version = "1.0.2" + Version = "2.0.0" Description = "A bounded named shared-memory key-value store for opaque binary values." PackageLicenseExpression = "MIT" PackageReadmeFile = "README.md" @@ -304,7 +304,7 @@ function Assert-PackageMetadata { } Assert-Contains "README.md" "SharedMemoryStore" "package README identity" - Assert-Contains "README.md" "1.0.2" "package version alignment" + Assert-Contains "README.md" "2.0.0" "package version alignment" Assert-Contains "README.md" "net10.0" "target framework alignment" Assert-Contains "README.md" "MIT" "license alignment" Assert-Contains "LICENSE" "MIT License" "license metadata alignment" diff --git a/scripts/validate-lock-free-os.ps1 b/scripts/validate-lock-free-os.ps1 new file mode 100644 index 0000000..490cee3 --- /dev/null +++ b/scripts/validate-lock-free-os.ps1 @@ -0,0 +1,2490 @@ +[CmdletBinding()] +param( + [ValidateSet( + 'self-test', 'architecture', 'atomic', 'raw', 'no-lock', 'crash', + 'release-tests', 'interop', 'samples', 'pack', 'all')] + [string]$Command = 'all', + [ValidateSet('Debug', 'Release')] + [string]$Configuration = 'Release', + [string]$OutputPath = '', + [string]$DockerRuntimeImage = 'mcr.microsoft.com/dotnet/runtime:10.0', + [ValidateRange(1, 86400)] + [int]$StepTimeoutSeconds = 21600, + [switch]$SkipDocker, + [switch]$ValidateOnly +) + +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) +$git = (Get-Command git -ErrorAction Stop).Source +if (-not $ValidateOnly) { + $earlyStatus = @(& $git -C $root ` + '-c' 'core.autocrlf=true' ` + '-c' 'core.safecrlf=false' ` + 'status' '--porcelain=v2' '--untracked-files=normal' 2>$null) + if ($LASTEXITCODE -ne 0) { + throw 'Executable OS qualification could not determine repository cleanliness.' + } + if ($earlyStatus.Count -ne 0) { + throw 'Executable OS qualification requires a clean working tree.' + } +} +$runStartedUtc = [DateTimeOffset]::UtcNow +$platform = if ($IsWindows) { 'windows' } elseif ($IsLinux) { 'linux' } else { 'unsupported' } +$architecture = [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant() +if ([string]::IsNullOrWhiteSpace($OutputPath)) { + $runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ') + '-' + + "$platform-$architecture-$Command-" + [Guid]::NewGuid().ToString('N').Substring(0, 8) + $OutputPath = "artifacts/lock-free-os-validation/$runId.json" +} +$resultPath = if ([IO.Path]::IsPathFullyQualified($OutputPath)) { + [IO.Path]::GetFullPath($OutputPath) +} +else { + [IO.Path]::GetFullPath((Join-Path $root $OutputPath)) +} +$artifactRoot = [IO.Path]::GetFullPath((Join-Path $root 'artifacts')) + [IO.Path]::DirectorySeparatorChar +if (-not ($resultPath.StartsWith($artifactRoot, [StringComparison]::OrdinalIgnoreCase))) { + throw "OS validation output must remain below '$artifactRoot'." +} +if (Test-Path -LiteralPath $resultPath) { + throw "Refusing to overwrite historical OS validation evidence '$resultPath'." +} +New-Item -ItemType Directory -Path (Split-Path -Parent $resultPath) -Force | Out-Null +$evidenceRoot = Join-Path (Split-Path -Parent $resultPath) ([IO.Path]::GetFileNameWithoutExtension($resultPath) + '.evidence') +if (Test-Path -LiteralPath $evidenceRoot) { + throw "Refusing to overwrite historical OS validation evidence directory '$evidenceRoot'." +} +New-Item -ItemType Directory -Path $evidenceRoot | Out-Null + +$results = [Collections.Generic.List[object]]::new() +$dotnet = (Get-Command dotnet -ErrorAction Stop).Source +$pwsh = (Get-Command pwsh -ErrorAction Stop).Source +$qualifiedArchitecture = $architecture -eq 'x64' -and $platform -in @('windows', 'linux') +$integrationProject = 'tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj' +$contractProject = 'tests/SharedMemoryStore.ContractTests/SharedMemoryStore.ContractTests.csproj' + +function Get-StringSha256 { + param([AllowEmptyString()][string]$Value) + + $bytes = [Text.Encoding]::UTF8.GetBytes($Value) + return [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($bytes)) +} + +function Get-FileSha256 { + param([Parameter(Mandatory)][string]$Path) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return $null + } + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash +} + +function Invoke-TextCommand { + param( + [Parameter(Mandatory)][string]$FileName, + [Parameter(Mandatory)][string[]]$Arguments) + + try { + $output = & $FileName @Arguments 2>$null + if ($LASTEXITCODE -ne 0) { + return 'unknown' + } + return (($output | ForEach-Object { [string]$_ }) -join "`n").Trim() + } + catch { + return 'unknown' + } +} + +function Get-SourceManifestSha256 { + $discoveredPaths = @(& $git ` + -c core.autocrlf=true ` + -c core.safecrlf=false ` + -C $root ` + ls-files --cached --others --exclude-standard 2>$null) + if ($LASTEXITCODE -ne 0) { + return 'unknown' + } + + # Source provenance must be identical for the same Git content on Windows + # and Linux regardless of either host's ambient core.autocrlf setting. + # Use an ordinal path set/order and an explicit clean-filter policy. + $pathSet = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($discoveredPath in $discoveredPaths) { + if (-not [string]::IsNullOrEmpty([string]$discoveredPath)) { + $null = $pathSet.Add([string]$discoveredPath) + } + } + $orderedPaths = [Collections.Generic.List[string]]::new($pathSet) + $orderedPaths.Sort([StringComparer]::Ordinal) + $paths = @($orderedPaths) + + $existing = @($paths | Where-Object { Test-Path -LiteralPath (Join-Path $root $_) -PathType Leaf }) + $hashes = if ($existing.Count -eq 0) { + @() + } + else { + @($existing | & $git ` + -c core.autocrlf=true ` + -c core.safecrlf=false ` + -C $root ` + hash-object --stdin-paths 2>$null) + } + if ($LASTEXITCODE -ne 0 -or $hashes.Count -ne $existing.Count) { + return 'unknown' + } + $hashByPath = @{} + for ($index = 0; $index -lt $existing.Count; $index++) { + $hashByPath[$existing[$index]] = [string]$hashes[$index] + } + $entries = foreach ($path in $paths) { + "$path`0$(if ($hashByPath.ContainsKey($path)) { $hashByPath[$path] } else { 'missing' })" + } + return Get-StringSha256 ($entries -join "`n") +} + +function Remove-SolutionProjectBuildOutputs { + param([Parameter(Mandatory)][string]$ReportPath) + + $solutionPath = Join-Path $root 'SharedMemoryStore.slnx' + [xml]$solution = Get-Content -LiteralPath $solutionPath -Raw + $projects = @( + $solution.SelectNodes('//Project[@Path]') | + ForEach-Object { [string]$_.Path } | + Where-Object { [IO.Path]::GetExtension($_).Equals('.csproj', [StringComparison]::OrdinalIgnoreCase) } | + Sort-Object -Unique) + if ($projects.Count -eq 0) { + throw 'SharedMemoryStore.slnx does not contain any C# projects.' + } + + $repositoryProjects = @( + & $git -C $root ls-files --cached --others --exclude-standard -- '*.csproj' 2>$null | + Sort-Object -Unique) + if ($LASTEXITCODE -ne 0 -or $repositoryProjects.Count -eq 0) { + throw 'Could not enumerate tracked and nonignored repository project files.' + } + + $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + $pathComparer = if ($IsWindows) { [StringComparer]::OrdinalIgnoreCase } else { [StringComparer]::Ordinal } + $repositoryProjectSet = [Collections.Generic.HashSet[string]]::new($pathComparer) + foreach ($repositoryProject in $repositoryProjects) { + $null = $repositoryProjectSet.Add($repositoryProject.Replace('\', '/')) + } + $normalizedProjects = @($projects | ForEach-Object { $_.Replace('\', '/') } | Sort-Object -Unique) + $solutionProjectSet = [Collections.Generic.HashSet[string]]::new($pathComparer) + foreach ($normalizedProjectPath in $normalizedProjects) { + $null = $solutionProjectSet.Add($normalizedProjectPath) + } + $outsideSolution = @($repositoryProjectSet | Where-Object { -not $solutionProjectSet.Contains($_) }) + $missingFromRepository = @($solutionProjectSet | Where-Object { -not $repositoryProjectSet.Contains($_) }) + if ($outsideSolution.Count -ne 0 -or $missingFromRepository.Count -ne 0) { + throw "SharedMemoryStore.slnx must contain every tracked or nonignored C# project; outsideSolution='$($outsideSolution -join ',')'; missingFromRepository='$($missingFromRepository -join ',')'." + } + + $normalizedRoot = [IO.Path]::GetFullPath($root).TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + $rootPrefix = $normalizedRoot + [IO.Path]::DirectorySeparatorChar + $removed = 0 + $targetRecords = [Collections.Generic.List[object]]::new() + foreach ($project in $projects) { + $normalizedProject = $project.Replace('\', '/') + if ([IO.Path]::IsPathRooted($project) -or -not $repositoryProjectSet.Contains($normalizedProject)) { + throw "Solution project is not a tracked or nonignored repository project: '$project'." + } + + $projectPath = [IO.Path]::GetFullPath((Join-Path $normalizedRoot $project)) + if (-not $projectPath.StartsWith($rootPrefix, $comparison) ` + -or -not (Test-Path -LiteralPath $projectPath -PathType Leaf)) { + throw "Solution project path is missing or outside the repository: '$projectPath'." + } + + $projectItem = Get-Item -LiteralPath $projectPath -Force + if (($projectItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to clean outputs for linked solution project '$projectPath'." + } + + $projectDirectory = [IO.Path]::GetDirectoryName($projectPath) + $ancestor = $projectDirectory + while ($true) { + $ancestorItem = Get-Item -LiteralPath $ancestor -Force + if (-not $ancestorItem.PSIsContainer ` + -or ($ancestorItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to clean through non-directory or linked ancestor '$ancestor'." + } + if ($ancestor.Equals($normalizedRoot, $comparison)) { + break + } + if (-not $ancestor.StartsWith($rootPrefix, $comparison)) { + throw "Project ancestor escaped the repository while checking '$projectPath'." + } + $parent = [IO.Directory]::GetParent($ancestor) + if ($null -eq $parent) { + throw "Repository root was not reached while checking '$projectPath'." + } + $ancestor = $parent.FullName + } + + foreach ($directoryName in @('bin', 'obj')) { + $target = [IO.Path]::GetFullPath((Join-Path $projectDirectory $directoryName)) + if (-not $target.StartsWith($rootPrefix, $comparison)) { + throw "Refusing to clean project output outside the repository: '$target'." + } + + $relativeTarget = [IO.Path]::GetRelativePath($normalizedRoot, $target).Replace('\', '/') + $protectedEntries = @( + & $git -C $normalizedRoot --literal-pathspecs ls-files --cached --others --exclude-standard -- $relativeTarget 2>$null) + if ($LASTEXITCODE -ne 0) { + throw "Could not prove that '$relativeTarget' contains only ignored build output." + } + if ($protectedEntries.Count -ne 0) { + throw "Refusing to clean '$relativeTarget' because it contains tracked or nonignored entry '$($protectedEntries[0])'." + } + $existed = Test-Path -LiteralPath $target + if (-not $existed) { + $targetRecords.Add([pscustomobject][ordered]@{ + path = $relativeTarget + existed = $false + removed = $false + verifiedAbsent = $false + trackedOrNonignoredFiles = 0 + }) + continue + } + + $item = Get-Item -LiteralPath $target -Force + if (-not $item.PSIsContainer ` + -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to recursively clean non-directory or linked output path '$target'." + } + $linkedDescendants = @(Get-ChildItem -LiteralPath $target -Recurse -Force -ErrorAction Stop | + Where-Object { ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 } | + Sort-Object { $_.FullName.Length } -Descending) + foreach ($linkedDescendant in $linkedDescendants) { + Remove-Item -LiteralPath $linkedDescendant.FullName -Force + if (Test-Path -LiteralPath $linkedDescendant.FullName) { + throw "Could not unlink project output entry '$($linkedDescendant.FullName)'." + } + } + Remove-Item -LiteralPath $target -Recurse -Force + if (Test-Path -LiteralPath $target) { + throw "Project output remained after recursive cleanup: '$target'." + } + $targetRecords.Add([pscustomobject][ordered]@{ + path = $relativeTarget + existed = $true + removed = $true + verifiedAbsent = $false + trackedOrNonignoredFiles = 0 + }) + $removed++ + } + } + + $stabilized = $false + for ($stabilizationPass = 1; $stabilizationPass -le 5; $stabilizationPass++) { + $recleaned = $false + foreach ($targetRecord in $targetRecords) { + $absoluteTarget = [IO.Path]::GetFullPath((Join-Path $normalizedRoot $targetRecord.path)) + $remainingItem = Get-Item -LiteralPath $absoluteTarget -Force -ErrorAction SilentlyContinue + if ($null -eq $remainingItem -and -not (Test-Path -LiteralPath $absoluteTarget)) { + continue + } + + $protectedEntries = @( + & $git -C $normalizedRoot --literal-pathspecs ls-files --cached --others --exclude-standard -- $targetRecord.path 2>$null) + if ($LASTEXITCODE -ne 0 -or $protectedEntries.Count -ne 0) { + throw "Refusing to re-clean recreated output '$($targetRecord.path)' without a zero-protected-file proof." + } + if ($null -eq $remainingItem ` + -or -not $remainingItem.PSIsContainer ` + -or ($remainingItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to re-clean non-directory or linked output '$absoluteTarget'." + } + $linkedDescendants = @(Get-ChildItem -LiteralPath $absoluteTarget -Recurse -Force -ErrorAction Stop | + Where-Object { ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 } | + Sort-Object { $_.FullName.Length } -Descending) + foreach ($linkedDescendant in $linkedDescendants) { + Remove-Item -LiteralPath $linkedDescendant.FullName -Force + } + Remove-Item -LiteralPath $absoluteTarget -Recurse -Force + $targetRecord.existed = $true + if (-not $targetRecord.removed) { + $targetRecord.removed = $true + $removed++ + } + $recleaned = $true + } + + Start-Sleep -Milliseconds 100 + $remainingTargets = @($targetRecords | Where-Object { + $candidate = [IO.Path]::GetFullPath((Join-Path $normalizedRoot $_.path)) + $null -ne (Get-Item -LiteralPath $candidate -Force -ErrorAction SilentlyContinue) ` + -or (Test-Path -LiteralPath $candidate) + }) + if ($remainingTargets.Count -eq 0) { + foreach ($targetRecord in $targetRecords) { + $targetRecord.verifiedAbsent = $true + } + $stabilized = $true + break + } + } + if (-not $stabilized) { + throw 'Project outputs did not converge to an absent state after five guarded cleanup passes.' + } + + $orderedTargets = @($targetRecords | Sort-Object path) + $expectedTargetCount = $normalizedProjects.Count * 2 + if ($orderedTargets.Count -ne $expectedTargetCount ` + -or @($orderedTargets | ForEach-Object path | Sort-Object -Unique).Count -ne $expectedTargetCount ` + -or @($orderedTargets | Where-Object { -not $_.verifiedAbsent }).Count -ne 0) { + throw 'Cross-platform pre-clean did not verify every unique solution project output target as absent.' + } + + if (Test-Path -LiteralPath $ReportPath) { + throw "Refusing to overwrite pre-clean evidence '$ReportPath'." + } + $report = [pscustomobject][ordered]@{ + schemaVersion = 1 + completedUtc = [DateTimeOffset]::UtcNow + solution = 'SharedMemoryStore.slnx' + solutionProjects = $normalizedProjects + solutionProjectCount = $normalizedProjects.Count + solutionProjectSetSha256 = Get-StringSha256 ($normalizedProjects -join "`n") + targets = $orderedTargets + targetCount = $orderedTargets.Count + targetSetSha256 = Get-StringSha256 (@($orderedTargets | ForEach-Object path) -join "`n") + existedCount = @($orderedTargets | Where-Object existed).Count + removedCount = @($orderedTargets | Where-Object removed).Count + verifiedAbsentCount = @($orderedTargets | Where-Object verifiedAbsent).Count + trackedOrNonignoredFileCount = [int64](@($orderedTargets | Measure-Object trackedOrNonignoredFiles -Sum).Sum) + } + New-Item -ItemType Directory -Path (Split-Path -Parent $ReportPath) -Force | Out-Null + [IO.File]::WriteAllText($ReportPath, ($report | ConvertTo-Json -Depth 6), [Text.UTF8Encoding]::new($false)) + $reportRelativePath = [IO.Path]::GetRelativePath($normalizedRoot, $ReportPath).Replace('\', '/') + $reportSha256 = Get-FileSha256 $ReportPath + $summary = [pscustomobject][ordered]@{ + schemaVersion = 1 + solutionProjectCount = $normalizedProjects.Count + uniqueSolutionProjectCount = $solutionProjectSet.Count + solutionProjectSetSha256 = $report.solutionProjectSetSha256 + targetCount = $orderedTargets.Count + uniqueTargetCount = @($orderedTargets | ForEach-Object path | Sort-Object -Unique).Count + targetSetSha256 = $report.targetSetSha256 + existedBeforeCount = $report.existedCount + removedCount = $report.removedCount + verifiedAbsentCount = $report.verifiedAbsentCount + protectedFileCount = $report.trackedOrNonignoredFileCount + reportPath = $reportRelativePath + reportSha256 = $reportSha256 + } + + return [pscustomobject]@{ + SolutionProjectCount = $normalizedProjects.Count + TargetCount = $orderedTargets.Count + RemovedDirectoryCount = $removed + VerifiedAbsentCount = $orderedTargets.Count + ReportPath = $ReportPath + ReportSha256 = $reportSha256 + Summary = $summary + } +} + +function Get-RepositoryProvenance { + $status = Invoke-TextCommand $git @( + '-c', 'core.autocrlf=true', + '-c', 'core.safecrlf=false', + '-C', $root, + 'status', '--porcelain=v2', '--untracked-files=normal') + return [ordered]@{ + repositoryCommit = Invoke-TextCommand $git @('-C', $root, 'rev-parse', 'HEAD') + headTree = Invoke-TextCommand $git @('-C', $root, 'rev-parse', 'HEAD^{tree}') + workingTreeState = if ([string]::IsNullOrWhiteSpace($status)) { 'clean' } elseif ($status -eq 'unknown') { 'unknown' } else { 'dirty' } + statusSha256 = Get-StringSha256 $status + sourceManifestSha256 = Get-SourceManifestSha256 + } +} + +$repositoryProvenance = Get-RepositoryProvenance +$completionProvenance = $null +$testedAssemblyManifest = @() +$completionAssemblyManifest = @() + +function Assert-KnownProvenance { + param( + [Parameter(Mandatory)]$Provenance, + [Parameter(Mandatory)][string]$Context) + + foreach ($property in @('repositoryCommit', 'headTree', 'workingTreeState', 'statusSha256', 'sourceManifestSha256')) { + $value = [string]$Provenance[$property] + if ([string]::IsNullOrWhiteSpace($value) -or $value -eq 'unknown') { + throw "$Context provenance property '$property' is unknown." + } + } +} + +function Assert-ProvenanceStable { + param( + [Parameter(Mandatory)]$Start, + [Parameter(Mandatory)]$End) + + Assert-KnownProvenance $Start 'start' + Assert-KnownProvenance $End 'completion' + foreach ($property in @('repositoryCommit', 'headTree', 'workingTreeState', 'statusSha256', 'sourceManifestSha256')) { + if ([string]$Start[$property] -ne [string]$End[$property]) { + throw "Repository provenance changed during OS validation: '$property'." + } + } +} + +function Get-TestedAssemblyManifest { + [xml]$solution = Get-Content -LiteralPath (Join-Path $root 'SharedMemoryStore.slnx') -Raw + $projectPaths = @($solution.SelectNodes("//*[local-name()='Project']") | ForEach-Object { [string]$_.Path }) + if ($projectPaths.Count -eq 0) { + throw 'Solution does not expose project paths for assembly provenance.' + } + + $files = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($projectPath in $projectPaths) { + $projectDirectory = Split-Path -Parent (Join-Path $root $projectPath) + $assemblyName = [IO.Path]::GetFileNameWithoutExtension($projectPath) + '.dll' + $outputDirectory = Join-Path $projectDirectory "bin/$Configuration/net10.0" + foreach ($fileName in @($assemblyName, 'SharedMemoryStore.dll') | Sort-Object -Unique) { + $fullPath = Join-Path $outputDirectory $fileName + if (Test-Path -LiteralPath $fullPath -PathType Leaf) { + [void]$files.Add([IO.Path]::GetFullPath($fullPath)) + } + elseif ($fileName -eq $assemblyName) { + throw "Expected freshly built assembly '$fullPath' is missing." + } + } + } + + return @($files | Sort-Object | ForEach-Object { + [pscustomobject][ordered]@{ + path = [IO.Path]::GetRelativePath($root, $_) + length = (Get-Item -LiteralPath $_).Length + sha256 = Get-FileSha256 $_ + } + }) +} + +function Get-TestedAssemblyHash { + param([Parameter(Mandatory)][string]$RelativePath) + + $matches = @($testedAssemblyManifest | Where-Object { + ([string]$_.path).Replace('\', '/') -ceq $RelativePath.Replace('\', '/') + }) + if ($matches.Count -ne 1) { + throw "Tested assembly manifest does not contain exactly one '$RelativePath' row." + } + $hash = [string]$matches[0].sha256 + if ($hash -notmatch '^[0-9A-F]{64}$') { + throw "Tested assembly '$RelativePath' has an invalid SHA-256 digest." + } + return $hash +} + +function Assert-AssemblyManifestStable { + param( + [Parameter(Mandatory)][object[]]$Start, + [Parameter(Mandatory)][object[]]$End) + + $startCanonical = @($Start | ForEach-Object { "$($_.path)|$($_.length)|$($_.sha256)" }) -join "`n" + $endCanonical = @($End | ForEach-Object { "$($_.path)|$($_.length)|$($_.sha256)" }) -join "`n" + if ([string]::IsNullOrWhiteSpace($startCanonical) -or $startCanonical -cne $endCanonical) { + throw 'Tested assembly manifest changed after the clean OS-validation build.' + } +} + +function Test-IsIntegerNumber { + param($Value) + + return $Value -is [byte] -or $Value -is [sbyte] ` + -or $Value -is [int16] -or $Value -is [uint16] ` + -or $Value -is [int32] -or $Value -is [uint32] ` + -or $Value -is [int64] -or $Value -is [uint64] +} + +function Test-IsNumericValue { + param($Value) + + return (Test-IsIntegerNumber $Value) -or $Value -is [single] ` + -or $Value -is [double] -or $Value -is [decimal] +} + +function Get-RequiredPropertyValue { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context) + + $member = $Object.PSObject.Properties[$Property] + if ($null -eq $member -or $null -eq $member.Value) { + throw "$Context is missing required property '$Property'." + } + return $member.Value +} + +function Get-StrictInt64 { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context, + [int64]$Minimum = [int64]::MinValue, + [int64]$Maximum = [int64]::MaxValue) + + $value = Get-RequiredPropertyValue $Object $Property $Context + if (-not (Test-IsIntegerNumber $value)) { + throw "$Context.$Property must be an integer JSON number." + } + try { + $converted = [Convert]::ToInt64($value, [Globalization.CultureInfo]::InvariantCulture) + } + catch { + throw "$Context.$Property is outside signed 64-bit range." + } + if ($converted -lt $Minimum -or $converted -gt $Maximum) { + throw "$Context.$Property=$converted is outside [$Minimum,$Maximum]." + } + return $converted +} + +function Get-StrictDouble { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context, + [double]$Minimum = -[double]::MaxValue, + [double]$Maximum = [double]::MaxValue, + [switch]$Positive) + + $value = Get-RequiredPropertyValue $Object $Property $Context + if (-not (Test-IsNumericValue $value)) { + throw "$Context.$Property must be a numeric JSON value." + } + $converted = [Convert]::ToDouble($value, [Globalization.CultureInfo]::InvariantCulture) + if (-not [double]::IsFinite($converted) -or $converted -lt $Minimum -or $converted -gt $Maximum ` + -or ($Positive -and $converted -le 0)) { + throw "$Context.$Property=$converted is not a valid finite value." + } + return $converted +} + +function Get-StrictString { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context) + + $value = Get-RequiredPropertyValue $Object $Property $Context + if ($value -isnot [string] -or [string]::IsNullOrWhiteSpace($value)) { + throw "$Context.$Property must be a nonempty JSON string." + } + return [string]$value +} + +function Get-StrictBoolean { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Property, + [Parameter(Mandatory)][string]$Context) + + $value = Get-RequiredPropertyValue $Object $Property $Context + if ($value -isnot [bool]) { + throw "$Context.$Property must be a JSON Boolean." + } + return [bool]$value +} + +function Assert-ExactStringArray { + param( + [Parameter(Mandatory)]$Actual, + [Parameter(Mandatory)][string[]]$Expected, + [Parameter(Mandatory)][string]$Context) + + $actualArray = @($Actual) + if ($actualArray.Count -ne $Expected.Count) { + throw "$Context must contain exactly [$($Expected -join ', ')]." + } + for ($index = 0; $index -lt $Expected.Count; $index++) { + if ($actualArray[$index] -isnot [string] -or [string]$actualArray[$index] -cne $Expected[$index]) { + throw "$Context must contain exactly [$($Expected -join ', ')] in canonical order." + } + } +} + +function Get-MedianValue { + param([Parameter(Mandatory)][double[]]$Values) + + if ($Values.Count -eq 0) { + throw 'Cannot compute a median for an empty evidence set.' + } + $sorted = @($Values | Sort-Object) + $middleIndex = [int][Math]::Floor($sorted.Count / 2.0) + if (($sorted.Count % 2) -eq 0) { + return ([double]$sorted[$middleIndex - 1] + [double]$sorted[$middleIndex]) / 2.0 + } + return [double]$sorted[$middleIndex] +} + +function Assert-DerivedDouble { + param( + [Parameter(Mandatory)][double]$Actual, + [Parameter(Mandatory)][double]$Expected, + [Parameter(Mandatory)][string]$Context) + + $tolerance = [Math]::Max(0.000000001, [Math]::Abs($Expected) * 0.000000000001) + if (-not [double]::IsFinite($Actual) -or -not [double]::IsFinite($Expected) ` + -or [Math]::Abs($Actual - $Expected) -gt $tolerance) { + throw "$Context is not reproducible from the raw evidence." + } +} + +function Assert-LinuxTinySyncTopology { + param( + [Parameter(Mandatory)]$Object, + [Parameter(Mandatory)][string]$Context) + + if ((Get-StrictInt64 $Object 'syncKeysPerWorker' $Context 2 2) -ne 2 ` + -or (Get-StrictInt64 $Object 'syncMaximumWorkerCount' $Context 12 12) -ne 12 ` + -or (Get-StrictInt64 $Object 'syncCanonicalBucketCount' $Context 16 16) -ne 16) { + throw "$Context does not describe the exact two-key/12-worker/16-bucket synchronization topology." + } + + $digest = Get-StrictString $Object 'syncKeyCatalogSha256' $Context + if ($digest -cne '9A7E93EB1382F2665155971C64C10D4C29039916CD9E314DB72B9906549656D2' ` + -or $digest -cnotmatch '^[0-9A-F]{64}$') { + throw "$Context has the wrong deterministic synchronization-key catalog digest." + } + + $assignments = @(Get-RequiredPropertyValue $Object 'syncKeyCanonicalBucketAssignments' $Context) + if ($assignments.Count -ne 24) { + throw "$Context must contain exactly 24 synchronization-key bucket assignments." + } + for ($index = 0; $index -lt $assignments.Count; $index++) { + [int64]$expected = [Math]::Floor($index / 2) + if (-not (Test-IsIntegerNumber $assignments[$index]) ` + -or [int64]$assignments[$index] -ne $expected) { + throw "$Context synchronization-key bucket assignment $index must be $expected." + } + } +} + +function Assert-RequiredBenchmarkHardwareMetadata { + param( + [Parameter(Mandatory)]$Environment, + [Parameter(Mandatory)][string]$Context) + + $logicalCount = Get-StrictInt64 $Environment 'logicalProcessorCount' $Context 1 ([int32]::MaxValue) + [void](Get-StrictInt64 $Environment 'physicalCoreCount' $Context 1 $logicalCount) + [void](Get-StrictInt64 $Environment 'totalMemoryBytes' $Context 1048576 ([int64]::MaxValue)) + $processorModel = Get-StrictString $Environment 'processorModel' $Context + $processorIdentifier = Get-StrictString $Environment 'processorIdentifier' $Context + foreach ($value in @($processorModel, $processorIdentifier)) { + if ($value.Trim() -match '^(?i:(?:unknown|unavailable|not[- ]available|n/?a)(?:\s+(?:cpu|processor|model))?)$') { + throw "$Context contains unknown processor-model evidence '$value'." + } + } +} + +function Assert-ExactBenchmarkStoreDimensions { + param( + [Parameter(Mandatory)]$Dimensions, + [Parameter(Mandatory)][string]$Context, + [Parameter(Mandatory)][int64]$SlotCount, + [Parameter(Mandatory)][int64]$MaxValueBytes, + [Parameter(Mandatory)][int64]$MaxDescriptorBytes, + [Parameter(Mandatory)][int64]$MaxKeyBytes, + [Parameter(Mandatory)][int64]$LeaseRecordCount, + [Parameter(Mandatory)][int64]$LockFreeParticipantRecordCount) + + $expected = [ordered]@{ + slotCount = $SlotCount + maxValueBytes = $MaxValueBytes + maxDescriptorBytes = $MaxDescriptorBytes + maxKeyBytes = $MaxKeyBytes + leaseRecordCount = $LeaseRecordCount + lockFreeParticipantRecordCount = $LockFreeParticipantRecordCount + } + foreach ($entry in $expected.GetEnumerator()) { + if ((Get-StrictInt64 $Dimensions $entry.Key $Context 0 ([int32]::MaxValue)) -ne $entry.Value) { + throw "$Context.$($entry.Key) does not match the exact benchmark store topology." + } + } +} + +function Assert-LinuxTinyStoreDimensions { + param( + [Parameter(Mandatory)]$Configuration, + [Parameter(Mandatory)][string]$Context) + + $allDimensions = Get-RequiredPropertyValue $Configuration 'scenarioStoreDimensions' $Context + if ((@($allDimensions.PSObject.Properties.Name) -join ',') -cne 'acquire-release,publish-remove') { + throw "$Context scenarioStoreDimensions must contain exactly acquire-release and publish-remove." + } + foreach ($scenario in @('acquire-release', 'publish-remove')) { + Assert-ExactBenchmarkStoreDimensions ` + $allDimensions.$scenario ` + "$Context scenarioStoreDimensions.$scenario" ` + 32 8 0 8 64 64 + } +} + +function Assert-LinuxTinyPerformanceConfiguration { + param([Parameter(Mandatory)]$Config) + + $tiny = Get-RequiredPropertyValue $Config 'linuxTinyPerformance' 'qualification config' + $expectedProperties = @( + 'mode', 'profiles', 'scenarios', 'processCounts', 'syncKeysPerWorker', + 'syncMaximumWorkerCount', 'syncCanonicalBucketCount', 'syncKeyCatalogSha256', + 'syncKeyCanonicalBucketAssignments', 'minimumThroughputRatio', + 'maximumUncontendedP99Ratio', 'maximumScaleP99Ratio', + 'maximumP99Microseconds', 'maximumStallMicroseconds') + $actualProperties = @($tiny.PSObject.Properties.Name) + if (($actualProperties -join ',') -cne ($expectedProperties -join ',')) { + throw "qualification config linuxTinyPerformance properties must be exactly [$($expectedProperties -join ', ')]." + } + if ((Get-StrictString $tiny 'mode' 'qualification config linuxTinyPerformance') -cne 'sync') { + throw 'qualification config linuxTinyPerformance.mode must be sync.' + } + Assert-ExactStringArray $tiny.profiles @('Legacy', 'LockFree') 'qualification config linuxTinyPerformance.profiles' + Assert-ExactStringArray $tiny.scenarios @('acquire-release', 'publish-remove') 'qualification config linuxTinyPerformance.scenarios' + $counts = @(Get-RequiredPropertyValue $tiny 'processCounts' 'qualification config linuxTinyPerformance') + if ($counts.Count -ne 2 ` + -or -not (Test-IsIntegerNumber $counts[0]) -or [int64]$counts[0] -ne 1 ` + -or -not (Test-IsIntegerNumber $counts[1]) -or [int64]$counts[1] -ne 8) { + throw 'qualification config linuxTinyPerformance.processCounts must be exactly [1, 8].' + } + [void](Assert-LinuxTinySyncTopology $tiny 'qualification config linuxTinyPerformance') + if ((Get-StrictDouble $tiny 'minimumThroughputRatio' 'qualification config linuxTinyPerformance' 1 1) -ne 1 ` + -or (Get-StrictDouble $tiny 'maximumUncontendedP99Ratio' 'qualification config linuxTinyPerformance' 1 1) -ne 1 ` + -or (Get-StrictDouble $tiny 'maximumScaleP99Ratio' 'qualification config linuxTinyPerformance' 3 3) -ne 3 ` + -or (Get-StrictDouble $tiny 'maximumP99Microseconds' 'qualification config linuxTinyPerformance' 10 10) -ne 10 ` + -or (Get-StrictDouble $tiny 'maximumStallMicroseconds' 'qualification config linuxTinyPerformance' 10000 10000) -ne 10000) { + throw 'qualification config linuxTinyPerformance gates must remain LF1/Legacy1 p99<=1, LF8/Legacy8 throughput>=1, LF8/LF1 p99<=3, LF8 p99<=10us, and every LF raw stall<=10000us.' + } + $release = Get-RequiredPropertyValue (Get-RequiredPropertyValue $Config 'tiers' 'qualification config') 'release' 'qualification config tiers' + if ((Get-StrictInt64 $release 'performanceWarmupSeconds' 'qualification config release' 10 10) -ne 10 ` + -or (Get-StrictInt64 $release 'performanceDurationSeconds' 'qualification config release' 60 60) -ne 60 ` + -or (Get-StrictInt64 $release 'performanceTrials' 'qualification config release' 3 3) -ne 3) { + throw 'Linux tiny release performance requires exactly 10s warmup, 60s measurement, and three trials.' + } + return $tiny +} + +function Test-LinuxTinyHostTuple { + param( + [Parameter(Mandatory)]$Environment, + [Parameter(Mandatory)][string]$ExpectedRepositoryCommit, + [Parameter(Mandatory)][string]$ExpectedOperatingSystem, + [Parameter(Mandatory)][string]$ExpectedOperatingSystemArchitecture, + [Parameter(Mandatory)][string]$ExpectedProcessArchitecture, + [Parameter(Mandatory)][int64]$ExpectedLogicalProcessorCount, + [Parameter(Mandatory)][bool]$LinuxHost) + + return $LinuxHost ` + -and [string]$Environment.repositoryCommit -ceq $ExpectedRepositoryCommit ` + -and [string]$Environment.repositoryWorkingTreeState -ceq 'clean' ` + -and [string]$Environment.operatingSystem -ceq $ExpectedOperatingSystem ` + -and [string]$Environment.operatingSystemArchitecture -ceq $ExpectedOperatingSystemArchitecture ` + -and [string]$Environment.processArchitecture -ceq $ExpectedProcessArchitecture ` + -and [int64]$Environment.logicalProcessorCount -eq $ExpectedLogicalProcessorCount +} + +function Assert-LinuxTinyPerformanceReport { + param( + [Parameter(Mandatory)]$Report, + [Parameter(Mandatory)]$TinyConfig, + [Parameter(Mandatory)]$ReleaseConfig, + [switch]$SkipEnvironmentBinding) + + if ((Get-StrictInt64 $Report 'schemaVersion' 'Linux tiny performance report' 8 8) -ne 8 ` + -or (Get-StrictInt64 $Report 'minimumCompatibleSchemaVersion' 'Linux tiny performance report' 8 8) -ne 8) { + throw 'Linux tiny performance report must be schema 8 with minimum-compatible schema 8.' + } + [void](Get-StrictString $Report 'schemaCompatibility' 'Linux tiny performance report') + $environment = Get-RequiredPropertyValue $Report 'environment' 'Linux tiny performance report' + foreach ($property in @( + 'repositoryCommit', 'repositoryWorkingTreeState', 'sharedMemoryStoreAssemblySha256', + 'probeAssemblySha256', 'operatingSystem', 'operatingSystemArchitecture', + 'processArchitecture', 'framework', 'runtimeVersion')) { + [void](Get-StrictString $environment $property 'Linux tiny performance environment') + } + Assert-RequiredBenchmarkHardwareMetadata $environment 'Linux tiny performance environment' + [void](Get-StrictInt64 $environment 'stopwatchFrequency' 'Linux tiny performance environment' 1 ([int64]::MaxValue)) + [void](Get-StrictBoolean $environment 'serverGarbageCollection' 'Linux tiny performance environment') + if (-not $SkipEnvironmentBinding) { + if (-not (Test-LinuxTinyHostTuple ` + $environment ` + ([string]$repositoryProvenance.repositoryCommit) ` + ([Runtime.InteropServices.RuntimeInformation]::OSDescription) ` + ([Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString()) ` + ([Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString()) ` + ([Environment]::ProcessorCount) ` + ([bool]$IsLinux))) { + throw 'Linux tiny performance environment does not match this clean Linux qualification host.' + } + $probePath = "benchmarks/SharedMemoryStore.SyncProbe/bin/$Configuration/net10.0/SharedMemoryStore.SyncProbe.dll" + $storePath = "benchmarks/SharedMemoryStore.SyncProbe/bin/$Configuration/net10.0/SharedMemoryStore.dll" + if ([string]$environment.probeAssemblySha256 -cne (Get-TestedAssemblyHash $probePath) ` + -or [string]$environment.sharedMemoryStoreAssemblySha256 -cne (Get-TestedAssemblyHash $storePath)) { + throw 'Linux tiny performance assembly hashes do not match the fresh tested-assembly manifest.' + } + } + + $configuration = Get-RequiredPropertyValue $Report 'configuration' 'Linux tiny performance report' + if ((Get-StrictString $configuration 'mode' 'Linux tiny performance configuration') -cne 'sync' ` + -or (Get-StrictInt64 $configuration 'warmupSeconds' 'Linux tiny performance configuration' 10 10) -ne + (Get-StrictInt64 $ReleaseConfig 'performanceWarmupSeconds' 'qualification config release' 10 10) ` + -or (Get-StrictInt64 $configuration 'durationSeconds' 'Linux tiny performance configuration' 60 60) -ne + (Get-StrictInt64 $ReleaseConfig 'performanceDurationSeconds' 'qualification config release' 60 60) ` + -or (Get-StrictInt64 $configuration 'durationBoundGraceSeconds' 'Linux tiny performance configuration' 60 60) -ne + (Get-StrictInt64 $ReleaseConfig 'performanceDurationBoundGraceSeconds' 'qualification config release' 60 60) ` + -or (Get-StrictInt64 $configuration 'trials' 'Linux tiny performance configuration' 3 3) -ne + (Get-StrictInt64 $ReleaseConfig 'performanceTrials' 'qualification config release' 3 3) ` + -or (Get-StrictInt64 $configuration 'warmupCycles' 'Linux tiny performance configuration' 0 0) -ne 0 ` + -or (Get-StrictInt64 $configuration 'samplingInterval' 'Linux tiny performance configuration' 64 64) -ne 64 ` + -or (Get-StrictInt64 $configuration 'maxLatencySamplesPerWorker' 'Linux tiny performance configuration' 65536 65536) -ne 65536 ` + -or -not (Get-StrictBoolean $configuration 'affinityRequested' 'Linux tiny performance configuration')) { + throw 'Linux tiny performance report configuration does not match the exact release workload.' + } + [void](Assert-LinuxTinySyncTopology $configuration 'Linux tiny performance configuration') + Assert-LinuxTinyStoreDimensions $configuration 'Linux tiny performance configuration' + Assert-ExactStringArray $configuration.profiles @('Legacy', 'LockFree') 'Linux tiny performance report profiles' + Assert-ExactStringArray $configuration.countBoundProfiles @('LockFree') 'Linux tiny performance report count-bound profiles' + Assert-ExactStringArray $configuration.scenarios @('acquire-release', 'publish-remove') 'Linux tiny performance report scenarios' + $scenarioCounts = Get-RequiredPropertyValue $configuration 'scenarioProcessCounts' 'Linux tiny performance configuration' + if ((@($scenarioCounts.PSObject.Properties.Name) -join ',') -cne 'acquire-release,publish-remove') { + throw 'Linux tiny performance scenarioProcessCounts has unexpected keys or order.' + } + foreach ($scenario in @('acquire-release', 'publish-remove')) { + $values = @($scenarioCounts.$scenario) + if ($values.Count -ne 2 ` + -or -not (Test-IsIntegerNumber $values[0]) -or [int64]$values[0] -ne 1 ` + -or -not (Test-IsIntegerNumber $values[1]) -or [int64]$values[1] -ne 8) { + throw "Linux tiny performance scenario '$scenario' must contain exactly process counts [1, 8]." + } + } + + $runs = @($Report.runs) + $summaries = @($Report.summary) + if ($runs.Count -ne 24 -or $summaries.Count -ne 8) { + throw "Linux tiny performance matrix must contain exactly 24 raw runs and 8 summaries; actual=$($runs.Count)/$($summaries.Count)." + } + $expectedRunKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + $expectedSummaryKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($profile in @('Legacy', 'LockFree')) { + foreach ($scenario in @('acquire-release', 'publish-remove')) { + foreach ($processCount in @(1, 8)) { + [void]$expectedSummaryKeys.Add("$profile|$scenario|$processCount") + foreach ($trial in 1..3) { + [void]$expectedRunKeys.Add("$profile|$scenario|$processCount|$trial") + } + } + } + } + $actualRunKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($run in $runs) { + $context = "Linux tiny performance run $($run.profile)/$($run.scenario)/$($run.processCount)/trial-$($run.trial)" + $profile = Get-StrictString $run 'profile' $context + $scenario = Get-StrictString $run 'scenario' $context + $processCount = Get-StrictInt64 $run 'processCount' $context 1 8 + if ($processCount -notin @(1, 8)) { + throw "$context has an unsupported process count." + } + $trial = Get-StrictInt64 $run 'trial' $context 1 3 + $key = "$profile|$scenario|$processCount|$trial" + if (-not $expectedRunKeys.Contains($key) -or -not $actualRunKeys.Add($key)) { + throw "$context is unexpected or duplicated." + } + if ((Get-StrictString $run 'qualification' $context) -cne 'qualification-measurement' ` + -or (Get-StrictInt64 $run 'failures' $context 0 0) -ne 0 ` + -or (Get-StrictInt64 $run 'operationTarget' $context 0 0) -ne 0 ` + -or (Get-StrictInt64 $run 'frameTarget' $context 0 0) -ne 0 ` + -or (Get-StrictBoolean $run 'oversubscribed' $context)) { + throw "$context is not a correctness-clean, non-oversubscribed qualification measurement." + } + $readerCount = Get-StrictInt64 $run 'readerProcessCount' $context 0 $processCount + $publisherCount = Get-StrictInt64 $run 'publisherProcessCount' $context 0 $processCount + $observerCount = Get-StrictInt64 $run 'observerProcessCount' $context 0 0 + if (($scenario -ceq 'acquire-release' -and ($readerCount -ne $processCount -or $publisherCount -ne 0)) ` + -or ($scenario -ceq 'publish-remove' -and ($readerCount -ne 0 -or $publisherCount -ne $processCount)) ` + -or $observerCount -ne 0) { + throw "$context has the wrong process-role topology." + } + $cycles = Get-StrictInt64 $run 'cycles' $context 1 ([int64]::MaxValue) + $operations = Get-StrictInt64 $run 'operations' $context 1 ([int64]::MaxValue) + if ([decimal]$operations -lt ([decimal]2 * [decimal]$cycles)) { + throw "$context has fewer than the two recorded store operations required per completed cycle." + } + $measuredSeconds = Get-StrictDouble $run 'measuredSeconds' $context ` + ([double]$ReleaseConfig.performanceDurationSeconds) ([double]::MaxValue) + $wallSeconds = Get-StrictDouble $run 'wallSeconds' $context $measuredSeconds ([double]::MaxValue) + [void]$wallSeconds + [int64]$minimumWindowSamples = [int64]$processCount * 1024 + [int64]$maximumWindowSamples = [int64]$processCount * 32768 + [int64]$minimumTotalSamples = $minimumWindowSamples * 2 + [int64]$maximumTotalSamples = $maximumWindowSamples * 2 + $earlySampleCount = Get-StrictInt64 $run 'earlySampleCount' $context $minimumWindowSamples $maximumWindowSamples + $lateSampleCount = Get-StrictInt64 $run 'lateSampleCount' $context $minimumWindowSamples $maximumWindowSamples + $sampleCount = Get-StrictInt64 $run 'sampleCount' $context $minimumTotalSamples $maximumTotalSamples + if ($sampleCount -ne ($earlySampleCount + $lateSampleCount) -or $sampleCount -gt $cycles) { + throw "$context sampleCount must equal its early/late windows and cannot exceed completed cycles." + } + $apiCallsPerSecond = Get-StrictDouble $run 'apiCallsPerSecond' $context 0 ([double]::MaxValue) -Positive + Assert-DerivedDouble $apiCallsPerSecond ([double]$operations / $measuredSeconds) "$context.apiCallsPerSecond" + $p50 = Get-StrictDouble $run 'p50Microseconds' $context 0 ([double]::MaxValue) + $p95 = Get-StrictDouble $run 'p95Microseconds' $context 0 ([double]::MaxValue) + $p99 = Get-StrictDouble $run 'p99Microseconds' $context 0 ([double]::MaxValue) + $maximum = Get-StrictDouble $run 'maxMicroseconds' $context 0 ([double]::MaxValue) + [void](Get-StrictDouble $run 'earlyP99Microseconds' $context 0 ([double]::MaxValue) -Positive) + [void](Get-StrictDouble $run 'lateP99Microseconds' $context 0 ([double]::MaxValue) -Positive) + if ($p50 -gt $p95 -or $p95 -gt $p99 -or $p99 -gt $maximum) { + throw "$context latency percentiles/maximum are not monotonic." + } + if ($profile -ceq 'LockFree' -and $maximum -gt + (Get-StrictDouble $TinyConfig 'maximumStallMicroseconds' 'qualification config linuxTinyPerformance' 10000 10000)) { + throw "$context exceeds the every-run 10000us maximum-stall gate: $maximum." + } + $assignedProcessors = @(Get-RequiredPropertyValue $run 'assignedProcessors' $context) + if ($assignedProcessors.Count -ne $processCount ` + -or @($assignedProcessors | Sort-Object -Unique).Count -ne $processCount ` + -or (Get-StrictInt64 $run 'affinityAppliedCount' $context $processCount $processCount) -ne $processCount) { + throw "$context lacks complete unique $processCount-process affinity evidence." + } + foreach ($processor in $assignedProcessors) { + if (-not (Test-IsIntegerNumber $processor) ` + -or [int64]$processor -lt 0 ` + -or [int64]$processor -gt 63) { + throw "$context has a processor assignment outside the probe's 64-bit affinity mask [0,63]." + } + } + $workerCycles = @(Get-RequiredPropertyValue $run 'workerCycles' $context) + if ($workerCycles.Count -ne $processCount) { + throw "$context must contain exactly $processCount worker-cycle rows." + } + [decimal]$cycleTotal = 0 + foreach ($workerCycle in $workerCycles) { + if (-not (Test-IsIntegerNumber $workerCycle) -or [int64]$workerCycle -lt 0) { + throw "$context has an invalid worker-cycle count." + } + $cycleTotal += [decimal]$workerCycle + } + if ($cycleTotal -ne [decimal]$cycles) { + throw "$context worker cycles do not sum to Cycles." + } + [decimal]$statusTotal = 0 + $histogram = Get-RequiredPropertyValue $run 'statusHistogram' $context + if (@($histogram.PSObject.Properties).Count -eq 0) { + throw "$context has an empty status histogram." + } + foreach ($entry in $histogram.PSObject.Properties) { + if (-not (Test-IsIntegerNumber $entry.Value) -or [int64]$entry.Value -lt 0) { + throw "$context status '$($entry.Name)' is not a nonnegative integer." + } + if ($entry.Name -ceq 'Validation.ChecksumMismatch' ` + -or $entry.Name -clike 'CorruptReason.*') { + throw "$context contains forbidden checksum/corruption evidence '$($entry.Name)'." + } + if ($entry.Name -match '^(Acquire|Release|Publish|Remove)\.') { + $statusTotal += [decimal]$entry.Value + } + } + if ($statusTotal -ne [decimal]$operations) { + throw "$context operation-status histogram does not sum to Operations." + } + $requiredSuccesses = if ($scenario -ceq 'acquire-release') { + @('Acquire.Success', 'Release.Success') + } + else { + @('Publish.Success', 'Remove.Success') + } + foreach ($successName in $requiredSuccesses) { + $successRows = @($histogram.PSObject.Properties | Where-Object { $_.Name -ceq $successName }) + if ($successRows.Count -ne 1 ` + -or -not (Test-IsIntegerNumber $successRows[0].Value) ` + -or [int64]$successRows[0].Value -ne $cycles) { + throw "$context must contain exactly $cycles '$successName' operations, one per completed cycle." + } + } + } + if (-not $actualRunKeys.SetEquals($expectedRunKeys)) { + throw 'Linux tiny performance raw run tuple set is incomplete.' + } + + $actualSummaryKeys = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + $metrics = [Collections.Generic.List[object]]::new() + foreach ($summary in $summaries) { + $context = "Linux tiny performance summary $($summary.profile)/$($summary.scenario)/$($summary.processCount)" + $profile = Get-StrictString $summary 'profile' $context + $scenario = Get-StrictString $summary 'scenario' $context + $processCount = Get-StrictInt64 $summary 'processCount' $context 1 8 + if ($processCount -notin @(1, 8)) { + throw "$context has an unsupported process count." + } + $key = "$profile|$scenario|$processCount" + if (-not $expectedSummaryKeys.Contains($key) -or -not $actualSummaryKeys.Add($key)) { + throw "$context is unexpected or duplicated." + } + $matchingRuns = @($runs | Where-Object { + [string]$_.profile -ceq $profile -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq $processCount + }) + if ($matchingRuns.Count -ne 3) { + throw "$context does not summarize exactly three raw trials." + } + if ((Get-StrictInt64 $summary 'totalFailures' $context 0 0) -ne 0) { + throw "$context has correctness failures." + } + foreach ($pair in @( + @('medianApiCallsPerSecond', 'apiCallsPerSecond'), + @('medianP99Microseconds', 'p99Microseconds'), + @('medianMaxMicroseconds', 'maxMicroseconds'))) { + $rawValues = [double[]]@($matchingRuns | ForEach-Object { + Get-StrictDouble $_ $pair[1] $context 0 ([double]::MaxValue) + }) + Assert-DerivedDouble ` + (Get-StrictDouble $summary $pair[0] $context 0 ([double]::MaxValue)) ` + (Get-MedianValue $rawValues) "$context.$($pair[0])" + } + $merged = [ordered]@{} + foreach ($run in $matchingRuns) { + foreach ($entry in $run.statusHistogram.PSObject.Properties) { + if (-not $merged.Contains($entry.Name)) { $merged[$entry.Name] = [int64]0 } + $merged[$entry.Name] = [int64]$merged[$entry.Name] + [int64]$entry.Value + } + } + $summaryHistogram = Get-RequiredPropertyValue $summary 'statusHistogram' $context + if ((@($summaryHistogram.PSObject.Properties.Name) -join ',') -cne (@($merged.Keys) -join ',')) { + throw "$context status histogram keys do not match the raw trials." + } + foreach ($entry in $summaryHistogram.PSObject.Properties) { + if (-not (Test-IsIntegerNumber $entry.Value) -or [int64]$entry.Value -ne [int64]$merged[$entry.Name]) { + throw "$context status '$($entry.Name)' is not the raw-trial total." + } + } + $metrics.Add([pscustomobject][ordered]@{ + profile = $profile + scenario = $scenario + processCount = $processCount + medianApiCallsPerSecond = [double]$summary.medianApiCallsPerSecond + medianP99Microseconds = [double]$summary.medianP99Microseconds + maximumRawStallMicroseconds = [double](($matchingRuns | Measure-Object maxMicroseconds -Maximum).Maximum) + }) + } + if (-not $actualSummaryKeys.SetEquals($expectedSummaryKeys)) { + throw 'Linux tiny performance summary tuple set is incomplete.' + } + foreach ($scenario in @('acquire-release', 'publish-remove')) { + $legacyOne = @($summaries | Where-Object { + [string]$_.profile -ceq 'Legacy' -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq 1 + })[0] + $lockFreeOne = @($summaries | Where-Object { + [string]$_.profile -ceq 'LockFree' -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq 1 + })[0] + $legacyEight = @($summaries | Where-Object { + [string]$_.profile -ceq 'Legacy' -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq 8 + })[0] + $lockFreeEight = @($summaries | Where-Object { + [string]$_.profile -ceq 'LockFree' -and [string]$_.scenario -ceq $scenario -and [int64]$_.processCount -eq 8 + })[0] + $legacyOneP99 = Get-StrictDouble $legacyOne 'medianP99Microseconds' "$scenario legacy/1p summary" 0 ([double]::MaxValue) -Positive + $lockFreeOneP99 = Get-StrictDouble $lockFreeOne 'medianP99Microseconds' "$scenario lock-free/1p summary" 0 ([double]::MaxValue) -Positive + $legacyEightRate = Get-StrictDouble $legacyEight 'medianApiCallsPerSecond' "$scenario legacy/8p summary" 0 ([double]::MaxValue) -Positive + $lockFreeEightRate = Get-StrictDouble $lockFreeEight 'medianApiCallsPerSecond' "$scenario lock-free/8p summary" 0 ([double]::MaxValue) -Positive + $lockFreeEightP99 = Get-StrictDouble $lockFreeEight 'medianP99Microseconds' "$scenario lock-free/8p summary" 0 ([double]::MaxValue) -Positive + $uncontendedP99Ratio = $lockFreeOneP99 / $legacyOneP99 + $throughputRatio = $lockFreeEightRate / $legacyEightRate + $scaleP99Ratio = $lockFreeEightP99 / $lockFreeOneP99 + if (-not [double]::IsFinite($uncontendedP99Ratio) ` + -or $uncontendedP99Ratio -gt [double]$TinyConfig.maximumUncontendedP99Ratio ` + -or -not [double]::IsFinite($throughputRatio) ` + -or $throughputRatio -lt [double]$TinyConfig.minimumThroughputRatio ` + -or -not [double]::IsFinite($scaleP99Ratio) ` + -or $scaleP99Ratio -gt [double]$TinyConfig.maximumScaleP99Ratio ` + -or $lockFreeEightP99 -gt [double]$TinyConfig.maximumP99Microseconds) { + throw "Linux tiny performance '$scenario' gate failed: uncontendedP99Ratio=$uncontendedP99Ratio throughputRatio=$throughputRatio scaleP99Ratio=$scaleP99Ratio lockFreeEightP99Microseconds=$lockFreeEightP99." + } + } + return [pscustomobject][ordered]@{ + schemaVersion = 2 + runCount = 24 + summaryCount = 8 + warmupSeconds = 10 + durationSeconds = 60 + trials = 3 + processCounts = @(1, 8) + minimumThroughputRatio = [double]$TinyConfig.minimumThroughputRatio + maximumUncontendedP99Ratio = [double]$TinyConfig.maximumUncontendedP99Ratio + maximumScaleP99Ratio = [double]$TinyConfig.maximumScaleP99Ratio + maximumP99Microseconds = [double]$TinyConfig.maximumP99Microseconds + maximumStallMicroseconds = [double]$TinyConfig.maximumStallMicroseconds + metrics = @($metrics) + } +} + +function Invoke-LinuxTinyPerformanceParserSelfTest { + param( + [Parameter(Mandatory)]$TinyConfig, + [Parameter(Mandatory)]$ReleaseConfig) + + $runs = [Collections.Generic.List[object]]::new() + $summaries = [Collections.Generic.List[object]]::new() + foreach ($profile in @('Legacy', 'LockFree')) { + foreach ($scenario in @('acquire-release', 'publish-remove')) { + foreach ($processCount in @(1, 8)) { + $api = if ($profile -ceq 'Legacy') { 1000.0 } else { 1100.0 } + $p99 = if ($processCount -eq 1) { + if ($profile -ceq 'Legacy') { 5.0 } else { 4.0 } + } + else { + if ($profile -ceq 'Legacy') { 3.0 } else { 8.0 } + } + $maximum = if ($profile -ceq 'Legacy') { 500.0 } else { 9000.0 } + [int64]$cycles = if ($profile -ceq 'Legacy') { 30000 } else { 33000 } + [int64]$operations = $cycles * 2 + [int64]$workerCycle = $cycles / $processCount + [int64]$windowSamples = [int64]$processCount * 1024 + $workerCycles = @() + for ($worker = 0; $worker -lt $processCount; $worker++) { + $workerCycles += $workerCycle + } + $histogram = if ($scenario -ceq 'acquire-release') { + [pscustomobject][ordered]@{ 'Acquire.Success' = $cycles; 'Release.Success' = $cycles } + } + else { + [pscustomobject][ordered]@{ 'Publish.Success' = $cycles; 'Remove.Success' = $cycles } + } + foreach ($trial in 1..3) { + $runs.Add([pscustomobject][ordered]@{ + Profile = $profile; Scenario = $scenario; ProcessCount = $processCount; Trial = $trial + ReaderProcessCount = $(if ($scenario -ceq 'acquire-release') { $processCount } else { 0 }) + PublisherProcessCount = $(if ($scenario -ceq 'publish-remove') { $processCount } else { 0 }) + ObserverProcessCount = 0; Cycles = $cycles; Operations = $operations + ApiCallsPerSecond = $api; P50Microseconds = 1.0; P95Microseconds = 2.0 + P99Microseconds = $p99; MaxMicroseconds = $maximum + EarlyP99Microseconds = $p99; LateP99Microseconds = $p99 + Failures = 0; MeasuredSeconds = 60.0; WallSeconds = 70.0 + EarlySampleCount = $windowSamples; LateSampleCount = $windowSamples + SampleCount = ($windowSamples * 2); AffinityAppliedCount = $processCount + AssignedProcessors = @(0..($processCount - 1)); Oversubscribed = $false + Qualification = 'qualification-measurement'; StatusHistogram = $histogram + WorkerCycles = @($workerCycles); OperationTarget = 0; FrameTarget = 0 + }) + } + $summaryHistogram = if ($scenario -ceq 'acquire-release') { + [pscustomobject][ordered]@{ + 'Acquire.Success' = ($cycles * 3); 'Release.Success' = ($cycles * 3) + } + } + else { + [pscustomobject][ordered]@{ + 'Publish.Success' = ($cycles * 3); 'Remove.Success' = ($cycles * 3) + } + } + $summaries.Add([pscustomobject][ordered]@{ + Profile = $profile; Scenario = $scenario; ProcessCount = $processCount + MedianApiCallsPerSecond = $api; MedianP99Microseconds = $p99 + MedianMaxMicroseconds = $maximum; TotalFailures = 0; StatusHistogram = $summaryHistogram + }) + } + } + } + $report = [pscustomobject][ordered]@{ + SchemaVersion = 8 + Environment = [pscustomobject][ordered]@{ + RepositoryCommit = 'synthetic'; RepositoryWorkingTreeState = 'clean' + SharedMemoryStoreAssemblySha256 = ('A' * 64); ProbeAssemblySha256 = ('B' * 64) + OperatingSystem = 'Ubuntu 24.04 synthetic'; OperatingSystemArchitecture = 'X64'; ProcessArchitecture = 'X64' + Framework = '.NET synthetic'; RuntimeVersion = 'synthetic'; LogicalProcessorCount = 8 + PhysicalCoreCount = 4; TotalMemoryBytes = 17179869184 + ProcessorModel = 'Synthetic CPU'; ProcessorIdentifier = 'Synthetic CPU' + ServerGarbageCollection = $false; StopwatchFrequency = 10000000 + } + Configuration = [pscustomobject][ordered]@{ + Mode = 'sync'; DurationSeconds = 60; DurationBoundGraceSeconds = 60 + Trials = 3; Profiles = @('Legacy', 'LockFree'); CountBoundProfiles = @('LockFree') + Scenarios = @('acquire-release', 'publish-remove') + ScenarioProcessCounts = [pscustomobject][ordered]@{ + 'acquire-release' = @(1, 8); 'publish-remove' = @(1, 8) + } + ScenarioStoreDimensions = [pscustomobject][ordered]@{ + 'acquire-release' = [pscustomobject][ordered]@{ + SlotCount = 32; MaxValueBytes = 8; MaxDescriptorBytes = 0; MaxKeyBytes = 8 + LeaseRecordCount = 64; LockFreeParticipantRecordCount = 64 + } + 'publish-remove' = [pscustomobject][ordered]@{ + SlotCount = 32; MaxValueBytes = 8; MaxDescriptorBytes = 0; MaxKeyBytes = 8 + LeaseRecordCount = 64; LockFreeParticipantRecordCount = 64 + } + } + WarmupCycles = 0; WarmupSeconds = 10; AffinityRequested = $true + SamplingInterval = 64; MaxLatencySamplesPerWorker = 65536 + SyncKeysPerWorker = [int]$TinyConfig.syncKeysPerWorker + SyncMaximumWorkerCount = [int]$TinyConfig.syncMaximumWorkerCount + SyncCanonicalBucketCount = [int]$TinyConfig.syncCanonicalBucketCount + SyncKeyCatalogSha256 = [string]$TinyConfig.syncKeyCatalogSha256 + SyncKeyCanonicalBucketAssignments = @($TinyConfig.syncKeyCanonicalBucketAssignments) + } + Runs = @($runs); Summary = @($summaries); MinimumCompatibleSchemaVersion = 8 + SchemaCompatibility = 'synthetic schema-v8 parser self-test' + } + [void](Assert-LinuxTinyPerformanceReport $report $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + if (-not (Test-LinuxTinyHostTuple ` + $report.Environment ` + 'synthetic' ` + 'Ubuntu 24.04 synthetic' ` + 'X64' ` + 'X64' ` + 8 ` + $true)) { + throw 'Linux tiny performance host-tuple self-test rejected an exact distro description without the word Linux.' + } + if (Test-LinuxTinyHostTuple ` + $report.Environment ` + 'synthetic' ` + 'Debian synthetic' ` + 'X64' ` + 'X64' ` + 8 ` + $true) { + throw 'Linux tiny performance host-tuple self-test accepted a different OS description.' + } + [int]$assertions = 3 + + $oddMedian = Get-MedianValue ([double[]]@(30.0, 10.0, 20.0)) + $evenMedian = Get-MedianValue ([double[]]@(40.0, 10.0, 30.0, 20.0)) + if ($oddMedian -ne 20.0 -or $evenMedian -ne 25.0) { + throw 'Linux tiny performance parser self-test did not compute canonical odd/even medians.' + } + $assertions++ + + $unknownHardware = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $unknownHardware.Environment.ProcessorModel = ' Unknown CPU ' + $rejected = $false + try { [void](Assert-LinuxTinyPerformanceReport $unknownHardware $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted unknown processor metadata.' + } + $assertions++ + + $missingHardware = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $missingHardware.Environment.PSObject.Properties.Remove('TotalMemoryBytes') + $rejected = $false + try { [void](Assert-LinuxTinyPerformanceReport $missingHardware $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted missing memory metadata.' + } + $assertions++ + + $wrongDimensions = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $wrongDimensions.Configuration.ScenarioStoreDimensions.'acquire-release'.SlotCount = 31 + $rejected = $false + try { [void](Assert-LinuxTinyPerformanceReport $wrongDimensions $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted the wrong store dimensions.' + } + $assertions++ + + $wrongCountPolicy = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $wrongCountPolicy.Configuration.CountBoundProfiles = @('Legacy') + $rejected = $false + try { [void](Assert-LinuxTinyPerformanceReport $wrongCountPolicy $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted a swapped count-bound policy.' + } + $assertions++ + + $wrongTarget = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $wrongTarget.Runs[0].OperationTarget = 1 + $rejected = $false + try { [void](Assert-LinuxTinyPerformanceReport $wrongTarget $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted a count target on a duration row.' + } + $assertions++ + + $shortDuration = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $shortDuration.Runs[0].MeasuredSeconds = 59.999 + $rejected = $false + try { [void](Assert-LinuxTinyPerformanceReport $shortDuration $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) } catch { $rejected = $true } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted a short duration row.' + } + $assertions++ + + $tampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $tamperedLockFreeRun = @($tampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq 'acquire-release' ` + -and [int64]$_.ProcessCount -eq 1 + })[0] + $tamperedLockFreeRun.MaxMicroseconds = 10001.0 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $tampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted an over-limit raw lock-free stall.' + } + $assertions++ + + $sampleTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $sampleTamperedOneProcessRun = @($sampleTampered.Runs | Where-Object { + [int64]$_.ProcessCount -eq 1 + })[0] + $sampleTamperedOneProcessRun.LateSampleCount = 1023 + $sampleTamperedOneProcessRun.SampleCount = 2047 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $sampleTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted incoherent early/late sample counts.' + } + $assertions++ + + foreach ($scenario in @('acquire-release', 'publish-remove')) { + $uncontendedTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($run in @($uncontendedTampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 1 + })) { + $run.P99Microseconds = 6.0 + $run.EarlyP99Microseconds = 6.0 + $run.LateP99Microseconds = 6.0 + } + @($uncontendedTampered.Summary | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 1 + })[0].MedianP99Microseconds = 6.0 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $uncontendedTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw "Linux tiny performance parser self-test accepted an over-limit '$scenario' uncontended p99 ratio." + } + $assertions++ + + $scaleTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($run in @($scaleTampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 1 + })) { + $run.P99Microseconds = 2.0 + $run.EarlyP99Microseconds = 2.0 + $run.LateP99Microseconds = 2.0 + } + @($scaleTampered.Summary | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 1 + })[0].MedianP99Microseconds = 2.0 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $scaleTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw "Linux tiny performance parser self-test accepted an over-limit '$scenario' scale p99 ratio." + } + $assertions++ + + $absoluteTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($run in @($absoluteTampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 8 + })) { + $run.P99Microseconds = 11.0 + $run.EarlyP99Microseconds = 11.0 + $run.LateP99Microseconds = 11.0 + } + @($absoluteTampered.Summary | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 8 + })[0].MedianP99Microseconds = 11.0 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $absoluteTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw "Linux tiny performance parser self-test accepted an over-limit '$scenario' absolute p99." + } + $assertions++ + + $throughputTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + foreach ($run in @($throughputTampered.Runs | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 8 + })) { + $run.MeasuredSeconds = 120.0 + $run.WallSeconds = 130.0 + $run.ApiCallsPerSecond = [double]$run.Operations / 120.0 + } + @($throughputTampered.Summary | Where-Object { + [string]$_.Profile -ceq 'LockFree' ` + -and [string]$_.Scenario -ceq $scenario ` + -and [int64]$_.ProcessCount -eq 8 + })[0].MedianApiCallsPerSecond = 550.0 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $throughputTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw "Linux tiny performance parser self-test accepted an under-limit '$scenario' 8-process throughput ratio." + } + $assertions++ + } + + $topologyTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $topologyTampered.Configuration.SyncKeyCanonicalBucketAssignments[1] = 1 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $topologyTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted a colliding synchronization-key topology.' + } + $assertions++ + + $affinityTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $affinityTamperedEightProcessRun = @($affinityTampered.Runs | Where-Object { + [int64]$_.ProcessCount -eq 8 + })[0] + $affinityTamperedEightProcessRun.AssignedProcessors = @(64, 65, 66, 67, 68, 69, 70, 71) + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $affinityTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted processor IDs outside its 64-bit affinity mask.' + } + $assertions++ + + $operationTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $operationTampered.Runs[0].Operations = [int64]$operationTampered.Runs[0].Cycles + $operationTampered.Runs[0].ApiCallsPerSecond = + [double]$operationTampered.Runs[0].Operations / [double]$operationTampered.Runs[0].MeasuredSeconds + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $operationTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted fewer than two operations per completed cycle.' + } + $assertions++ + + $successTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $successTampered.Runs[0].StatusHistogram = [pscustomobject][ordered]@{ + 'Acquire.NotFound' = [int64]$successTampered.Runs[0].Operations + } + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $successTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted a cycle set without exact paired success counts.' + } + $assertions++ + + $corruptionTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $corruptionTampered.Runs[0].StatusHistogram | Add-Member ` + -NotePropertyName 'Validation.ChecksumMismatch' -NotePropertyValue 1 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $corruptionTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted checksum/corruption evidence.' + } + $assertions++ + + $corruptReasonTampered = $report | ConvertTo-Json -Depth 20 | ConvertFrom-Json + $corruptReasonTampered.Runs[0].StatusHistogram | Add-Member ` + -NotePropertyName 'CorruptReason.Forged' -NotePropertyValue 1 + $rejected = $false + try { + [void](Assert-LinuxTinyPerformanceReport $corruptReasonTampered $TinyConfig $ReleaseConfig -SkipEnvironmentBinding) + } + catch { + $rejected = $true + } + if (-not $rejected) { + throw 'Linux tiny performance parser self-test accepted a corruption-reason row.' + } + $assertions++ + return $assertions +} + +# Each externally selectable validation has a structural self-test. This makes +# a stale path/filter addition fail the cheap `self-test` command rather than +# being discovered only in a release qualification job. +$definitions = [ordered]@{ + 'architecture' = @( + 'tests/SharedMemoryStore.ContractTests/SharedMemoryStore.ContractTests.csproj', + 'tests/SharedMemoryStore.ContractTests/LockFreeProfileApiContractTests.cs', + 'tests/SharedMemoryStore.ContractTests/LockFreeLayoutContractTests.cs') + 'atomic' = @( + 'tests/SharedMemoryStore.IntegrationTests/MappedAtomicLitmusIntegrationTests.cs', + 'tests/SharedMemoryStore.LockFreeAgent/Program.cs') + 'raw' = @( + 'tests/SharedMemoryStore.IntegrationTests/LockFreeRawVisibilityIntegrationTests.cs', + 'tests/SharedMemoryStore.LockFreeAgent/RawVisibilityCommands.cs') + 'no-lock' = @( + 'tests/SharedMemoryStore.IntegrationTests/LockFreeNoOperationLockIntegrationTests.cs', + 'tests/SharedMemoryStore.IntegrationTests/LockFreeOsTraceIntegrationTests.cs', + 'tests/SharedMemoryStore.LockFreeAgent/SteadyNoLockCommands.cs') + 'crash' = @( + 'tests/SharedMemoryStore.IntegrationTests/LockFreeCrashRecoveryIntegrationTests.cs', + 'tests/SharedMemoryStore.IntegrationTests/LockFreeOsTraceIntegrationTests.cs', + 'tests/SharedMemoryStore.LockFreeAgent/CheckpointCrashCommands.cs') + 'release-tests' = @( + 'SharedMemoryStore.slnx', + 'benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj') + 'interop' = @( + 'scripts/validate-native.ps1', + 'scripts/validate-python.ps1', + 'scripts/validate-docker-shared-memory.ps1') + 'samples' = @('samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj') + 'pack' = @('src/SharedMemoryStore/SharedMemoryStore.csproj') +} + +function Assert-ResultCommandEvidence { + param( + [Parameter(Mandatory)]$Result, + [string]$Context = 'OS validation result') + + $members = @{} + foreach ($property in @('command', 'stdout', 'stderr', 'stdoutSha256', 'stderrSha256')) { + $member = $Result.PSObject.Properties[$property] + if ($null -eq $member) { + throw "$Context is missing nullable execution-evidence property '$property'." + } + $members[$property] = $member.Value + } + + if ($null -eq $members.command) { + foreach ($property in @('stdout', 'stderr', 'stdoutSha256', 'stderrSha256')) { + if ($null -ne $members[$property]) { + throw "$Context has '$property' evidence without a command." + } + } + return $false + } + + if ($members.command -isnot [string] -or [string]::IsNullOrWhiteSpace($members.command)) { + throw "$Context has an empty or non-string command." + } + foreach ($stream in @('stdout', 'stderr')) { + if ($members[$stream] -isnot [string] -or [string]::IsNullOrWhiteSpace($members[$stream])) { + throw "$Context has an empty or non-string $stream path." + } + $digest = $members[$stream + 'Sha256'] + if ($digest -isnot [string] -or $digest -notmatch '^[0-9A-F]{64}$') { + throw "$Context has no SHA-256-bound $stream evidence." + } + } + return $true +} + +function Add-Result { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][ValidateSet('pass', 'fail', 'not-qualified')][string]$Status, + [Parameter(Mandatory)][string]$Detail, + [bool]$Required = $true, + [AllowNull()][object]$CommandLine = $null, + [AllowNull()][object]$Stdout = $null, + [AllowNull()][object]$Stderr = $null, + [Nullable[int]]$ExitCode = $null, + [double]$ElapsedSeconds = 0, + [int]$TimeoutSeconds = 0, + [bool]$TimedOut = $false, + [Nullable[DateTimeOffset]]$StartedUtc = $null) + + foreach ($field in @( + @{ Name = 'command'; Value = $CommandLine }, + @{ Name = 'stdout'; Value = $Stdout }, + @{ Name = 'stderr'; Value = $Stderr })) { + if ($null -ne $field.Value ` + -and ($field.Value -isnot [string] -or [string]::IsNullOrWhiteSpace($field.Value))) { + throw "OS validation result '$Name' has an empty or non-string $($field.Name) field." + } + } + $providedFieldCount = @($CommandLine, $Stdout, $Stderr | Where-Object { $null -ne $_ }).Count + if ($providedFieldCount -notin @(0, 3)) { + throw "OS validation result '$Name' must provide command, stdout, and stderr together or omit all three." + } + + $result = [pscustomobject][ordered]@{ + name = $Name + status = $Status + required = $Required + detail = $Detail + command = $CommandLine + startedUtc = $StartedUtc + exitCode = $ExitCode + elapsedSeconds = $ElapsedSeconds + timeoutSeconds = $TimeoutSeconds + timedOut = $TimedOut + stdout = $Stdout + stderr = $Stderr + stdoutSha256 = if ($null -ne $Stdout) { Get-FileSha256 (Join-Path $root $Stdout) } else { $null } + stderrSha256 = if ($null -ne $Stderr) { Get-FileSha256 (Join-Path $root $Stderr) } else { $null } + } + [void](Assert-ResultCommandEvidence $result "OS validation result '$Name'") + $results.Add($result) +} + +function Invoke-ResultCommandEvidenceSelfTest { + [int]$assertions = 0 + + foreach ($definition in @( + @{ Name = '__shape-structural'; Status = 'pass'; Required = $true }, + @{ Name = '__shape-optional'; Status = 'not-qualified'; Required = $false })) { + $before = $results.Count + Add-Result $definition.Name $definition.Status 'synthetic result-shape self-test' $definition.Required + try { + $row = $results[$before] + [void](Assert-ResultCommandEvidence $row "Synthetic result '$($definition.Name)'") + foreach ($property in @('command', 'stdout', 'stderr', 'stdoutSha256', 'stderrSha256')) { + if ($null -ne $row.$property) { + throw "Synthetic result '$($definition.Name)' did not retain JSON-null '$property'." + } + } + } + finally { + while ($results.Count -gt $before) { + $results.RemoveAt($results.Count - 1) + } + } + $assertions++ + } + + $executedStdoutPath = Join-Path $evidenceRoot '__shape-executed.stdout.log' + $executedStderrPath = Join-Path $evidenceRoot '__shape-executed.stderr.log' + [IO.File]::WriteAllText($executedStdoutPath, "synthetic stdout`n") + [IO.File]::WriteAllText($executedStderrPath, "synthetic stderr`n") + $executedStdout = [IO.Path]::GetRelativePath($root, $executedStdoutPath) + $executedStderr = [IO.Path]::GetRelativePath($root, $executedStderrPath) + $before = $results.Count + try { + Add-Result '__shape-executed' 'pass' 'synthetic result-shape self-test' $true ` + -CommandLine 'synthetic command' -Stdout $executedStdout -Stderr $executedStderr + $row = $results[$before] + if (-not (Assert-ResultCommandEvidence $row "Synthetic result '__shape-executed'") ` + -or [string]$row.command -cne 'synthetic command' ` + -or [string]$row.stdout -cne $executedStdout ` + -or [string]$row.stderr -cne $executedStderr ` + -or [string]$row.stdoutSha256 -cne (Get-FileSha256 $executedStdoutPath) ` + -or [string]$row.stderrSha256 -cne (Get-FileSha256 $executedStderrPath)) { + throw 'Synthetic executed result did not preserve its command/log evidence and hashes.' + } + } + finally { + while ($results.Count -gt $before) { + $results.RemoveAt($results.Count - 1) + } + Remove-Item -LiteralPath $executedStdoutPath, $executedStderrPath -Force -ErrorAction SilentlyContinue + } + $assertions++ + + $invalidCases = @( + @{ Name = 'empty-command'; Command = ''; Stdout = $null; Stderr = $null }, + @{ Name = 'whitespace-command'; Command = ' '; Stdout = $null; Stderr = $null }, + @{ Name = 'empty-stdout'; Command = 'synthetic command'; Stdout = ''; Stderr = 'synthetic.stderr.log' }, + @{ Name = 'whitespace-stderr'; Command = 'synthetic command'; Stdout = 'synthetic.stdout.log'; Stderr = "`t" }) + foreach ($case in $invalidCases) { + $before = $results.Count + $rejected = $false + try { + Add-Result "__shape-$($case.Name)" 'not-qualified' 'synthetic invalid result-shape self-test' $false ` + -CommandLine $case.Command -Stdout $case.Stdout -Stderr $case.Stderr + } + catch { + $rejected = $true + } + finally { + while ($results.Count -gt $before) { + $results.RemoveAt($results.Count - 1) + } + } + if (-not $rejected) { + throw "Result-shape self-test accepted $($case.Name) pseudo-evidence." + } + $assertions++ + } + return $assertions +} + +function Test-Definition { + param([Parameter(Mandatory)][string]$Name) + + if (-not $definitions.Contains($Name)) { + throw "No structural self-test is registered for '$Name'." + } + + $missing = @($definitions[$Name] | Where-Object { -not (Test-Path -LiteralPath (Join-Path $root $_)) }) + if ($missing.Count -ne 0) { + throw "Validation '$Name' has missing inputs: $($missing -join ', ')." + } + + Add-Result "self-test-$Name" 'pass' ($definitions[$Name] -join ',') +} + +function Invoke-Required { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$FileName, + [Parameter(Mandatory)][string[]]$Arguments, + [bool]$Required = $true) + + $safeName = $Name -replace '[^A-Za-z0-9_.-]', '-' + $stdoutPath = Join-Path $evidenceRoot ($safeName + '.stdout.log') + $stderrPath = Join-Path $evidenceRoot ($safeName + '.stderr.log') + $start = [Diagnostics.ProcessStartInfo]::new() + $start.FileName = $FileName + $start.WorkingDirectory = $root + $start.UseShellExecute = $false + $start.CreateNoWindow = $true + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + foreach ($argument in $Arguments) { + $start.ArgumentList.Add($argument) + } + + $process = [Diagnostics.Process]::new() + $process.StartInfo = $start + $startedUtc = [DateTimeOffset]::UtcNow + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + try { + if (-not $process.Start()) { + throw "Could not start required validation '$Name'." + } + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + $completed = $process.WaitForExit([int]([int64]$StepTimeoutSeconds * 1000)) + $terminationSucceeded = $true + $terminationDetail = $null + if (-not $completed) { + try { + $process.Kill($true) + } + catch { + $terminationSucceeded = $false + $terminationDetail = "process-tree kill failed: $($_.Exception.Message)" + } + if ($terminationSucceeded -and -not $process.WaitForExit(30000)) { + $terminationSucceeded = $false + $terminationDetail = 'process tree did not terminate within 30 seconds of Kill' + } + } + + $streamsDrained = $false + $streamDrainDetail = $null + try { + $streamsDrained = [Threading.Tasks.Task]::WaitAll( + [Threading.Tasks.Task[]]@($stdoutTask, $stderrTask), + 30000) + } + catch { + $streamDrainDetail = "redirected-stream drain failed: $($_.Exception.GetBaseException().Message)" + } + if (-not $streamsDrained -and [string]::IsNullOrWhiteSpace($streamDrainDetail)) { + $streamDrainDetail = 'redirected streams did not close within 30 seconds' + } + $stdout = if ($stdoutTask.IsCompletedSuccessfully) { + $stdoutTask.GetAwaiter().GetResult() + } + else { + "[os-validator] stdout unavailable: $streamDrainDetail" + } + $stderr = if ($stderrTask.IsCompletedSuccessfully) { + $stderrTask.GetAwaiter().GetResult() + } + else { + "[os-validator] stderr unavailable: $streamDrainDetail" + } + if (-not $terminationSucceeded) { + $stderr += [Environment]::NewLine + "[os-validator] $terminationDetail" + } + [IO.File]::WriteAllText($stdoutPath, $stdout) + [IO.File]::WriteAllText($stderrPath, $stderr) + $stopwatch.Stop() + + $relativeStdout = [IO.Path]::GetRelativePath($root, $stdoutPath) + $relativeStderr = [IO.Path]::GetRelativePath($root, $stderrPath) + $commandLine = $FileName + ' ' + ($Arguments -join ' ') + $hasExited = $false + try { + $hasExited = $process.HasExited + } + catch { + $hasExited = $false + } + $exitCode = if ($hasExited -and $completed) { $process.ExitCode } else { -1 } + $executionSucceeded = $completed -and $terminationSucceeded -and $streamsDrained -and $exitCode -eq 0 + if (-not $executionSucceeded) { + $failureDetail = if (-not $completed) { + "timeout=$StepTimeoutSeconds seconds; $($(if ($terminationSucceeded) { 'process tree killed' } else { $terminationDetail }))" + } + elseif (-not $streamsDrained) { + $streamDrainDetail + } + else { + "exit=$exitCode" + } + Add-Result -Name $Name -Status 'fail' ` + -Detail $failureDetail ` + -Required $Required -CommandLine $commandLine -Stdout $relativeStdout -Stderr $relativeStderr ` + -ExitCode $exitCode -ElapsedSeconds $stopwatch.Elapsed.TotalSeconds ` + -TimeoutSeconds $StepTimeoutSeconds -TimedOut (-not $completed) -StartedUtc $startedUtc + if (-not $completed) { + throw "Validation '$Name' timed out after $StepTimeoutSeconds seconds; $failureDetail." + } + if (-not $streamsDrained) { + throw "Validation '$Name' could not prove complete redirected output: $streamDrainDetail." + } + throw "Validation '$Name' failed with exit code $exitCode." + } + Add-Result -Name $Name -Status 'pass' -Detail 'command completed successfully' ` + -Required $Required -CommandLine $commandLine -Stdout $relativeStdout -Stderr $relativeStderr ` + -ExitCode $exitCode -ElapsedSeconds $stopwatch.Elapsed.TotalSeconds ` + -TimeoutSeconds $StepTimeoutSeconds -TimedOut $false -StartedUtc $startedUtc + } + finally { + $process.Dispose() + } +} + +function Test-NativeCommand { + param( + [Parameter(Mandatory)][string]$FileName, + [string[]]$Arguments = @(), + [int]$TimeoutMilliseconds = 30000) + + try { + $resolved = (Get-Command $FileName -ErrorAction Stop).Source + $start = [Diagnostics.ProcessStartInfo]::new($resolved) + $start.UseShellExecute = $false + $start.CreateNoWindow = $true + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + foreach ($argument in $Arguments) { + $start.ArgumentList.Add($argument) + } + + $process = [Diagnostics.Process]::Start($start) + if ($null -eq $process) { + return $false + } + try { + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit($TimeoutMilliseconds)) { + try { + $process.Kill($true) + } + catch { + return $false + } + if (-not $process.WaitForExit(30000)) { + return $false + } + try { + [void][Threading.Tasks.Task]::WaitAll( + [Threading.Tasks.Task[]]@($stdout, $stderr), + 5000) + } + catch { + return $false + } + return $false + } + if (-not [Threading.Tasks.Task]::WaitAll( + [Threading.Tasks.Task[]]@($stdout, $stderr), + 30000)) { + return $false + } + if (-not $stdout.IsCompletedSuccessfully -or -not $stderr.IsCompletedSuccessfully) { + return $false + } + $stdout.GetAwaiter().GetResult() | Out-Null + $stderr.GetAwaiter().GetResult() | Out-Null + return $process.ExitCode -eq 0 + } + finally { + $process.Dispose() + } + } + catch { + return $false + } +} + +function Get-LinuxProcessStartIdentity { + param([Parameter(Mandatory)][int]$ProcessId) + + if (-not $IsLinux -or $ProcessId -le 0) { + return $null + } + + try { + $stat = Get-Content -LiteralPath "/proc/$ProcessId/stat" -Raw + $commandEnd = $stat.LastIndexOf(')') + if ($commandEnd -lt 0 -or $commandEnd + 2 -ge $stat.Length) { + return $null + } + + $fields = @($stat.Substring($commandEnd + 2).Split( + ' ', + [StringSplitOptions]::RemoveEmptyEntries)) + if ($fields.Count -le 19 -or $fields[19] -notmatch '^\d+$') { + return $null + } + + return [string]$fields[19] + } + catch { + return $null + } +} + +function Test-DockerHostPidIdentityVisible { + param([Parameter(Mandatory)][string]$Image) + + $startIdentity = Get-LinuxProcessStartIdentity $PID + if ([string]::IsNullOrWhiteSpace($startIdentity)) { + return $false + } + + # Linux region ownership is fenced by both PID and /proc field 22. Merely + # finding a reused numeric PID inside an unrelated namespace is not proof + # that a container can safely join the existing owner sidecar. + $probe = 'pid="$1"; expected="$2"; line=$(cat "/proc/$pid/stat") || exit 21; rest=${line##*) }; set -- $rest; shift 19 || exit 22; [ "$1" = "$expected" ]' + return Test-NativeCommand 'docker' @( + 'run', '--rm', '--pid=host', + $Image, + 'sh', '-c', $probe, 'sms-owner-identity-probe', + [string]$PID, $startIdentity) +} + +function Invoke-OptionalScript { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string[]]$Prerequisites, + [Parameter(Mandatory)][string]$ScriptPath, + [string[]]$Arguments = @()) + + $missing = @($Prerequisites | Where-Object { $null -eq (Get-Command $_ -ErrorAction SilentlyContinue) }) + if ($missing.Count -ne 0) { + Add-Result $Name 'not-qualified' "missing prerequisite: $($missing -join ',')" + return + } + + Invoke-Required $Name $pwsh (@('-NoProfile', '-File', $ScriptPath) + $Arguments) +} + +function Test-Selected { + param([Parameter(Mandatory)][string]$Name) + return $Command -eq 'all' -or $Command -eq $Name +} + +function Assert-ExactAllResultShape { + if ($Command -ne 'all' -or -not $qualifiedArchitecture) { + return + } + $names = @( + 'self-test-architecture', 'self-test-atomic', 'self-test-raw', + 'self-test-no-lock', 'self-test-crash', 'self-test-release-tests', + 'self-test-interop', 'self-test-samples', 'self-test-pack', + 'dotnet-info', 'clean', 'restore', 'build', + 'architecture', 'atomic', 'raw', 'no-lock-held', 'no-lock-linux-strace', + 'linux-tiny-performance', + 'crash-checkpoint-kill', 'crash-linux-sigstop', 'crash-linux-docker-pause', + 'release-tests', 'native', 'python', 'docker', 'sample-6', 'sample-12', 'pack') + $requiredByName = [ordered]@{} + foreach ($name in $names) { + $requiredByName[$name] = $true + } + $requiredByName['crash-linux-docker-pause'] = $false + if ($platform -eq 'windows' -and $architecture -eq 'x64') { + $requiredByName['no-lock-linux-strace'] = $false + $requiredByName['crash-linux-sigstop'] = $false + $requiredByName['linux-tiny-performance'] = $false + } + + if ($results.Count -ne $requiredByName.Count ` + -or @($results | Group-Object name | Where-Object Count -ne 1).Count -ne 0) { + throw "command=all must emit exactly $($requiredByName.Count) unique result rows." + } + foreach ($entry in $requiredByName.GetEnumerator()) { + $rows = @($results | Where-Object { [string]$_.name -ceq [string]$entry.Key }) + if ($rows.Count -ne 1 -or [bool]$rows[0].required -ne [bool]$entry.Value) { + throw "command=all row '$($entry.Key)' is missing, duplicated, or has the wrong required flag." + } + } +} + +function Invoke-DotNetTest { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$Project, + [Parameter(Mandatory)][string]$Filter, + [bool]$Required = $true) + + $trxDirectory = Join-Path $evidenceRoot ('trx/' + ($Name -replace '[^A-Za-z0-9_.-]', '-')) + New-Item -ItemType Directory -Path $trxDirectory -Force | Out-Null + Invoke-Required $Name $dotnet @( + 'test', $Project, '-c', $Configuration, '--nologo', '--no-build', '--no-restore', + '--filter', $Filter, '--logger', 'trx', '--results-directory', $trxDirectory) $Required + Assert-TrxPassed $Name $trxDirectory +} + +function Assert-TrxPassed { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$Directory) + + $files = @(Get-ChildItem -LiteralPath $Directory -Recurse -Filter '*.trx' -File -ErrorAction SilentlyContinue) + $total = 0 + $passed = 0 + $outcomes = @{} + foreach ($file in $files) { + [xml]$document = Get-Content -LiteralPath $file.FullName -Raw + foreach ($node in @($document.SelectNodes("//*[local-name()='UnitTestResult']"))) { + $outcome = [string]$node.outcome + $total++ + if (-not $outcomes.ContainsKey($outcome)) { + $outcomes[$outcome] = 0 + } + $outcomes[$outcome]++ + if ($outcome -ceq 'Passed') { + $passed++ + } + } + } + $nonPassed = $total - $passed + if ($files.Count -eq 0 -or $passed -eq 0 -or $nonPassed -ne 0) { + $outcomeDetail = @($outcomes.GetEnumerator() | Sort-Object Key | ForEach-Object { + "$($_.Key)=$($_.Value)" + }) -join ', ' + throw "Validation '$Name' TRX proof is invalid: files=$($files.Count), passed=$passed, nonPassed=$nonPassed, outcomes=[$outcomeDetail]." + } + $result = @($results | Where-Object name -eq $Name | Select-Object -Last 1) + if ($result.Count -ne 1) { + throw "Validation '$Name' has no unique executable result to attach TRX evidence." + } + $result[0].detail = "TRX passed=$passed nonPassed=0 files=$($files.Count)" +} + +function Add-NotQualifiedPlatform { + param([Parameter(Mandatory)][string[]]$Names) + foreach ($name in $Names) { + Add-Result $name 'not-qualified' "$platform-$architecture; layout v2 requires Windows/Linux x64" + } +} + +try { + Assert-KnownProvenance $repositoryProvenance 'start' + if (-not $ValidateOnly -and $repositoryProvenance.workingTreeState -ne 'clean') { + throw 'Executable OS qualification requires a clean working tree.' + } + $definitionNames = if ($Command -in @('self-test', 'all')) { + @($definitions.Keys) + } + else { + @($Command) + } + foreach ($name in $definitionNames) { + Test-Definition $name + } + + $qualificationConfig = Join-Path $root 'specs/009-lock-free-publish-read/qualification-config.json' + if (-not (Test-Path -LiteralPath $qualificationConfig)) { + throw 'OS validation qualification config is missing.' + } + $parsedConfig = Get-Content -LiteralPath $qualificationConfig -Raw | ConvertFrom-Json + if (-not (Test-IsIntegerNumber $parsedConfig.schemaVersion) ` + -or [int64]$parsedConfig.schemaVersion -ne 5) { + throw 'OS validation requires qualification config schema 5.' + } + $linuxTinyConfig = Assert-LinuxTinyPerformanceConfiguration $parsedConfig + $releaseConfig = Get-RequiredPropertyValue ` + (Get-RequiredPropertyValue $parsedConfig 'tiers' 'qualification config') ` + 'release' 'qualification config tiers' + + if ($ValidateOnly) { + $performanceParserAssertions = Invoke-LinuxTinyPerformanceParserSelfTest $linuxTinyConfig $releaseConfig + $resultCommandEvidenceAssertions = Invoke-ResultCommandEvidenceSelfTest + Add-Result 'validation-plan' 'pass' ` + "configuration and structural inputs validated; linuxTinyPerformanceParserAssertions=$performanceParserAssertions; resultCommandEvidenceAssertions=$resultCommandEvidenceAssertions; no restore, build, tests, interop, sample, performance workload, or pack executed" + $completionProvenance = Get-RepositoryProvenance + Assert-ProvenanceStable $repositoryProvenance $completionProvenance + } + else { + Invoke-Required 'dotnet-info' $dotnet @('--info') + $preclean = Remove-SolutionProjectBuildOutputs (Join-Path $evidenceRoot 'preclean.json') + Invoke-Required 'clean' $dotnet @( + 'clean', 'SharedMemoryStore.slnx', '-c', $Configuration, '--nologo', '--disable-build-servers') + $cleanResult = @($results | Where-Object name -eq 'clean') + if ($cleanResult.Count -ne 1) { + throw 'OS validation did not record exactly one clean result.' + } + $cleanResult[0] | Add-Member -NotePropertyName preclean -NotePropertyValue $preclean.Summary + $cleanResult[0].detail += "; solutionProjects=$($preclean.SolutionProjectCount); outputTargets=$($preclean.TargetCount); precleanedOutputDirectories=$($preclean.RemovedDirectoryCount); verifiedAbsent=$($preclean.VerifiedAbsentCount); precleanReport=$([IO.Path]::GetRelativePath($root, $preclean.ReportPath)); precleanReportSha256=$($preclean.ReportSha256)" + Invoke-Required 'restore' $dotnet @('restore', 'SharedMemoryStore.slnx', '--nologo', '--disable-build-servers') + Invoke-Required 'build' $dotnet @( + 'build', 'SharedMemoryStore.slnx', '-c', $Configuration, '--nologo', '--no-restore', '--disable-build-servers') + $testedAssemblyManifest = @(Get-TestedAssemblyManifest) + + if ($Command -eq 'all') { + if (-not $IsLinux) { + Add-Result 'linux-tiny-performance' 'not-qualified' ` + 'Linux-only SC-006 tiny-operation release matrix; optional on Windows' $false + } + elseif (-not $qualifiedArchitecture) { + Add-Result 'linux-tiny-performance' 'not-qualified' ` + "$platform-$architecture; Linux tiny performance requires Linux x64" + } + elseif ([Environment]::ProcessorCount -lt 8) { + Add-Result 'linux-tiny-performance' 'not-qualified' ` + "Linux tiny performance requires at least 8 logical processors for complete affinity; actual=$([Environment]::ProcessorCount)" + } + else { + $performancePath = Join-Path $evidenceRoot 'linux-tiny-performance.json' + Invoke-Required 'linux-tiny-performance' $dotnet @( + 'run', '-c', $Configuration, '--no-build', '--no-restore', + '--project', 'benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj', '--', + '--mode', [string]$linuxTinyConfig.mode, + '--profile', 'both', + '--count-bound-profiles', 'v2', + '--scenario', 'acquire-release,publish-remove', + '--process-counts', '1,8', + '--warmup', [string]$releaseConfig.performanceWarmupSeconds, + '--duration', [string]$releaseConfig.performanceDurationSeconds, + '--duration-bound-grace', [string]$releaseConfig.performanceDurationBoundGraceSeconds, + '--trials', [string]$releaseConfig.performanceTrials, + '--repository-commit', [string]$repositoryProvenance.repositoryCommit, + '--repository-working-tree-state', [string]$repositoryProvenance.workingTreeState, + '--output', $performancePath) + $performanceRow = @($results | Where-Object name -eq 'linux-tiny-performance') + if ($performanceRow.Count -ne 1) { + throw 'Linux tiny performance command did not emit exactly one result row.' + } + try { + if (-not (Test-Path -LiteralPath $performancePath -PathType Leaf)) { + throw 'Linux tiny performance command did not emit its raw JSON report.' + } + $performanceReport = Get-Content -LiteralPath $performancePath -Raw | ConvertFrom-Json -Depth 30 + $performanceValidation = Assert-LinuxTinyPerformanceReport ` + $performanceReport $linuxTinyConfig $releaseConfig + $relativePerformancePath = [IO.Path]::GetRelativePath($root, $performancePath) + $performanceRow[0].detail = + 'schema8 exact 2-profile x 2-scenario x (1,8)-process x 3-trial Linux matrix passed; ' + + 'LF1 p99<=Legacy1, LF8 throughput>=Legacy8, LF8/LF1 p99<=3, LF8 p99<=10us, ' + + 'and every lock-free raw MaxMicroseconds<=10000' + $performanceRow[0] | Add-Member -NotePropertyName performanceEvidence -NotePropertyValue ` + ([pscustomobject][ordered]@{ + schemaVersion = 1 + reportPath = $relativePerformancePath + reportSha256 = Get-FileSha256 $performancePath + validation = $performanceValidation + }) + } + catch { + $performanceRow[0].status = 'fail' + $performanceRow[0].detail = $_.Exception.Message + throw + } + } + } + + if ($Command -eq 'self-test') { + Invoke-DotNetTest 'self-test-trace-classifier' $integrationProject 'FullyQualifiedName=SharedMemoryStore.IntegrationTests.LockFreeOsTraceIntegrationTests.TraceClassifierSeparatesColdAndMarkedStoreLockCalls|FullyQualifiedName=SharedMemoryStore.IntegrationTests.LockFreeOsTraceIntegrationTests.DockerCheckpointPrefixSharesTheHostPidNamespace' + } + + if (Test-Selected 'architecture') { + if (-not $qualifiedArchitecture) { + Add-NotQualifiedPlatform @('architecture') + } + else { + Invoke-DotNetTest 'architecture' $contractProject 'FullyQualifiedName~LockFreeProfileApiContractTests|FullyQualifiedName~LockFreeLayoutContractTests' + } + } + + if (Test-Selected 'atomic') { + if (-not $qualifiedArchitecture) { + Add-NotQualifiedPlatform @('atomic') + } + else { + Invoke-DotNetTest 'atomic' $integrationProject 'FullyQualifiedName~MappedAtomicLitmusIntegrationTests' + } + } + + if (Test-Selected 'raw') { + if (-not $qualifiedArchitecture) { + Add-NotQualifiedPlatform @('raw') + } + else { + Invoke-DotNetTest 'raw' $integrationProject 'FullyQualifiedName~LockFreeRawVisibilityIntegrationTests' + } + } + + if (Test-Selected 'no-lock') { + if (-not $qualifiedArchitecture) { + Add-NotQualifiedPlatform @('no-lock-held', 'no-lock-linux-strace') + } + else { + Invoke-DotNetTest 'no-lock-held' $integrationProject 'FullyQualifiedName~LockFreeNoOperationLockIntegrationTests' + + if (-not $IsLinux) { + Add-Result 'no-lock-linux-strace' 'not-qualified' 'not applicable on this non-Linux platform; held-lock evidence is reported separately' $false + } + elseif (-not (Test-NativeCommand 'strace' @('--version'))) { + Add-Result 'no-lock-linux-strace' 'not-qualified' 'strace command unavailable or unusable' + } + else { + Invoke-DotNetTest 'no-lock-linux-strace' $integrationProject 'FullyQualifiedName=SharedMemoryStore.IntegrationTests.LockFreeOsTraceIntegrationTests.LinuxMarkedSteadyIntervalDoesNotUseStoreOperationLock' + } + } + } + + if (Test-Selected 'crash') { + if (-not $qualifiedArchitecture) { + Add-NotQualifiedPlatform @('crash-checkpoint-kill', 'crash-linux-sigstop', 'crash-linux-docker-pause') + } + else { + Invoke-DotNetTest 'crash-checkpoint-kill' $integrationProject 'FullyQualifiedName~LockFreeCrashRecoveryIntegrationTests' + + if (-not $IsLinux) { + Add-Result 'crash-linux-sigstop' 'not-qualified' 'not applicable on this non-Linux platform' $false + } + elseif (-not (Test-NativeCommand 'kill' @('--version'))) { + Add-Result 'crash-linux-sigstop' 'not-qualified' 'kill command unavailable or unusable' + } + else { + Invoke-DotNetTest 'crash-linux-sigstop' $integrationProject 'FullyQualifiedName=SharedMemoryStore.IntegrationTests.LockFreeOsTraceIntegrationTests.LinuxSigStopAtProtocolCheckpointDoesNotBlockUnrelatedProgress' + } + + if ($SkipDocker) { + Add-Result 'crash-linux-docker-pause' 'not-qualified' 'optional Docker checkpoint-pause coverage explicitly skipped' $false + } + elseif (-not $IsLinux) { + Add-Result 'crash-linux-docker-pause' 'not-qualified' 'optional coverage is not applicable on this non-Linux platform' $false + } + elseif (-not (Test-NativeCommand 'docker' @('info', '--format', '{{.ServerVersion}}'))) { + Add-Result 'crash-linux-docker-pause' 'not-qualified' 'optional Docker command or daemon unavailable' $false + } + elseif (-not (Test-NativeCommand 'docker' @('image', 'inspect', $DockerRuntimeImage))) { + Add-Result 'crash-linux-docker-pause' 'not-qualified' "optional runtime image unavailable (not pulled automatically): $DockerRuntimeImage" $false + } + elseif (-not (Test-DockerHostPidIdentityVisible $DockerRuntimeImage)) { + Add-Result 'crash-linux-docker-pause' 'not-qualified' 'optional Docker host PID namespace does not expose the invoking Linux PID with its exact /proc start identity' $false + } + else { + $previousRun = $env:SMS_RUN_LOCK_FREE_DOCKER_PAUSE_VALIDATION + $previousImage = $env:SMS_LOCK_FREE_DOCKER_IMAGE + try { + $env:SMS_RUN_LOCK_FREE_DOCKER_PAUSE_VALIDATION = '1' + $env:SMS_LOCK_FREE_DOCKER_IMAGE = $DockerRuntimeImage + Invoke-DotNetTest 'crash-linux-docker-pause' $integrationProject 'FullyQualifiedName=SharedMemoryStore.IntegrationTests.LockFreeOsTraceIntegrationTests.LinuxDockerPauseAtProtocolCheckpointDoesNotBlockUnrelatedProgress' $false + } + finally { + $env:SMS_RUN_LOCK_FREE_DOCKER_PAUSE_VALIDATION = $previousRun + $env:SMS_LOCK_FREE_DOCKER_IMAGE = $previousImage + } + } + } + } + + if (Test-Selected 'release-tests') { + $releaseTrx = Join-Path $evidenceRoot 'trx/release-tests' + New-Item -ItemType Directory -Path $releaseTrx -Force | Out-Null + Invoke-Required 'release-tests' $dotnet @( + 'test', 'SharedMemoryStore.slnx', '-c', $Configuration, '--nologo', '--no-build', '--no-restore', + '--logger', 'trx', '--results-directory', $releaseTrx) + Assert-TrxPassed 'release-tests' $releaseTrx + } + + if (Test-Selected 'interop') { + Invoke-OptionalScript 'native' @('cmake') (Join-Path $PSScriptRoot 'validate-native.ps1') @('-Configuration', $Configuration) + Invoke-OptionalScript 'python' @('python', 'cmake') (Join-Path $PSScriptRoot 'validate-python.ps1') @('-Configuration', $Configuration) + + if ($SkipDocker) { + Add-Result 'docker' 'not-qualified' 'explicitly skipped' + } + elseif (-not (Test-NativeCommand 'docker' @('info', '--format', '{{.ServerVersion}}'))) { + Add-Result 'docker' 'not-qualified' 'Docker command or daemon unavailable' + } + else { + Invoke-Required 'docker' $pwsh @( + '-NoProfile', '-File', (Join-Path $PSScriptRoot 'validate-docker-shared-memory.ps1'), + '-Profile', 'All', '-Configuration', $Configuration) + } + } + + if (Test-Selected 'samples') { + foreach ($workers in @(6, 12)) { + Invoke-Required "sample-$workers" $dotnet @( + 'run', '-c', $Configuration, '--no-build', '--no-restore', + '--project', 'samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj', '--', + '--workers', [string]$workers, '--frames', [string]($workers * 4)) + } + } + + if (Test-Selected 'pack') { + Invoke-Required 'pack' $dotnet @( + 'pack', 'src/SharedMemoryStore/SharedMemoryStore.csproj', + '-c', $Configuration, '--nologo', '--no-build', '--no-restore') + } + $completionAssemblyManifest = @(Get-TestedAssemblyManifest) + Assert-AssemblyManifestStable $testedAssemblyManifest $completionAssemblyManifest + $completionProvenance = Get-RepositoryProvenance + Assert-ProvenanceStable $repositoryProvenance $completionProvenance + Assert-ExactAllResultShape + } +} +catch { + Add-Result 'validation-script' 'fail' $_.Exception.Message + throw +} +finally { + if ($null -eq $completionProvenance) { + $completionProvenance = Get-RepositoryProvenance + } + if ($testedAssemblyManifest.Count -gt 0 -and $completionAssemblyManifest.Count -eq 0) { + try { + $completionAssemblyManifest = @(Get-TestedAssemblyManifest) + } + catch { + $completionAssemblyManifest = @() + } + } + foreach ($result in $results) { + [void](Assert-ResultCommandEvidence $result "OS validation result '$($result.name)' before serialization") + } + $requiredStatuses = @($results | Where-Object required | ForEach-Object status) + $overallStatus = if ($ValidateOnly -and $requiredStatuses -notcontains 'fail') { + 'validation-only' + } + elseif ($requiredStatuses -contains 'fail') { + 'fail' + } + elseif ($requiredStatuses -contains 'not-qualified') { + 'not-qualified' + } + elseif ($requiredStatuses.Count -gt 0) { + 'pass' + } + else { + 'not-qualified' + } + + $manifest = @(Get-ChildItem -LiteralPath $evidenceRoot -Recurse -File | Sort-Object FullName | ForEach-Object { + [pscustomobject][ordered]@{ + path = [IO.Path]::GetRelativePath($root, $_.FullName) + length = $_.Length + sha256 = Get-FileSha256 $_.FullName + } + }) + $dotnetInfo = @($results | Where-Object name -eq 'dotnet-info' | Select-Object -First 1) + [pscustomobject][ordered]@{ + schemaVersion = 3 + validationOnly = [bool]$ValidateOnly + startedUtc = $runStartedUtc + completedUtc = [DateTimeOffset]::UtcNow + command = $Command + configuration = $Configuration + platform = $platform + architecture = $architecture + qualifiedArchitecture = $qualifiedArchitecture + overallStatus = $overallStatus + provenance = $repositoryProvenance + completionProvenance = $completionProvenance + testedAssemblies = $testedAssemblyManifest + completionTestedAssemblies = $completionAssemblyManifest + host = [ordered]@{ + operatingSystem = [Runtime.InteropServices.RuntimeInformation]::OSDescription + operatingSystemArchitecture = [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() + processArchitecture = [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString() + framework = [Runtime.InteropServices.RuntimeInformation]::FrameworkDescription + runtimeVersion = [Environment]::Version.ToString() + logicalProcessorCount = [Environment]::ProcessorCount + powershellVersion = $PSVersionTable.PSVersion.ToString() + } + toolchain = [ordered]@{ + dotnetPath = $dotnet + dotnetVersion = Invoke-TextCommand $dotnet @('--version') + dotnetInfoSha256 = if ($dotnetInfo.Count -eq 1) { $dotnetInfo[0].stdoutSha256 } else { $null } + powershellPath = $pwsh + gitPath = $git + gitVersion = Invoke-TextCommand $git @('--version') + cmakePath = (Get-Command cmake -ErrorAction SilentlyContinue).Source + pythonPath = (Get-Command python -ErrorAction SilentlyContinue).Source + dockerPath = (Get-Command docker -ErrorAction SilentlyContinue).Source + stracePath = (Get-Command strace -ErrorAction SilentlyContinue).Source + } + inputs = [ordered]@{ + script = [IO.Path]::GetRelativePath($root, $PSCommandPath) + scriptSha256 = Get-FileSha256 $PSCommandPath + qualificationConfig = [IO.Path]::GetRelativePath($root, $qualificationConfig) + qualificationConfigSha256 = Get-FileSha256 $qualificationConfig + solutionSha256 = Get-FileSha256 (Join-Path $root 'SharedMemoryStore.slnx') + } + results = $results + evidenceManifest = $manifest + } | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $resultPath +} + +if ($overallStatus -eq 'validation-only') { + Write-Host "OS validation orchestration '$Command' validated without executing workloads. Evidence: $resultPath" +} +elseif ($overallStatus -eq 'not-qualified') { + Write-Warning "OS validation '$Command' is NOT QUALIFIED. Evidence: $resultPath" + exit 2 +} +else { + Write-Host "OS validation '$Command' completed. Evidence: $resultPath" +} diff --git a/scripts/validate-native.ps1 b/scripts/validate-native.ps1 index 2c7fa7e..00f22a2 100644 --- a/scripts/validate-native.ps1 +++ b/scripts/validate-native.ps1 @@ -11,8 +11,18 @@ param( $ErrorActionPreference = 'Stop' $repositoryRoot = Split-Path -Parent $PSScriptRoot -$buildPath = [System.IO.Path]::GetFullPath((Join-Path $repositoryRoot $BuildDirectory)) -$installPath = [System.IO.Path]::GetFullPath((Join-Path $repositoryRoot $InstallDirectory)) +$requestedBuildPath = if ([IO.Path]::IsPathRooted($BuildDirectory)) { + [IO.Path]::GetFullPath($BuildDirectory) +} +else { + [IO.Path]::GetFullPath((Join-Path $repositoryRoot $BuildDirectory)) +} +$requestedInstallPath = if ([IO.Path]::IsPathRooted($InstallDirectory)) { + [IO.Path]::GetFullPath($InstallDirectory) +} +else { + [IO.Path]::GetFullPath((Join-Path $repositoryRoot $InstallDirectory)) +} function Invoke-Checked { param( @@ -28,6 +38,128 @@ function Invoke-Checked { } } +function Test-MultiConfigBuild { + param([Parameter(Mandatory)][string]$BuildPath) + + $cachePath = Join-Path $BuildPath 'CMakeCache.txt' + $entry = Get-Content -LiteralPath $cachePath | + Where-Object { $_.StartsWith('CMAKE_CONFIGURATION_TYPES:', [StringComparison]::Ordinal) } | + Select-Object -First 1 + if ($null -eq $entry) { + return $false + } + + $separator = $entry.IndexOf('=') + return $separator -ge 0 -and -not [string]::IsNullOrWhiteSpace($entry.Substring($separator + 1)) +} + +function Reset-RepositoryScratchDirectory { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][string]$Description) + + $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + $normalizedRoot = [IO.Path]::GetFullPath($repositoryRoot).TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + $rootPrefix = $normalizedRoot + [IO.Path]::DirectorySeparatorChar + $target = [IO.Path]::GetFullPath($Path).TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + if (-not $target.StartsWith($rootPrefix, $comparison)) { + throw "$Description must be a scratch directory inside the repository: '$target'." + } + + $relativeTarget = [IO.Path]::GetRelativePath($normalizedRoot, $target).Replace('\', '/') + $git = (Get-Command git -ErrorAction Stop).Source + & $git -C $normalizedRoot check-ignore --quiet --no-index -- $relativeTarget 2>$null + $ignoreExitCode = $LASTEXITCODE + if ($ignoreExitCode -eq 1) { + throw "$Description must be covered by repository ignore rules: '$relativeTarget'." + } + if ($ignoreExitCode -ne 0) { + throw "Could not verify repository ignore coverage for $Description '$relativeTarget'." + } + $protectedEntries = @( + & $git -C $normalizedRoot --literal-pathspecs ls-files --cached --others --exclude-standard -- $relativeTarget 2>$null) + if ($LASTEXITCODE -ne 0) { + throw "Could not prove that $Description '$relativeTarget' contains only ignored scratch output." + } + if ($protectedEntries.Count -ne 0) { + throw "Refusing to reset $Description '$relativeTarget' because it contains tracked or nonignored entry '$($protectedEntries[0])'." + } + + $ancestor = [IO.Path]::GetDirectoryName($target) + while (-not (Test-Path -LiteralPath $ancestor)) { + $parent = [IO.Directory]::GetParent($ancestor) + if ($null -eq $parent) { + throw "Repository root was not reached while checking $Description '$target'." + } + $ancestor = $parent.FullName + } + while ($true) { + $ancestorItem = Get-Item -LiteralPath $ancestor -Force + if (-not $ancestorItem.PSIsContainer ` + -or ($ancestorItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to reset $Description through non-directory or linked ancestor '$ancestor'." + } + if ($ancestor.Equals($normalizedRoot, $comparison)) { + break + } + if (-not $ancestor.StartsWith($rootPrefix, $comparison)) { + throw "$Description ancestor escaped the repository while checking '$target'." + } + $parent = [IO.Directory]::GetParent($ancestor) + if ($null -eq $parent) { + throw "Repository root was not reached while checking $Description '$target'." + } + $ancestor = $parent.FullName + } + + $item = Get-Item -LiteralPath $target -Force -ErrorAction SilentlyContinue + if ($null -ne $item -or (Test-Path -LiteralPath $target)) { + if ($null -eq $item) { + throw "Refusing to reset an uninspectable $Description '$target'." + } + if (-not $item.PSIsContainer ` + -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to reset non-directory or linked $Description '$target'." + } + $linkedDescendants = @(Get-ChildItem -LiteralPath $target -Recurse -Force -ErrorAction Stop | + Where-Object { ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 } | + Sort-Object { $_.FullName.Length } -Descending) + foreach ($linkedDescendant in $linkedDescendants) { + Remove-Item -LiteralPath $linkedDescendant.FullName -Force + if (Test-Path -LiteralPath $linkedDescendant.FullName) { + throw "Could not unlink $Description entry '$($linkedDescendant.FullName)'." + } + } + Remove-Item -LiteralPath $target -Recurse -Force + $remainingItem = Get-Item -LiteralPath $target -Force -ErrorAction SilentlyContinue + if ($null -ne $remainingItem -or (Test-Path -LiteralPath $target)) { + throw "$Description remained after guarded reset: '$target'." + } + } + + return (New-Item -ItemType Directory -Path $target -Force).FullName +} + +$comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } +$buildPrefix = $requestedBuildPath.TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar +$installPrefix = $requestedInstallPath.TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar +if ($requestedBuildPath.Equals($requestedInstallPath, $comparison) ` + -or $requestedBuildPath.StartsWith($installPrefix, $comparison) ` + -or $requestedInstallPath.StartsWith($buildPrefix, $comparison)) { + throw 'Native build and install scratch directories must be separate and non-nested.' +} + +$buildPath = Reset-RepositoryScratchDirectory $requestedBuildPath 'native build directory' +$installPath = Reset-RepositoryScratchDirectory $requestedInstallPath 'native install directory' + $cmakeCommand = (Get-Command $CMakeExecutable -ErrorAction Stop).Source $ctestName = if ($IsWindows) { 'ctest.exe' } else { 'ctest' } $ctestCommand = Join-Path (Split-Path -Parent $cmakeCommand) $ctestName @@ -38,44 +170,66 @@ if (-not (Test-Path -LiteralPath $ctestCommand)) { $configureArguments = @( '-S', $repositoryRoot, '-B', $buildPath, + "-DCMAKE_BUILD_TYPE=$Configuration", '-DSMS_BUILD_TESTS=ON', '-DSMS_BUILD_SAMPLES=ON', '-DSMS_BUILD_STATIC=ON' ) -if (-not $IsWindows) { - $configureArguments += "-DCMAKE_BUILD_TYPE=$Configuration" -} Invoke-Checked $cmakeCommand @configureArguments -Invoke-Checked $cmakeCommand '--build' $buildPath '--config' $Configuration '--parallel' +$multiConfig = Test-MultiConfigBuild $buildPath +$buildArguments = @('--build', $buildPath) +if ($multiConfig) { + $buildArguments += @('--config', $Configuration) +} +$buildArguments += '--parallel' +Invoke-Checked $cmakeCommand @buildArguments if (-not $SkipTests) { - Invoke-Checked $ctestCommand '--test-dir' $buildPath '-C' $Configuration '--output-on-failure' + $testArguments = @('--test-dir', $buildPath) + if ($multiConfig) { + $testArguments += @('-C', $Configuration) + } + $testArguments += '--output-on-failure' + Invoke-Checked $ctestCommand @testArguments } -Invoke-Checked $cmakeCommand '--install' $buildPath '--config' $Configuration '--prefix' $installPath +$installArguments = @('--install', $buildPath) +if ($multiConfig) { + $installArguments += @('--config', $Configuration) +} +$installArguments += @('--prefix', $installPath) +Invoke-Checked $cmakeCommand @installArguments if (-not $SkipConsumer) { $consumerBuild = Join-Path $buildPath 'package-consumer' $consumerArguments = @( '-S', (Join-Path $repositoryRoot 'tests/cpp/package_consumer'), '-B', $consumerBuild, + "-DCMAKE_BUILD_TYPE=$Configuration", "-DCMAKE_PREFIX_PATH=$installPath" ) - if (-not $IsWindows) { - $consumerArguments += "-DCMAKE_BUILD_TYPE=$Configuration" - } Invoke-Checked $cmakeCommand @consumerArguments - Invoke-Checked $cmakeCommand '--build' $consumerBuild '--config' $Configuration '--parallel' - $consumer = Get-ChildItem -LiteralPath $consumerBuild -Recurse -File | - Where-Object { $_.Name -in @('shared_memory_store_package_consumer', 'shared_memory_store_package_consumer.exe') } | - Select-Object -First 1 - if ($null -eq $consumer) { + $consumerIsMultiConfig = Test-MultiConfigBuild $consumerBuild + $consumerBuildArguments = @('--build', $consumerBuild) + if ($consumerIsMultiConfig) { + $consumerBuildArguments += @('--config', $Configuration) + } + $consumerBuildArguments += '--parallel' + Invoke-Checked $cmakeCommand @consumerBuildArguments + $consumerName = if ($IsWindows) { 'shared_memory_store_package_consumer.exe' } else { 'shared_memory_store_package_consumer' } + $consumerPath = if ($consumerIsMultiConfig) { + Join-Path (Join-Path $consumerBuild $Configuration) $consumerName + } + else { + Join-Path $consumerBuild $consumerName + } + if (-not (Test-Path -LiteralPath $consumerPath -PathType Leaf)) { throw 'The installed-package consumer executable was not produced.' } - Invoke-Checked $consumer.FullName + Invoke-Checked $consumerPath } Write-Host 'Native SharedMemoryStore validation passed.' diff --git a/scripts/validate-package-consumption.ps1 b/scripts/validate-package-consumption.ps1 index 84bdfe0..6ed1b04 100644 --- a/scripts/validate-package-consumption.ps1 +++ b/scripts/validate-package-consumption.ps1 @@ -7,6 +7,8 @@ $root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path $artifactRoot = Join-Path $root "artifacts" $packageDir = Join-Path $root "artifacts/package" $consumerDir = Join-Path $root "artifacts/consumer-smoke" +$consumerCacheDir = Join-Path $root ("artifacts/consumer-smoke-packages-" + [Guid]::NewGuid().ToString('N')) +$previousNuGetPackages = $env:NUGET_PACKAGES function Assert-UnderArtifactRoot { param([string]$Path) @@ -25,6 +27,11 @@ function Invoke-DotNet { } } +try { + Assert-UnderArtifactRoot $consumerCacheDir + New-Item -ItemType Directory -Force -Path $consumerCacheDir | Out-Null + $env:NUGET_PACKAGES = $consumerCacheDir + if (Test-Path -LiteralPath $packageDir) { Assert-UnderArtifactRoot $packageDir Remove-Item -LiteralPath $packageDir -Recurse -Force @@ -45,6 +52,7 @@ $program = @' using SharedMemoryStore; using System.Buffers; using System.Buffers.Binary; +using System.Runtime.InteropServices; var options = new SharedMemoryStoreOptions { @@ -108,6 +116,51 @@ var disposed = store.TryPublish([1], [6]); Console.WriteLine($"disposed publish: {disposed}"); if (disposed != StoreStatus.StoreDisposed) return 19; +if ((OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64) +{ + var lockFreeOptions = SharedMemoryStoreOptions.CreateLockFree( + $"sms-consumer-v2-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 32, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 1, + openMode: OpenMode.CreateNew); + var lockFreeOpen = MemoryStore.TryCreateOrOpen(lockFreeOptions, out var lockFreeStore); + Console.WriteLine($"lock-free open: {lockFreeOpen}"); + if (lockFreeOpen != StoreOpenStatus.Success || lockFreeStore is null) return 20; + + using (lockFreeStore) + { + if (lockFreeStore.Profile != StoreProfile.LockFree) return 21; + if (lockFreeStore.ProtocolInfo.LayoutMajorVersion != 2 + || lockFreeStore.ProtocolInfo.LayoutMinorVersion != 0 + || lockFreeStore.ProtocolInfo.ResourceProtocolVersion != 2) return 22; + if (lockFreeStore.TryPublish([1], [7, 8, 9], [4]) != StoreStatus.Success) return 23; + if (lockFreeStore.TryAcquire([1], out var lockFreeLease) != StoreStatus.Success) return 24; + using (lockFreeLease) + { + if (!new byte[] { 7, 8, 9 }.AsSpan().SequenceEqual(lockFreeLease.ValueSpan)) return 25; + } + + if (lockFreeStore.TryRemove([1]) != StoreStatus.Success) return 26; + var lockFreeOpenExisting = SharedMemoryStoreOptions.CreateLockFree( + lockFreeOptions.Name, + slotCount: 2, + maxValueBytes: 32, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 1, + openMode: OpenMode.OpenExisting); + if (MemoryStore.TryCreateOrOpen(lockFreeOpenExisting, out var exhausted) + != StoreOpenStatus.ParticipantTableFull) return 27; + exhausted?.Dispose(); + } +} + return 0; static async Task PublishLengthPrefixedFrameAsync(MemoryStore store, byte[] key, byte[] frame, byte[] descriptor) @@ -168,3 +221,17 @@ static byte[] CreateLengthPrefixedFrame(byte[] payload) Set-Content -LiteralPath (Join-Path $consumerDir "Program.cs") -Value $program -Encoding UTF8 Invoke-DotNet run --project (Join-Path $consumerDir "SharedMemoryStore.ConsumerSmoke.csproj") -c $Configuration +} +finally { + if ($null -eq $previousNuGetPackages) { + Remove-Item Env:NUGET_PACKAGES -ErrorAction SilentlyContinue + } + else { + $env:NUGET_PACKAGES = $previousNuGetPackages + } + + if (Test-Path -LiteralPath $consumerCacheDir) { + Assert-UnderArtifactRoot $consumerCacheDir + Remove-Item -LiteralPath $consumerCacheDir -Recurse -Force + } +} diff --git a/scripts/validate-python.ps1 b/scripts/validate-python.ps1 index 7a46a5a..56dd1d1 100644 --- a/scripts/validate-python.ps1 +++ b/scripts/validate-python.ps1 @@ -10,7 +10,12 @@ param( $ErrorActionPreference = 'Stop' $repositoryRoot = Split-Path -Parent $PSScriptRoot $artifactsRoot = [IO.Path]::GetFullPath((Join-Path $repositoryRoot 'artifacts')) -$workPath = [IO.Path]::GetFullPath((Join-Path $repositoryRoot $ArtifactsDirectory)) +$requestedWorkPath = if ([IO.Path]::IsPathRooted($ArtifactsDirectory)) { + [IO.Path]::GetFullPath($ArtifactsDirectory) +} +else { + [IO.Path]::GetFullPath((Join-Path $repositoryRoot $ArtifactsDirectory)) +} function Invoke-Checked { param( @@ -24,27 +29,112 @@ function Invoke-Checked { } } -function Assert-ArtifactPath { - param([Parameter(Mandatory)][string]$Path) +function Test-MultiConfigBuild { + param([Parameter(Mandatory)][string]$BuildPath) - $fullPath = [IO.Path]::GetFullPath($Path) - $prefix = $artifactsRoot + [IO.Path]::DirectorySeparatorChar - if (-not $fullPath.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { - throw "Validation output must stay below '$artifactsRoot'; received '$fullPath'." + $cachePath = Join-Path $BuildPath 'CMakeCache.txt' + $entry = Get-Content -LiteralPath $cachePath | + Where-Object { $_.StartsWith('CMAKE_CONFIGURATION_TYPES:', [StringComparison]::Ordinal) } | + Select-Object -First 1 + if ($null -eq $entry) { + return $false } - return $fullPath + $separator = $entry.IndexOf('=') + return $separator -ge 0 -and -not [string]::IsNullOrWhiteSpace($entry.Substring($separator + 1)) } -function Reset-Directory { +function Reset-ArtifactScratchDirectory { param([Parameter(Mandatory)][string]$Path) - $fullPath = Assert-ArtifactPath $Path - if (Test-Path -LiteralPath $fullPath) { - Remove-Item -LiteralPath $fullPath -Recurse -Force + $comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal } + $normalizedRoot = [IO.Path]::GetFullPath($repositoryRoot).TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + $rootPrefix = $normalizedRoot + [IO.Path]::DirectorySeparatorChar + $normalizedArtifactsRoot = [IO.Path]::GetFullPath($artifactsRoot).TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + $artifactsPrefix = $normalizedArtifactsRoot + [IO.Path]::DirectorySeparatorChar + $target = [IO.Path]::GetFullPath($Path).TrimEnd( + [IO.Path]::DirectorySeparatorChar, + [IO.Path]::AltDirectorySeparatorChar) + if (-not $target.StartsWith($artifactsPrefix, $comparison)) { + throw "Python validation output must stay below '$normalizedArtifactsRoot'; received '$target'." + } + + $relativeTarget = [IO.Path]::GetRelativePath($normalizedRoot, $target).Replace('\', '/') + $git = (Get-Command git -ErrorAction Stop).Source + & $git -C $normalizedRoot check-ignore --quiet --no-index -- $relativeTarget 2>$null + $ignoreExitCode = $LASTEXITCODE + if ($ignoreExitCode -eq 1) { + throw "Python validation scratch path must be covered by repository ignore rules: '$relativeTarget'." + } + if ($ignoreExitCode -ne 0) { + throw "Could not verify repository ignore coverage for Python validation scratch path '$relativeTarget'." + } + $protectedEntries = @( + & $git -C $normalizedRoot --literal-pathspecs ls-files --cached --others --exclude-standard -- $relativeTarget 2>$null) + if ($LASTEXITCODE -ne 0) { + throw "Could not prove that Python validation scratch path '$relativeTarget' contains only ignored output." + } + if ($protectedEntries.Count -ne 0) { + throw "Refusing to reset Python validation scratch path '$relativeTarget' because it contains tracked or nonignored entry '$($protectedEntries[0])'." + } + + $ancestor = [IO.Path]::GetDirectoryName($target) + while (-not (Test-Path -LiteralPath $ancestor)) { + $parent = [IO.Directory]::GetParent($ancestor) + if ($null -eq $parent) { + throw "Repository root was not reached while checking Python validation scratch path '$target'." + } + $ancestor = $parent.FullName + } + while ($true) { + $ancestorItem = Get-Item -LiteralPath $ancestor -Force + if (-not $ancestorItem.PSIsContainer ` + -or ($ancestorItem.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to reset Python validation through non-directory or linked ancestor '$ancestor'." + } + if ($ancestor.Equals($normalizedRoot, $comparison)) { + break + } + if (-not $ancestor.StartsWith($rootPrefix, $comparison)) { + throw "Python validation ancestor escaped the repository while checking '$target'." + } + $parent = [IO.Directory]::GetParent($ancestor) + if ($null -eq $parent) { + throw "Repository root was not reached while checking Python validation scratch path '$target'." + } + $ancestor = $parent.FullName + } + + $item = Get-Item -LiteralPath $target -Force -ErrorAction SilentlyContinue + if ($null -ne $item -or (Test-Path -LiteralPath $target)) { + if ($null -eq $item) { + throw "Refusing to reset an uninspectable Python validation scratch path '$target'." + } + if (-not $item.PSIsContainer ` + -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Refusing to reset non-directory or linked Python validation scratch path '$target'." + } + $linkedDescendants = @(Get-ChildItem -LiteralPath $target -Recurse -Force -ErrorAction Stop | + Where-Object { ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 } | + Sort-Object { $_.FullName.Length } -Descending) + foreach ($linkedDescendant in $linkedDescendants) { + Remove-Item -LiteralPath $linkedDescendant.FullName -Force + if (Test-Path -LiteralPath $linkedDescendant.FullName) { + throw "Could not unlink Python validation entry '$($linkedDescendant.FullName)'." + } + } + Remove-Item -LiteralPath $target -Recurse -Force + $remainingItem = Get-Item -LiteralPath $target -Force -ErrorAction SilentlyContinue + if ($null -ne $remainingItem -or (Test-Path -LiteralPath $target)) { + throw "Python validation scratch path remained after guarded reset: '$target'." + } } - New-Item -ItemType Directory -Path $fullPath -Force | Out-Null - return $fullPath + + return (New-Item -ItemType Directory -Path $target -Force).FullName } function Get-VenvPython { @@ -88,7 +178,7 @@ function Assert-ArchiveContainsSuffix { } } -$workPath = Reset-Directory $workPath +$workPath = Reset-ArtifactScratchDirectory $requestedWorkPath $nativeBuildPath = Join-Path $workPath 'native-build' $stagedSourcePath = Join-Path $workPath 'source-package' $buildEnvironmentPath = Join-Path $workPath 'build-environment' @@ -105,18 +195,27 @@ $expectedPythonFiles = @('__init__.py', '_native.py', 'enums.py', 'store.py') $configureArguments = @( '-S', $repositoryRoot, '-B', $nativeBuildPath, + "-DCMAKE_BUILD_TYPE=$Configuration", '-DSMS_BUILD_TESTS=OFF', '-DSMS_BUILD_SAMPLES=OFF', '-DSMS_INSTALL=ON', '-DSMS_PYTHON_INSTALL_DIR=shared_memory_store' ) -if (-not $IsWindows) { - $configureArguments += "-DCMAKE_BUILD_TYPE=$Configuration" -} Invoke-Checked $cmake @configureArguments -Invoke-Checked $cmake '--build' $nativeBuildPath '--config' $Configuration '--parallel' -Invoke-Checked $cmake '--install' $nativeBuildPath '--config' $Configuration '--component' 'Python' '--prefix' $stagedSourcePath +$multiConfig = Test-MultiConfigBuild $nativeBuildPath +$buildArguments = @('--build', $nativeBuildPath) +if ($multiConfig) { + $buildArguments += @('--config', $Configuration) +} +$buildArguments += '--parallel' +Invoke-Checked $cmake @buildArguments +$installArguments = @('--install', $nativeBuildPath) +if ($multiConfig) { + $installArguments += @('--config', $Configuration) +} +$installArguments += @('--component', 'Python', '--prefix', $stagedSourcePath) +Invoke-Checked $cmake @installArguments $stagedPackagePath = Join-Path $stagedSourcePath 'shared_memory_store' foreach ($file in $expectedPythonFiles) { diff --git a/specs/009-lock-free-publish-read/atomic-litmus-results.md b/specs/009-lock-free-publish-read/atomic-litmus-results.md new file mode 100644 index 0000000..1d216b4 --- /dev/null +++ b/specs/009-lock-free-publish-read/atomic-litmus-results.md @@ -0,0 +1,33 @@ +# Layout-v2 mapped atomic qualification + +Date: 2026-07-13 + +## Development host + +- OS: Windows 10.0.26200 +- Architecture/RID: x64 / `win-x64` +- .NET SDK: 10.0.201 +- .NET runtime: 10.0.5 +- Configuration: Release + +## Command + +```powershell +dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj ` + -c Release --filter FullyQualifiedName~MappedAtomicLitmusIntegrationTests --nologo +``` + +## Result + +- Passed: 4 +- Failed: 0 +- Skipped: 0 +- Duration: 11 seconds + +The aligned mapped publication, cross-view CAS, and two-word sequentially +consistent Dekker tests all passed across child processes. The forbidden +old/old Dekker outcome was not observed. The x64 development gate therefore +allows local layout-v2 implementation to continue. + +This result does not replace the separate Windows/Linux x64 release gate in +T086. diff --git a/specs/009-lock-free-publish-read/baseline.md b/specs/009-lock-free-publish-read/baseline.md new file mode 100644 index 0000000..4b6de86 --- /dev/null +++ b/specs/009-lock-free-publish-read/baseline.md @@ -0,0 +1,81 @@ +# Legacy Release Baseline + +**Captured**: 2026-07-13 +**Branch**: `codex/lock-free-csharp` +**Source commit**: `0cf7a43f9c39de1691b237a9761035339edd0964` +**Environment**: Microsoft Windows 10.0.26200, x64, .NET SDK 10.0.201 + +The working tree contained only feature-009 specification/context changes when +this baseline was executed; production and existing test sources matched the +source commit. + +## Release tests + +Command: + +```powershell +dotnet test SharedMemoryStore.slnx -c Release --nologo +``` + +Result: PASS, 226 passed, 0 failed, 0 skipped. + +| Suite | Passed | Duration reported by test host | +|---|---:|---:| +| SharedMemoryStore.UnitTests | 62 | 1 s | +| SharedMemoryStore.ContractTests | 43 | 200 ms | +| SharedMemoryStore.IntegrationTests | 49 | 8 s | +| SharedMemoryStore.InteropTests | 72 | 2 s | + +## Release package + +Command: + +```powershell +dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release --nologo --no-restore +``` + +Result: PASS. + +| Artifact | SHA-256 | +|---|---| +| `SharedMemoryStore.1.0.2.nupkg` | `7061A7CC16761E3920DD94BB527468AD96BA6D4B0CA038727503CA03423D2E59` | +| `SharedMemoryStore.1.0.2.snupkg` | `4D07328C95B7D960DC06BF7669D11196D58F45BE5A782D156B3F5753ABFC5B9F` | + +## Public contract snapshot + +The legacy package exposes: + +- `MemoryStore` with the two `TryCreateOrOpen` overloads; simple/segmented + publish; reservation; acquire/release; remove; lease/reservation recovery; + diagnostics; wait-policy overloads; and disposal. +- `SharedMemoryStoreOptions` with the existing five-dimension + `CalculateRequiredBytes(...)` and `Create(...)` signatures. +- `OpenMode`, `StoreOpenStatus` numeric values 0-10, and `StoreStatus` numeric + values 0-22. +- Value-type `ValueReservation`, `ValueLease`, `StoreWaitOptions`, recovery + options/reports, and `DiagnosticsSnapshot`. +- Mapped layout 1.2/resource naming 1 and package version 1.0.2. + +Existing contract/API reflection tests are the executable detailed snapshot and +must remain green for the legacy profile throughout the feature. + +## Post-facade compatibility checkpoint + +**Captured**: 2026-07-13 after T013/T016 platform extraction + +The legacy profile was rerun after introducing the engine-neutral facade, +opaque public-token handles, the CAS lifetime gate, and actual-capacity mapping +discovery: + +| Gate | Result | +|---|---:| +| Legacy contract tests | 43 passed | +| Unit tests (legacy plus facade/lifetime contracts) | 71 passed | +| Legacy integration tests | 49 passed | +| Interop tests | 72 passed | +| Package-consumption integration | 2 passed | +| Release pack | succeeded | + +There were no failures or skips. The existing five-argument sizing helper, +`Create(...)`, layout-v1.2 behavior, resource protocol, and status numeric +assignments remain unchanged. diff --git a/specs/009-lock-free-publish-read/benchmark-results/cr-m06-versioned-spill-summary.md b/specs/009-lock-free-publish-read/benchmark-results/cr-m06-versioned-spill-summary.md new file mode 100644 index 0000000..b58b13d --- /dev/null +++ b/specs/009-lock-free-publish-read/benchmark-results/cr-m06-versioned-spill-summary.md @@ -0,0 +1,81 @@ +# CR-M06 Versioned Spill-Summary Qualification + +**Result: Passed on Windows x64** + +This qualification compares the original sticky overflow hint with the +required-feature-bit, exact-generation versioned-empty `SpillSummary`. Both raw +reports record the same repository commit, host, runtime, store size, collision +set, churn count, missing-key sample count, and trial count. The working tree is +intentionally dirty because the feature is not committed; the final after +artifact therefore also records exact managed-assembly hashes. + +## Environment and command + +- Repository commit: `0cf7a43f9c39de1691b237a9761035339edd0964` +- OS: Microsoft Windows `10.0.26200`, x64 +- Runtime: .NET `10.0.5`, x64 +- CPU: Intel Family 6 Model 183, 32 logical processors +- Store: 4,096 slots +- Workload: 17 published exact bucket-pair collisions, 10,000 complete + publish/remove cycles, 16,384 early and 16,384 late missing-key samples +- Trials: 3 + +```powershell +dotnet run -c Release --no-build ` + --project benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj -- ` + --mode overflow --profile v2 --overflow-slot-count 4096 ` + --overflow-churn-cycles 10000 --overflow-missing-samples 16384 ` + --trials 3 ` + --output artifacts/sync-probe-cr-m06-versioned-overflow-qualification.json +``` + +## Raw artifacts + +| Artifact | SHA-256 | +|---|---| +| [Before: sticky hint](../../../artifacts/sync-probe-cr-m06-sticky-overflow-qualification.json) | `45FD4E953B691CCFBCD7863449CE78A824699961BAFDE676FDC85ACE551637DB` | +| [After: versioned empty](../../../artifacts/sync-probe-cr-m06-versioned-overflow-qualification.json) | `54ECECAFB0807839ECC5AFDB5EA7A346739D23DBAF967B49AAC89D46136EC477` | + +The after artifact is report schema 6, 1,586,081 bytes, written at +`2026-07-13T13:30:28.9165932+03:00`. It records +`RepositoryWorkingTreeState=dirty`, `SharedMemoryStore.dll` SHA-256 +`8501A6A9F71A08B2901DA3F41B7F945AB7E557D12DC092183A9573CA769F45BF`, +and `SharedMemoryStore.SyncProbe.dll` SHA-256 +`2A67E54D4C7EF793486B35DDD0D9F5923A6DC3FE7C225CE358208D7EB8108F79`. + +## After-fix trial evidence + +| Trial | Failures | Spill during / after first cleanup / after churn | Overflow occupancy during / after first cleanup / after churn | Cleanup scans before -> after | Full-scan max | Late-window scans before -> after | Early p99 (us) | Late p99 (us) | Ratio | +|---:|---:|---|---|---|---:|---|---:|---:|---:| +| 1 | 0 | `1 / 0 / 0` | `1 / 0 / 0` | `1 -> 3` | 4,096 | `30,000 -> 30,000` | 0.5 | 0.2 | 0.4x | +| 2 | 0 | `1 / 0 / 0` | `1 / 0 / 0` | `1 -> 3` | 4,096 | `30,000 -> 30,000` | 0.2 | 0.2 | 1.0x | +| 3 | 0 | `1 / 0 / 0` | `1 / 0 / 0` | `1 -> 3` | 4,096 | `30,000 -> 30,000` | 0.2 | 0.2 | 1.0x | + +All three diagnostics gates and latency gates are true. The late window added +exactly zero overflow scans in every trial, and the median late/early p99 ratio +was `1.0x`, within the `2.0x` limit. The exact current summary witness fast path +also reduced churn scan invocations from the independently reviewed +pre-fast-path run's 200,000 per trial to 30,000; primary-only mutations retain +Present without scanning the overflow table. + +## Before/after conclusion + +The original artifact retained one logical spilled bucket after overflow +occupancy returned to zero. Its three late/early p99 ratios were `61.6x`, +`148.5x`, and `262x` (median `148.5x`), and all diagnostics qualifications +failed. With versioned-empty cleanup, the summary reaches logical Empty after a +stable real full scan, the late missing-key path performs no overflow scan, and +the median ratio falls to `1.0x` with zero correctness failures. CR-M06 is +therefore resolved for this qualified Windows x64 environment. + +## Focused regression evidence + +- Spill summary, operation-budget equality, and checkpoint unit tests: 26/26 + passed in Release. +- Layout and diagnostics contract tests: 36/36 passed in Release. +- Full canonical checkpoint kill/recovery matrix: 45/45 passed in Release, + including immediate post-recovery empty spill/occupancy and zero-scan checks + for spill-summary IDs 41-44, plus participant-recovery ID 45. +- Full solution regression: 696/696 passed in Release (unit 232, contract 113, + integration 229, interop 75, linearizability 47). +- SyncProbe Release build: 0 warnings, 0 errors. diff --git a/specs/009-lock-free-publish-read/benchmark-results/final-broker-20s.json b/specs/009-lock-free-publish-read/benchmark-results/final-broker-20s.json new file mode 100644 index 0000000..f595579 --- /dev/null +++ b/specs/009-lock-free-publish-read/benchmark-results/final-broker-20s.json @@ -0,0 +1,623 @@ +{ + "SchemaVersion": 6, + "TimestampUtc": "2026-07-13T20:13:42.6861237+00:00", + "Environment": { + "RepositoryCommit": "0cf7a43f9c39de1691b237a9761035339edd0964", + "RepositoryWorkingTreeState": "dirty", + "SharedMemoryStoreAssemblySha256": "C67996B06CF518CB04CB2A6BD366155364B475900D1087133979C41F8E04406D", + "ProbeAssemblySha256": "FCF8BF9B300D6F3A4902F5E4B3FDA530CCC85D2B5B3ED161ED663514C9EBB052", + "OperatingSystem": "Microsoft Windows 10.0.26200", + "OperatingSystemArchitecture": "X64", + "ProcessArchitecture": "X64", + "Framework": ".NET 10.0.5", + "RuntimeVersion": "10.0.5", + "LogicalProcessorCount": 32, + "ProcessorIdentifier": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", + "ServerGarbageCollection": false, + "StopwatchFrequency": 10000000 + }, + "Configuration": { + "Mode": "broker", + "DurationSeconds": 20, + "Trials": 3, + "Profiles": [ + "LockFree" + ], + "Scenarios": [ + "broker-directed" + ], + "ScenarioProcessCounts": { + "broker-directed": [ + 1, + 12 + ] + }, + "ReaderKeyCount": 256, + "ReaderPayloadBytes": 256, + "BrokerRotatingKeyCount": 256, + "LargeFrameBytes": 1363148, + "LargeFrames": 256, + "MixedOperationTarget": 0, + "MixedCollisionKeyCount": 512, + "MixedPrimaryBucketCount": 512, + "WarmupCycles": 0, + "WarmupSeconds": 10, + "BrokerObserverSamplingInterval": 16, + "SamplingInterval": 64, + "MaxLatencySamplesPerWorker": 65536, + "AffinityRequested": true, + "AffinityPolicy": "physical-core-first-then-siblings", + "StickyOverflowSlotCount": 4096, + "StickyOverflowChurnCycles": 10000, + "StickyOverflowMissingSamplesPerWindow": 16384, + "LegacyFullPayloadCopiesFieldSemantics": "Retained for v3-v5 readers. Consult FullPayloadCopyCountIsInstrumented and FullPayloadCopyEvidenceKind before interpreting the value as a measured event count." + }, + "Runs": [ + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 110382, + "Operations": 675834, + "ApiCallsPerSecond": 33791.476807295694, + "P50Microseconds": 211.6, + "P95Microseconds": 354.4, + "P99Microseconds": 570, + "MaxMicroseconds": 4998.3, + "EarlyP99Microseconds": 672.9, + "LateP99Microseconds": 353.2, + "LateToEarlyP99Ratio": 0.524892257393372, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 1725, + "EarlyP99Microseconds": 672.9, + "LateP99Microseconds": 353.2, + "LateToEarlyP99Ratio": 0.524892257393372 + } + ], + "Frames": 110382, + "FramesPerSecond": 5519.063546585276, + "BytesWritten": 150467002536, + "BytesRead": 159871360588, + "BytesPerSecond": 15516815667.632517, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 99488640, + "Failures": 0, + "MeasuredSeconds": 20.0001321, + "WallSeconds": 20.0001321, + "SampleCount": 1725, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 5519.063546585276, + "MaxWorkerApiCallsPerSecond": 5519.063546585276, + "WorstWorkerP99Microseconds": 5518.781428342049, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 117281, + "Advance.Success": 110382, + "Commit.Success": 110382, + "Release.Success": 117281, + "Remove.Success": 110126, + "Reserve.Success": 110382 + }, + "WorkerCycles": [ + 110382 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 2, + "Cycles": 113134, + "Operations": 692690, + "ApiCallsPerSecond": 34634.321979585024, + "P50Microseconds": 214, + "P95Microseconds": 300.7, + "P99Microseconds": 583, + "MaxMicroseconds": 6989.7, + "EarlyP99Microseconds": 576.5, + "LateP99Microseconds": 583, + "LateToEarlyP99Ratio": 1.0112749349522983, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 1768, + "EarlyP99Microseconds": 576.5, + "LateP99Microseconds": 583, + "LateToEarlyP99Ratio": 1.0112749349522983 + } + ], + "Frames": 113134, + "FramesPerSecond": 5656.670924711447, + "BytesWritten": 154218385832, + "BytesRead": 163857205340, + "BytesPerSecond": 15903697813.593237, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 101988688, + "Failures": 0, + "MeasuredSeconds": 20.0001028, + "WallSeconds": 20.0001028, + "SampleCount": 1768, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 5656.670924711447, + "MaxWorkerApiCallsPerSecond": 5656.670924711447, + "WorstWorkerP99Microseconds": 5656.566391901823, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 120205, + "Advance.Success": 113134, + "Commit.Success": 113134, + "Release.Success": 120205, + "Remove.Success": 112878, + "Reserve.Success": 113134 + }, + "WorkerCycles": [ + 113134 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 3, + "Cycles": 110443, + "Operations": 676208, + "ApiCallsPerSecond": 33810.215058123635, + "P50Microseconds": 207.9, + "P95Microseconds": 331.3, + "P99Microseconds": 427.2, + "MaxMicroseconds": 5289.2, + "EarlyP99Microseconds": 552.1, + "LateP99Microseconds": 348.3, + "LateToEarlyP99Ratio": 0.6308639739177685, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 1726, + "EarlyP99Microseconds": 552.1, + "LateP99Microseconds": 348.3, + "LateToEarlyP99Ratio": 0.6308639739177685 + } + ], + "Frames": 110443, + "FramesPerSecond": 5522.119794004727, + "BytesWritten": 150550154564, + "BytesRead": 159959965208, + "BytesPerSecond": 15525421064.546778, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 99543976, + "Failures": 0, + "MeasuredSeconds": 20.0001094, + "WallSeconds": 20.0001094, + "SampleCount": 1726, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 5522.119794004727, + "MaxWorkerApiCallsPerSecond": 5522.119794004727, + "WorstWorkerP99Microseconds": 5521.672402945507, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 117346, + "Advance.Success": 110443, + "Commit.Success": 110443, + "Release.Success": 117346, + "Remove.Success": 110187, + "Reserve.Success": 110443 + }, + "WorkerCycles": [ + 110443 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 88284, + "Operations": 540484, + "ApiCallsPerSecond": 27024.009480733162, + "P50Microseconds": 1468.4, + "P95Microseconds": 2578.6, + "P99Microseconds": 4196.3, + "MaxMicroseconds": 129623, + "EarlyP99Microseconds": 5704, + "LateP99Microseconds": 3385.6, + "LateToEarlyP99Ratio": 0.5935483870967742, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 1380, + "EarlyP99Microseconds": 5704, + "LateP99Microseconds": 3385.6, + "LateToEarlyP99Ratio": 0.5935483870967742 + } + ], + "Frames": 88284, + "FramesPerSecond": 4414.168880109395, + "BytesWritten": 120344158032, + "BytesRead": 127866008696, + "BytesPerSecond": 12410420842.933058, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 79587320, + "Failures": 0, + "MeasuredSeconds": 20.000141, + "WallSeconds": 20.000141, + "SampleCount": 1380, + "FairnessIndex": 1.0000000000000004, + "MinWorkerApiCallsPerSecond": 367.84740667578296, + "MaxWorkerApiCallsPerSecond": 367.84740667578296, + "WorstWorkerP99Microseconds": 367.828079285615, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 93802, + "Advance.Success": 88284, + "Commit.Success": 88284, + "Release.Success": 93802, + "Remove.Success": 88028, + "Reserve.Success": 88284 + }, + "WorkerCycles": [ + 7357, + 7357, + 7357, + 7357, + 7357, + 7357, + 7357, + 7357, + 7357, + 7357, + 7357, + 7357 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 2, + "Cycles": 93948, + "Operations": 575176, + "ApiCallsPerSecond": 28756.931230823964, + "P50Microseconds": 1435.2, + "P95Microseconds": 2495, + "P99Microseconds": 3888.9, + "MaxMicroseconds": 114516.1, + "EarlyP99Microseconds": 8337.5, + "LateP99Microseconds": 3372.1, + "LateToEarlyP99Ratio": 0.4044497751124438, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 1468, + "EarlyP99Microseconds": 8337.5, + "LateP99Microseconds": 3372.1, + "LateToEarlyP99Ratio": 0.4044497751124438 + } + ], + "Frames": 93948, + "FramesPerSecond": 4697.094759297067, + "BytesWritten": 128065028304, + "BytesRead": 136069433360, + "BytesPerSecond": 13205864900.06947, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 84675600, + "Failures": 0, + "MeasuredSeconds": 20.0012997, + "WallSeconds": 20.0012997, + "SampleCount": 1468, + "FairnessIndex": 0.9999999999999998, + "MinWorkerApiCallsPerSecond": 391.4245632747556, + "MaxWorkerApiCallsPerSecond": 391.4245632747556, + "WorstWorkerP99Microseconds": 391.4066810655731, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 99820, + "Advance.Success": 93948, + "Commit.Success": 93948, + "Release.Success": 99820, + "Remove.Success": 93692, + "Reserve.Success": 93948 + }, + "WorkerCycles": [ + 7829, + 7829, + 7829, + 7829, + 7829, + 7829, + 7829, + 7829, + 7829, + 7829, + 7829, + 7829 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 3, + "Cycles": 86568, + "Operations": 529974, + "ApiCallsPerSecond": 26496.605045922042, + "P50Microseconds": 1451.2, + "P95Microseconds": 2749.6, + "P99Microseconds": 5972.4, + "MaxMicroseconds": 237200.5, + "EarlyP99Microseconds": 24569.2, + "LateP99Microseconds": 3671.2, + "LateToEarlyP99Ratio": 0.1494228546310014, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 1353, + "EarlyP99Microseconds": 24569.2, + "LateP99Microseconds": 3671.2, + "LateToEarlyP99Ratio": 0.1494228546310014 + } + ], + "Frames": 86568, + "FramesPerSecond": 4328.057802109875, + "BytesWritten": 118004996064, + "BytesRead": 125380989892, + "BytesPerSecond": 12168337208.218632, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 78023072, + "Failures": 0, + "MeasuredSeconds": 20.0015813, + "WallSeconds": 20.0015813, + "SampleCount": 1353, + "FairnessIndex": 0.9999999999999994, + "MinWorkerApiCallsPerSecond": 360.6714835091563, + "MaxWorkerApiCallsPerSecond": 360.6714835091563, + "WorstWorkerP99Microseconds": 360.658071695875, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 91979, + "Advance.Success": 86568, + "Commit.Success": 86568, + "Release.Success": 91979, + "Remove.Success": 86312, + "Reserve.Success": 86568 + }, + "WorkerCycles": [ + 7214, + 7214, + 7214, + 7214, + 7214, + 7214, + 7214, + 7214, + 7214, + 7214, + 7214, + 7214 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + } + ], + "Summary": [ + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 33810.215058123635, + "MedianP50Microseconds": 211.6, + "MedianP95Microseconds": 331.3, + "MedianP99Microseconds": 570, + "MedianMaxMicroseconds": 5289.2, + "MedianEarlyP99Microseconds": 576.5, + "MedianLateP99Microseconds": 353.2, + "MedianLateToEarlyP99Ratio": 0.6308639739177685, + "MedianFramesPerSecond": 5522.119794004727, + "MedianBytesPerSecond": 15525421064.546778, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 5521.672402945507, + "TotalFrames": 333959, + "TotalBytesWritten": 455235542932, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 301021304, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 354832, + "Advance.Success": 333959, + "Commit.Success": 333959, + "Release.Success": 354832, + "Remove.Success": 333191, + "Reserve.Success": 333959 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "structural-direct-reservation-write-and-borrowed-lease-read" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval" + ] + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 27024.009480733162, + "MedianP50Microseconds": 1451.2, + "MedianP95Microseconds": 2578.6, + "MedianP99Microseconds": 4196.3, + "MedianMaxMicroseconds": 129623, + "MedianEarlyP99Microseconds": 8337.5, + "MedianLateP99Microseconds": 3385.6, + "MedianLateToEarlyP99Ratio": 0.4044497751124438, + "MedianFramesPerSecond": 4414.168880109395, + "MedianBytesPerSecond": 12410420842.933058, + "MedianFairnessIndex": 0.9999999999999998, + "MedianWorstWorkerP99Microseconds": 367.828079285615, + "TotalFrames": 268800, + "TotalBytesWritten": 366414182400, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 242285992, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 285601, + "Advance.Success": 268800, + "Commit.Success": 268800, + "Release.Success": 285601, + "Remove.Success": 268032, + "Reserve.Success": 268800 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "structural-direct-reservation-write-and-borrowed-lease-read" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval" + ] + } + ], + "MinimumCompatibleSchemaVersion": 3, + "SchemaCompatibility": "Schema v6 is additive over v3-v5: all existing property names and meanings are retained; new evidence tags disambiguate structural assertions from measured counters, and overflow qualification fields expose the spill/cleanup/late-window transitions." +} \ No newline at end of file diff --git a/specs/009-lock-free-publish-read/benchmark-results/final-broker-release-windows.json b/specs/009-lock-free-publish-read/benchmark-results/final-broker-release-windows.json new file mode 100644 index 0000000..0f77e16 --- /dev/null +++ b/specs/009-lock-free-publish-read/benchmark-results/final-broker-release-windows.json @@ -0,0 +1,623 @@ +{ + "SchemaVersion": 6, + "TimestampUtc": "2026-07-13T20:21:13.9748106+00:00", + "Environment": { + "RepositoryCommit": "0cf7a43f9c39de1691b237a9761035339edd0964", + "RepositoryWorkingTreeState": "dirty", + "SharedMemoryStoreAssemblySha256": "C67996B06CF518CB04CB2A6BD366155364B475900D1087133979C41F8E04406D", + "ProbeAssemblySha256": "FCF8BF9B300D6F3A4902F5E4B3FDA530CCC85D2B5B3ED161ED663514C9EBB052", + "OperatingSystem": "Microsoft Windows 10.0.26200", + "OperatingSystemArchitecture": "X64", + "ProcessArchitecture": "X64", + "Framework": ".NET 10.0.5", + "RuntimeVersion": "10.0.5", + "LogicalProcessorCount": 32, + "ProcessorIdentifier": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", + "ServerGarbageCollection": false, + "StopwatchFrequency": 10000000 + }, + "Configuration": { + "Mode": "broker", + "DurationSeconds": 60, + "Trials": 3, + "Profiles": [ + "LockFree" + ], + "Scenarios": [ + "broker-directed" + ], + "ScenarioProcessCounts": { + "broker-directed": [ + 1, + 12 + ] + }, + "ReaderKeyCount": 256, + "ReaderPayloadBytes": 256, + "BrokerRotatingKeyCount": 256, + "LargeFrameBytes": 1363148, + "LargeFrames": 256, + "MixedOperationTarget": 0, + "MixedCollisionKeyCount": 512, + "MixedPrimaryBucketCount": 512, + "WarmupCycles": 0, + "WarmupSeconds": 10, + "BrokerObserverSamplingInterval": 16, + "SamplingInterval": 64, + "MaxLatencySamplesPerWorker": 65536, + "AffinityRequested": true, + "AffinityPolicy": "physical-core-first-then-siblings", + "StickyOverflowSlotCount": 4096, + "StickyOverflowChurnCycles": 10000, + "StickyOverflowMissingSamplesPerWindow": 16384, + "LegacyFullPayloadCopiesFieldSemantics": "Retained for v3-v5 readers. Consult FullPayloadCopyCountIsInstrumented and FullPayloadCopyEvidenceKind before interpreting the value as a measured event count." + }, + "Runs": [ + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 348955, + "Operations": 2137094, + "ApiCallsPerSecond": 35618.184833238316, + "P50Microseconds": 210.4, + "P95Microseconds": 283, + "P99Microseconds": 473.1, + "MaxMicroseconds": 14749.3, + "EarlyP99Microseconds": 421.1, + "LateP99Microseconds": 520.2, + "LateToEarlyP99Ratio": 1.2353360246972216, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 5453, + "EarlyP99Microseconds": 421.1, + "LateP99Microseconds": 520.2, + "LateToEarlyP99Ratio": 1.2353360246972216 + } + ], + "Frames": 348955, + "FramesPerSecond": 5815.9087473375885, + "BytesWritten": 475677310340, + "BytesRead": 505407568220, + "BytesPerSecond": 16351392377.520712, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 314675592, + "Failures": 0, + "MeasuredSeconds": 60.0000817, + "WallSeconds": 60.0000817, + "SampleCount": 5453, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 5815.9087473375885, + "MaxWorkerApiCallsPerSecond": 5815.9087473375885, + "WorstWorkerP99Microseconds": 5815.795630266608, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "qualification-measurement", + "StatusHistogram": { + "Acquire.Success": 370765, + "Advance.Success": 348955, + "Commit.Success": 348955, + "Release.Success": 370765, + "Remove.Success": 348699, + "Reserve.Success": 348955 + }, + "WorkerCycles": [ + 348955 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 2, + "Cycles": 336423, + "Operations": 2060336, + "ApiCallsPerSecond": 34338.92938435645, + "P50Microseconds": 219.4, + "P95Microseconds": 332.5, + "P99Microseconds": 478.9, + "MaxMicroseconds": 10996, + "EarlyP99Microseconds": 621.8, + "LateP99Microseconds": 433.8, + "LateToEarlyP99Ratio": 0.6976519781280155, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 5257, + "EarlyP99Microseconds": 621.8, + "LateP99Microseconds": 433.8, + "LateToEarlyP99Ratio": 0.6976519781280155 + } + ], + "Frames": 336423, + "FramesPerSecond": 5607.049355189324, + "BytesWritten": 458594339604, + "BytesRead": 487257252600, + "BytesPerSecond": 15764191390.51799, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 303373512, + "Failures": 0, + "MeasuredSeconds": 60.0000069, + "WallSeconds": 60.0000069, + "SampleCount": 5257, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 5607.049355189324, + "MaxWorkerApiCallsPerSecond": 5607.049355189324, + "WorstWorkerP99Microseconds": 5607.00264886263, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "qualification-measurement", + "StatusHistogram": { + "Acquire.Success": 357450, + "Advance.Success": 336423, + "Commit.Success": 336423, + "Release.Success": 357450, + "Remove.Success": 336167, + "Reserve.Success": 336423 + }, + "WorkerCycles": [ + 336423 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 3, + "Cycles": 348628, + "Operations": 2135092, + "ApiCallsPerSecond": 35584.84804392952, + "P50Microseconds": 206.7, + "P95Microseconds": 300.6, + "P99Microseconds": 427.5, + "MaxMicroseconds": 3488.1, + "EarlyP99Microseconds": 438.2, + "LateP99Microseconds": 416.3, + "LateToEarlyP99Ratio": 0.9500228206298494, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 5448, + "EarlyP99Microseconds": 438.2, + "LateP99Microseconds": 416.3, + "LateToEarlyP99Ratio": 0.9500228206298494 + } + ], + "Frames": 348628, + "FramesPerSecond": 5810.463625857369, + "BytesWritten": 475231560944, + "BytesRead": 504934555864, + "BytesPerSecond": 16336093397.57779, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 314379872, + "Failures": 0, + "MeasuredSeconds": 60.0000314, + "WallSeconds": 60.0000314, + "SampleCount": 5448, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 5810.463625857369, + "MaxWorkerApiCallsPerSecond": 5810.463625857369, + "WorstWorkerP99Microseconds": 5810.3271316606, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "qualification-measurement", + "StatusHistogram": { + "Acquire.Success": 370418, + "Advance.Success": 348628, + "Commit.Success": 348628, + "Release.Success": 370418, + "Remove.Success": 348372, + "Reserve.Success": 348628 + }, + "WorkerCycles": [ + 348628 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 307908, + "Operations": 1885682, + "ApiCallsPerSecond": 31427.961887099977, + "P50Microseconds": 1410.3, + "P95Microseconds": 2518.6, + "P99Microseconds": 4097.1, + "MaxMicroseconds": 239067.2, + "EarlyP99Microseconds": 6391.3, + "LateP99Microseconds": 3329.2, + "LateToEarlyP99Ratio": 0.5208955924459812, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 4812, + "EarlyP99Microseconds": 6391.3, + "LateP99Microseconds": 3329.2, + "LateToEarlyP99Ratio": 0.5208955924459812 + } + ], + "Frames": 307908, + "FramesPerSecond": 5131.788333734521, + "BytesWritten": 419724174384, + "BytesRead": 445957957644, + "BytesPerSecond": 14428002734.140451, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 277796048, + "Failures": 0, + "MeasuredSeconds": 60.0001364, + "WallSeconds": 60.0001364, + "SampleCount": 4812, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 427.6490278112101, + "MaxWorkerApiCallsPerSecond": 427.6490278112101, + "WorstWorkerP99Microseconds": 427.63938242686663, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "qualification-measurement", + "StatusHistogram": { + "Acquire.Success": 327153, + "Advance.Success": 307908, + "Commit.Success": 307908, + "Release.Success": 327153, + "Remove.Success": 307652, + "Reserve.Success": 307908 + }, + "WorkerCycles": [ + 25659, + 25659, + 25659, + 25659, + 25659, + 25659, + 25659, + 25659, + 25659, + 25659, + 25659, + 25659 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 2, + "Cycles": 272748, + "Operations": 1670326, + "ApiCallsPerSecond": 27838.033691239576, + "P50Microseconds": 1584.1, + "P95Microseconds": 3011.3, + "P99Microseconds": 4862.8, + "MaxMicroseconds": 60914.2, + "EarlyP99Microseconds": 7367.3, + "LateP99Microseconds": 4311.5, + "LateToEarlyP99Ratio": 0.5852211800795406, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 4262, + "EarlyP99Microseconds": 7367.3, + "LateP99Microseconds": 4311.5, + "LateToEarlyP99Ratio": 0.5852211800795406 + } + ], + "Frames": 272748, + "FramesPerSecond": 4545.6803122373785, + "BytesWritten": 371795890704, + "BytesRead": 395033474660, + "BytesPerSecond": 12780152921.306915, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 246083128, + "Failures": 0, + "MeasuredSeconds": 60.0015798, + "WallSeconds": 60.0015798, + "SampleCount": 4262, + "FairnessIndex": 1.0000000000000002, + "MinWorkerApiCallsPerSecond": 378.80669268644823, + "MaxWorkerApiCallsPerSecond": 378.80669268644823, + "WorstWorkerP99Microseconds": 378.80081385260695, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "qualification-measurement", + "StatusHistogram": { + "Acquire.Success": 289795, + "Advance.Success": 272748, + "Commit.Success": 272748, + "Release.Success": 289795, + "Remove.Success": 272492, + "Reserve.Success": 272748 + }, + "WorkerCycles": [ + 22729, + 22729, + 22729, + 22729, + 22729, + 22729, + 22729, + 22729, + 22729, + 22729, + 22729, + 22729 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 3, + "Cycles": 283260, + "Operations": 1734712, + "ApiCallsPerSecond": 28911.49115458239, + "P50Microseconds": 1505.4, + "P95Microseconds": 2849.7, + "P99Microseconds": 4055.7, + "MaxMicroseconds": 112440.1, + "EarlyP99Microseconds": 4759.3, + "LateP99Microseconds": 3639.6, + "LateToEarlyP99Ratio": 0.764734309667388, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 4426, + "EarlyP99Microseconds": 4759.3, + "LateP99Microseconds": 3639.6, + "LateToEarlyP99Ratio": 0.764734309667388 + } + ], + "Frames": 283260, + "FramesPerSecond": 4720.938682874741, + "BytesWritten": 386125302480, + "BytesRead": 410258474672, + "BytesPerSecond": 13272890559.806446, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 255549288, + "Failures": 0, + "MeasuredSeconds": 60.0007793, + "WallSeconds": 60.0007793, + "SampleCount": 4426, + "FairnessIndex": 0.9999999999999998, + "MinWorkerApiCallsPerSecond": 393.4115569062284, + "MaxWorkerApiCallsPerSecond": 393.4115569062284, + "WorstWorkerP99Microseconds": 393.40349748458834, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "qualification-measurement", + "StatusHistogram": { + "Acquire.Success": 300964, + "Advance.Success": 283260, + "Commit.Success": 283260, + "Release.Success": 300964, + "Remove.Success": 283004, + "Reserve.Success": 283260 + }, + "WorkerCycles": [ + 23605, + 23605, + 23605, + 23605, + 23605, + 23605, + 23605, + 23605, + 23605, + 23605, + 23605, + 23605 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + } + ], + "Summary": [ + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 35584.84804392952, + "MedianP50Microseconds": 210.4, + "MedianP95Microseconds": 300.6, + "MedianP99Microseconds": 473.1, + "MedianMaxMicroseconds": 10996, + "MedianEarlyP99Microseconds": 438.2, + "MedianLateP99Microseconds": 433.8, + "MedianLateToEarlyP99Ratio": 0.9500228206298494, + "MedianFramesPerSecond": 5810.463625857369, + "MedianBytesPerSecond": 16336093397.57779, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 5810.3271316606, + "TotalFrames": 1034006, + "TotalBytesWritten": 1409503210888, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 932428976, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 1098633, + "Advance.Success": 1034006, + "Commit.Success": 1034006, + "Release.Success": 1098633, + "Remove.Success": 1033238, + "Reserve.Success": 1034006 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "structural-direct-reservation-write-and-borrowed-lease-read" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval" + ] + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 28911.49115458239, + "MedianP50Microseconds": 1505.4, + "MedianP95Microseconds": 2849.7, + "MedianP99Microseconds": 4097.1, + "MedianMaxMicroseconds": 112440.1, + "MedianEarlyP99Microseconds": 6391.3, + "MedianLateP99Microseconds": 3639.6, + "MedianLateToEarlyP99Ratio": 0.5852211800795406, + "MedianFramesPerSecond": 4720.938682874741, + "MedianBytesPerSecond": 13272890559.806446, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 393.40349748458834, + "TotalFrames": 863916, + "TotalBytesWritten": 1177645367568, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 779428464, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 917912, + "Advance.Success": 863916, + "Commit.Success": 863916, + "Release.Success": 917912, + "Remove.Success": 863148, + "Reserve.Success": 863916 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "structural-direct-reservation-write-and-borrowed-lease-read" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval" + ] + } + ], + "MinimumCompatibleSchemaVersion": 3, + "SchemaCompatibility": "Schema v6 is additive over v3-v5: all existing property names and meanings are retained; new evidence tags disambiguate structural assertions from measured counters, and overflow qualification fields expose the spill/cleanup/late-window transitions." +} \ No newline at end of file diff --git a/specs/009-lock-free-publish-read/benchmark-results/final-short-broker.json b/specs/009-lock-free-publish-read/benchmark-results/final-short-broker.json new file mode 100644 index 0000000..c3a347c --- /dev/null +++ b/specs/009-lock-free-publish-read/benchmark-results/final-short-broker.json @@ -0,0 +1,623 @@ +{ + "SchemaVersion": 6, + "TimestampUtc": "2026-07-13T20:10:06.3142886+00:00", + "Environment": { + "RepositoryCommit": "0cf7a43f9c39de1691b237a9761035339edd0964", + "RepositoryWorkingTreeState": "dirty", + "SharedMemoryStoreAssemblySha256": "C67996B06CF518CB04CB2A6BD366155364B475900D1087133979C41F8E04406D", + "ProbeAssemblySha256": "FCF8BF9B300D6F3A4902F5E4B3FDA530CCC85D2B5B3ED161ED663514C9EBB052", + "OperatingSystem": "Microsoft Windows 10.0.26200", + "OperatingSystemArchitecture": "X64", + "ProcessArchitecture": "X64", + "Framework": ".NET 10.0.5", + "RuntimeVersion": "10.0.5", + "LogicalProcessorCount": 32, + "ProcessorIdentifier": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", + "ServerGarbageCollection": false, + "StopwatchFrequency": 10000000 + }, + "Configuration": { + "Mode": "broker", + "DurationSeconds": 5, + "Trials": 3, + "Profiles": [ + "LockFree" + ], + "Scenarios": [ + "broker-directed" + ], + "ScenarioProcessCounts": { + "broker-directed": [ + 1, + 12 + ] + }, + "ReaderKeyCount": 256, + "ReaderPayloadBytes": 256, + "BrokerRotatingKeyCount": 256, + "LargeFrameBytes": 1363148, + "LargeFrames": 256, + "MixedOperationTarget": 0, + "MixedCollisionKeyCount": 512, + "MixedPrimaryBucketCount": 512, + "WarmupCycles": 0, + "WarmupSeconds": 2, + "BrokerObserverSamplingInterval": 16, + "SamplingInterval": 64, + "MaxLatencySamplesPerWorker": 65536, + "AffinityRequested": true, + "AffinityPolicy": "physical-core-first-then-siblings", + "StickyOverflowSlotCount": 4096, + "StickyOverflowChurnCycles": 10000, + "StickyOverflowMissingSamplesPerWindow": 16384, + "LegacyFullPayloadCopiesFieldSemantics": "Retained for v3-v5 readers. Consult FullPayloadCopyCountIsInstrumented and FullPayloadCopyEvidenceKind before interpreting the value as a measured event count." + }, + "Runs": [ + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 18077, + "Operations": 110466, + "ApiCallsPerSecond": 22093.04844168769, + "P50Microseconds": 358.3, + "P95Microseconds": 516.6, + "P99Microseconds": 852.1, + "MaxMicroseconds": 32708.7, + "EarlyP99Microseconds": 2398.2, + "LateP99Microseconds": 651.2, + "LateToEarlyP99Ratio": 0.27153698607288806, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 283, + "EarlyP99Microseconds": 2398.2, + "LateP99Microseconds": 651.2, + "LateToEarlyP99Ratio": 0.27153698607288806 + } + ], + "Frames": 18077, + "FramesPerSecond": 3615.375198526138, + "BytesWritten": 24641626396, + "BytesRead": 26181983636, + "BytesPerSecond": 10164652276.88538, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 16293928, + "Failures": 0, + "MeasuredSeconds": 5.0000343, + "WallSeconds": 5.0000343, + "SampleCount": 283, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 3615.375198526138, + "MaxWorkerApiCallsPerSecond": 3615.375198526138, + "WorstWorkerP99Microseconds": 3614.6533572025364, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 19207, + "Advance.Success": 18077, + "Commit.Success": 18077, + "Release.Success": 19207, + "Remove.Success": 17821, + "Reserve.Success": 18077 + }, + "WorkerCycles": [ + 18077 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 2, + "Cycles": 20454, + "Operations": 125026, + "ApiCallsPerSecond": 25005.1634924613, + "P50Microseconds": 343.7, + "P95Microseconds": 546.4, + "P99Microseconds": 779, + "MaxMicroseconds": 1150.9, + "EarlyP99Microseconds": 797.8, + "LateP99Microseconds": 779, + "LateToEarlyP99Ratio": 0.9764351967911757, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 320, + "EarlyP99Microseconds": 797.8, + "LateP99Microseconds": 779, + "LateToEarlyP99Ratio": 0.9764351967911757 + } + ], + "Frames": 20454, + "FramesPerSecond": 4090.7940274407197, + "BytesWritten": 27881829192, + "BytesRead": 29625295484, + "BytesPerSecond": 11501408143.144112, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 18437048, + "Failures": 0, + "MeasuredSeconds": 5.0000073, + "WallSeconds": 5.0000073, + "SampleCount": 320, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 4090.7940274407197, + "MaxWorkerApiCallsPerSecond": 4090.7940274407197, + "WorstWorkerP99Microseconds": 4090.605123571913, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 21733, + "Advance.Success": 20454, + "Commit.Success": 20454, + "Release.Success": 21733, + "Remove.Success": 20198, + "Reserve.Success": 20454 + }, + "WorkerCycles": [ + 20454 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 3, + "Cycles": 20078, + "Operations": 122722, + "ApiCallsPerSecond": 24543.786405339866, + "P50Microseconds": 340.5, + "P95Microseconds": 515.9, + "P99Microseconds": 618.3, + "MaxMicroseconds": 2088.9, + "EarlyP99Microseconds": 666.1, + "LateP99Microseconds": 476.2, + "LateToEarlyP99Ratio": 0.7149076715207926, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 314, + "EarlyP99Microseconds": 666.1, + "LateP99Microseconds": 476.2, + "LateToEarlyP99Ratio": 0.7149076715207926 + } + ], + "Frames": 20078, + "FramesPerSecond": 4015.4996125096873, + "BytesWritten": 27369285544, + "BytesRead": 29080036284, + "BytesPerSecond": 11289582126.04685, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 18080216, + "Failures": 0, + "MeasuredSeconds": 5.000125, + "WallSeconds": 5.000125, + "SampleCount": 314, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 4015.4996125096873, + "MaxWorkerApiCallsPerSecond": 4015.4996125096873, + "WorstWorkerP99Microseconds": 4015.074587339501, + "AffinityAppliedCount": 3, + "AssignedProcessors": [ + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 21333, + "Advance.Success": 20078, + "Commit.Success": 20078, + "Release.Success": 21333, + "Remove.Success": 19822, + "Reserve.Success": 20078 + }, + "WorkerCycles": [ + 20078 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 14124, + "Operations": 86254, + "ApiCallsPerSecond": 17247.6185042907, + "P50Microseconds": 2430.7, + "P95Microseconds": 6003.6, + "P99Microseconds": 13599.2, + "MaxMicroseconds": 18299, + "EarlyP99Microseconds": 15839.4, + "LateP99Microseconds": 13292.7, + "LateToEarlyP99Ratio": 0.8392173945982803, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 221, + "EarlyP99Microseconds": 15839.4, + "LateP99Microseconds": 13292.7, + "LateToEarlyP99Ratio": 0.8392173945982803 + } + ], + "Frames": 14124, + "FramesPerSecond": 2824.2790334894826, + "BytesWritten": 19253102352, + "BytesRead": 20456762036, + "BytesPerSecond": 7940508171.462692, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 12714640, + "Failures": 0, + "MeasuredSeconds": 5.0009223, + "WallSeconds": 5.0009223, + "SampleCount": 221, + "FairnessIndex": 1.0000000000000004, + "MinWorkerApiCallsPerSecond": 235.35658612412354, + "MaxWorkerApiCallsPerSecond": 235.35658612412354, + "WorstWorkerP99Microseconds": 235.3215249778504, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 15007, + "Advance.Success": 14124, + "Commit.Success": 14124, + "Release.Success": 15007, + "Remove.Success": 13868, + "Reserve.Success": 14124 + }, + "WorkerCycles": [ + 1177, + 1177, + 1177, + 1177, + 1177, + 1177, + 1177, + 1177, + 1177, + 1177, + 1177, + 1177 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 2, + "Cycles": 16140, + "Operations": 98602, + "ApiCallsPerSecond": 19715.58190609379, + "P50Microseconds": 2063.1, + "P95Microseconds": 6520.4, + "P99Microseconds": 17456.4, + "MaxMicroseconds": 43490, + "EarlyP99Microseconds": 43490, + "LateP99Microseconds": 11869.7, + "LateToEarlyP99Ratio": 0.27292940905955393, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 253, + "EarlyP99Microseconds": 43490, + "LateP99Microseconds": 11869.7, + "LateToEarlyP99Ratio": 0.27292940905955393 + } + ], + "Frames": 16140, + "FramesPerSecond": 3227.211334094174, + "BytesWritten": 22001208720, + "BytesRead": 23376625052, + "BytesPerSecond": 9073349409.271362, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 14532008, + "Failures": 0, + "MeasuredSeconds": 5.0012219, + "WallSeconds": 5.0012219, + "SampleCount": 253, + "FairnessIndex": 0.9999999999999994, + "MinWorkerApiCallsPerSecond": 268.9342778411812, + "MaxWorkerApiCallsPerSecond": 268.9342778411812, + "WorstWorkerP99Microseconds": 268.8804718750327, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 17149, + "Advance.Success": 16140, + "Commit.Success": 16140, + "Release.Success": 17149, + "Remove.Success": 15884, + "Reserve.Success": 16140 + }, + "WorkerCycles": [ + 1345, + 1345, + 1345, + 1345, + 1345, + 1345, + 1345, + 1345, + 1345, + 1345, + 1345, + 1345 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 3, + "Cycles": 15624, + "Operations": 95442, + "ApiCallsPerSecond": 19080.358010705648, + "P50Microseconds": 2122.8, + "P95Microseconds": 4388.6, + "P99Microseconds": 8972.4, + "MaxMicroseconds": 77717.6, + "EarlyP99Microseconds": 8972.4, + "LateP99Microseconds": 4912.6, + "LateToEarlyP99Ratio": 0.547523516561901, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 245, + "EarlyP99Microseconds": 8972.4, + "LateP99Microseconds": 4912.6, + "LateToEarlyP99Ratio": 0.547523516561901 + } + ], + "Frames": 15624, + "FramesPerSecond": 3123.4835141684484, + "BytesWritten": 21297824352, + "BytesRead": 22629619948, + "BytesPerSecond": 8781787512.199358, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 14084240, + "Failures": 0, + "MeasuredSeconds": 5.0021074, + "WallSeconds": 5.0021074, + "SampleCount": 245, + "FairnessIndex": 1.0000000000000002, + "MinWorkerApiCallsPerSecond": 260.2902928473707, + "MaxWorkerApiCallsPerSecond": 260.2902928473707, + "WorstWorkerP99Microseconds": 260.2407274699738, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 16601, + "Advance.Success": 15624, + "Commit.Success": 15624, + "Release.Success": 16601, + "Remove.Success": 15368, + "Reserve.Success": 15624 + }, + "WorkerCycles": [ + 1302, + 1302, + 1302, + 1302, + 1302, + 1302, + 1302, + 1302, + 1302, + 1302, + 1302, + 1302 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "structural-direct-reservation-write-and-borrowed-lease-read", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval", + "StickyOverflow": null + } + ], + "Summary": [ + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 24543.786405339866, + "MedianP50Microseconds": 343.7, + "MedianP95Microseconds": 516.6, + "MedianP99Microseconds": 779, + "MedianMaxMicroseconds": 2088.9, + "MedianEarlyP99Microseconds": 797.8, + "MedianLateP99Microseconds": 651.2, + "MedianLateToEarlyP99Ratio": 0.7149076715207926, + "MedianFramesPerSecond": 4015.4996125096873, + "MedianBytesPerSecond": 11289582126.04685, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 4015.074587339501, + "TotalFrames": 58609, + "TotalBytesWritten": 79892741132, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 52811192, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 62273, + "Advance.Success": 58609, + "Commit.Success": 58609, + "Release.Success": 62273, + "Remove.Success": 57841, + "Reserve.Success": 58609 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "structural-direct-reservation-write-and-borrowed-lease-read" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval" + ] + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 19080.358010705648, + "MedianP50Microseconds": 2122.8, + "MedianP95Microseconds": 6003.6, + "MedianP99Microseconds": 13599.2, + "MedianMaxMicroseconds": 43490, + "MedianEarlyP99Microseconds": 15839.4, + "MedianLateP99Microseconds": 11869.7, + "MedianLateToEarlyP99Ratio": 0.547523516561901, + "MedianFramesPerSecond": 3123.4835141684484, + "MedianBytesPerSecond": 8781787512.199358, + "MedianFairnessIndex": 1.0000000000000002, + "MedianWorstWorkerP99Microseconds": 260.2407274699738, + "TotalFrames": 45888, + "TotalBytesWritten": 62552135424, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 41330888, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 48757, + "Advance.Success": 45888, + "Commit.Success": 45888, + "Release.Success": 48757, + "Remove.Success": 45120, + "Reserve.Success": 45888 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "structural-direct-reservation-write-and-borrowed-lease-read" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "dedicated-producer-and-broker-coordinator-thread-entire-measured-interval" + ] + } + ], + "MinimumCompatibleSchemaVersion": 3, + "SchemaCompatibility": "Schema v6 is additive over v3-v5: all existing property names and meanings are retained; new evidence tags disambiguate structural assertions from measured counters, and overflow qualification fields expose the spill/cleanup/late-window transitions." +} \ No newline at end of file diff --git a/specs/009-lock-free-publish-read/benchmark-results/final-short-sync-8p.json b/specs/009-lock-free-publish-read/benchmark-results/final-short-sync-8p.json new file mode 100644 index 0000000..2fbf415 --- /dev/null +++ b/specs/009-lock-free-publish-read/benchmark-results/final-short-sync-8p.json @@ -0,0 +1,1137 @@ +{ + "SchemaVersion": 6, + "TimestampUtc": "2026-07-13T20:08:44.1625463+00:00", + "Environment": { + "RepositoryCommit": "0cf7a43f9c39de1691b237a9761035339edd0964", + "RepositoryWorkingTreeState": "dirty", + "SharedMemoryStoreAssemblySha256": "C67996B06CF518CB04CB2A6BD366155364B475900D1087133979C41F8E04406D", + "ProbeAssemblySha256": "FCF8BF9B300D6F3A4902F5E4B3FDA530CCC85D2B5B3ED161ED663514C9EBB052", + "OperatingSystem": "Microsoft Windows 10.0.26200", + "OperatingSystemArchitecture": "X64", + "ProcessArchitecture": "X64", + "Framework": ".NET 10.0.5", + "RuntimeVersion": "10.0.5", + "LogicalProcessorCount": 32, + "ProcessorIdentifier": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", + "ServerGarbageCollection": false, + "StopwatchFrequency": 10000000 + }, + "Configuration": { + "Mode": "sync", + "DurationSeconds": 5, + "Trials": 3, + "Profiles": [ + "Legacy", + "LockFree" + ], + "Scenarios": [ + "acquire-release", + "publish-remove" + ], + "ScenarioProcessCounts": { + "acquire-release": [ + 8 + ], + "publish-remove": [ + 8 + ] + }, + "ReaderKeyCount": 256, + "ReaderPayloadBytes": 256, + "BrokerRotatingKeyCount": 256, + "LargeFrameBytes": 1363148, + "LargeFrames": 256, + "MixedOperationTarget": 0, + "MixedCollisionKeyCount": 512, + "MixedPrimaryBucketCount": 512, + "WarmupCycles": 0, + "WarmupSeconds": 2, + "BrokerObserverSamplingInterval": 16, + "SamplingInterval": 64, + "MaxLatencySamplesPerWorker": 65536, + "AffinityRequested": true, + "AffinityPolicy": "physical-core-first-then-siblings", + "StickyOverflowSlotCount": 4096, + "StickyOverflowChurnCycles": 10000, + "StickyOverflowMissingSamplesPerWindow": 16384, + "LegacyFullPayloadCopiesFieldSemantics": "Retained for v3-v5 readers. Consult FullPayloadCopyCountIsInstrumented and FullPayloadCopyEvidenceKind before interpreting the value as a measured event count." + }, + "Runs": [ + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 32314, + "Operations": 64628, + "ApiCallsPerSecond": 12896.780596144245, + "P50Microseconds": 296.4, + "P95Microseconds": 6322.7, + "P99Microseconds": 19811.3, + "MaxMicroseconds": 34736.2, + "EarlyP99Microseconds": 15808.8, + "LateP99Microseconds": 32175.6, + "LateToEarlyP99Ratio": 2.0352967967208135, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 508, + "EarlyP99Microseconds": 15808.8, + "LateP99Microseconds": 32175.6, + "LateToEarlyP99Ratio": 2.0352967967208135 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 32314, + "BytesPerSecond": 6448.390298072122, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0111731, + "WallSeconds": 5.1069839, + "SampleCount": 508, + "FairnessIndex": 0.9994826545880484, + "MinWorkerApiCallsPerSecond": 1598.4759412399765, + "MaxWorkerApiCallsPerSecond": 1710.577509286199, + "WorstWorkerP99Microseconds": 34736.2, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 32314, + "Release.Success": 32314 + }, + "WorkerCycles": [ + 4286, + 4006, + 4005, + 4004, + 4003, + 4004, + 4003, + 4003 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 2, + "Cycles": 29280, + "Operations": 58560, + "ApiCallsPerSecond": 11663.359359394766, + "P50Microseconds": 287.9, + "P95Microseconds": 1980.4, + "P99Microseconds": 8876.7, + "MaxMicroseconds": 15088.1, + "EarlyP99Microseconds": 10495.4, + "LateP99Microseconds": 6100, + "LateToEarlyP99Ratio": 0.5812070049736076, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 464, + "EarlyP99Microseconds": 10495.4, + "LateP99Microseconds": 6100, + "LateToEarlyP99Ratio": 0.5812070049736076 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 29280, + "BytesPerSecond": 5831.679679697383, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0208519, + "WallSeconds": 5.0764125, + "SampleCount": 464, + "FairnessIndex": 0.9999972747048195, + "MinWorkerApiCallsPerSecond": 1457.5887291702543, + "MaxWorkerApiCallsPerSecond": 1463.2512835832163, + "WorstWorkerP99Microseconds": 15088.1, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 29280, + "Release.Success": 29280 + }, + "WorkerCycles": [ + 3660, + 3661, + 3661, + 3660, + 3660, + 3660, + 3659, + 3659 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 3, + "Cycles": 35595, + "Operations": 71190, + "ApiCallsPerSecond": 14231.454384870223, + "P50Microseconds": 288.5, + "P95Microseconds": 3479.1, + "P99Microseconds": 21495.2, + "MaxMicroseconds": 34952.4, + "EarlyP99Microseconds": 15052.8, + "LateP99Microseconds": 25780.2, + "LateToEarlyP99Ratio": 1.7126514668367347, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 560, + "EarlyP99Microseconds": 15052.8, + "LateP99Microseconds": 25780.2, + "LateToEarlyP99Ratio": 1.7126514668367347 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 35595, + "BytesPerSecond": 7115.727192435112, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0022997, + "WallSeconds": 5.0522976, + "SampleCount": 560, + "FairnessIndex": 0.9999999198981423, + "MinWorkerApiCallsPerSecond": 1778.0673295761749, + "MaxWorkerApiCallsPerSecond": 1779.581677161574, + "WorstWorkerP99Microseconds": 34952.4, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 35595, + "Release.Success": 35595 + }, + "WorkerCycles": [ + 4451, + 4451, + 4450, + 4450, + 4449, + 4449, + 4448, + 4447 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 8, + "ReaderProcessCount": 0, + "PublisherProcessCount": 8, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 134978, + "Operations": 269956, + "ApiCallsPerSecond": 53989.80274390499, + "P50Microseconds": 74.3, + "P95Microseconds": 668.4, + "P99Microseconds": 5014.8, + "MaxMicroseconds": 14362.9, + "EarlyP99Microseconds": 5108.7, + "LateP99Microseconds": 5014.8, + "LateToEarlyP99Ratio": 0.9816195901109872, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 2112, + "EarlyP99Microseconds": 5108.7, + "LateP99Microseconds": 5014.8, + "LateToEarlyP99Ratio": 0.9816195901109872 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 134978, + "BytesRead": 0, + "BytesPerSecond": 26994.901371952496, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0001294, + "WallSeconds": 5.0490223, + "SampleCount": 2112, + "FairnessIndex": 0.9999999896919907, + "MinWorkerApiCallsPerSecond": 6747.941697783731, + "MaxWorkerApiCallsPerSecond": 6749.893216689312, + "WorstWorkerP99Microseconds": 8068.5, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Publish.Success": 134978, + "Remove.Success": 134978 + }, + "WorkerCycles": [ + 16873, + 16875, + 16872, + 16870, + 16873, + 16874, + 16870, + 16871 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 8, + "ReaderProcessCount": 0, + "PublisherProcessCount": 8, + "ObserverProcessCount": 0, + "Trial": 2, + "Cycles": 106398, + "Operations": 212796, + "ApiCallsPerSecond": 42558.529277578586, + "P50Microseconds": 74.9, + "P95Microseconds": 779.8, + "P99Microseconds": 4993.2, + "MaxMicroseconds": 18086.3, + "EarlyP99Microseconds": 4494.4, + "LateP99Microseconds": 6132.2, + "LateToEarlyP99Ratio": 1.3644090423638306, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 1665, + "EarlyP99Microseconds": 4494.4, + "LateP99Microseconds": 6132.2, + "LateToEarlyP99Ratio": 1.3644090423638306 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 106398, + "BytesRead": 0, + "BytesPerSecond": 21279.264638789293, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0000788, + "WallSeconds": 5.062486, + "SampleCount": 1665, + "FairnessIndex": 0.9999998280260451, + "MinWorkerApiCallsPerSecond": 5318.31618333695, + "MaxWorkerApiCallsPerSecond": 5325.518200040447, + "WorstWorkerP99Microseconds": 9065.9, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Publish.Success": 106398, + "Remove.Success": 106398 + }, + "WorkerCycles": [ + 13314, + 13300, + 13299, + 13296, + 13298, + 13297, + 13298, + 13296 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 8, + "ReaderProcessCount": 0, + "PublisherProcessCount": 8, + "ObserverProcessCount": 0, + "Trial": 3, + "Cycles": 218285, + "Operations": 436570, + "ApiCallsPerSecond": 87300.48413904558, + "P50Microseconds": 75.8, + "P95Microseconds": 342.2, + "P99Microseconds": 3879.8, + "MaxMicroseconds": 37894.8, + "EarlyP99Microseconds": 5167, + "LateP99Microseconds": 2444.9, + "LateToEarlyP99Ratio": 0.47317592413392684, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 3416, + "EarlyP99Microseconds": 5167, + "LateP99Microseconds": 2444.9, + "LateToEarlyP99Ratio": 0.47317592413392684 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 218285, + "BytesRead": 0, + "BytesPerSecond": 43650.24206952279, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0007741, + "WallSeconds": 5.0793214, + "SampleCount": 3416, + "FairnessIndex": 0.9999999614356678, + "MinWorkerApiCallsPerSecond": 10907.904795384597, + "MaxWorkerApiCallsPerSecond": 10915.227770591924, + "WorstWorkerP99Microseconds": 5162.9, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Publish.Success": 218285, + "Remove.Success": 218285 + }, + "WorkerCycles": [ + 27292, + 27273, + 27283, + 27288, + 27290, + 27288, + 27287, + 27284 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 74634299, + "Operations": 149268598, + "ApiCallsPerSecond": 29852522.513847195, + "P50Microseconds": 0.5, + "P95Microseconds": 0.6, + "P99Microseconds": 0.7, + "MaxMicroseconds": 1824, + "EarlyP99Microseconds": 0.7, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.7142857142857143, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 524288, + "EarlyP99Microseconds": 0.7, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.7142857142857143 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 74634299, + "BytesPerSecond": 14926261.256923597, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0002005, + "WallSeconds": 5.1789597, + "SampleCount": 524288, + "FairnessIndex": 0.9945369629015075, + "MinWorkerApiCallsPerSecond": 3204057.468253855, + "MaxWorkerApiCallsPerSecond": 4114383.8132090904, + "WorstWorkerP99Microseconds": 0.8, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 74634299, + "Release.Success": 74634299 + }, + "WorkerCycles": [ + 9104918, + 9474685, + 8010309, + 9302829, + 9287459, + 10278783, + 10286372, + 8888944 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 2, + "Cycles": 70825708, + "Operations": 141651416, + "ApiCallsPerSecond": 28296739.6788499, + "P50Microseconds": 0.5, + "P95Microseconds": 0.6, + "P99Microseconds": 0.7, + "MaxMicroseconds": 8158.6, + "EarlyP99Microseconds": 0.7, + "LateP99Microseconds": 0.6, + "LateToEarlyP99Ratio": 0.8571428571428572, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 524288, + "EarlyP99Microseconds": 0.7, + "LateP99Microseconds": 0.6, + "LateToEarlyP99Ratio": 0.8571428571428572 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 70825708, + "BytesPerSecond": 14148369.83942495, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0059271, + "WallSeconds": 5.1820834, + "SampleCount": 524288, + "FairnessIndex": 0.9907905004204844, + "MinWorkerApiCallsPerSecond": 2825524.5666681803, + "MaxWorkerApiCallsPerSecond": 4007705.3572444636, + "WorstWorkerP99Microseconds": 0.8, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 70825708, + "Release.Success": 70825708 + }, + "WorkerCycles": [ + 8758340, + 8776656, + 7072185, + 8737587, + 8910651, + 9937765, + 10019371, + 8613153 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 3, + "Cycles": 73670766, + "Operations": 147341532, + "ApiCallsPerSecond": 29386744.665650275, + "P50Microseconds": 0.5, + "P95Microseconds": 0.6, + "P99Microseconds": 0.7, + "MaxMicroseconds": 13029.8, + "EarlyP99Microseconds": 0.7, + "LateP99Microseconds": 0.6, + "LateToEarlyP99Ratio": 0.8571428571428572, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 524288, + "EarlyP99Microseconds": 0.7, + "LateP99Microseconds": 0.6, + "LateToEarlyP99Ratio": 0.8571428571428572 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 73670766, + "BytesPerSecond": 14693372.332825137, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0138773, + "WallSeconds": 5.1881612, + "SampleCount": 524288, + "FairnessIndex": 0.996015400562295, + "MinWorkerApiCallsPerSecond": 3330217.1156043247, + "MaxWorkerApiCallsPerSecond": 4049473.497202203, + "WorstWorkerP99Microseconds": 0.8, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Acquire.Success": 73670766, + "Release.Success": 73670766 + }, + "WorkerCycles": [ + 8954791, + 9489733, + 8348650, + 9042441, + 9068588, + 9984362, + 10123803, + 8658398 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 8, + "ReaderProcessCount": 0, + "PublisherProcessCount": 8, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 15379958, + "Operations": 30759916, + "ApiCallsPerSecond": 6150918.968000156, + "P50Microseconds": 1.9, + "P95Microseconds": 6, + "P99Microseconds": 8, + "MaxMicroseconds": 10742.3, + "EarlyP99Microseconds": 8.1, + "LateP99Microseconds": 7.9, + "LateToEarlyP99Ratio": 0.9753086419753088, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 240315, + "EarlyP99Microseconds": 8.1, + "LateP99Microseconds": 7.9, + "LateToEarlyP99Ratio": 0.9753086419753088 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 15379958, + "BytesRead": 0, + "BytesPerSecond": 3075459.484000078, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0008651, + "WallSeconds": 5.1637616, + "SampleCount": 240315, + "FairnessIndex": 0.8402895009602035, + "MinWorkerApiCallsPerSecond": 251257.08843799835, + "MaxWorkerApiCallsPerSecond": 1103969.7911467357, + "WorstWorkerP99Microseconds": 10, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Publish.Success": 15379958, + "Remove.Success": 15379958 + }, + "WorkerCycles": [ + 2691038, + 2760402, + 628146, + 2191927, + 988256, + 2485439, + 2644364, + 990386 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 8, + "ReaderProcessCount": 0, + "PublisherProcessCount": 8, + "ObserverProcessCount": 0, + "Trial": 2, + "Cycles": 15837078, + "Operations": 31674156, + "ApiCallsPerSecond": 6334381.332237784, + "P50Microseconds": 1.8, + "P95Microseconds": 6.1, + "P99Microseconds": 8.1, + "MaxMicroseconds": 5679.5, + "EarlyP99Microseconds": 8.2, + "LateP99Microseconds": 8, + "LateToEarlyP99Ratio": 0.9756097560975611, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 247458, + "EarlyP99Microseconds": 8.2, + "LateP99Microseconds": 8, + "LateToEarlyP99Ratio": 0.9756097560975611 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 15837078, + "BytesRead": 0, + "BytesPerSecond": 3167190.666118892, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0003551, + "WallSeconds": 5.1233736, + "SampleCount": 247458, + "FairnessIndex": 0.8358281167490414, + "MinWorkerApiCallsPerSecond": 260258.71642595943, + "MaxWorkerApiCallsPerSecond": 1125840.3985413515, + "WorstWorkerP99Microseconds": 10.8, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Publish.Success": 15837078, + "Remove.Success": 15837078 + }, + "WorkerCycles": [ + 2740930, + 2814631, + 650693, + 2320866, + 958039, + 2666805, + 2692607, + 992507 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 8, + "ReaderProcessCount": 0, + "PublisherProcessCount": 8, + "ObserverProcessCount": 0, + "Trial": 3, + "Cycles": 15805302, + "Operations": 31610604, + "ApiCallsPerSecond": 6322027.992629069, + "P50Microseconds": 1.8, + "P95Microseconds": 6.1, + "P99Microseconds": 8.4, + "MaxMicroseconds": 5288.3, + "EarlyP99Microseconds": 8.4, + "LateP99Microseconds": 8.4, + "LateToEarlyP99Ratio": 1, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 246961, + "EarlyP99Microseconds": 8.4, + "LateP99Microseconds": 8.4, + "LateToEarlyP99Ratio": 1 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 15805302, + "BytesRead": 0, + "BytesPerSecond": 3161013.9963145345, + "FullPayloadCopies": 0, + "MeasuredThreadAllocatedBytes": 0, + "Failures": 0, + "MeasuredSeconds": 5.0000734, + "WallSeconds": 5.1773677, + "SampleCount": 246961, + "FairnessIndex": 0.8314533192752664, + "MinWorkerApiCallsPerSecond": 210799.8471846907, + "MaxWorkerApiCallsPerSecond": 1145099.5290812931, + "WorstWorkerP99Microseconds": 12.7, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only-insufficient-warmup", + "StatusHistogram": { + "Publish.Success": 15805302, + "Remove.Success": 15805302 + }, + "WorkerCycles": [ + 2677695, + 2862781, + 527002, + 2270444, + 1000777, + 2708548, + 2703168, + 1054887 + ], + "FullPayloadCopyCountIsInstrumented": false, + "FullPayloadCopyEvidenceKind": "not-instrumented-legacy-field-do-not-interpret-as-count", + "ProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScope": "sum-of-dedicated-worker-thread-measured-regions", + "StickyOverflow": null + } + ], + "Summary": [ + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 12896.780596144245, + "MedianP50Microseconds": 288.5, + "MedianP95Microseconds": 3479.1, + "MedianP99Microseconds": 19811.3, + "MedianMaxMicroseconds": 34736.2, + "MedianEarlyP99Microseconds": 15052.8, + "MedianLateP99Microseconds": 25780.2, + "MedianLateToEarlyP99Ratio": 1.7126514668367347, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 6448.390298072122, + "MedianFairnessIndex": 0.9999972747048195, + "MedianWorstWorkerP99Microseconds": 34736.2, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 97189, + "Release.Success": 97189 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "not-instrumented-legacy-field-do-not-interpret-as-count" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "sum-of-dedicated-worker-thread-measured-regions" + ] + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 53989.80274390499, + "MedianP50Microseconds": 74.9, + "MedianP95Microseconds": 668.4, + "MedianP99Microseconds": 4993.2, + "MedianMaxMicroseconds": 18086.3, + "MedianEarlyP99Microseconds": 5108.7, + "MedianLateP99Microseconds": 5014.8, + "MedianLateToEarlyP99Ratio": 0.9816195901109872, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 26994.901371952496, + "MedianFairnessIndex": 0.9999999614356678, + "MedianWorstWorkerP99Microseconds": 8068.5, + "TotalFrames": 0, + "TotalBytesWritten": 459661, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 459661, + "Remove.Success": 459661 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "not-instrumented-legacy-field-do-not-interpret-as-count" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "sum-of-dedicated-worker-thread-measured-regions" + ] + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 29386744.665650275, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 0.6, + "MedianP99Microseconds": 0.7, + "MedianMaxMicroseconds": 8158.6, + "MedianEarlyP99Microseconds": 0.7, + "MedianLateP99Microseconds": 0.6, + "MedianLateToEarlyP99Ratio": 0.8571428571428572, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 14693372.332825137, + "MedianFairnessIndex": 0.9945369629015075, + "MedianWorstWorkerP99Microseconds": 0.8, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 219130773, + "Release.Success": 219130773 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "not-instrumented-legacy-field-do-not-interpret-as-count" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "sum-of-dedicated-worker-thread-measured-regions" + ] + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 6322027.992629069, + "MedianP50Microseconds": 1.8, + "MedianP95Microseconds": 6.1, + "MedianP99Microseconds": 8.1, + "MedianMaxMicroseconds": 5679.5, + "MedianEarlyP99Microseconds": 8.2, + "MedianLateP99Microseconds": 8, + "MedianLateToEarlyP99Ratio": 0.9756097560975611, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 3161013.9963145345, + "MedianFairnessIndex": 0.8358281167490414, + "MedianWorstWorkerP99Microseconds": 10.8, + "TotalFrames": 0, + "TotalBytesWritten": 47022338, + "TotalFullPayloadCopies": 0, + "TotalMeasuredThreadAllocatedBytes": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 47022338, + "Remove.Success": 47022338 + }, + "FullPayloadCopyCountsAreInstrumented": false, + "FullPayloadCopyEvidenceKinds": [ + "not-instrumented-legacy-field-do-not-interpret-as-count" + ], + "TotalProducerStoreOperationAllocatedBytes": 0, + "AllocationMeasurementScopes": [ + "sum-of-dedicated-worker-thread-measured-regions" + ] + } + ], + "MinimumCompatibleSchemaVersion": 3, + "SchemaCompatibility": "Schema v6 is additive over v3-v5: all existing property names and meanings are retained; new evidence tags disambiguate structural assertions from measured counters, and overflow qualification fields expose the spill/cleanup/late-window transitions." +} \ No newline at end of file diff --git a/specs/009-lock-free-publish-read/benchmark-results/short-matrix.json b/specs/009-lock-free-publish-read/benchmark-results/short-matrix.json new file mode 100644 index 0000000..e17d9c7 --- /dev/null +++ b/specs/009-lock-free-publish-read/benchmark-results/short-matrix.json @@ -0,0 +1,5218 @@ +{ + "SchemaVersion": 3, + "TimestampUtc": "2026-07-13T07:03:56.3693759+00:00", + "Environment": { + "RepositoryCommit": "0cf7a43f9c39de1691b237a9761035339edd0964", + "OperatingSystem": "Microsoft Windows 10.0.26200", + "OperatingSystemArchitecture": "X64", + "ProcessArchitecture": "X64", + "Framework": ".NET 10.0.5", + "RuntimeVersion": "10.0.5", + "LogicalProcessorCount": 32, + "ProcessorIdentifier": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", + "ServerGarbageCollection": false, + "StopwatchFrequency": 10000000 + }, + "Configuration": { + "Mode": "full", + "DurationSeconds": 1, + "Trials": 1, + "Profiles": [ + "Legacy", + "LockFree" + ], + "Scenarios": [ + "acquire-release", + "publish-remove", + "same-key-read", + "distributed-key-read", + "broker-directed", + "mixed-churn", + "large-ingest" + ], + "ScenarioProcessCounts": { + "acquire-release": [ + 1, + 2, + 4, + 8, + 12 + ], + "broker-directed": [ + 1, + 12 + ], + "distributed-key-read": [ + 1, + 2, + 4, + 6, + 8, + 12 + ], + "large-ingest": [ + 1, + 12 + ], + "mixed-churn": [ + 12 + ], + "publish-remove": [ + 1, + 2, + 4, + 8, + 12 + ], + "same-key-read": [ + 1, + 2, + 4, + 6, + 8, + 12 + ] + }, + "ReaderKeyCount": 256, + "ReaderPayloadBytes": 256, + "BrokerRotatingKeyCount": 256, + "LargeFrameBytes": 1363148, + "LargeFrames": 100, + "MixedOperationTarget": 100000, + "MixedCollisionKeyCount": 512, + "MixedPrimaryBucketCount": 512, + "WarmupCycles": 10000, + "SamplingInterval": 64, + "MaxLatencySamplesPerWorker": 65536, + "AffinityRequested": true, + "AffinityPolicy": "physical-core-first-then-siblings" + }, + "Runs": [ + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 202387, + "Operations": 404774, + "ApiCallsPerSecond": 404764.9332654949, + "P50Microseconds": 3.6, + "P95Microseconds": 7.5, + "P99Microseconds": 8.4, + "MaxMicroseconds": 284.4, + "EarlyP99Microseconds": 10.4, + "LateP99Microseconds": 5.8, + "LateToEarlyP99Ratio": 0.5576923076923077, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 3163, + "EarlyP99Microseconds": 10.4, + "LateP99Microseconds": 5.8, + "LateToEarlyP99Ratio": 0.5576923076923077 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 202387, + "BytesPerSecond": 202382.46663274744, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000224, + "WallSeconds": 1.0757979, + "SampleCount": 3163, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 404764.9332654949, + "MaxWorkerApiCallsPerSecond": 404764.9332654949, + "WorstWorkerP99Microseconds": 8.4, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 202387, + "Release.Success": 202387 + }, + "WorkerCycles": [ + 202387 + ] + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 2, + "ReaderProcessCount": 2, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 25888, + "Operations": 51776, + "ApiCallsPerSecond": 51771.99284775359, + "P50Microseconds": 68.2, + "P95Microseconds": 83.8, + "P99Microseconds": 215.2, + "MaxMicroseconds": 6069.8, + "EarlyP99Microseconds": 151.4, + "LateP99Microseconds": 215.3, + "LateToEarlyP99Ratio": 1.4220607661822986, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 405, + "EarlyP99Microseconds": 151.4, + "LateP99Microseconds": 215.3, + "LateToEarlyP99Ratio": 1.4220607661822986 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 25888, + "BytesPerSecond": 25885.996423876793, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000774, + "WallSeconds": 1.0346208, + "SampleCount": 405, + "FairnessIndex": 0.9999467945779156, + "MinWorkerApiCallsPerSecond": 25698.01097395062, + "MaxWorkerApiCallsPerSecond": 26075.668839005742, + "WorstWorkerP99Microseconds": 215.2, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 25888, + "Release.Success": 25888 + }, + "WorkerCycles": [ + 12850, + 13038 + ] + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 4, + "ReaderProcessCount": 4, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 25665, + "Operations": 51330, + "ApiCallsPerSecond": 51325.51928216667, + "P50Microseconds": 134.7, + "P95Microseconds": 143.6, + "P99Microseconds": 373.5, + "MaxMicroseconds": 1340.3, + "EarlyP99Microseconds": 1331.6, + "LateP99Microseconds": 144.4, + "LateToEarlyP99Ratio": 0.10844097326524484, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 404, + "EarlyP99Microseconds": 1331.6, + "LateP99Microseconds": 144.4, + "LateToEarlyP99Ratio": 0.10844097326524484 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 25665, + "BytesPerSecond": 25662.759641083336, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000873, + "WallSeconds": 1.0372186, + "SampleCount": 404, + "FairnessIndex": 0.9999999918341387, + "MinWorkerApiCallsPerSecond": 12830.879864187858, + "MaxWorkerApiCallsPerSecond": 12833.591891777842, + "WorstWorkerP99Microseconds": 373.5, + "AffinityAppliedCount": 4, + "AssignedProcessors": [ + 0, + 2, + 4, + 6 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 25665, + "Release.Success": 25665 + }, + "WorkerCycles": [ + 6416, + 6417, + 6416, + 6416 + ] + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 12429, + "Operations": 24858, + "ApiCallsPerSecond": 24841.482898021102, + "P50Microseconds": 275.1, + "P95Microseconds": 3207.5, + "P99Microseconds": 7847.7, + "MaxMicroseconds": 9305.9, + "EarlyP99Microseconds": 8996.8, + "LateP99Microseconds": 3446.4, + "LateToEarlyP99Ratio": 0.38306953583496356, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 200, + "EarlyP99Microseconds": 8996.8, + "LateP99Microseconds": 3446.4, + "LateToEarlyP99Ratio": 0.38306953583496356 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 12429, + "BytesPerSecond": 12420.741449010551, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0006649, + "WallSeconds": 1.0612537, + "SampleCount": 200, + "FairnessIndex": 0.9999999279756707, + "MinWorkerApiCallsPerSecond": 3104.260992991726, + "MaxWorkerApiCallsPerSecond": 3106.2635986483556, + "WorstWorkerP99Microseconds": 9305.9, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 12429, + "Release.Success": 12429 + }, + "WorkerCycles": [ + 1554, + 1554, + 1554, + 1553, + 1554, + 1554, + 1553, + 1553 + ] + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 12132, + "Operations": 24264, + "ApiCallsPerSecond": 24199.320057350713, + "P50Microseconds": 587, + "P95Microseconds": 1904.2, + "P99Microseconds": 10344.6, + "MaxMicroseconds": 10362.9, + "EarlyP99Microseconds": 10362.9, + "LateP99Microseconds": 1904.2, + "LateToEarlyP99Ratio": 0.1837516525296973, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 192, + "EarlyP99Microseconds": 10362.9, + "LateP99Microseconds": 1904.2, + "LateToEarlyP99Ratio": 0.1837516525296973 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 12132, + "BytesPerSecond": 12099.660028675356, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0026728, + "WallSeconds": 1.0657069, + "SampleCount": 192, + "FairnessIndex": 0.9999985799942068, + "MinWorkerApiCallsPerSecond": 2015.3338974272865, + "MaxWorkerApiCallsPerSecond": 2022.710926326652, + "WorstWorkerP99Microseconds": 10362.9, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 12132, + "Release.Success": 12132 + }, + "WorkerCycles": [ + 1012, + 1012, + 1012, + 1011, + 1011, + 1011, + 1011, + 1011, + 1011, + 1010, + 1010, + 1010 + ] + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 1, + "ReaderProcessCount": 0, + "PublisherProcessCount": 1, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 844966, + "Operations": 1689932, + "ApiCallsPerSecond": 1689885.866115855, + "P50Microseconds": 0.9, + "P95Microseconds": 2, + "P99Microseconds": 3, + "MaxMicroseconds": 221.9, + "EarlyP99Microseconds": 3.2, + "LateP99Microseconds": 1.3, + "LateToEarlyP99Ratio": 0.40625, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 13203, + "EarlyP99Microseconds": 3.2, + "LateP99Microseconds": 1.3, + "LateToEarlyP99Ratio": 0.40625 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 844966, + "BytesRead": 0, + "BytesPerSecond": 844942.9330579275, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000273, + "WallSeconds": 1.0395428, + "SampleCount": 13203, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 1689885.866115855, + "MaxWorkerApiCallsPerSecond": 1689885.866115855, + "WorstWorkerP99Microseconds": 3, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 844966, + "Remove.Success": 844966 + }, + "WorkerCycles": [ + 844966 + ] + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 2, + "ReaderProcessCount": 0, + "PublisherProcessCount": 2, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 95945, + "Operations": 191890, + "ApiCallsPerSecond": 191871.82973772383, + "P50Microseconds": 18.5, + "P95Microseconds": 19.3, + "P99Microseconds": 40.3, + "MaxMicroseconds": 3920, + "EarlyP99Microseconds": 37.4, + "LateP99Microseconds": 42.8, + "LateToEarlyP99Ratio": 1.1443850267379678, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 1500, + "EarlyP99Microseconds": 37.4, + "LateP99Microseconds": 42.8, + "LateToEarlyP99Ratio": 1.1443850267379678 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 95945, + "BytesRead": 0, + "BytesPerSecond": 95935.91486886192, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000947, + "WallSeconds": 1.0353729, + "SampleCount": 1500, + "FairnessIndex": 0.999991677588926, + "MinWorkerApiCallsPerSecond": 95659.36195961504, + "MaxWorkerApiCallsPerSecond": 96212.88863944584, + "WorstWorkerP99Microseconds": 42.8, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 95945, + "Remove.Success": 95945 + }, + "WorkerCycles": [ + 48111, + 47834 + ] + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 4, + "ReaderProcessCount": 0, + "PublisherProcessCount": 4, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 100013, + "Operations": 200026, + "ApiCallsPerSecond": 200017.73926736828, + "P50Microseconds": 35.3, + "P95Microseconds": 40, + "P99Microseconds": 102.6, + "MaxMicroseconds": 1852.8, + "EarlyP99Microseconds": 165, + "LateP99Microseconds": 69.2, + "LateToEarlyP99Ratio": 0.41939393939393943, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 1565, + "EarlyP99Microseconds": 165, + "LateP99Microseconds": 69.2, + "LateToEarlyP99Ratio": 0.41939393939393943 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 100013, + "BytesRead": 0, + "BytesPerSecond": 100008.86963368414, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000413, + "WallSeconds": 1.0428175, + "SampleCount": 1565, + "FairnessIndex": 0.9999995726519106, + "MinWorkerApiCallsPerSecond": 49971.93615903664, + "MaxWorkerApiCallsPerSecond": 50059.299169811624, + "WorstWorkerP99Microseconds": 338.2, + "AffinityAppliedCount": 4, + "AssignedProcessors": [ + 0, + 2, + 4, + 6 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 100013, + "Remove.Success": 100013 + }, + "WorkerCycles": [ + 24987, + 25000, + 25030, + 24996 + ] + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 8, + "ReaderProcessCount": 0, + "PublisherProcessCount": 8, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 55261, + "Operations": 110522, + "ApiCallsPerSecond": 110515.55694303021, + "P50Microseconds": 70.2, + "P95Microseconds": 155.4, + "P99Microseconds": 3354.1, + "MaxMicroseconds": 11263.6, + "EarlyP99Microseconds": 6441.7, + "LateP99Microseconds": 3299.4, + "LateToEarlyP99Ratio": 0.5121939860595806, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 866, + "EarlyP99Microseconds": 6441.7, + "LateP99Microseconds": 3299.4, + "LateToEarlyP99Ratio": 0.5121939860595806 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 55261, + "BytesRead": 0, + "BytesPerSecond": 55257.77847151511, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000583, + "WallSeconds": 1.045861, + "SampleCount": 866, + "FairnessIndex": 0.9999996937635796, + "MinWorkerApiCallsPerSecond": 13805.195157122338, + "MaxWorkerApiCallsPerSecond": 13827.650160450941, + "WorstWorkerP99Microseconds": 3617.6, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 55261, + "Remove.Success": 55261 + }, + "WorkerCycles": [ + 6903, + 6907, + 6905, + 6905, + 6914, + 6905, + 6909, + 6913 + ] + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 12, + "ReaderProcessCount": 0, + "PublisherProcessCount": 12, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 57190, + "Operations": 114380, + "ApiCallsPerSecond": 114364.86952776148, + "P50Microseconds": 140.6, + "P95Microseconds": 215.6, + "P99Microseconds": 1745, + "MaxMicroseconds": 6647, + "EarlyP99Microseconds": 481.8, + "LateP99Microseconds": 2161.8, + "LateToEarlyP99Ratio": 4.486924034869241, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 900, + "EarlyP99Microseconds": 481.8, + "LateP99Microseconds": 2161.8, + "LateToEarlyP99Ratio": 4.486924034869241 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 57190, + "BytesRead": 0, + "BytesPerSecond": 57182.43476388074, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0001323, + "WallSeconds": 1.064077, + "SampleCount": 900, + "FairnessIndex": 0.9999991196368768, + "MinWorkerApiCallsPerSecond": 9523.426689713278, + "MaxWorkerApiCallsPerSecond": 9557.25166719446, + "WorstWorkerP99Microseconds": 6647, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 57190, + "Remove.Success": 57190 + }, + "WorkerCycles": [ + 4767, + 4779, + 4768, + 4768, + 4765, + 4765, + 4765, + 4765, + 4762, + 4762, + 4762, + 4762 + ] + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 237426, + "Operations": 474852, + "ApiCallsPerSecond": 474843.97513682017, + "P50Microseconds": 3, + "P95Microseconds": 9.1, + "P99Microseconds": 11.6, + "MaxMicroseconds": 1504.2, + "EarlyP99Microseconds": 12.4, + "LateP99Microseconds": 5.3, + "LateToEarlyP99Ratio": 0.42741935483870963, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 3710, + "EarlyP99Microseconds": 12.4, + "LateP99Microseconds": 5.3, + "LateToEarlyP99Ratio": 0.42741935483870963 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 60781056, + "BytesPerSecond": 60780028.81751298, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000169, + "WallSeconds": 1.0382877, + "SampleCount": 3710, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 474843.97513682017, + "MaxWorkerApiCallsPerSecond": 474843.97513682017, + "WorstWorkerP99Microseconds": 11.6, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 237426, + "Release.Success": 237426 + }, + "WorkerCycles": [ + 237426 + ] + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 2, + "ReaderProcessCount": 2, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 35221, + "Operations": 70442, + "ApiCallsPerSecond": 70439.6614032414, + "P50Microseconds": 50.9, + "P95Microseconds": 59.6, + "P99Microseconds": 161.7, + "MaxMicroseconds": 5197.7, + "EarlyP99Microseconds": 219.6, + "LateP99Microseconds": 144, + "LateToEarlyP99Ratio": 0.6557377049180328, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 552, + "EarlyP99Microseconds": 219.6, + "LateP99Microseconds": 144, + "LateToEarlyP99Ratio": 0.6557377049180328 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 9016576, + "BytesPerSecond": 9016276.6596149, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000332, + "WallSeconds": 1.0344099, + "SampleCount": 552, + "FairnessIndex": 0.9996868768315358, + "MinWorkerApiCallsPerSecond": 34596.85138453403, + "MaxWorkerApiCallsPerSecond": 35843.50894392747, + "WorstWorkerP99Microseconds": 1799.9, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 35221, + "Release.Success": 35221 + }, + "WorkerCycles": [ + 17299, + 17922 + ] + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 4, + "ReaderProcessCount": 4, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 35560, + "Operations": 71120, + "ApiCallsPerSecond": 71115.05750350351, + "P50Microseconds": 100.6, + "P95Microseconds": 119.3, + "P99Microseconds": 212.9, + "MaxMicroseconds": 1737.3, + "EarlyP99Microseconds": 212.9, + "LateP99Microseconds": 213.7, + "LateToEarlyP99Ratio": 1.0037576326914044, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 556, + "EarlyP99Microseconds": 212.9, + "LateP99Microseconds": 213.7, + "LateToEarlyP99Ratio": 1.0037576326914044 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 9103360, + "BytesPerSecond": 9102727.36044845, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000695, + "WallSeconds": 1.0372804, + "SampleCount": 556, + "FairnessIndex": 0.9999999622795166, + "MinWorkerApiCallsPerSecond": 17775.001044941277, + "MaxWorkerApiCallsPerSecond": 17783.747470785915, + "WorstWorkerP99Microseconds": 673.8, + "AffinityAppliedCount": 4, + "AssignedProcessors": [ + 0, + 2, + 4, + 6 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 35560, + "Release.Success": 35560 + }, + "WorkerCycles": [ + 8888, + 8891, + 8889, + 8892 + ] + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 6, + "ReaderProcessCount": 6, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 16902, + "Operations": 33804, + "ApiCallsPerSecond": 33206.24117019879, + "P50Microseconds": 154.3, + "P95Microseconds": 848.3, + "P99Microseconds": 18621, + "MaxMicroseconds": 20369.9, + "EarlyP99Microseconds": 848.3, + "LateP99Microseconds": 18658.3, + "LateToEarlyP99Ratio": 21.994931038547683, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 270, + "EarlyP99Microseconds": 848.3, + "LateP99Microseconds": 18658.3, + "LateToEarlyP99Ratio": 21.994931038547683 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 4326912, + "BytesPerSecond": 4250398.869785445, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0180014, + "WallSeconds": 1.0537143, + "SampleCount": 270, + "FairnessIndex": 0.9999927569725131, + "MinWorkerApiCallsPerSecond": 5534.373528366465, + "MaxWorkerApiCallsPerSecond": 5573.364580051332, + "WorstWorkerP99Microseconds": 20369.9, + "AffinityAppliedCount": 6, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 16902, + "Release.Success": 16902 + }, + "WorkerCycles": [ + 2817, + 2817, + 2817, + 2817, + 2817, + 2817 + ] + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 18397, + "Operations": 36794, + "ApiCallsPerSecond": 36288.27928226648, + "P50Microseconds": 208.5, + "P95Microseconds": 1507.3, + "P99Microseconds": 34203.2, + "MaxMicroseconds": 40797.2, + "EarlyP99Microseconds": 1865.1, + "LateP99Microseconds": 40797.2, + "LateToEarlyP99Ratio": 21.87400139402713, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 288, + "EarlyP99Microseconds": 1865.1, + "LateP99Microseconds": 40797.2, + "LateToEarlyP99Ratio": 21.87400139402713 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 4709632, + "BytesPerSecond": 4644899.748130109, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0139362, + "WallSeconds": 1.0527901, + "SampleCount": 288, + "FairnessIndex": 0.9999824267217524, + "MinWorkerApiCallsPerSecond": 4535.80504127405, + "MaxWorkerApiCallsPerSecond": 4596.910991786046, + "WorstWorkerP99Microseconds": 40797.2, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 18397, + "Release.Success": 18397 + }, + "WorkerCycles": [ + 2300, + 2300, + 2301, + 2300, + 2299, + 2299, + 2299, + 2299 + ] + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 21190, + "Operations": 42380, + "ApiCallsPerSecond": 42366.379209084276, + "P50Microseconds": 417, + "P95Microseconds": 796.1, + "P99Microseconds": 5855.7, + "MaxMicroseconds": 7725.7, + "EarlyP99Microseconds": 5924.6, + "LateP99Microseconds": 924.6, + "LateToEarlyP99Ratio": 0.15606116868649358, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 336, + "EarlyP99Microseconds": 5924.6, + "LateP99Microseconds": 924.6, + "LateToEarlyP99Ratio": 0.15606116868649358 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 5424640, + "BytesPerSecond": 5422896.538762787, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0003215, + "WallSeconds": 1.0666198, + "SampleCount": 336, + "FairnessIndex": 0.999998509231126, + "MinWorkerApiCallsPerSecond": 3529.241565987469, + "MaxWorkerApiCallsPerSecond": 3545.1179746479074, + "WorstWorkerP99Microseconds": 7725.7, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 21190, + "Release.Success": 21190 + }, + "WorkerCycles": [ + 1773, + 1766, + 1766, + 1765, + 1765, + 1765, + 1765, + 1765, + 1765, + 1765, + 1765, + 1765 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 237786, + "Operations": 475572, + "ApiCallsPerSecond": 475561.63275640586, + "P50Microseconds": 3, + "P95Microseconds": 9.9, + "P99Microseconds": 13, + "MaxMicroseconds": 137.3, + "EarlyP99Microseconds": 13.5, + "LateP99Microseconds": 3.5, + "LateToEarlyP99Ratio": 0.25925925925925924, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 3716, + "EarlyP99Microseconds": 13.5, + "LateP99Microseconds": 3.5, + "LateToEarlyP99Ratio": 0.25925925925925924 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 60873216, + "BytesPerSecond": 60871888.99281995, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000218, + "WallSeconds": 1.0358717, + "SampleCount": 3716, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 475561.63275640586, + "MaxWorkerApiCallsPerSecond": 475561.63275640586, + "WorstWorkerP99Microseconds": 13, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 237786, + "Release.Success": 237786 + }, + "WorkerCycles": [ + 237786 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 2, + "ReaderProcessCount": 2, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 31652, + "Operations": 63304, + "ApiCallsPerSecond": 63301.765447679696, + "P50Microseconds": 54.7, + "P95Microseconds": 59, + "P99Microseconds": 183.8, + "MaxMicroseconds": 1821.4, + "EarlyP99Microseconds": 231.5, + "LateP99Microseconds": 81.4, + "LateToEarlyP99Ratio": 0.3516198704103672, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 495, + "EarlyP99Microseconds": 231.5, + "LateP99Microseconds": 81.4, + "LateToEarlyP99Ratio": 0.3516198704103672 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 8102912, + "BytesPerSecond": 8102625.977303001, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000353, + "WallSeconds": 1.0341154, + "SampleCount": 495, + "FairnessIndex": 0.9999986791834468, + "MinWorkerApiCallsPerSecond": 31614.88399459499, + "MaxWorkerApiCallsPerSecond": 31687.635592190687, + "WorstWorkerP99Microseconds": 231.5, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 31652, + "Release.Success": 31652 + }, + "WorkerCycles": [ + 15808, + 15844 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 4, + "ReaderProcessCount": 4, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 32479, + "Operations": 64958, + "ApiCallsPerSecond": 64502.33614675846, + "P50Microseconds": 102.5, + "P95Microseconds": 110, + "P99Microseconds": 220.8, + "MaxMicroseconds": 547.3, + "EarlyP99Microseconds": 234.4, + "LateP99Microseconds": 220.8, + "LateToEarlyP99Ratio": 0.9419795221843004, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 508, + "EarlyP99Microseconds": 234.4, + "LateP99Microseconds": 220.8, + "LateToEarlyP99Ratio": 0.9419795221843004 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 8314624, + "BytesPerSecond": 8256299.026785083, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0070643, + "WallSeconds": 1.0449965, + "SampleCount": 508, + "FairnessIndex": 0.999999948479593, + "MinWorkerApiCallsPerSecond": 16122.401558617976, + "MaxWorkerApiCallsPerSecond": 16132.038440842358, + "WorstWorkerP99Microseconds": 537.9, + "AffinityAppliedCount": 4, + "AssignedProcessors": [ + 0, + 2, + 4, + 6 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 32479, + "Release.Success": 32479 + }, + "WorkerCycles": [ + 8118, + 8118, + 8123, + 8120 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 6, + "ReaderProcessCount": 6, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 20349, + "Operations": 40698, + "ApiCallsPerSecond": 40539.30483728396, + "P50Microseconds": 156.1, + "P95Microseconds": 334.2, + "P99Microseconds": 2897.2, + "MaxMicroseconds": 4773.7, + "EarlyP99Microseconds": 3370.7, + "LateP99Microseconds": 2897.2, + "LateToEarlyP99Ratio": 0.8595247278013469, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 318, + "EarlyP99Microseconds": 3370.7, + "LateP99Microseconds": 2897.2, + "LateToEarlyP99Ratio": 0.8595247278013469 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 5209344, + "BytesPerSecond": 5189031.019172347, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0039146, + "WallSeconds": 1.0450941, + "SampleCount": 318, + "FairnessIndex": 0.9999999721217228, + "MinWorkerApiCallsPerSecond": 6754.935810674493, + "MaxWorkerApiCallsPerSecond": 6758.089484914837, + "WorstWorkerP99Microseconds": 4773.7, + "AffinityAppliedCount": 6, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 20349, + "Release.Success": 20349 + }, + "WorkerCycles": [ + 3391, + 3392, + 3392, + 3392, + 3390, + 3392 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 27287, + "Operations": 54574, + "ApiCallsPerSecond": 54507.3919670163, + "P50Microseconds": 217.5, + "P95Microseconds": 805, + "P99Microseconds": 3368.7, + "MaxMicroseconds": 13323.6, + "EarlyP99Microseconds": 13309.8, + "LateP99Microseconds": 1859.3, + "LateToEarlyP99Ratio": 0.13969406001592813, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 432, + "EarlyP99Microseconds": 13309.8, + "LateP99Microseconds": 1859.3, + "LateToEarlyP99Ratio": 0.13969406001592813 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 6985472, + "BytesPerSecond": 6976946.171778087, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.001222, + "WallSeconds": 1.0482315, + "SampleCount": 432, + "FairnessIndex": 0.999999977063214, + "MinWorkerApiCallsPerSecond": 6812.228609602166, + "MaxWorkerApiCallsPerSecond": 6815.671249732826, + "WorstWorkerP99Microseconds": 13323.6, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 27287, + "Release.Success": 27287 + }, + "WorkerCycles": [ + 3411, + 3410, + 3412, + 3411, + 3410, + 3411, + 3411, + 3411 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 15324, + "Operations": 30648, + "ApiCallsPerSecond": 30639.473034654457, + "P50Microseconds": 469.5, + "P95Microseconds": 3634.5, + "P99Microseconds": 4612, + "MaxMicroseconds": 5600.9, + "EarlyP99Microseconds": 5600.9, + "LateP99Microseconds": 558.8, + "LateToEarlyP99Ratio": 0.09976967987287758, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 240, + "EarlyP99Microseconds": 5600.9, + "LateP99Microseconds": 558.8, + "LateToEarlyP99Ratio": 0.09976967987287758 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 3922944, + "BytesPerSecond": 3921852.5484357704, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0002783, + "WallSeconds": 1.0680084, + "SampleCount": 240, + "FairnessIndex": 0.9999999950032947, + "MinWorkerApiCallsPerSecond": 2553.289419554538, + "MaxWorkerApiCallsPerSecond": 2553.8646451738055, + "WorstWorkerP99Microseconds": 5600.9, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 15324, + "Release.Success": 15324 + }, + "WorkerCycles": [ + 1277, + 1277, + 1277, + 1277, + 1277, + 1277, + 1277, + 1277, + 1277, + 1277, + 1277, + 1277 + ] + }, + { + "Profile": "Legacy", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 3389, + "Operations": 26856, + "ApiCallsPerSecond": 26852.734707459575, + "P50Microseconds": 251.1, + "P95Microseconds": 516, + "P99Microseconds": 1079.7, + "MaxMicroseconds": 1079.7, + "EarlyP99Microseconds": 1079.7, + "LateP99Microseconds": 324, + "LateToEarlyP99Ratio": 0.3000833564879133, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 53, + "EarlyP99Microseconds": 1079.7, + "LateP99Microseconds": 324, + "LateToEarlyP99Ratio": 0.3000833564879133 + } + ], + "Frames": 3389, + "FramesPerSecond": 3388.587947705559, + "BytesWritten": 4619708572, + "BytesRead": 9239417144, + "BytesPerSecond": 13857440651.216812, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0001216, + "WallSeconds": 1.0001216, + "SampleCount": 53, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 3388.587947705559, + "MaxWorkerApiCallsPerSecond": 3388.587947705559, + "WorstWorkerP99Microseconds": 3386.663540823186, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 6778, + "Advance.Success": 3389, + "Commit.Success": 3389, + "Release.Success": 6778, + "Remove.Success": 3133, + "Reserve.Success": 3389 + }, + "WorkerCycles": [ + 3389 + ] + }, + { + "Profile": "Legacy", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 2197, + "Operations": 17320, + "ApiCallsPerSecond": 17318.668194415848, + "P50Microseconds": 415.5, + "P95Microseconds": 977.1, + "P99Microseconds": 1107.2, + "MaxMicroseconds": 1107.2, + "EarlyP99Microseconds": 1107.2, + "LateP99Microseconds": 863.6, + "LateToEarlyP99Ratio": 0.7799855491329479, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 35, + "EarlyP99Microseconds": 1107.2, + "LateP99Microseconds": 863.6, + "LateToEarlyP99Ratio": 0.7799855491329479 + } + ], + "Frames": 2197, + "FramesPerSecond": 2196.831063691202, + "BytesWritten": 2994836156, + "BytesRead": 5989672312, + "BytesPerSecond": 8983817612.425604, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000769, + "WallSeconds": 1.0000769, + "SampleCount": 35, + "FairnessIndex": 0.9999977210668722, + "MinWorkerApiCallsPerSecond": 182.9859283821074, + "MaxWorkerApiCallsPerSecond": 183.98585148802056, + "WorstWorkerP99Microseconds": 183.89133860801653, + "AffinityAppliedCount": 13, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19, + 20 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 4394, + "Advance.Success": 2197, + "Commit.Success": 2197, + "Release.Success": 4394, + "Remove.Success": 1941, + "Reserve.Success": 2197 + }, + "WorkerCycles": [ + 184, + 183, + 183, + 183, + 183, + 183, + 183, + 183, + 183, + 183, + 183, + 183 + ] + }, + { + "Profile": "Legacy", + "Scenario": "mixed-churn", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 2, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 46405, + "Operations": 100010, + "ApiCallsPerSecond": 14159.661202249388, + "P50Microseconds": 899.7, + "P95Microseconds": 3018.3, + "P99Microseconds": 22042.5, + "MaxMicroseconds": 37396.9, + "EarlyP99Microseconds": 24246.6, + "LateP99Microseconds": 7061.4, + "LateToEarlyP99Ratio": 0.2912325851872015, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 56, + "EarlyP99Microseconds": 32008.9, + "LateP99Microseconds": 22042.5, + "LateToEarlyP99Ratio": 0.6886365979461961 + }, + { + "Role": "reader", + "SampleCount": 672, + "EarlyP99Microseconds": 23735.9, + "LateP99Microseconds": 3446.3, + "LateToEarlyP99Ratio": 0.1451935675495768 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 904448, + "BytesRead": 10969344, + "BytesPerSecond": 1681120.6069990918, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 7.0630221, + "WallSeconds": 7.1270526, + "SampleCount": 728, + "FairnessIndex": 0.8104599529594054, + "MinWorkerApiCallsPerSecond": 1011.3432416559581, + "MaxWorkerApiCallsPerSecond": 2755.433486115126, + "WorstWorkerP99Microseconds": 37396.9, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.NotFound": 23, + "Acquire.Success": 42849, + "Advance.Success": 3533, + "Commit.Success": 3533, + "RecoverLeases.Success": 2, + "RecoverReservations.Success": 2, + "Release.Success": 42849, + "Remove.RemovePending": 65, + "Remove.Success": 3468, + "Reserve.DuplicateKey": 153, + "Reserve.Success": 3533 + }, + "WorkerCycles": [ + 3574, + 3573, + 3573, + 3572, + 3572, + 3572, + 3572, + 3572, + 3574, + 3573, + 3573, + 3572, + 1762, + 1771 + ] + }, + { + "Profile": "Legacy", + "Scenario": "large-ingest", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 100, + "Operations": 500, + "ApiCallsPerSecond": 11278.254847957845, + "P50Microseconds": 421.7, + "P95Microseconds": 527.6, + "P99Microseconds": 527.6, + "MaxMicroseconds": 527.6, + "EarlyP99Microseconds": 527.6, + "LateP99Microseconds": 421.7, + "LateToEarlyP99Ratio": 0.7992797573919636, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 2, + "EarlyP99Microseconds": 527.6, + "LateP99Microseconds": 421.7, + "LateToEarlyP99Ratio": 0.7992797573919636 + } + ], + "Frames": 100, + "FramesPerSecond": 2255.6509695915693, + "BytesWritten": 136314800, + "BytesRead": 136314800, + "BytesPerSecond": 6149572215.793617, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 0.0443331, + "WallSeconds": 0.0443331, + "SampleCount": 2, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 2255.6509695915693, + "MaxWorkerApiCallsPerSecond": 2255.6509695915693, + "WorstWorkerP99Microseconds": 2247.9184275361017, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 100, + "Advance.Success": 100, + "Commit.Success": 100, + "Release.Success": 100, + "Reserve.Success": 100 + }, + "WorkerCycles": [ + 100 + ] + }, + { + "Profile": "Legacy", + "Scenario": "large-ingest", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 100, + "Operations": 500, + "ApiCallsPerSecond": 6947.117154794275, + "P50Microseconds": 663.6, + "P95Microseconds": 812.7, + "P99Microseconds": 812.7, + "MaxMicroseconds": 812.7, + "EarlyP99Microseconds": 812.7, + "LateP99Microseconds": 663.6, + "LateToEarlyP99Ratio": 0.8165374677002584, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 2, + "EarlyP99Microseconds": 812.7, + "LateP99Microseconds": 663.6, + "LateToEarlyP99Ratio": 0.8165374677002584 + } + ], + "Frames": 100, + "FramesPerSecond": 1389.4234309588549, + "BytesWritten": 136314800, + "BytesRead": 136314800, + "BytesPerSecond": 3787979542.1294026, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 0.0719723, + "WallSeconds": 0.0719723, + "SampleCount": 2, + "FairnessIndex": 0.9968102073365229, + "MinWorkerApiCallsPerSecond": 111.1538744767084, + "MaxWorkerApiCallsPerSecond": 125.04810878629695, + "WorstWorkerP99Microseconds": 124.70210053760461, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 100, + "Advance.Success": 100, + "Commit.Success": 100, + "Release.Success": 100, + "Reserve.Success": 100 + }, + "WorkerCycles": [ + 9, + 9, + 9, + 9, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8 + ] + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 2372433, + "Operations": 4744866, + "ApiCallsPerSecond": 4738841.510787336, + "P50Microseconds": 0.3, + "P95Microseconds": 1.4, + "P99Microseconds": 1.8, + "MaxMicroseconds": 61.5, + "EarlyP99Microseconds": 3.4, + "LateP99Microseconds": 0.3, + "LateToEarlyP99Ratio": 0.08823529411764706, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 37070, + "EarlyP99Microseconds": 3.4, + "LateP99Microseconds": 0.3, + "LateToEarlyP99Ratio": 0.08823529411764706 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 2372433, + "BytesPerSecond": 2369420.755393668, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0012713, + "WallSeconds": 1.0507145, + "SampleCount": 37070, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 4738841.510787336, + "MaxWorkerApiCallsPerSecond": 4738841.510787336, + "WorstWorkerP99Microseconds": 1.8, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 2372433, + "Release.Success": 2372433 + }, + "WorkerCycles": [ + 2372433 + ] + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 2, + "ReaderProcessCount": 2, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 3489315, + "Operations": 6978630, + "ApiCallsPerSecond": 6978482.056180409, + "P50Microseconds": 0.4, + "P95Microseconds": 1.4, + "P99Microseconds": 1.8, + "MaxMicroseconds": 1240.2, + "EarlyP99Microseconds": 3.4, + "LateP99Microseconds": 0.7, + "LateToEarlyP99Ratio": 0.20588235294117646, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 54521, + "EarlyP99Microseconds": 3.4, + "LateP99Microseconds": 0.7, + "LateToEarlyP99Ratio": 0.20588235294117646 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 3489315, + "BytesPerSecond": 3489241.0280902046, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000212, + "WallSeconds": 1.0530084, + "SampleCount": 54521, + "FairnessIndex": 0.9940148924329332, + "MinWorkerApiCallsPerSecond": 3218493.1704790783, + "MaxWorkerApiCallsPerSecond": 3759996.2880786927, + "WorstWorkerP99Microseconds": 1.8, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 3489315, + "Release.Success": 3489315 + }, + "WorkerCycles": [ + 1880038, + 1609277 + ] + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 4, + "ReaderProcessCount": 4, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 7550156, + "Operations": 15100312, + "ApiCallsPerSecond": 15099997.920043264, + "P50Microseconds": 0.3, + "P95Microseconds": 1.6, + "P99Microseconds": 2, + "MaxMicroseconds": 32109.3, + "EarlyP99Microseconds": 3.8, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.13157894736842105, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 117973, + "EarlyP99Microseconds": 3.8, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.13157894736842105 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 7550156, + "BytesPerSecond": 7549998.960021632, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000208, + "WallSeconds": 1.0780792, + "SampleCount": 117973, + "FairnessIndex": 0.9982544615137126, + "MinWorkerApiCallsPerSecond": 3536571.2929165377, + "MaxWorkerApiCallsPerSecond": 3957069.692950387, + "WorstWorkerP99Microseconds": 2.1, + "AffinityAppliedCount": 4, + "AssignedProcessors": [ + 0, + 2, + 4, + 6 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 7550156, + "Release.Success": 7550156 + }, + "WorkerCycles": [ + 1933426, + 1978576, + 1869839, + 1768315 + ] + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 9775298, + "Operations": 19550596, + "ApiCallsPerSecond": 19550009.499715008, + "P50Microseconds": 0.4, + "P95Microseconds": 2.3, + "P99Microseconds": 3.8, + "MaxMicroseconds": 9821.3, + "EarlyP99Microseconds": 5.5, + "LateP99Microseconds": 0.9, + "LateToEarlyP99Ratio": 0.16363636363636364, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 152744, + "EarlyP99Microseconds": 5.5, + "LateP99Microseconds": 0.9, + "LateToEarlyP99Ratio": 0.16363636363636364 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 9775298, + "BytesPerSecond": 9775004.749857504, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.00003, + "WallSeconds": 1.1481369, + "SampleCount": 152744, + "FairnessIndex": 0.9960589678415701, + "MinWorkerApiCallsPerSecond": 2165099.802292392, + "MaxWorkerApiCallsPerSecond": 2648810.5208116984, + "WorstWorkerP99Microseconds": 4.5, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 9775298, + "Release.Success": 9775298 + }, + "WorkerCycles": [ + 1216523, + 1321069, + 1214838, + 1324434, + 1141469, + 1252821, + 1221581, + 1082563 + ] + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 11644329, + "Operations": 23288658, + "ApiCallsPerSecond": 23087812.501487065, + "P50Microseconds": 0.5, + "P95Microseconds": 2.5, + "P99Microseconds": 5.4, + "MaxMicroseconds": 32148.3, + "EarlyP99Microseconds": 6.7, + "LateP99Microseconds": 1.2, + "LateToEarlyP99Ratio": 0.1791044776119403, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 181950, + "EarlyP99Microseconds": 6.7, + "LateP99Microseconds": 1.2, + "LateToEarlyP99Ratio": 0.1791044776119403 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 11644329, + "BytesPerSecond": 11543906.250743533, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0086992, + "WallSeconds": 1.1780537, + "SampleCount": 181950, + "FairnessIndex": 0.9429126417545635, + "MinWorkerApiCallsPerSecond": 1109150.8293331885, + "MaxWorkerApiCallsPerSecond": 2436227.6211061464, + "WorstWorkerP99Microseconds": 7, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 11644329, + "Release.Success": 11644329 + }, + "WorkerCycles": [ + 1101595, + 1063631, + 1212865, + 1218140, + 1156802, + 1029443, + 1188574, + 1091967, + 554590, + 689761, + 717206, + 619755 + ] + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 1, + "ReaderProcessCount": 0, + "PublisherProcessCount": 1, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 865695, + "Operations": 1731390, + "ApiCallsPerSecond": 1731354.853496474, + "P50Microseconds": 0.7, + "P95Microseconds": 3.9, + "P99Microseconds": 5, + "MaxMicroseconds": 271.5, + "EarlyP99Microseconds": 8, + "LateP99Microseconds": 1.1, + "LateToEarlyP99Ratio": 0.1375, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 13527, + "EarlyP99Microseconds": 8, + "LateP99Microseconds": 1.1, + "LateToEarlyP99Ratio": 0.1375 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 865695, + "BytesRead": 0, + "BytesPerSecond": 865677.426748237, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000203, + "WallSeconds": 1.0440665, + "SampleCount": 13527, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 1731354.853496474, + "MaxWorkerApiCallsPerSecond": 1731354.853496474, + "WorstWorkerP99Microseconds": 5, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 865695, + "Remove.Success": 865695 + }, + "WorkerCycles": [ + 865695 + ] + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 2, + "ReaderProcessCount": 0, + "PublisherProcessCount": 2, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 1407319, + "Operations": 2814638, + "ApiCallsPerSecond": 2814572.420462603, + "P50Microseconds": 0.9, + "P95Microseconds": 4.5, + "P99Microseconds": 7.8, + "MaxMicroseconds": 245, + "EarlyP99Microseconds": 9, + "LateP99Microseconds": 1.4, + "LateToEarlyP99Ratio": 0.15555555555555556, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 21990, + "EarlyP99Microseconds": 9, + "LateP99Microseconds": 1.4, + "LateToEarlyP99Ratio": 0.15555555555555556 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 1407319, + "BytesRead": 0, + "BytesPerSecond": 1407286.2102313016, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000233, + "WallSeconds": 1.0554422, + "SampleCount": 21990, + "FairnessIndex": 0.9963768462253689, + "MinWorkerApiCallsPerSecond": 1322425.1874931313, + "MaxWorkerApiCallsPerSecond": 1492149.6203532384, + "WorstWorkerP99Microseconds": 8.1, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 1407319, + "Remove.Success": 1407319 + }, + "WorkerCycles": [ + 661228, + 746091 + ] + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 4, + "ReaderProcessCount": 0, + "PublisherProcessCount": 4, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 2387259, + "Operations": 4774518, + "ApiCallsPerSecond": 4774392.910905735, + "P50Microseconds": 1.1, + "P95Microseconds": 5.3, + "P99Microseconds": 7.6, + "MaxMicroseconds": 24.8, + "EarlyP99Microseconds": 11.8, + "LateP99Microseconds": 1.5, + "LateToEarlyP99Ratio": 0.1271186440677966, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 37303, + "EarlyP99Microseconds": 11.8, + "LateP99Microseconds": 1.5, + "LateToEarlyP99Ratio": 0.1271186440677966 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 2387259, + "BytesRead": 0, + "BytesPerSecond": 2387196.4554528673, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000262, + "WallSeconds": 1.0662507, + "SampleCount": 37303, + "FairnessIndex": 0.9932850533597223, + "MinWorkerApiCallsPerSecond": 1095323.874457736, + "MaxWorkerApiCallsPerSecond": 1302527.8737697073, + "WorstWorkerP99Microseconds": 8.4, + "AffinityAppliedCount": 4, + "AssignedProcessors": [ + 0, + 2, + 4, + 6 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 2387259, + "Remove.Success": 2387259 + }, + "WorkerCycles": [ + 640179, + 651281, + 547673, + 548126 + ] + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 8, + "ReaderProcessCount": 0, + "PublisherProcessCount": 8, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 2896886, + "Operations": 5793772, + "ApiCallsPerSecond": 5644743.435145687, + "P50Microseconds": 1.4, + "P95Microseconds": 7.9, + "P99Microseconds": 12.9, + "MaxMicroseconds": 84.2, + "EarlyP99Microseconds": 15, + "LateP99Microseconds": 8.5, + "LateToEarlyP99Ratio": 0.5666666666666667, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 45267, + "EarlyP99Microseconds": 15, + "LateP99Microseconds": 8.5, + "LateToEarlyP99Ratio": 0.5666666666666667 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 2896886, + "BytesRead": 0, + "BytesPerSecond": 2822371.7175728437, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0264013, + "WallSeconds": 1.0973472, + "SampleCount": 45267, + "FairnessIndex": 0.8151495288165751, + "MinWorkerApiCallsPerSecond": 230825.45273858105, + "MaxWorkerApiCallsPerSecond": 1087076.7587413704, + "WorstWorkerP99Microseconds": 17.6, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 2896886, + "Remove.Success": 2896886 + }, + "WorkerCycles": [ + 493701, + 543548, + 197738, + 463032, + 115415, + 470048, + 493494, + 119910 + ] + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 12, + "ReaderProcessCount": 0, + "PublisherProcessCount": 12, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 3048270, + "Operations": 6096540, + "ApiCallsPerSecond": 6096311.388322938, + "P50Microseconds": 2.1, + "P95Microseconds": 11, + "P99Microseconds": 16, + "MaxMicroseconds": 32160.9, + "EarlyP99Microseconds": 19.5, + "LateP99Microseconds": 8.2, + "LateToEarlyP99Ratio": 0.42051282051282046, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 47635, + "EarlyP99Microseconds": 19.5, + "LateP99Microseconds": 8.2, + "LateToEarlyP99Ratio": 0.42051282051282046 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 3048270, + "BytesRead": 0, + "BytesPerSecond": 3048155.694161469, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000375, + "WallSeconds": 1.1457726, + "SampleCount": 47635, + "FairnessIndex": 0.8392807303451494, + "MinWorkerApiCallsPerSecond": 196333.11130552847, + "MaxWorkerApiCallsPerSecond": 850190.4333051435, + "WorstWorkerP99Microseconds": 21.4, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Publish.Success": 3048270, + "Remove.Success": 3048270 + }, + "WorkerCycles": [ + 402536, + 425099, + 143670, + 350021, + 143173, + 239909, + 399870, + 98169, + 139921, + 238763, + 192207, + 274932 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 1837987, + "Operations": 3675974, + "ApiCallsPerSecond": 3675897.541331141, + "P50Microseconds": 0.4, + "P95Microseconds": 2, + "P99Microseconds": 2.4, + "MaxMicroseconds": 55.5, + "EarlyP99Microseconds": 3.8, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.13157894736842105, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 28719, + "EarlyP99Microseconds": 3.8, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.13157894736842105 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 470524672, + "BytesPerSecond": 470514885.290386, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000208, + "WallSeconds": 1.0445344, + "SampleCount": 28719, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 3675897.541331141, + "MaxWorkerApiCallsPerSecond": 3675897.541331141, + "WorstWorkerP99Microseconds": 2.4, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 1837987, + "Release.Success": 1837987 + }, + "WorkerCycles": [ + 1837987 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 2, + "ReaderProcessCount": 2, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 2917639, + "Operations": 5835278, + "ApiCallsPerSecond": 5835126.870214061, + "P50Microseconds": 0.4, + "P95Microseconds": 2, + "P99Microseconds": 2.6, + "MaxMicroseconds": 2155.7, + "EarlyP99Microseconds": 4.2, + "LateP99Microseconds": 1.2, + "LateToEarlyP99Ratio": 0.2857142857142857, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 45589, + "EarlyP99Microseconds": 4.2, + "LateP99Microseconds": 1.2, + "LateToEarlyP99Ratio": 0.2857142857142857 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 746915584, + "BytesPerSecond": 746896239.3873998, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000259, + "WallSeconds": 1.0517342, + "SampleCount": 45589, + "FairnessIndex": 0.9857203793763621, + "MinWorkerApiCallsPerSecond": 2566420.636531725, + "MaxWorkerApiCallsPerSecond": 3268739.339651103, + "WorstWorkerP99Microseconds": 2.7, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 2917639, + "Release.Success": 2917639 + }, + "WorkerCycles": [ + 1283227, + 1634412 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 4, + "ReaderProcessCount": 4, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 5320264, + "Operations": 10640528, + "ApiCallsPerSecond": 10640324.769796897, + "P50Microseconds": 0.4, + "P95Microseconds": 2.1, + "P99Microseconds": 2.7, + "MaxMicroseconds": 30.4, + "EarlyP99Microseconds": 4.4, + "LateP99Microseconds": 1.2, + "LateToEarlyP99Ratio": 0.2727272727272727, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 83131, + "EarlyP99Microseconds": 4.4, + "LateP99Microseconds": 1.2, + "LateToEarlyP99Ratio": 0.2727272727272727 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 1361987584, + "BytesPerSecond": 1361961570.5340028, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000191, + "WallSeconds": 1.0709846, + "SampleCount": 83131, + "FairnessIndex": 0.9956347219834573, + "MinWorkerApiCallsPerSecond": 2508613.848396268, + "MaxWorkerApiCallsPerSecond": 2957685.508206793, + "WorstWorkerP99Microseconds": 2.8, + "AffinityAppliedCount": 4, + "AssignedProcessors": [ + 0, + 2, + 4, + 6 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 5320264, + "Release.Success": 5320264 + }, + "WorkerCycles": [ + 1254329, + 1478871, + 1278060, + 1309004 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 6, + "ReaderProcessCount": 6, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 6647694, + "Operations": 13295388, + "ApiCallsPerSecond": 13295215.162202891, + "P50Microseconds": 0.4, + "P95Microseconds": 2.5, + "P99Microseconds": 3.2, + "MaxMicroseconds": 852.1, + "EarlyP99Microseconds": 5.1, + "LateP99Microseconds": 1.6, + "LateToEarlyP99Ratio": 0.3137254901960785, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 103873, + "EarlyP99Microseconds": 5.1, + "LateP99Microseconds": 1.6, + "LateToEarlyP99Ratio": 0.3137254901960785 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 1701809664, + "BytesPerSecond": 1701787540.76197, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.000013, + "WallSeconds": 1.0867922, + "SampleCount": 103873, + "FairnessIndex": 0.9978543635100015, + "MinWorkerApiCallsPerSecond": 2049480.8833223467, + "MaxWorkerApiCallsPerSecond": 2379737.0634181756, + "WorstWorkerP99Microseconds": 3.2, + "AffinityAppliedCount": 6, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 6647694, + "Release.Success": 6647694 + }, + "WorkerCycles": [ + 1079430, + 1102989, + 1024747, + 1105813, + 1189884, + 1144831 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 7817063, + "Operations": 15634126, + "ApiCallsPerSecond": 15633869.604538485, + "P50Microseconds": 0.5, + "P95Microseconds": 2.6, + "P99Microseconds": 3.4, + "MaxMicroseconds": 46.9, + "EarlyP99Microseconds": 5.5, + "LateP99Microseconds": 1.9, + "LateToEarlyP99Ratio": 0.34545454545454546, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 122144, + "EarlyP99Microseconds": 5.5, + "LateP99Microseconds": 1.9, + "LateToEarlyP99Ratio": 0.34545454545454546 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 2001168128, + "BytesPerSecond": 2001135309.3809261, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000164, + "WallSeconds": 1.1127572, + "SampleCount": 122144, + "FairnessIndex": 0.9972547295142269, + "MinWorkerApiCallsPerSecond": 1799973.1801314175, + "MaxWorkerApiCallsPerSecond": 2123272.6646191655, + "WorstWorkerP99Microseconds": 3.4, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 7817063, + "Release.Success": 7817063 + }, + "WorkerCycles": [ + 1061653, + 998457, + 983801, + 1031661, + 945213, + 977823, + 918464, + 899991 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 8711250, + "Operations": 17422500, + "ApiCallsPerSecond": 17421806.61209684, + "P50Microseconds": 0.5, + "P95Microseconds": 3.6, + "P99Microseconds": 6.2, + "MaxMicroseconds": 32160.6, + "EarlyP99Microseconds": 9.1, + "LateP99Microseconds": 3.2, + "LateToEarlyP99Ratio": 0.3516483516483517, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 136120, + "EarlyP99Microseconds": 9.1, + "LateP99Microseconds": 3.2, + "LateToEarlyP99Ratio": 0.3516483516483517 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 2230080000, + "BytesPerSecond": 2229991246.3483953, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000398, + "WallSeconds": 1.1790948, + "SampleCount": 136120, + "FairnessIndex": 0.9086590271329934, + "MinWorkerApiCallsPerSecond": 765005.6292765177, + "MaxWorkerApiCallsPerSecond": 1928894.6399482407, + "WorstWorkerP99Microseconds": 9.1, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 8711250, + "Release.Success": 8711250 + }, + "WorkerCycles": [ + 963505, + 920933, + 964456, + 795409, + 740112, + 813152, + 961553, + 892291, + 438658, + 420039, + 382518, + 418624 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 1503625, + "Operations": 3007250, + "ApiCallsPerSecond": 3007184.443379134, + "P50Microseconds": 0.4, + "P95Microseconds": 2.1, + "P99Microseconds": 2.7, + "MaxMicroseconds": 138.3, + "EarlyP99Microseconds": 4.5, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.1111111111111111, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 23495, + "EarlyP99Microseconds": 4.5, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.1111111111111111 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 384928000, + "BytesPerSecond": 384919608.75252914, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000218, + "WallSeconds": 1.0408996, + "SampleCount": 23495, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 3007184.443379134, + "MaxWorkerApiCallsPerSecond": 3007184.443379134, + "WorstWorkerP99Microseconds": 2.7, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 1503625, + "Release.Success": 1503625 + }, + "WorkerCycles": [ + 1503625 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 2, + "ReaderProcessCount": 2, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 3020281, + "Operations": 6040562, + "ApiCallsPerSecond": 6040428.5065300055, + "P50Microseconds": 0.4, + "P95Microseconds": 1.9, + "P99Microseconds": 2.5, + "MaxMicroseconds": 481.9, + "EarlyP99Microseconds": 4.3, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.11627906976744186, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 47193, + "EarlyP99Microseconds": 4.3, + "LateP99Microseconds": 0.5, + "LateToEarlyP99Ratio": 0.11627906976744186 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 773191936, + "BytesPerSecond": 773174848.8358407, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000221, + "WallSeconds": 1.0540961, + "SampleCount": 47193, + "FairnessIndex": 0.9997397004091367, + "MinWorkerApiCallsPerSecond": 2971486.216099489, + "MaxWorkerApiCallsPerSecond": 3068954.1761127077, + "WorstWorkerP99Microseconds": 2.5, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 3020281, + "Release.Success": 3020281 + }, + "WorkerCycles": [ + 1485770, + 1534511 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 4, + "ReaderProcessCount": 4, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 5004742, + "Operations": 10009484, + "ApiCallsPerSecond": 10009247.78175235, + "P50Microseconds": 0.5, + "P95Microseconds": 2.3, + "P99Microseconds": 3.2, + "MaxMicroseconds": 32113.8, + "EarlyP99Microseconds": 5, + "LateP99Microseconds": 1.3, + "LateToEarlyP99Ratio": 0.26, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 78201, + "EarlyP99Microseconds": 5, + "LateP99Microseconds": 1.3, + "LateToEarlyP99Ratio": 0.26 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 1281213952, + "BytesPerSecond": 1281183716.0643008, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000236, + "WallSeconds": 1.0677962, + "SampleCount": 78201, + "FairnessIndex": 0.9897544272683813, + "MinWorkerApiCallsPerSecond": 2353507.2900982406, + "MaxWorkerApiCallsPerSecond": 2943296.5382016986, + "WorstWorkerP99Microseconds": 3.4, + "AffinityAppliedCount": 4, + "AssignedProcessors": [ + 0, + 2, + 4, + 6 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 5004742, + "Release.Success": 5004742 + }, + "WorkerCycles": [ + 1179483, + 1176762, + 1176814, + 1471683 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 6, + "ReaderProcessCount": 6, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 5352193, + "Operations": 10704386, + "ApiCallsPerSecond": 10704256.478496611, + "P50Microseconds": 0.5, + "P95Microseconds": 2.7, + "P99Microseconds": 3.7, + "MaxMicroseconds": 449, + "EarlyP99Microseconds": 5.5, + "LateP99Microseconds": 1.8, + "LateToEarlyP99Ratio": 0.32727272727272727, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 83631, + "EarlyP99Microseconds": 5.5, + "LateP99Microseconds": 1.8, + "LateToEarlyP99Ratio": 0.32727272727272727 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 1370161408, + "BytesPerSecond": 1370144829.2475662, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000121, + "WallSeconds": 1.1072406, + "SampleCount": 83631, + "FairnessIndex": 0.9975812607971184, + "MinWorkerApiCallsPerSecond": 1707770.3606408525, + "MaxWorkerApiCallsPerSecond": 1963163.276275428, + "WorstWorkerP99Microseconds": 4.3, + "AffinityAppliedCount": 6, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 5352193, + "Release.Success": 5352193 + }, + "WorkerCycles": [ + 853895, + 910649, + 859814, + 873867, + 872379, + 981589 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 8, + "ReaderProcessCount": 8, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 6910388, + "Operations": 13820776, + "ApiCallsPerSecond": 13820600.478373924, + "P50Microseconds": 0.6, + "P95Microseconds": 3.1, + "P99Microseconds": 4.4, + "MaxMicroseconds": 221.8, + "EarlyP99Microseconds": 6.7, + "LateP99Microseconds": 2, + "LateToEarlyP99Ratio": 0.29850746268656714, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 107978, + "EarlyP99Microseconds": 6.7, + "LateP99Microseconds": 2, + "LateToEarlyP99Ratio": 0.29850746268656714 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 1769059328, + "BytesPerSecond": 1769036861.2318623, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000127, + "WallSeconds": 1.1090611, + "SampleCount": 107978, + "FairnessIndex": 0.9982110823492648, + "MinWorkerApiCallsPerSecond": 1611757.6259630641, + "MaxWorkerApiCallsPerSecond": 1837144.0168470535, + "WorstWorkerP99Microseconds": 4.8, + "AffinityAppliedCount": 8, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 6910388, + "Release.Success": 6910388 + }, + "WorkerCycles": [ + 918580, + 847006, + 907881, + 857368, + 849333, + 892674, + 805888, + 831658 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 0, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 8260179, + "Operations": 16520358, + "ApiCallsPerSecond": 16519655.91462363, + "P50Microseconds": 0.6, + "P95Microseconds": 3.6, + "P99Microseconds": 6.4, + "MaxMicroseconds": 443.9, + "EarlyP99Microseconds": 8.7, + "LateP99Microseconds": 3.1, + "LateToEarlyP99Ratio": 0.35632183908045983, + "RoleLatency": [ + { + "Role": "reader", + "SampleCount": 129071, + "EarlyP99Microseconds": 8.7, + "LateP99Microseconds": 3.1, + "LateToEarlyP99Ratio": 0.35632183908045983 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 0, + "BytesRead": 2114605824, + "BytesPerSecond": 2114515957.0718246, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0000425, + "WallSeconds": 1.1951414, + "SampleCount": 129071, + "FairnessIndex": 0.9425906438594458, + "MinWorkerApiCallsPerSecond": 802536.4856809695, + "MaxWorkerApiCallsPerSecond": 1747030.396919729, + "WorstWorkerP99Microseconds": 8.8, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 8260179, + "Release.Success": 8260179 + }, + "WorkerCycles": [ + 699494, + 871958, + 831572, + 817112, + 772755, + 809743, + 738325, + 873529, + 543951, + 445744, + 454716, + 401280 + ] + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 4499, + "Operations": 35736, + "ApiCallsPerSecond": 35721.8327211428, + "P50Microseconds": 195.3, + "P95Microseconds": 424.1, + "P99Microseconds": 597.6, + "MaxMicroseconds": 597.6, + "EarlyP99Microseconds": 597.6, + "LateP99Microseconds": 355, + "LateToEarlyP99Ratio": 0.5940428380187416, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 71, + "EarlyP99Microseconds": 597.6, + "LateP99Microseconds": 355, + "LateToEarlyP99Ratio": 0.5940428380187416 + } + ], + "Frames": 4499, + "FramesPerSecond": 4497.216403974184, + "BytesWritten": 6132802852, + "BytesRead": 12265605704, + "BytesPerSecond": 18391114639.933804, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0003966, + "WallSeconds": 1.0003966, + "SampleCount": 71, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 4497.216403974184, + "MaxWorkerApiCallsPerSecond": 4497.216403974184, + "WorstWorkerP99Microseconds": 4496.436581504884, + "AffinityAppliedCount": 2, + "AssignedProcessors": [ + 0, + 2 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 8998, + "Advance.Success": 4499, + "Commit.Success": 4499, + "Release.Success": 8998, + "Remove.Success": 4243, + "Reserve.Success": 4499 + }, + "WorkerCycles": [ + 4499 + ] + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 1, + "Trial": 1, + "Cycles": 2538, + "Operations": 20048, + "ApiCallsPerSecond": 20044.58440281776, + "P50Microseconds": 282.1, + "P95Microseconds": 756.6, + "P99Microseconds": 1143.8, + "MaxMicroseconds": 1143.8, + "EarlyP99Microseconds": 1143.8, + "LateP99Microseconds": 426.2, + "LateToEarlyP99Ratio": 0.37261759048784754, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 40, + "EarlyP99Microseconds": 1143.8, + "LateP99Microseconds": 426.2, + "LateToEarlyP99Ratio": 0.37261759048784754 + } + ], + "Frames": 2538, + "FramesPerSecond": 2537.5675984812187, + "BytesWritten": 3459669624, + "BytesRead": 6919339248, + "BytesPerSecond": 10377240590.203428, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 1.0001704, + "WallSeconds": 1.0001704, + "SampleCount": 40, + "FairnessIndex": 0.999994411222266, + "MinWorkerApiCallsPerSecond": 210.96405172558596, + "MaxWorkerApiCallsPerSecond": 211.96388135461717, + "WorstWorkerP99Microseconds": 211.69412316144403, + "AffinityAppliedCount": 13, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19, + 20 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 5076, + "Advance.Success": 2538, + "Commit.Success": 2538, + "Release.Success": 5076, + "Remove.Success": 2282, + "Reserve.Success": 2538 + }, + "WorkerCycles": [ + 212, + 212, + 212, + 212, + 212, + 212, + 211, + 211, + 211, + 211, + 211, + 211 + ] + }, + { + "Profile": "LockFree", + "Scenario": "mixed-churn", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 2, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 46496, + "Operations": 100011, + "ApiCallsPerSecond": 2139487.8201658777, + "P50Microseconds": 6, + "P95Microseconds": 22.9, + "P99Microseconds": 33, + "MaxMicroseconds": 38.3, + "EarlyP99Microseconds": 35.6, + "LateP99Microseconds": 28.1, + "LateToEarlyP99Ratio": 0.7893258426966292, + "RoleLatency": [ + { + "Role": "publisher", + "SampleCount": 56, + "EarlyP99Microseconds": 36.9, + "LateP99Microseconds": 36.5, + "LateToEarlyP99Ratio": 0.989159891598916 + }, + { + "Role": "reader", + "SampleCount": 672, + "EarlyP99Microseconds": 34.6, + "LateP99Microseconds": 12.7, + "LateToEarlyP99Ratio": 0.36705202312138724 + } + ], + "Frames": 0, + "FramesPerSecond": 0, + "BytesWritten": 910336, + "BytesRead": 10952192, + "BytesPerSecond": 253769427.08678734, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 0.0467453, + "WallSeconds": 0.124848, + "SampleCount": 728, + "FairnessIndex": 0.9054130573398514, + "MinWorkerApiCallsPerSecond": 152828.19877078553, + "MaxWorkerApiCallsPerSecond": 415883.198761199, + "WorstWorkerP99Microseconds": 38.3, + "AffinityAppliedCount": 14, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.NotFound": 158, + "Acquire.Success": 42782, + "Advance.Success": 3556, + "Commit.Success": 3556, + "RecoverLeases.Success": 2, + "RecoverReservations.Success": 2, + "Release.Success": 42782, + "Remove.RemovePending": 36, + "Remove.Success": 3520, + "Reserve.DuplicateKey": 61, + "Reserve.Success": 3556 + }, + "WorkerCycles": [ + 3580, + 3578, + 3580, + 3578, + 3578, + 3576, + 3578, + 3578, + 3580, + 3577, + 3579, + 3578, + 1775, + 1781 + ] + }, + { + "Profile": "LockFree", + "Scenario": "large-ingest", + "ProcessCount": 1, + "ReaderProcessCount": 1, + "PublisherProcessCount": 1, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 100, + "Operations": 500, + "ApiCallsPerSecond": 10784.694361761787, + "P50Microseconds": 436.2, + "P95Microseconds": 578.4, + "P99Microseconds": 578.4, + "MaxMicroseconds": 578.4, + "EarlyP99Microseconds": 578.4, + "LateP99Microseconds": 436.2, + "LateToEarlyP99Ratio": 0.754149377593361, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 2, + "EarlyP99Microseconds": 578.4, + "LateP99Microseconds": 436.2, + "LateToEarlyP99Ratio": 0.754149377593361 + } + ], + "Frames": 100, + "FramesPerSecond": 2156.9388723523575, + "BytesWritten": 136314800, + "BytesRead": 136314800, + "BytesPerSecond": 5880453819.938743, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 0.046362, + "WallSeconds": 0.046362, + "SampleCount": 2, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 2156.9388723523575, + "MaxWorkerApiCallsPerSecond": 2156.9388723523575, + "WorstWorkerP99Microseconds": 2150.4636399607757, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 100, + "Advance.Success": 100, + "Commit.Success": 100, + "Release.Success": 100, + "Reserve.Success": 100 + }, + "WorkerCycles": [ + 100 + ] + }, + { + "Profile": "LockFree", + "Scenario": "large-ingest", + "ProcessCount": 12, + "ReaderProcessCount": 12, + "PublisherProcessCount": 1, + "ObserverProcessCount": 0, + "Trial": 1, + "Cycles": 100, + "Operations": 500, + "ApiCallsPerSecond": 6935.793968078702, + "P50Microseconds": 599.6, + "P95Microseconds": 639.9, + "P99Microseconds": 639.9, + "MaxMicroseconds": 639.9, + "EarlyP99Microseconds": 599.6, + "LateP99Microseconds": 639.9, + "LateToEarlyP99Ratio": 1.0672114743162107, + "RoleLatency": [ + { + "Role": "broker-end-to-end", + "SampleCount": 2, + "EarlyP99Microseconds": 599.6, + "LateP99Microseconds": 639.9, + "LateToEarlyP99Ratio": 1.0672114743162107 + } + ], + "Frames": 100, + "FramesPerSecond": 1387.1587936157405, + "BytesWritten": 136314800, + "BytesRead": 136314800, + "BytesPerSecond": 3781805470.399419, + "FullPayloadCopies": 0, + "Failures": 0, + "MeasuredSeconds": 0.0720898, + "WallSeconds": 0.0720898, + "SampleCount": 2, + "FairnessIndex": 0.9968102073365224, + "MinWorkerApiCallsPerSecond": 110.97270348925923, + "MaxWorkerApiCallsPerSecond": 124.84429142541664, + "WorstWorkerP99Microseconds": 123.9114721371287, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 17, + 18, + 19 + ], + "AffinityStrategies": [ + "windows-physical-core-first" + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 100, + "Advance.Success": 100, + "Commit.Success": 100, + "Release.Success": 100, + "Reserve.Success": 100 + }, + "WorkerCycles": [ + 9, + 9, + 9, + 9, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8 + ] + } + ], + "Summary": [ + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 404764.9332654949, + "MedianP50Microseconds": 3.6, + "MedianP95Microseconds": 7.5, + "MedianP99Microseconds": 8.4, + "MedianMaxMicroseconds": 284.4, + "MedianEarlyP99Microseconds": 10.4, + "MedianLateP99Microseconds": 5.8, + "MedianLateToEarlyP99Ratio": 0.5576923076923077, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 202382.46663274744, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 8.4, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 202387, + "Release.Success": 202387 + } + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 2, + "MedianApiCallsPerSecond": 51771.99284775359, + "MedianP50Microseconds": 68.2, + "MedianP95Microseconds": 83.8, + "MedianP99Microseconds": 215.2, + "MedianMaxMicroseconds": 6069.8, + "MedianEarlyP99Microseconds": 151.4, + "MedianLateP99Microseconds": 215.3, + "MedianLateToEarlyP99Ratio": 1.4220607661822986, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 25885.996423876793, + "MedianFairnessIndex": 0.9999467945779156, + "MedianWorstWorkerP99Microseconds": 215.2, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 25888, + "Release.Success": 25888 + } + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 4, + "MedianApiCallsPerSecond": 51325.51928216667, + "MedianP50Microseconds": 134.7, + "MedianP95Microseconds": 143.6, + "MedianP99Microseconds": 373.5, + "MedianMaxMicroseconds": 1340.3, + "MedianEarlyP99Microseconds": 1331.6, + "MedianLateP99Microseconds": 144.4, + "MedianLateToEarlyP99Ratio": 0.10844097326524484, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 25662.759641083336, + "MedianFairnessIndex": 0.9999999918341387, + "MedianWorstWorkerP99Microseconds": 373.5, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 25665, + "Release.Success": 25665 + } + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 24841.482898021102, + "MedianP50Microseconds": 275.1, + "MedianP95Microseconds": 3207.5, + "MedianP99Microseconds": 7847.7, + "MedianMaxMicroseconds": 9305.9, + "MedianEarlyP99Microseconds": 8996.8, + "MedianLateP99Microseconds": 3446.4, + "MedianLateToEarlyP99Ratio": 0.38306953583496356, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 12420.741449010551, + "MedianFairnessIndex": 0.9999999279756707, + "MedianWorstWorkerP99Microseconds": 9305.9, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 12429, + "Release.Success": 12429 + } + }, + { + "Profile": "Legacy", + "Scenario": "acquire-release", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 24199.320057350713, + "MedianP50Microseconds": 587, + "MedianP95Microseconds": 1904.2, + "MedianP99Microseconds": 10344.6, + "MedianMaxMicroseconds": 10362.9, + "MedianEarlyP99Microseconds": 10362.9, + "MedianLateP99Microseconds": 1904.2, + "MedianLateToEarlyP99Ratio": 0.1837516525296973, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 12099.660028675356, + "MedianFairnessIndex": 0.9999985799942068, + "MedianWorstWorkerP99Microseconds": 10362.9, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 12132, + "Release.Success": 12132 + } + }, + { + "Profile": "Legacy", + "Scenario": "broker-directed", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 26852.734707459575, + "MedianP50Microseconds": 251.1, + "MedianP95Microseconds": 516, + "MedianP99Microseconds": 1079.7, + "MedianMaxMicroseconds": 1079.7, + "MedianEarlyP99Microseconds": 1079.7, + "MedianLateP99Microseconds": 324, + "MedianLateToEarlyP99Ratio": 0.3000833564879133, + "MedianFramesPerSecond": 3388.587947705559, + "MedianBytesPerSecond": 13857440651.216812, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 3386.663540823186, + "TotalFrames": 3389, + "TotalBytesWritten": 4619708572, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 6778, + "Advance.Success": 3389, + "Commit.Success": 3389, + "Release.Success": 6778, + "Remove.Success": 3133, + "Reserve.Success": 3389 + } + }, + { + "Profile": "Legacy", + "Scenario": "broker-directed", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 17318.668194415848, + "MedianP50Microseconds": 415.5, + "MedianP95Microseconds": 977.1, + "MedianP99Microseconds": 1107.2, + "MedianMaxMicroseconds": 1107.2, + "MedianEarlyP99Microseconds": 1107.2, + "MedianLateP99Microseconds": 863.6, + "MedianLateToEarlyP99Ratio": 0.7799855491329479, + "MedianFramesPerSecond": 2196.831063691202, + "MedianBytesPerSecond": 8983817612.425604, + "MedianFairnessIndex": 0.9999977210668722, + "MedianWorstWorkerP99Microseconds": 183.89133860801653, + "TotalFrames": 2197, + "TotalBytesWritten": 2994836156, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 4394, + "Advance.Success": 2197, + "Commit.Success": 2197, + "Release.Success": 4394, + "Remove.Success": 1941, + "Reserve.Success": 2197 + } + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 475561.63275640586, + "MedianP50Microseconds": 3, + "MedianP95Microseconds": 9.9, + "MedianP99Microseconds": 13, + "MedianMaxMicroseconds": 137.3, + "MedianEarlyP99Microseconds": 13.5, + "MedianLateP99Microseconds": 3.5, + "MedianLateToEarlyP99Ratio": 0.25925925925925924, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 60871888.99281995, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 13, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 237786, + "Release.Success": 237786 + } + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 2, + "MedianApiCallsPerSecond": 63301.765447679696, + "MedianP50Microseconds": 54.7, + "MedianP95Microseconds": 59, + "MedianP99Microseconds": 183.8, + "MedianMaxMicroseconds": 1821.4, + "MedianEarlyP99Microseconds": 231.5, + "MedianLateP99Microseconds": 81.4, + "MedianLateToEarlyP99Ratio": 0.3516198704103672, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 8102625.977303001, + "MedianFairnessIndex": 0.9999986791834468, + "MedianWorstWorkerP99Microseconds": 231.5, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 31652, + "Release.Success": 31652 + } + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 4, + "MedianApiCallsPerSecond": 64502.33614675846, + "MedianP50Microseconds": 102.5, + "MedianP95Microseconds": 110, + "MedianP99Microseconds": 220.8, + "MedianMaxMicroseconds": 547.3, + "MedianEarlyP99Microseconds": 234.4, + "MedianLateP99Microseconds": 220.8, + "MedianLateToEarlyP99Ratio": 0.9419795221843004, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 8256299.026785083, + "MedianFairnessIndex": 0.999999948479593, + "MedianWorstWorkerP99Microseconds": 537.9, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 32479, + "Release.Success": 32479 + } + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 6, + "MedianApiCallsPerSecond": 40539.30483728396, + "MedianP50Microseconds": 156.1, + "MedianP95Microseconds": 334.2, + "MedianP99Microseconds": 2897.2, + "MedianMaxMicroseconds": 4773.7, + "MedianEarlyP99Microseconds": 3370.7, + "MedianLateP99Microseconds": 2897.2, + "MedianLateToEarlyP99Ratio": 0.8595247278013469, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 5189031.019172347, + "MedianFairnessIndex": 0.9999999721217228, + "MedianWorstWorkerP99Microseconds": 4773.7, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 20349, + "Release.Success": 20349 + } + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 54507.3919670163, + "MedianP50Microseconds": 217.5, + "MedianP95Microseconds": 805, + "MedianP99Microseconds": 3368.7, + "MedianMaxMicroseconds": 13323.6, + "MedianEarlyP99Microseconds": 13309.8, + "MedianLateP99Microseconds": 1859.3, + "MedianLateToEarlyP99Ratio": 0.13969406001592813, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 6976946.171778087, + "MedianFairnessIndex": 0.999999977063214, + "MedianWorstWorkerP99Microseconds": 13323.6, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 27287, + "Release.Success": 27287 + } + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 30639.473034654457, + "MedianP50Microseconds": 469.5, + "MedianP95Microseconds": 3634.5, + "MedianP99Microseconds": 4612, + "MedianMaxMicroseconds": 5600.9, + "MedianEarlyP99Microseconds": 5600.9, + "MedianLateP99Microseconds": 558.8, + "MedianLateToEarlyP99Ratio": 0.09976967987287758, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 3921852.5484357704, + "MedianFairnessIndex": 0.9999999950032947, + "MedianWorstWorkerP99Microseconds": 5600.9, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 15324, + "Release.Success": 15324 + } + }, + { + "Profile": "Legacy", + "Scenario": "large-ingest", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 11278.254847957845, + "MedianP50Microseconds": 421.7, + "MedianP95Microseconds": 527.6, + "MedianP99Microseconds": 527.6, + "MedianMaxMicroseconds": 527.6, + "MedianEarlyP99Microseconds": 527.6, + "MedianLateP99Microseconds": 421.7, + "MedianLateToEarlyP99Ratio": 0.7992797573919636, + "MedianFramesPerSecond": 2255.6509695915693, + "MedianBytesPerSecond": 6149572215.793617, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 2247.9184275361017, + "TotalFrames": 100, + "TotalBytesWritten": 136314800, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 100, + "Advance.Success": 100, + "Commit.Success": 100, + "Release.Success": 100, + "Reserve.Success": 100 + } + }, + { + "Profile": "Legacy", + "Scenario": "large-ingest", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 6947.117154794275, + "MedianP50Microseconds": 663.6, + "MedianP95Microseconds": 812.7, + "MedianP99Microseconds": 812.7, + "MedianMaxMicroseconds": 812.7, + "MedianEarlyP99Microseconds": 812.7, + "MedianLateP99Microseconds": 663.6, + "MedianLateToEarlyP99Ratio": 0.8165374677002584, + "MedianFramesPerSecond": 1389.4234309588549, + "MedianBytesPerSecond": 3787979542.1294026, + "MedianFairnessIndex": 0.9968102073365229, + "MedianWorstWorkerP99Microseconds": 124.70210053760461, + "TotalFrames": 100, + "TotalBytesWritten": 136314800, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 100, + "Advance.Success": 100, + "Commit.Success": 100, + "Release.Success": 100, + "Reserve.Success": 100 + } + }, + { + "Profile": "Legacy", + "Scenario": "mixed-churn", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 14159.661202249388, + "MedianP50Microseconds": 899.7, + "MedianP95Microseconds": 3018.3, + "MedianP99Microseconds": 22042.5, + "MedianMaxMicroseconds": 37396.9, + "MedianEarlyP99Microseconds": 24246.6, + "MedianLateP99Microseconds": 7061.4, + "MedianLateToEarlyP99Ratio": 0.2912325851872015, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 1681120.6069990918, + "MedianFairnessIndex": 0.8104599529594054, + "MedianWorstWorkerP99Microseconds": 37396.9, + "TotalFrames": 0, + "TotalBytesWritten": 904448, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.NotFound": 23, + "Acquire.Success": 42849, + "Advance.Success": 3533, + "Commit.Success": 3533, + "RecoverLeases.Success": 2, + "RecoverReservations.Success": 2, + "Release.Success": 42849, + "Remove.RemovePending": 65, + "Remove.Success": 3468, + "Reserve.DuplicateKey": 153, + "Reserve.Success": 3533 + } + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 1689885.866115855, + "MedianP50Microseconds": 0.9, + "MedianP95Microseconds": 2, + "MedianP99Microseconds": 3, + "MedianMaxMicroseconds": 221.9, + "MedianEarlyP99Microseconds": 3.2, + "MedianLateP99Microseconds": 1.3, + "MedianLateToEarlyP99Ratio": 0.40625, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 844942.9330579275, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 3, + "TotalFrames": 0, + "TotalBytesWritten": 844966, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 844966, + "Remove.Success": 844966 + } + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 2, + "MedianApiCallsPerSecond": 191871.82973772383, + "MedianP50Microseconds": 18.5, + "MedianP95Microseconds": 19.3, + "MedianP99Microseconds": 40.3, + "MedianMaxMicroseconds": 3920, + "MedianEarlyP99Microseconds": 37.4, + "MedianLateP99Microseconds": 42.8, + "MedianLateToEarlyP99Ratio": 1.1443850267379678, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 95935.91486886192, + "MedianFairnessIndex": 0.999991677588926, + "MedianWorstWorkerP99Microseconds": 42.8, + "TotalFrames": 0, + "TotalBytesWritten": 95945, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 95945, + "Remove.Success": 95945 + } + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 4, + "MedianApiCallsPerSecond": 200017.73926736828, + "MedianP50Microseconds": 35.3, + "MedianP95Microseconds": 40, + "MedianP99Microseconds": 102.6, + "MedianMaxMicroseconds": 1852.8, + "MedianEarlyP99Microseconds": 165, + "MedianLateP99Microseconds": 69.2, + "MedianLateToEarlyP99Ratio": 0.41939393939393943, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 100008.86963368414, + "MedianFairnessIndex": 0.9999995726519106, + "MedianWorstWorkerP99Microseconds": 338.2, + "TotalFrames": 0, + "TotalBytesWritten": 100013, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 100013, + "Remove.Success": 100013 + } + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 110515.55694303021, + "MedianP50Microseconds": 70.2, + "MedianP95Microseconds": 155.4, + "MedianP99Microseconds": 3354.1, + "MedianMaxMicroseconds": 11263.6, + "MedianEarlyP99Microseconds": 6441.7, + "MedianLateP99Microseconds": 3299.4, + "MedianLateToEarlyP99Ratio": 0.5121939860595806, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 55257.77847151511, + "MedianFairnessIndex": 0.9999996937635796, + "MedianWorstWorkerP99Microseconds": 3617.6, + "TotalFrames": 0, + "TotalBytesWritten": 55261, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 55261, + "Remove.Success": 55261 + } + }, + { + "Profile": "Legacy", + "Scenario": "publish-remove", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 114364.86952776148, + "MedianP50Microseconds": 140.6, + "MedianP95Microseconds": 215.6, + "MedianP99Microseconds": 1745, + "MedianMaxMicroseconds": 6647, + "MedianEarlyP99Microseconds": 481.8, + "MedianLateP99Microseconds": 2161.8, + "MedianLateToEarlyP99Ratio": 4.486924034869241, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 57182.43476388074, + "MedianFairnessIndex": 0.9999991196368768, + "MedianWorstWorkerP99Microseconds": 6647, + "TotalFrames": 0, + "TotalBytesWritten": 57190, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 57190, + "Remove.Success": 57190 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 474843.97513682017, + "MedianP50Microseconds": 3, + "MedianP95Microseconds": 9.1, + "MedianP99Microseconds": 11.6, + "MedianMaxMicroseconds": 1504.2, + "MedianEarlyP99Microseconds": 12.4, + "MedianLateP99Microseconds": 5.3, + "MedianLateToEarlyP99Ratio": 0.42741935483870963, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 60780028.81751298, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 11.6, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 237426, + "Release.Success": 237426 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 2, + "MedianApiCallsPerSecond": 70439.6614032414, + "MedianP50Microseconds": 50.9, + "MedianP95Microseconds": 59.6, + "MedianP99Microseconds": 161.7, + "MedianMaxMicroseconds": 5197.7, + "MedianEarlyP99Microseconds": 219.6, + "MedianLateP99Microseconds": 144, + "MedianLateToEarlyP99Ratio": 0.6557377049180328, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 9016276.6596149, + "MedianFairnessIndex": 0.9996868768315358, + "MedianWorstWorkerP99Microseconds": 1799.9, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 35221, + "Release.Success": 35221 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 4, + "MedianApiCallsPerSecond": 71115.05750350351, + "MedianP50Microseconds": 100.6, + "MedianP95Microseconds": 119.3, + "MedianP99Microseconds": 212.9, + "MedianMaxMicroseconds": 1737.3, + "MedianEarlyP99Microseconds": 212.9, + "MedianLateP99Microseconds": 213.7, + "MedianLateToEarlyP99Ratio": 1.0037576326914044, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 9102727.36044845, + "MedianFairnessIndex": 0.9999999622795166, + "MedianWorstWorkerP99Microseconds": 673.8, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 35560, + "Release.Success": 35560 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 6, + "MedianApiCallsPerSecond": 33206.24117019879, + "MedianP50Microseconds": 154.3, + "MedianP95Microseconds": 848.3, + "MedianP99Microseconds": 18621, + "MedianMaxMicroseconds": 20369.9, + "MedianEarlyP99Microseconds": 848.3, + "MedianLateP99Microseconds": 18658.3, + "MedianLateToEarlyP99Ratio": 21.994931038547683, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 4250398.869785445, + "MedianFairnessIndex": 0.9999927569725131, + "MedianWorstWorkerP99Microseconds": 20369.9, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 16902, + "Release.Success": 16902 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 36288.27928226648, + "MedianP50Microseconds": 208.5, + "MedianP95Microseconds": 1507.3, + "MedianP99Microseconds": 34203.2, + "MedianMaxMicroseconds": 40797.2, + "MedianEarlyP99Microseconds": 1865.1, + "MedianLateP99Microseconds": 40797.2, + "MedianLateToEarlyP99Ratio": 21.87400139402713, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 4644899.748130109, + "MedianFairnessIndex": 0.9999824267217524, + "MedianWorstWorkerP99Microseconds": 40797.2, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 18397, + "Release.Success": 18397 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 42366.379209084276, + "MedianP50Microseconds": 417, + "MedianP95Microseconds": 796.1, + "MedianP99Microseconds": 5855.7, + "MedianMaxMicroseconds": 7725.7, + "MedianEarlyP99Microseconds": 5924.6, + "MedianLateP99Microseconds": 924.6, + "MedianLateToEarlyP99Ratio": 0.15606116868649358, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 5422896.538762787, + "MedianFairnessIndex": 0.999998509231126, + "MedianWorstWorkerP99Microseconds": 7725.7, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 21190, + "Release.Success": 21190 + } + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 4738841.510787336, + "MedianP50Microseconds": 0.3, + "MedianP95Microseconds": 1.4, + "MedianP99Microseconds": 1.8, + "MedianMaxMicroseconds": 61.5, + "MedianEarlyP99Microseconds": 3.4, + "MedianLateP99Microseconds": 0.3, + "MedianLateToEarlyP99Ratio": 0.08823529411764706, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 2369420.755393668, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 1.8, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 2372433, + "Release.Success": 2372433 + } + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 2, + "MedianApiCallsPerSecond": 6978482.056180409, + "MedianP50Microseconds": 0.4, + "MedianP95Microseconds": 1.4, + "MedianP99Microseconds": 1.8, + "MedianMaxMicroseconds": 1240.2, + "MedianEarlyP99Microseconds": 3.4, + "MedianLateP99Microseconds": 0.7, + "MedianLateToEarlyP99Ratio": 0.20588235294117646, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 3489241.0280902046, + "MedianFairnessIndex": 0.9940148924329332, + "MedianWorstWorkerP99Microseconds": 1.8, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 3489315, + "Release.Success": 3489315 + } + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 4, + "MedianApiCallsPerSecond": 15099997.920043264, + "MedianP50Microseconds": 0.3, + "MedianP95Microseconds": 1.6, + "MedianP99Microseconds": 2, + "MedianMaxMicroseconds": 32109.3, + "MedianEarlyP99Microseconds": 3.8, + "MedianLateP99Microseconds": 0.5, + "MedianLateToEarlyP99Ratio": 0.13157894736842105, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 7549998.960021632, + "MedianFairnessIndex": 0.9982544615137126, + "MedianWorstWorkerP99Microseconds": 2.1, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 7550156, + "Release.Success": 7550156 + } + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 19550009.499715008, + "MedianP50Microseconds": 0.4, + "MedianP95Microseconds": 2.3, + "MedianP99Microseconds": 3.8, + "MedianMaxMicroseconds": 9821.3, + "MedianEarlyP99Microseconds": 5.5, + "MedianLateP99Microseconds": 0.9, + "MedianLateToEarlyP99Ratio": 0.16363636363636364, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 9775004.749857504, + "MedianFairnessIndex": 0.9960589678415701, + "MedianWorstWorkerP99Microseconds": 4.5, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 9775298, + "Release.Success": 9775298 + } + }, + { + "Profile": "LockFree", + "Scenario": "acquire-release", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 23087812.501487065, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 2.5, + "MedianP99Microseconds": 5.4, + "MedianMaxMicroseconds": 32148.3, + "MedianEarlyP99Microseconds": 6.7, + "MedianLateP99Microseconds": 1.2, + "MedianLateToEarlyP99Ratio": 0.1791044776119403, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 11543906.250743533, + "MedianFairnessIndex": 0.9429126417545635, + "MedianWorstWorkerP99Microseconds": 7, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 11644329, + "Release.Success": 11644329 + } + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 35721.8327211428, + "MedianP50Microseconds": 195.3, + "MedianP95Microseconds": 424.1, + "MedianP99Microseconds": 597.6, + "MedianMaxMicroseconds": 597.6, + "MedianEarlyP99Microseconds": 597.6, + "MedianLateP99Microseconds": 355, + "MedianLateToEarlyP99Ratio": 0.5940428380187416, + "MedianFramesPerSecond": 4497.216403974184, + "MedianBytesPerSecond": 18391114639.933804, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 4496.436581504884, + "TotalFrames": 4499, + "TotalBytesWritten": 6132802852, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 8998, + "Advance.Success": 4499, + "Commit.Success": 4499, + "Release.Success": 8998, + "Remove.Success": 4243, + "Reserve.Success": 4499 + } + }, + { + "Profile": "LockFree", + "Scenario": "broker-directed", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 20044.58440281776, + "MedianP50Microseconds": 282.1, + "MedianP95Microseconds": 756.6, + "MedianP99Microseconds": 1143.8, + "MedianMaxMicroseconds": 1143.8, + "MedianEarlyP99Microseconds": 1143.8, + "MedianLateP99Microseconds": 426.2, + "MedianLateToEarlyP99Ratio": 0.37261759048784754, + "MedianFramesPerSecond": 2537.5675984812187, + "MedianBytesPerSecond": 10377240590.203428, + "MedianFairnessIndex": 0.999994411222266, + "MedianWorstWorkerP99Microseconds": 211.69412316144403, + "TotalFrames": 2538, + "TotalBytesWritten": 3459669624, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 5076, + "Advance.Success": 2538, + "Commit.Success": 2538, + "Release.Success": 5076, + "Remove.Success": 2282, + "Reserve.Success": 2538 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 3007184.443379134, + "MedianP50Microseconds": 0.4, + "MedianP95Microseconds": 2.1, + "MedianP99Microseconds": 2.7, + "MedianMaxMicroseconds": 138.3, + "MedianEarlyP99Microseconds": 4.5, + "MedianLateP99Microseconds": 0.5, + "MedianLateToEarlyP99Ratio": 0.1111111111111111, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 384919608.75252914, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 2.7, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 1503625, + "Release.Success": 1503625 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 2, + "MedianApiCallsPerSecond": 6040428.5065300055, + "MedianP50Microseconds": 0.4, + "MedianP95Microseconds": 1.9, + "MedianP99Microseconds": 2.5, + "MedianMaxMicroseconds": 481.9, + "MedianEarlyP99Microseconds": 4.3, + "MedianLateP99Microseconds": 0.5, + "MedianLateToEarlyP99Ratio": 0.11627906976744186, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 773174848.8358407, + "MedianFairnessIndex": 0.9997397004091367, + "MedianWorstWorkerP99Microseconds": 2.5, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 3020281, + "Release.Success": 3020281 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 4, + "MedianApiCallsPerSecond": 10009247.78175235, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 2.3, + "MedianP99Microseconds": 3.2, + "MedianMaxMicroseconds": 32113.8, + "MedianEarlyP99Microseconds": 5, + "MedianLateP99Microseconds": 1.3, + "MedianLateToEarlyP99Ratio": 0.26, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 1281183716.0643008, + "MedianFairnessIndex": 0.9897544272683813, + "MedianWorstWorkerP99Microseconds": 3.4, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 5004742, + "Release.Success": 5004742 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 6, + "MedianApiCallsPerSecond": 10704256.478496611, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 2.7, + "MedianP99Microseconds": 3.7, + "MedianMaxMicroseconds": 449, + "MedianEarlyP99Microseconds": 5.5, + "MedianLateP99Microseconds": 1.8, + "MedianLateToEarlyP99Ratio": 0.32727272727272727, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 1370144829.2475662, + "MedianFairnessIndex": 0.9975812607971184, + "MedianWorstWorkerP99Microseconds": 4.3, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 5352193, + "Release.Success": 5352193 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 13820600.478373924, + "MedianP50Microseconds": 0.6, + "MedianP95Microseconds": 3.1, + "MedianP99Microseconds": 4.4, + "MedianMaxMicroseconds": 221.8, + "MedianEarlyP99Microseconds": 6.7, + "MedianLateP99Microseconds": 2, + "MedianLateToEarlyP99Ratio": 0.29850746268656714, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 1769036861.2318623, + "MedianFairnessIndex": 0.9982110823492648, + "MedianWorstWorkerP99Microseconds": 4.8, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 6910388, + "Release.Success": 6910388 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 16519655.91462363, + "MedianP50Microseconds": 0.6, + "MedianP95Microseconds": 3.6, + "MedianP99Microseconds": 6.4, + "MedianMaxMicroseconds": 443.9, + "MedianEarlyP99Microseconds": 8.7, + "MedianLateP99Microseconds": 3.1, + "MedianLateToEarlyP99Ratio": 0.35632183908045983, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 2114515957.0718246, + "MedianFairnessIndex": 0.9425906438594458, + "MedianWorstWorkerP99Microseconds": 8.8, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 8260179, + "Release.Success": 8260179 + } + }, + { + "Profile": "LockFree", + "Scenario": "large-ingest", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 10784.694361761787, + "MedianP50Microseconds": 436.2, + "MedianP95Microseconds": 578.4, + "MedianP99Microseconds": 578.4, + "MedianMaxMicroseconds": 578.4, + "MedianEarlyP99Microseconds": 578.4, + "MedianLateP99Microseconds": 436.2, + "MedianLateToEarlyP99Ratio": 0.754149377593361, + "MedianFramesPerSecond": 2156.9388723523575, + "MedianBytesPerSecond": 5880453819.938743, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 2150.4636399607757, + "TotalFrames": 100, + "TotalBytesWritten": 136314800, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 100, + "Advance.Success": 100, + "Commit.Success": 100, + "Release.Success": 100, + "Reserve.Success": 100 + } + }, + { + "Profile": "LockFree", + "Scenario": "large-ingest", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 6935.793968078702, + "MedianP50Microseconds": 599.6, + "MedianP95Microseconds": 639.9, + "MedianP99Microseconds": 639.9, + "MedianMaxMicroseconds": 639.9, + "MedianEarlyP99Microseconds": 599.6, + "MedianLateP99Microseconds": 639.9, + "MedianLateToEarlyP99Ratio": 1.0672114743162107, + "MedianFramesPerSecond": 1387.1587936157405, + "MedianBytesPerSecond": 3781805470.399419, + "MedianFairnessIndex": 0.9968102073365224, + "MedianWorstWorkerP99Microseconds": 123.9114721371287, + "TotalFrames": 100, + "TotalBytesWritten": 136314800, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 100, + "Advance.Success": 100, + "Commit.Success": 100, + "Release.Success": 100, + "Reserve.Success": 100 + } + }, + { + "Profile": "LockFree", + "Scenario": "mixed-churn", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 2139487.8201658777, + "MedianP50Microseconds": 6, + "MedianP95Microseconds": 22.9, + "MedianP99Microseconds": 33, + "MedianMaxMicroseconds": 38.3, + "MedianEarlyP99Microseconds": 35.6, + "MedianLateP99Microseconds": 28.1, + "MedianLateToEarlyP99Ratio": 0.7893258426966292, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 253769427.08678734, + "MedianFairnessIndex": 0.9054130573398514, + "MedianWorstWorkerP99Microseconds": 38.3, + "TotalFrames": 0, + "TotalBytesWritten": 910336, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.NotFound": 158, + "Acquire.Success": 42782, + "Advance.Success": 3556, + "Commit.Success": 3556, + "RecoverLeases.Success": 2, + "RecoverReservations.Success": 2, + "Release.Success": 42782, + "Remove.RemovePending": 36, + "Remove.Success": 3520, + "Reserve.DuplicateKey": 61, + "Reserve.Success": 3556 + } + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 1731354.853496474, + "MedianP50Microseconds": 0.7, + "MedianP95Microseconds": 3.9, + "MedianP99Microseconds": 5, + "MedianMaxMicroseconds": 271.5, + "MedianEarlyP99Microseconds": 8, + "MedianLateP99Microseconds": 1.1, + "MedianLateToEarlyP99Ratio": 0.1375, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 865677.426748237, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 5, + "TotalFrames": 0, + "TotalBytesWritten": 865695, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 865695, + "Remove.Success": 865695 + } + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 2, + "MedianApiCallsPerSecond": 2814572.420462603, + "MedianP50Microseconds": 0.9, + "MedianP95Microseconds": 4.5, + "MedianP99Microseconds": 7.8, + "MedianMaxMicroseconds": 245, + "MedianEarlyP99Microseconds": 9, + "MedianLateP99Microseconds": 1.4, + "MedianLateToEarlyP99Ratio": 0.15555555555555556, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 1407286.2102313016, + "MedianFairnessIndex": 0.9963768462253689, + "MedianWorstWorkerP99Microseconds": 8.1, + "TotalFrames": 0, + "TotalBytesWritten": 1407319, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 1407319, + "Remove.Success": 1407319 + } + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 4, + "MedianApiCallsPerSecond": 4774392.910905735, + "MedianP50Microseconds": 1.1, + "MedianP95Microseconds": 5.3, + "MedianP99Microseconds": 7.6, + "MedianMaxMicroseconds": 24.8, + "MedianEarlyP99Microseconds": 11.8, + "MedianLateP99Microseconds": 1.5, + "MedianLateToEarlyP99Ratio": 0.1271186440677966, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 2387196.4554528673, + "MedianFairnessIndex": 0.9932850533597223, + "MedianWorstWorkerP99Microseconds": 8.4, + "TotalFrames": 0, + "TotalBytesWritten": 2387259, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 2387259, + "Remove.Success": 2387259 + } + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 5644743.435145687, + "MedianP50Microseconds": 1.4, + "MedianP95Microseconds": 7.9, + "MedianP99Microseconds": 12.9, + "MedianMaxMicroseconds": 84.2, + "MedianEarlyP99Microseconds": 15, + "MedianLateP99Microseconds": 8.5, + "MedianLateToEarlyP99Ratio": 0.5666666666666667, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 2822371.7175728437, + "MedianFairnessIndex": 0.8151495288165751, + "MedianWorstWorkerP99Microseconds": 17.6, + "TotalFrames": 0, + "TotalBytesWritten": 2896886, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 2896886, + "Remove.Success": 2896886 + } + }, + { + "Profile": "LockFree", + "Scenario": "publish-remove", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 6096311.388322938, + "MedianP50Microseconds": 2.1, + "MedianP95Microseconds": 11, + "MedianP99Microseconds": 16, + "MedianMaxMicroseconds": 32160.9, + "MedianEarlyP99Microseconds": 19.5, + "MedianLateP99Microseconds": 8.2, + "MedianLateToEarlyP99Ratio": 0.42051282051282046, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 3048155.694161469, + "MedianFairnessIndex": 0.8392807303451494, + "MedianWorstWorkerP99Microseconds": 21.4, + "TotalFrames": 0, + "TotalBytesWritten": 3048270, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Publish.Success": 3048270, + "Remove.Success": 3048270 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 3675897.541331141, + "MedianP50Microseconds": 0.4, + "MedianP95Microseconds": 2, + "MedianP99Microseconds": 2.4, + "MedianMaxMicroseconds": 55.5, + "MedianEarlyP99Microseconds": 3.8, + "MedianLateP99Microseconds": 0.5, + "MedianLateToEarlyP99Ratio": 0.13157894736842105, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 470514885.290386, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 2.4, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 1837987, + "Release.Success": 1837987 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 2, + "MedianApiCallsPerSecond": 5835126.870214061, + "MedianP50Microseconds": 0.4, + "MedianP95Microseconds": 2, + "MedianP99Microseconds": 2.6, + "MedianMaxMicroseconds": 2155.7, + "MedianEarlyP99Microseconds": 4.2, + "MedianLateP99Microseconds": 1.2, + "MedianLateToEarlyP99Ratio": 0.2857142857142857, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 746896239.3873998, + "MedianFairnessIndex": 0.9857203793763621, + "MedianWorstWorkerP99Microseconds": 2.7, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 2917639, + "Release.Success": 2917639 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 4, + "MedianApiCallsPerSecond": 10640324.769796897, + "MedianP50Microseconds": 0.4, + "MedianP95Microseconds": 2.1, + "MedianP99Microseconds": 2.7, + "MedianMaxMicroseconds": 30.4, + "MedianEarlyP99Microseconds": 4.4, + "MedianLateP99Microseconds": 1.2, + "MedianLateToEarlyP99Ratio": 0.2727272727272727, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 1361961570.5340028, + "MedianFairnessIndex": 0.9956347219834573, + "MedianWorstWorkerP99Microseconds": 2.8, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 5320264, + "Release.Success": 5320264 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 6, + "MedianApiCallsPerSecond": 13295215.162202891, + "MedianP50Microseconds": 0.4, + "MedianP95Microseconds": 2.5, + "MedianP99Microseconds": 3.2, + "MedianMaxMicroseconds": 852.1, + "MedianEarlyP99Microseconds": 5.1, + "MedianLateP99Microseconds": 1.6, + "MedianLateToEarlyP99Ratio": 0.3137254901960785, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 1701787540.76197, + "MedianFairnessIndex": 0.9978543635100015, + "MedianWorstWorkerP99Microseconds": 3.2, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 6647694, + "Release.Success": 6647694 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 8, + "MedianApiCallsPerSecond": 15633869.604538485, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 2.6, + "MedianP99Microseconds": 3.4, + "MedianMaxMicroseconds": 46.9, + "MedianEarlyP99Microseconds": 5.5, + "MedianLateP99Microseconds": 1.9, + "MedianLateToEarlyP99Ratio": 0.34545454545454546, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 2001135309.3809261, + "MedianFairnessIndex": 0.9972547295142269, + "MedianWorstWorkerP99Microseconds": 3.4, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 7817063, + "Release.Success": 7817063 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 17421806.61209684, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 3.6, + "MedianP99Microseconds": 6.2, + "MedianMaxMicroseconds": 32160.6, + "MedianEarlyP99Microseconds": 9.1, + "MedianLateP99Microseconds": 3.2, + "MedianLateToEarlyP99Ratio": 0.3516483516483517, + "MedianFramesPerSecond": 0, + "MedianBytesPerSecond": 2229991246.3483953, + "MedianFairnessIndex": 0.9086590271329934, + "MedianWorstWorkerP99Microseconds": 9.1, + "TotalFrames": 0, + "TotalBytesWritten": 0, + "TotalFullPayloadCopies": 0, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 8711250, + "Release.Success": 8711250 + } + } + ] +} \ No newline at end of file diff --git a/specs/009-lock-free-publish-read/benchmark-results/short-report.md b/specs/009-lock-free-publish-read/benchmark-results/short-report.md new file mode 100644 index 0000000..ed93931 --- /dev/null +++ b/specs/009-lock-free-publish-read/benchmark-results/short-report.md @@ -0,0 +1,100 @@ +# Short Convergence Report + +## Frozen conditional conclusion + +This report is frozen before the one-shot runs and is not edited afterward. +The short convergence conclusion is **PASS if and only if** the final PR +[`summary.json`](../../../artifacts/lock-free-qualification/009-final-r6-pr/summary.json) +has schema 4, tier `pr`, `validationOnly: false`, and `overallStatus: passed`, +and its hashed +[`sync-probe.json`](../../../artifacts/lock-free-qualification/009-final-r6-pr/sync-probe.json) +passes every exact row, count, provenance, correctness, raw-visibility, and +short-performance check enforced by the runner. Its start/completion provenance +must be identical and clean, and must match the final nightly and release +summaries. Otherwise the short conclusion is **FAIL** or **NOT QUALIFIED** as +reported by the raw JSON; this tracked report cannot promote it. + +The same clean commit then has to pass the immutable +[`009-final-r6-nightly`](../../../artifacts/lock-free-qualification/009-final-r6-nightly/summary.json) +and +[`009-final-r6-release`](../../../artifacts/lock-free-qualification/009-final-r6-release/summary.json) +gates before the short result can contribute to release qualification. Raw +JSON remains in the ignored immutable `artifacts/` tree; no copy is maintained +under tracked `specs/` and no tracked post-run edit is permitted. + +## Current diagnostic evidence + +Historical snapshots have passed Windows OS release tests, bounded SC-011 +production races and production-generated histories, Windows native/Python +validation, and focused Linux tests. Their exact counts are snapshot-specific +and are not asserted for the current tree. These are diagnostic results only: +they are not one provenance-matched final sequence and do not establish any +final performance threshold. + +The earlier clean R2 short-tier diagnostic, `009-r2-pre-final4-pr`, passed all 24 +qualification rows on commit `ca200238423877044d841a3b92f93edb37385d46`: +1,014/1,014 tests, the one exact 10,000-cycle churn row, all 50 directory +configurations, and 108/108 suspension rows. Its summary SHA-256 is +`3D543FBF5E4C4A5C514C1C3309565F0B2575126E5B176B79E347802AC46E331A` +and its synchronization-probe SHA-256 is +`9788BF7222BFE8B77E4512EECFEE9788BD1D898763AABBA0417F16D2AF8DE3EC`. +This is a successful convergence diagnostic, not the frozen final PR result. + +The attempted R2 Linux final is preserved with SHA-256 +`090651C119CADD7DAC2C545D04F595FD9E65F5CEFAFC1B9B79EDA009F66EAA7C`. +It was launched on Windows, correctly reported `not-qualified`, and contributes +no final Linux evidence. The corrected Ubuntu invocation passed Linux-x64 +validation-only orchestration with SHA-256 +`AE16F3AED1A9E0FE5113F20AD3AB2F28AE194E5CF0717F6372BF87362A51666C`; +that is an invocation check only, not a benchmark or final result. + +The R3 Linux attempt is also preserved, SHA-256 +`59419F78D559829A0E3B47AE1FAF334B5B79E83CAF8004646A1FE3687C5B86D5`. +It reached Linux x64 but stopped at `dotnet --info` before workloads because a +stale user-local workload manifest set remained after an SDK package upgrade. +After updating that empty-workload set, the executable Linux architecture +preflight passed restore/build and 45/45 tests; its SHA-256 is +`799A41E8181AA299E0E0E925E3F2818935F2F1842EA94FC66C1AC6E1D6EA7F0A`. +Both remain diagnostic only. + +The R4 Linux candidate passed all 28 required rows and the complete 1,014-test +suite on commit `b19d982b76f2f54ebfb9f72e0f2aef85eb2632b8`. Its report and raw +Linux tiny-performance SHA-256 values are +`1346A080DFC7B397437F26088C16C074B9BA1AE3A4E3515D2E07D2DDAA7E2BDE` +and `2716C25672FDB332345059F83FAFDD0063E9CE8C6635B7FE198E9A4053FF6D41`. +The same-source R4 PR summary then failed with 1,013/1,014 tests because the +disposal race used a current-process lease-recovery override without its +required process-wide quiescence. The failed summary SHA-256 is +`7A3754C670A3622C569CB5E6AFFF479C9C668D20975E31641113337213FAD177`; +it emitted no `sync-probe.json`, and R4 nightly/release were not run. The +test-only correction uses normal concurrent recovery for leases and +reservations and avoids dereferencing borrowed spans during a disposal race. +The subsequent clean `009-r5-pre-final-pr` diagnostic passed all 24 gates and +1,014/1,014 tests on commit `527d451bd124dbbe8880fcb909b6e1bd70ad222a`. +R5 then passed Linux, PR, and nightly, but its release probe could not converge: +the harness imposed the 100-million-operation target on every Legacy +mixed-churn trial before reaching the LockFree qualification, and would have +done the same with the 100,000-frame Legacy ingest rows. Read-only liveness +sampling proved slow semaphore-serialized progress, not deadlock. R6 preserves +that attempt, keeps Legacy rows duration-bound, and applies count targets only +to LockFree with schema-v8/minimum-v8 per-run evidence. Its final pre-freeze +worktree passed 1,028/1,028 Release tests with zero skips, 9/9 completion-policy +cases, 5/5 watchdog race cases, both validation-only importers, documentation +validation, and a real four-row Legacy/LockFree duration/count smoke with zero +failures. Independent final review closed H0/M0 with no lower correctness +finding. All earlier results remain +diagnostic; only the R6 final paths linked above determine this conclusion. + +Two still-earlier one-second smoke artifacts are also retained: + +| Artifact | Shape | Failures | SHA-256 | +|---|---|---:|---| +| `short-matrix.json` | schema 3, both profiles, 7 scenarios, 54 run rows | 0 | `1EE8F7108912F0B440A72CCE14F2C5239159C5EF1AEDB3758E14802520C7F8DA` | +| `smoke-readers.json` | schema 2, both profiles, 2 reader scenarios, 12 run rows | 0 | `85F171400F3C642F81F22EFC66C452525A63F1985E49C45F8DB9B34BDFD62971` | + +They identify commit `0cf7a43f9c39de1691b237a9761035339edd0964`, +predate later protocol corrections, and are smoke history only. The preserved +failed PR run at +`artifacts/lock-free-qualification/20260713T173242Z-pr/` has summary SHA-256 +`436C4777B057472A448F0F925C3A16A778834977182ACF88C75E84C6B1F607E1`; +its `CorruptStore` result is the regression origin, not a final verdict. diff --git a/specs/009-lock-free-publish-read/benchmark-results/smoke-readers.json b/specs/009-lock-free-publish-read/benchmark-results/smoke-readers.json new file mode 100644 index 0000000..816eb30 --- /dev/null +++ b/specs/009-lock-free-publish-read/benchmark-results/smoke-readers.json @@ -0,0 +1,772 @@ +{ + "SchemaVersion": 2, + "TimestampUtc": "2026-07-12T23:05:38.0746798+00:00", + "Environment": { + "RepositoryCommit": "0cf7a43f9c39de1691b237a9761035339edd0964", + "OperatingSystem": "Microsoft Windows 10.0.26200", + "OperatingSystemArchitecture": "X64", + "ProcessArchitecture": "X64", + "Framework": ".NET 10.0.5", + "RuntimeVersion": "10.0.5", + "LogicalProcessorCount": 32, + "ProcessorIdentifier": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", + "ServerGarbageCollection": false, + "StopwatchFrequency": 10000000 + }, + "Configuration": { + "Mode": "readers", + "DurationSeconds": 1, + "Trials": 1, + "Profiles": [ + "Legacy", + "LockFree" + ], + "Scenarios": [ + "same-key-read", + "distributed-key-read" + ], + "ProcessCounts": [ + 1, + 6, + 12 + ], + "ReaderKeyCount": 256, + "ReaderPayloadBytes": 256, + "WarmupCycles": 10000, + "SamplingInterval": 64, + "MaxLatencySamplesPerWorker": 65536, + "AffinityRequested": true + }, + "Runs": [ + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 1, + "Trial": 1, + "Cycles": 322965, + "ApiCallsPerSecond": 645917.1462487896, + "P50Microseconds": 2.1, + "P95Microseconds": 7.4, + "P99Microseconds": 10.4, + "MaxMicroseconds": 120.6, + "Failures": 0, + "MeasuredSeconds": 1.0000199, + "WallSeconds": 1.0763231, + "SampleCount": 5047, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 645917.1462487896, + "MaxWorkerApiCallsPerSecond": 645917.1462487896, + "WorstWorkerP99Microseconds": 10.4, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 322965, + "Release.Success": 322965 + }, + "WorkerCycles": [ + 322965 + ] + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 6, + "Trial": 1, + "Cycles": 41255, + "ApiCallsPerSecond": 82461.36428734333, + "P50Microseconds": 113.8, + "P95Microseconds": 244.1, + "P99Microseconds": 1251.9, + "MaxMicroseconds": 2554.3, + "Failures": 0, + "MeasuredSeconds": 1.0005898, + "WallSeconds": 1.0560255, + "SampleCount": 648, + "FairnessIndex": 0.999999864837961, + "MinWorkerApiCallsPerSecond": 13733.96837529413, + "MaxWorkerApiCallsPerSecond": 13748.483703959047, + "WorstWorkerP99Microseconds": 1259.1, + "AffinityAppliedCount": 6, + "AssignedProcessors": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 41255, + "Release.Success": 41255 + }, + "WorkerCycles": [ + 6871, + 6875, + 6877, + 6877, + 6877, + 6878 + ] + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 12, + "Trial": 1, + "Cycles": 32478, + "ApiCallsPerSecond": 64896.78817047326, + "P50Microseconds": 219.1, + "P95Microseconds": 368, + "P99Microseconds": 708.7, + "MaxMicroseconds": 1468.5, + "Failures": 0, + "MeasuredSeconds": 1.0009124, + "WallSeconds": 1.0674693, + "SampleCount": 516, + "FairnessIndex": 0.9999999187818723, + "MinWorkerApiCallsPerSecond": 5407.480114266751, + "MaxWorkerApiCallsPerSecond": 5412.614409686741, + "WorstWorkerP99Microseconds": 1468.5, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 32478, + "Release.Success": 32478 + }, + "WorkerCycles": [ + 2708, + 2708, + 2706, + 2707, + 2707, + 2707, + 2706, + 2706, + 2706, + 2705, + 2706, + 2706 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 1, + "Trial": 1, + "Cycles": 278892, + "ApiCallsPerSecond": 557773.9042923324, + "P50Microseconds": 2.1, + "P95Microseconds": 8.1, + "P99Microseconds": 10.6, + "MaxMicroseconds": 30.3, + "Failures": 0, + "MeasuredSeconds": 1.0000181, + "WallSeconds": 1.0379606, + "SampleCount": 4358, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 557773.9042923324, + "MaxWorkerApiCallsPerSecond": 557773.9042923324, + "WorstWorkerP99Microseconds": 10.6, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 278892, + "Release.Success": 278892 + }, + "WorkerCycles": [ + 278892 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 6, + "Trial": 1, + "Cycles": 51921, + "ApiCallsPerSecond": 103831.65836682667, + "P50Microseconds": 110.6, + "P95Microseconds": 117.7, + "P99Microseconds": 129.6, + "MaxMicroseconds": 230.7, + "Failures": 0, + "MeasuredSeconds": 1.0000996, + "WallSeconds": 1.0538927, + "SampleCount": 816, + "FairnessIndex": 0.9999969825737371, + "MinWorkerApiCallsPerSecond": 17291.273766501807, + "MaxWorkerApiCallsPerSecond": 17372.79953955182, + "WorstWorkerP99Microseconds": 131.1, + "AffinityAppliedCount": 6, + "AssignedProcessors": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 51921, + "Release.Success": 51921 + }, + "WorkerCycles": [ + 8687, + 8647, + 8647, + 8646, + 8647, + 8647 + ] + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 12, + "Trial": 1, + "Cycles": 32631, + "ApiCallsPerSecond": 65250.17014415287, + "P50Microseconds": 223.6, + "P95Microseconds": 585.9, + "P99Microseconds": 3779.6, + "MaxMicroseconds": 6852.1, + "Failures": 0, + "MeasuredSeconds": 1.0001813, + "WallSeconds": 1.0637495, + "SampleCount": 516, + "FairnessIndex": 0.9999999867797692, + "MinWorkerApiCallsPerSecond": 5437.523129221568, + "MaxWorkerApiCallsPerSecond": 5439.198806015874, + "WorstWorkerP99Microseconds": 6852.1, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 32631, + "Release.Success": 32631 + }, + "WorkerCycles": [ + 2719, + 2720, + 2720, + 2719, + 2719, + 2719, + 2719, + 2720, + 2719, + 2719, + 2719, + 2719 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 1, + "Trial": 1, + "Cycles": 2944943, + "ApiCallsPerSecond": 5889770.560497015, + "P50Microseconds": 0.2, + "P95Microseconds": 1.3, + "P99Microseconds": 1.7, + "MaxMicroseconds": 19.3, + "Failures": 0, + "MeasuredSeconds": 1.0000196, + "WallSeconds": 1.0600165, + "SampleCount": 46015, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 5889770.560497015, + "MaxWorkerApiCallsPerSecond": 5889770.560497015, + "WorstWorkerP99Microseconds": 1.7, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 2944943, + "Release.Success": 2944943 + }, + "WorkerCycles": [ + 2944943 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 6, + "Trial": 1, + "Cycles": 8561205, + "ApiCallsPerSecond": 17121922.025222283, + "P50Microseconds": 0.5, + "P95Microseconds": 2.6, + "P99Microseconds": 3, + "MaxMicroseconds": 391.4, + "Failures": 0, + "MeasuredSeconds": 1.0000285, + "WallSeconds": 1.0954311, + "SampleCount": 133771, + "FairnessIndex": 0.999289467690636, + "MinWorkerApiCallsPerSecond": 2770705.069950209, + "MaxWorkerApiCallsPerSecond": 2956687.3351726895, + "WorstWorkerP99Microseconds": 3.2, + "AffinityAppliedCount": 6, + "AssignedProcessors": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 8561205, + "Release.Success": 8561205 + }, + "WorkerCycles": [ + 1396754, + 1469434, + 1389738, + 1478379, + 1385388, + 1441512 + ] + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 12, + "Trial": 1, + "Cycles": 13091243, + "ApiCallsPerSecond": 26181292.13307873, + "P50Microseconds": 0.6, + "P95Microseconds": 2.9, + "P99Microseconds": 3.8, + "MaxMicroseconds": 6278.7, + "Failures": 0, + "MeasuredSeconds": 1.0000456, + "WallSeconds": 1.1608522, + "SampleCount": 204555, + "FairnessIndex": 0.992488044913211, + "MinWorkerApiCallsPerSecond": 1796062.8551200284, + "MaxWorkerApiCallsPerSecond": 2411962.8683261117, + "WorstWorkerP99Microseconds": 4.3, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 13091243, + "Release.Success": 13091243 + }, + "WorkerCycles": [ + 1128539, + 1091536, + 1081705, + 962413, + 898044, + 1196215, + 1138296, + 1206031, + 1122685, + 1166473, + 1139640, + 959666 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 1, + "Trial": 1, + "Cycles": 3058843, + "ApiCallsPerSecond": 6117564.87221553, + "P50Microseconds": 0.2, + "P95Microseconds": 0.3, + "P99Microseconds": 1.6, + "MaxMicroseconds": 284.8, + "Failures": 0, + "MeasuredSeconds": 1.0000198, + "WallSeconds": 1.0520635, + "SampleCount": 47795, + "FairnessIndex": 1, + "MinWorkerApiCallsPerSecond": 6117564.87221553, + "MaxWorkerApiCallsPerSecond": 6117564.87221553, + "WorstWorkerP99Microseconds": 1.6, + "AffinityAppliedCount": 1, + "AssignedProcessors": [ + 0 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 3058843, + "Release.Success": 3058843 + }, + "WorkerCycles": [ + 3058843 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 6, + "Trial": 1, + "Cycles": 8817349, + "ApiCallsPerSecond": 17634131.944364585, + "P50Microseconds": 0.5, + "P95Microseconds": 2.6, + "P99Microseconds": 3.3, + "MaxMicroseconds": 53.3, + "Failures": 0, + "MeasuredSeconds": 1.0000321, + "WallSeconds": 1.0786938, + "SampleCount": 137775, + "FairnessIndex": 0.9980324763504409, + "MinWorkerApiCallsPerSecond": 2660870.5860541873, + "MaxWorkerApiCallsPerSecond": 3034958.896679665, + "WorstWorkerP99Microseconds": 3.4, + "AffinityAppliedCount": 6, + "AssignedProcessors": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 8817349, + "Release.Success": 8817349 + }, + "WorkerCycles": [ + 1330478, + 1517523, + 1507022, + 1515342, + 1459923, + 1487061 + ] + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 12, + "Trial": 1, + "Cycles": 14806220, + "ApiCallsPerSecond": 29611178.563793182, + "P50Microseconds": 0.5, + "P95Microseconds": 2.9, + "P99Microseconds": 3.5, + "MaxMicroseconds": 340.2, + "Failures": 0, + "MeasuredSeconds": 1.0000426, + "WallSeconds": 1.1167662, + "SampleCount": 231355, + "FairnessIndex": 0.9991852477028011, + "MinWorkerApiCallsPerSecond": 2336495.5506694093, + "MaxWorkerApiCallsPerSecond": 2653952.532392073, + "WorstWorkerP99Microseconds": 3.9, + "AffinityAppliedCount": 12, + "AssignedProcessors": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11 + ], + "Oversubscribed": false, + "Qualification": "smoke-only", + "StatusHistogram": { + "Acquire.Success": 14806220, + "Release.Success": 14806220 + }, + "WorkerCycles": [ + 1239249, + 1213100, + 1221067, + 1255337, + 1235950, + 1213783, + 1220169, + 1235079, + 1168266, + 1248394, + 1327004, + 1228822 + ] + } + ], + "Summary": [ + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 557773.9042923324, + "MedianP50Microseconds": 2.1, + "MedianP95Microseconds": 8.1, + "MedianP99Microseconds": 10.6, + "MedianMaxMicroseconds": 30.3, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 10.6, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 278892, + "Release.Success": 278892 + } + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 6, + "MedianApiCallsPerSecond": 103831.65836682667, + "MedianP50Microseconds": 110.6, + "MedianP95Microseconds": 117.7, + "MedianP99Microseconds": 129.6, + "MedianMaxMicroseconds": 230.7, + "MedianFairnessIndex": 0.9999969825737371, + "MedianWorstWorkerP99Microseconds": 131.1, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 51921, + "Release.Success": 51921 + } + }, + { + "Profile": "Legacy", + "Scenario": "distributed-key-read", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 65250.17014415287, + "MedianP50Microseconds": 223.6, + "MedianP95Microseconds": 585.9, + "MedianP99Microseconds": 3779.6, + "MedianMaxMicroseconds": 6852.1, + "MedianFairnessIndex": 0.9999999867797692, + "MedianWorstWorkerP99Microseconds": 6852.1, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 32631, + "Release.Success": 32631 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 645917.1462487896, + "MedianP50Microseconds": 2.1, + "MedianP95Microseconds": 7.4, + "MedianP99Microseconds": 10.4, + "MedianMaxMicroseconds": 120.6, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 10.4, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 322965, + "Release.Success": 322965 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 6, + "MedianApiCallsPerSecond": 82461.36428734333, + "MedianP50Microseconds": 113.8, + "MedianP95Microseconds": 244.1, + "MedianP99Microseconds": 1251.9, + "MedianMaxMicroseconds": 2554.3, + "MedianFairnessIndex": 0.999999864837961, + "MedianWorstWorkerP99Microseconds": 1259.1, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 41255, + "Release.Success": 41255 + } + }, + { + "Profile": "Legacy", + "Scenario": "same-key-read", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 64896.78817047326, + "MedianP50Microseconds": 219.1, + "MedianP95Microseconds": 368, + "MedianP99Microseconds": 708.7, + "MedianMaxMicroseconds": 1468.5, + "MedianFairnessIndex": 0.9999999187818723, + "MedianWorstWorkerP99Microseconds": 1468.5, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 32478, + "Release.Success": 32478 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 6117564.87221553, + "MedianP50Microseconds": 0.2, + "MedianP95Microseconds": 0.3, + "MedianP99Microseconds": 1.6, + "MedianMaxMicroseconds": 284.8, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 1.6, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 3058843, + "Release.Success": 3058843 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 6, + "MedianApiCallsPerSecond": 17634131.944364585, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 2.6, + "MedianP99Microseconds": 3.3, + "MedianMaxMicroseconds": 53.3, + "MedianFairnessIndex": 0.9980324763504409, + "MedianWorstWorkerP99Microseconds": 3.4, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 8817349, + "Release.Success": 8817349 + } + }, + { + "Profile": "LockFree", + "Scenario": "distributed-key-read", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 29611178.563793182, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 2.9, + "MedianP99Microseconds": 3.5, + "MedianMaxMicroseconds": 340.2, + "MedianFairnessIndex": 0.9991852477028011, + "MedianWorstWorkerP99Microseconds": 3.9, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 14806220, + "Release.Success": 14806220 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 1, + "MedianApiCallsPerSecond": 5889770.560497015, + "MedianP50Microseconds": 0.2, + "MedianP95Microseconds": 1.3, + "MedianP99Microseconds": 1.7, + "MedianMaxMicroseconds": 19.3, + "MedianFairnessIndex": 1, + "MedianWorstWorkerP99Microseconds": 1.7, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 2944943, + "Release.Success": 2944943 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 6, + "MedianApiCallsPerSecond": 17121922.025222283, + "MedianP50Microseconds": 0.5, + "MedianP95Microseconds": 2.6, + "MedianP99Microseconds": 3, + "MedianMaxMicroseconds": 391.4, + "MedianFairnessIndex": 0.999289467690636, + "MedianWorstWorkerP99Microseconds": 3.2, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 8561205, + "Release.Success": 8561205 + } + }, + { + "Profile": "LockFree", + "Scenario": "same-key-read", + "ProcessCount": 12, + "MedianApiCallsPerSecond": 26181292.13307873, + "MedianP50Microseconds": 0.6, + "MedianP95Microseconds": 2.9, + "MedianP99Microseconds": 3.8, + "MedianMaxMicroseconds": 6278.7, + "MedianFairnessIndex": 0.992488044913211, + "MedianWorstWorkerP99Microseconds": 4.3, + "TotalFailures": 0, + "StatusHistogram": { + "Acquire.Success": 13091243, + "Release.Success": 13091243 + } + } + ] +} \ No newline at end of file diff --git a/specs/009-lock-free-publish-read/checklists/concurrency.md b/specs/009-lock-free-publish-read/checklists/concurrency.md new file mode 100644 index 0000000..a55afd2 --- /dev/null +++ b/specs/009-lock-free-publish-read/checklists/concurrency.md @@ -0,0 +1,59 @@ +# Concurrency Requirements Checklist: Lock-Free Shared-Memory KV Store + +**Purpose**: Assess whether the concurrency, capacity, recovery, and release requirements are complete and unambiguous for final peer review +**Created**: 2026-07-13 +**Feature**: [spec.md](../spec.md) + +**Note**: This checklist evaluates the written requirements, not the implementation or test results. + +## Requirement Completeness + +- [x] CHK001 Are the key-value-store scope and the explicit exclusion of stream, queue, broker, delivery, and load-balancing responsibilities stated for every public workflow? [Completeness, Spec §Scope/FR-001] +- [x] CHK002 Are requirements defined for the producer, 6–12 load-balanced workers, and unrelated observer/administrative processes without assigning worker identity to the store? [Completeness, Spec §US1/FR-046] +- [x] CHK003 Are lock-free progress requirements present for publish, reserve, commit, acquire, project, release, remove, diagnostics, recovery, and disposal rather than only the primary publish/read path? [Coverage, Spec §FR-025..FR-041] +- [x] CHK004 Are all physical-capacity states—including tentative initialization, explicit/atomic reservation, publication, removal, abort, reclaim, and retirement—covered by the `StoreFull` requirements? [Completeness, Spec §FR-031] +- [x] CHK005 Are requirements defined for failed pre-metadata claims, stale owners, live paused owners, and same/future-generation helpers without introducing a global recovery owner? [Recovery Coverage, Spec §FR-032..FR-036] + +## Requirement Clarity + +- [x] CHK006 Is “lock-free” distinguished from wait-free completion and from the absence of named/OS synchronization in measurable terms? [Clarity, Spec §FR-025..FR-030] +- [x] CHK007 Is every public ordering point named precisely enough to decide cancellation, duplicate-key, and same-key race outcomes? [Clarity, Spec §Concurrency Outcome Contract/FR-028] +- [x] CHK008 Is `StoreFull` distinguished from provisional scan exhaustion, rotating free capacity, local proof-buffer contention, and caller-budget exhaustion? [Clarity, Spec §FR-029..FR-031] +- [x] CHK009 Is the confirmed between-collect `StoreFull` candidate distinguished from the later confirmation/checkpoint timestamp? [Clarity, Spec §FR-031] +- [x] CHK010 Are `NoWait`, finite, infinite, timeout, and cancellation outcomes specified at both pre-ordering and post-ordering boundaries? [Clarity, Spec §FR-029..FR-030] +- [x] CHK011 Is the per-open proof-buffer cost quantified per slot and at the maximum supported slot count, including its process-local/non-mapped ownership? [Clarity, Spec §FR-037/LC-015] + +## Requirement Consistency + +- [x] CHK012 Are the intent-specific duplicate witnesses consistent between the concurrency table, functional requirements, data model, and public API contract? [Consistency, Spec §Concurrency Outcome Contract/FR-005..FR-009] +- [x] CHK013 Is the rule that genuine physical exhaustion may precede final same-key duplicate arbitration consistent across all publish and reserve scenarios? [Consistency, Spec §FR-031] +- [x] CHK014 Are the forward-only generation lifecycle and no-same-generation-rollback requirement consistent with recovery, stale-token, and retirement requirements? [Consistency, Spec §FR-033..FR-035/FR-047] +- [x] CHK015 Are the zero-per-operation-allocation requirement and the eager per-open scratch-memory allowance expressed without contradiction? [Consistency, Spec §FR-037/SC-008] +- [x] CHK016 Are platform-specific x64 atomic assumptions consistent with the portable-core and unsupported-platform requirements? [Consistency, Spec §FR-043..FR-045/Assumptions] + +## Acceptance Criteria Quality + +- [x] CHK017 Can the no-global-operation-owner requirement be objectively assessed for every public v2 operation and across multiple processes? [Measurability, Spec §SC-006] +- [x] CHK018 Are correctness, progress, allocation, latency, throughput, fairness, payload, participant-count, duration, trial-count, and environment fields quantified for each required workload? [Acceptance Criteria, Spec §Benchmark Workload Matrix/SC-001..SC-018] +- [x] CHK019 Are pass, fail, and environment-qualified not-qualified outcomes defined separately for Windows x64 and Linux x64? [Acceptance Criteria, Spec §SC-010/SC-013] +- [x] CHK020 Is the non-convergence stop rule tied to a repeatable invariant and a fixed number of evidence-driven protocol corrections? [Acceptance Criteria, Spec §Non-Convergence Gate] + +## Scenario and Edge-Case Coverage + +- [x] CHK021 Are rotating-hole, between-collect movement, same-handle proof contention, and independent-handle progress scenarios addressed by the capacity requirements? [Coverage, Spec §FR-029..FR-031] +- [x] CHK022 Are malformed first/second snapshot words and unknown publication intent assigned explicit fail-closed outcomes? [Exception Flow, Spec §FR-029/FR-047..FR-052] +- [x] CHK023 Are terminal generation retirement, participant-token reuse, stale directory reference, and delayed helper scenarios bounded before identity fields can wrap? [Edge Cases, Spec §FR-035/FR-047..FR-052] +- [x] CHK024 Are suspended/crashed-process effects bounded to owned keys, slots, leases, and participant records while unrelated processes continue? [Recovery Coverage, Spec §FR-032/SC-005] +- [x] CHK025 Are cleanup outcomes specified when cancellation or expiry occurs after caller ownership is relinquished but before physical reclaim completes? [Exception Flow, Spec §FR-029/FR-033] + +## Dependencies and Assumptions + +- [x] CHK026 Is the external message broker dependency limited to distributing keys, with no broker state, worker assignment, or delivery semantics entering the mapped protocol? [Boundary, Spec §Scope/FR-046] +- [x] CHK027 Are the trust boundary, mapped-atomic alignment assumptions, owner-liveness limitations, and lack of cross-host durability documented as explicit constraints? [Assumption, Spec §Assumptions/FR-043..FR-045] +- [x] CHK028 Are package compatibility, required-feature bits, rollout, rollback, and old-runtime fail-closed behavior specified for the revised layout without implying C++/Python v2 support? [Dependency, Spec §FR-042..FR-045/LC-001..LC-016] +- [x] CHK029 Are Linux same-PID managed/native exclusion, stable lock-inode lifetime, synchronization-before-region teardown, OFD unsupported behavior, and the old same-PID compatibility boundary explicit and testable? [Concurrency/Compatibility, Spec §FR-057] + +## Notes + +- Check items off only after inspecting the cited requirements and their linked contracts. +- Record any ambiguity or conflict inline and route implementation gaps through the convergence workflow. diff --git a/specs/009-lock-free-publish-read/checklists/implementation.md b/specs/009-lock-free-publish-read/checklists/implementation.md new file mode 100644 index 0000000..a1cbfc8 --- /dev/null +++ b/specs/009-lock-free-publish-read/checklists/implementation.md @@ -0,0 +1,80 @@ +# Implementation Evidence Checklist + +## Conditional interpretation + +This checklist is frozen before final execution. A checked statement means its +machine predicate is defined and is true **if and only if** the linked JSON +reports the required pass on one identical clean commit/source manifest. It is +not a claim that an artifact already exists. Missing, failed, not-qualified, +validation-only, skipped, dirty, stale, or provenance-mismatched evidence makes +the corresponding statement false and the release **NOT QUALIFIED**. The raw +ignored artifacts are authoritative; no tracked post-run checkbox edit is +allowed. See the complete predicate and SC mapping in +[`release-qualification.md`](../release-qualification.md). + +- [x] PR is conditionally passed iff + [`009-final-r6-pr/summary.json`](../../../artifacts/lock-free-qualification/009-final-r6-pr/summary.json) + is schema 4 `passed` and its hashed + [`sync-probe.json`](../../../artifacts/lock-free-qualification/009-final-r6-pr/sync-probe.json) + passes exact short-matrix, correctness, and provenance validation. +- [x] Nightly is conditionally passed iff + [`009-final-r6-nightly/summary.json`](../../../artifacts/lock-free-qualification/009-final-r6-nightly/summary.json) + is schema 4 `passed`, exact configured counts pass, and its + [`sync-probe.json`](../../../artifacts/lock-free-qualification/009-final-r6-nightly/sync-probe.json) + is hash-bound to the same clean source. +- [x] Release is conditionally passed iff + [`009-final-r6-release/summary.json`](../../../artifacts/lock-free-qualification/009-final-r6-release/summary.json) + is schema 4 `passed` and exact long counts, three trials (60-second duration + rows or exact count targets), waits, + recovery, churn, race, allocation, copy, and threshold gates all pass. +- [x] Windows x64 is conditionally passed iff the runner-created + [`os-validation.json`](../../../artifacts/lock-free-qualification/009-final-r6-release/os-validation.json) + is schema 3 `pass`, qualified x64, and every required row passes. +- [x] Linux x64 is conditionally passed iff + [`009-final-r6-linux-x64.json`](../../../artifacts/lock-free-os-validation/009-final-r6-linux-x64.json) + is schema 3 `pass`, qualified x64, every required row passes, and its required + [`linux-tiny-performance.json`](../../../artifacts/lock-free-os-validation/009-final-r6-linux-x64.evidence/linux-tiny-performance.json) + is schema 8 with the exact 2-profile/2-scenario/1-and-8-process/3-trial release + matrix, zero failures, `[0,63]` mask-valid complete affinity, coherent paired + successes per completed cycle, no checksum/corruption evidence, reproducible + medians, one-process intrinsic-p99 non-regression, eight-process throughput + non-regression, at most 3x lock-free p99 amplification, at most 10 us + eight-process p99, and every raw lock-free stall at most 10 ms. +- [x] Profile-aware completion is conditionally passed iff config schema 5 and + report schema 8 name only `LockFree` as count-bound; every LockFree mixed row + records/reaches 100,000,000 operations, every LockFree large-ingest row + records/reaches 100,000 frames, every Legacy counterpart has both targets + zero and measures at least 60 seconds, no row has dual targets, and the + duration-bound controller grace is exactly 60 seconds. +- [x] Cross-platform provenance is conditionally passed iff PR, nightly, + release, both OS reports, every sync probe, and start/completion assembly + manifests identify the identical clean commit, tree, status hash, and source- + manifest hash. +- [x] SC-001 through SC-018 are conditionally passed iff every named field/gate + in the release qualification mapping passes; no averaging or narrative + substitution is permitted. +- [x] Full tests and compatibility are conditionally passed iff full-solution + TRX has zero non-passed tests and required package, native, Python, Docker, + samples, release-tests, and pack rows all pass. +- [x] Leak freedom and bounded waits are conditionally passed iff all three + executable owner/leak mappings pass and every selected wait/cancellation case + completes within its limit plus 250 ms. +- [x] Review is conditionally accepted iff `code-review.md` has no unresolved + High or Medium finding for the identical committed source proven by release + JSON. +- [x] Evidence immutability is conditionally passed iff the fixed paths were + absent before execution, scripts did not overwrite evidence, evidence + manifests are the exact normalized non-reparse file sets below their sibling + `.evidence` roots, every command log and raw performance file is hash-bound, + accepted OS report/tree digests revalidate at runner completion, and tracked + files were not edited during or after the qualifying sequence. + +Historical Windows/Linux tests, bounded SC-011 races/generated histories, and +native/Python runs are explicitly diagnostic for their producing snapshots. +The original `009-final-*` pass/failure pair, every `009-r2-pre-final*` +artifact, the rejected Windows-host `009-final-r2-linux-x64` attempt, the +stale-workload-set `009-final-r3-linux-x64` attempt, the R4 Linux pass/R4 PR +test-contract failure pair, the R5 Linux/PR/nightly passes plus incomplete +non-convergent R5 release attempt, and the unbound reservation timing observation are +likewise diagnostic only. Their stale counts are not current-tree claims and +none can make an R6 conditional checkbox true by itself. diff --git a/specs/009-lock-free-publish-read/checklists/requirements.md b/specs/009-lock-free-publish-read/checklists/requirements.md new file mode 100644 index 0000000..36bacbd --- /dev/null +++ b/specs/009-lock-free-publish-read/checklists/requirements.md @@ -0,0 +1,66 @@ +# Specification Quality Checklist: Lock-Free Shared-Memory Key-Value Store + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-07-12 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- Validation iteration 1 removed the incorrect work-stream model and restored + general key-value addressing, shared read leases, external broker ownership, + explicit removal, and multi-process access. +- Validation iteration 1 identified a recovery coordination loophole, loosely + bounded race outcomes, and under-specified benchmark inputs. +- Validation iteration 2 closed the recovery loophole, added the concurrency + outcome contract, and added a reproducible benchmark workload matrix. All + checklist items now pass. +- Phase-0 planning research refined SC-005 to compare the same healthy + participant set and made SC-006 platform-relative so its Windows and Linux + thresholds are both reproducible and achievable without weakening the + lock-free contract. +- Convergence profiling later split Linux SC-006 into uncontended latency, + eight-process throughput, lock-free self-amplification, absolute p99, and + sampled-maximum gates. This avoids treating the serialized legacy file-lock + incumbent distribution as a parallel latency oracle while retaining stricter + intrinsic, scaling, absolute-tail, and stall evidence. +- Cross-artifact analysis found that post-claim owner metadata left an + unrecoverable crash instruction. With user approval, FR-031/FR-035 and the + assumptions now make configurable participant/open-handle capacity explicit + and require the first slot/lease claim to contain a recoverable incarnation. +- Compatibility analysis narrowed immutable already-released clients to the + enforceable fail-closed/no-layout-access guarantee; current clients that can + validate the header retain the precise incompatible-layout outcome. +- C#/.NET-first delivery, current public operation names, layout compatibility, + zero-copy behavior, and lock-free progress are explicit product and contract + constraints. The specification does not choose an atomic algorithm, data + structure, memory layout, or operating-system notification primitive; those + decisions remain for planning. +- The approved convergence redesign adds an exact stale-helper outcome contract + and an explicit lock-free-profile value-slot limit. Both are measurable, + profile-scoped, cross-runtime compatibility requirements; the generation-bit + allocation and atomic transition algorithm remain planning decisions. diff --git a/specs/009-lock-free-publish-read/code-review.md b/specs/009-lock-free-publish-read/code-review.md new file mode 100644 index 0000000..2b146ea --- /dev/null +++ b/specs/009-lock-free-publish-read/code-review.md @@ -0,0 +1,661 @@ +# Final independent concurrency and code review + +**Feature**: `009-lock-free-publish-read` + +**Branch**: `codex/lock-free-csharp` + +**Review date**: 2026-07-15 + +**Decision**: **Code approved; immutable final release evidence pending** + +## Verdict + +The tracked implementation is approved on code, protocol, focused-test, and +pre-final qualification grounds. Independent re-review of the reclamation, +cold-open, Linux record-lock lifetime, benchmark, and evidence-importer deltas +found no unresolved High or Medium issue in the production implementation or +release harness. R5 passed Linux, PR, and nightly on one clean source state but +its release workload was rejected as non-convergent because the harness applied +lock-free durability counts to the semaphore-serialized Legacy comparison rows. +R6 corrects only the benchmark completion/evidence policy, preserves the full +store matrix, and adds a hard monotonic deadline around duration-bound trials; +it does not change production store code. This is not yet release approval: +every immutable artifact and +provenance condition in [Final evidence gate](#final-evidence-gate) must still +pass on one identical clean source state. A missing, failed, provenance-mismatched, +or incomplete gate leaves the feature unapproved for release. + +## Review closure + +The final independent review covered public compatibility, atomic/control-word +encodings, directory and slot generations, participant incarnation and +recovery, helping and progress, operation budgets, disposal, Linux lifecycle +artifacts, PID namespaces, zero-allocation paths, production race oracles, and +qualification harnesses. + +- Original High findings `CR-H01` through `CR-H10`: **resolved**. +- Original Medium findings `CR-M01` through `CR-M08`: **resolved**. +- SC-011 production-race and generated-history re-review: **approved**, with no + High or Medium finding. +- Exact Linux PID-namespace publication, classification, and mixed-namespace + recovery rules: **reviewed clean**. +- Linux owner-anchor cleanup and orphan sweep: **reviewed clean** after the + FIFO-safe/statx artifact-classification correction. +- Final directory/recovery hardening: **reviewed clean** after exact CAS-loss + cleanup validation, stable reservation-metadata tuple validation, and the + legal canceling zero-location window were covered by deterministic tests. +- Final lifecycle successor hardening: **reviewed clean** after lease, + participant, remove/reclaim, and abort/reclaim transitions rejected stable + same-generation regressions while accepting exact or later-incarnation + successors and transient movement. +- Historical Windows Release and focused Linux owner-anchor, PID-namespace, and + profile-open sets passed on their producing snapshots; stale counts are not + asserted for the current tree. +- Qualification-harness remediation: **independently reviewed clean**, with no + unresolved High or Medium finding. The review covered the exact raw Linux + tiny matrix, every-trial stall gate, mask-valid sparse affinity, + paired-success/checksum/corruption coherence, exact metric tuples, + manifest/file-set/log/path binding, reparse rejection, and completion + revalidation. +- Linux record-lock lifetime, load-context/native same-PID exclusion, stable + pathname identity, and teardown ordering: remediated with OFD locks, + persistent rendezvous inodes, descriptor-before-region cleanup, and focused + same/foreign-process regressions; final independent re-review is recorded + below. +- Pre-final evidence recomputation: **reviewed clean** after both PowerShell + median helpers adopted explicit midpoint-floor semantics and distinct + odd/even self-tests; the real three-trial raw report recomputes exactly. +- R4 disposal-test correction: **reviewed clean**. The concurrent theory now + uses normal non-override lease/reservation recovery and inspects only borrowed + span shape after racing disposal; exact content remains covered on the live + second handle and in dedicated data-path suites. No production code changed. +- R6 completion-policy and convergence correction: **reviewed clean**. Config + schema 5 selects only `LockFree` as count-bound; probe schema 8/minimum 8 + records the selected profiles and exact per-run targets. Legacy mixed-churn + and large-ingest remain three duration-bound comparisons, while every + LockFree mixed/large trial retains its 100,000,000-operation/100,000-frame + durability target. An atomic monotonic-deadline watchdog spans setup through + final cleanup, rejects delayed timer dispatch, bounds child-kill effort, and + then unconditionally fail-fast terminates the isolated probe on timeout. +- R6 independent re-review: **no unresolved High or Medium finding** after the + initial watchdog, cleanup, schema-compatibility, identity, and exact-grace + findings were corrected. Target-policy and watchdog tests, both validation- + only importers, documentation validation, and a real four-row Legacy/LockFree + duration/count smoke passed on the final pre-freeze worktree. The complete + Release solution passed 1,028/1,028 tests with zero skips: Contract 117, + Linearizability 83, Interop 75, Unit 451, and Integration 302. + +### Cold-open initialization-authority closure + +A final structural review found a High-severity cold-path race: an opener could +map an already-created zeroed region before acquiring cross-process lifecycle +coordination, infer initialization authority from its open mode, and overwrite +the physical creator's eventual header. This did not put an OS lock on any hot +operation, but it made concurrent create/open unsafe. + +The closed implementation now treats physical creation as the only +initialization authority. `SharedStoreOpenScope` owns the ordered cold +transaction and carries an explicit `CreatedNew`/`OpenedExisting` disposition +through header initialization or validation and participant registration. The +Windows named gate is acquired before mapping. Linux acquires `.lifecycle`, +reconciles and removes only proven-stale artifacts, then acquires `.lock` +before mapping, owner publication, and engine construction. The original +wait/cancellation budget covers the whole transaction, and failure releases the +ordered gates before mapped-owner cleanup can re-enter lifecycle coordination. + +The profile/open matrix now proves that every legacy/lock-free and open-mode +combination preserves an existing zero header, that a held cold scope blocks +before any physical mapping or owner publication, that an opposite-profile +sentinel is preserved, and that cleanup transfers/disposes ownership exactly +once. Focused Linux owner-marker, profile-open, and wait-policy tests passed. +Independent re-review found no unresolved High, Medium, or concrete Low issue. + +### Linux OFD-lock, inode, and teardown closure + +The first Linux audit found that one descriptor per wrapper was unsafe with +traditional process-associated `F_SETLK`: same-PID contenders did not conflict, +and closing any sibling descriptor could release a live lock. A shared managed +descriptor registry fixed that case inside one loaded assembly, but final review +found two further High-severity holes. The registry was static per +`AssemblyLoadContext`, not per OS process, so another package copy or native +module could bypass it. Separately, store and failed-open teardown disposed the +mapped region before the synchronization descriptor; final-owner cleanup could +unlink `.lock`, a same-process reopen could reuse the old descriptor/inode, and +a foreign process could create and lock a replacement inode. + +Current C# and C++ adapters now use direct nonblocking `F_OFD_SETLK` calls. +Each C# wrapper owns one descriptor and a non-reentrant local gate. C++ wrappers +inside one module share their existing per-path `FileState`/descriptor and timed +mutex; distinct modules/load contexts/descriptors contend in the kernel. +Unsupported commands/filesystems fail closed, and unlock failure closes/retires +the affected descriptor before its local gate is reopened. Current cleanup retains `.lock` +as an empty mode-`0600` stable rendezvous, matching `.lifecycle`. Lock-free, +legacy, initialized, uninitialized, platform-failure, and failed-scope teardown +all dispose ordinary synchronization before region/owner cleanup may re-enter +`.lifecycle`; the native store uses the same order. + +The accepted compatibility boundary is explicit: traditional and OFD locks +remain mutually exclusive across processes, and current managed/native OFD +implementations coexist inside one PID. Concurrently mixing an older +process-associated-`F_SETLK` package with a current adapter inside the same PID +is unsupported because closing any sibling descriptor can invalidate the old +lock. + +Focused Linux evidence covers both lock paths, same-thread and timed contenders, +foreign exclusion, two independently loaded current assemblies, a same-PID +native-style OFD descriptor in both directions, and concurrent final close/reopen. +The last test locks the persistent pathname and proves the reopened legacy +store's actual operation returns `StoreBusy`, detecting the former inode split; +failed-open event recording proves synchronization disposal precedes owner +cleanup. A held gate for one store still does not block a different store name. +Final re-review also found that both adapters cleared their wrapper-local held +flag after waking the next local waiter; the old releaser could overwrite the +new holder's state and strand the local semaphore/mutex. Both now clear private +ownership before publishing the unlock, and a 4,000-handoff C# stress regression +proves exclusivity plus immediate wrapper reuse; the rebuilt native suite passes. + +### Benchmark reproducibility-metadata closure + +The final harness audit found one Medium evidence-contract gap: the benchmark +methodology required CPU model, logical/physical processor counts, total memory, +and exact store dimensions, but schema 6 omitted several fields and Linux could +record `processorIdentifier: unknown`. Both importers accepted any nonempty +identifier. + +At that stage, schema 7 added portable processor model, logical/physical counts, total host +memory, and exact per-scenario slot/value/descriptor/key/lease/participant +dimensions. Windows uses processor-topology and physical-memory APIs; Linux uses +the process-visible CPU topology, `/proc/cpuinfo`, and `/proc/meminfo`. Detection +uncertainty emits an invalid zero/unknown value so qualification fails closed. +The then-current importers required schema 7, rejected missing/unknown/implausible metadata, +verified the exact selected scenario-dimension set, and included negative +self-tests for unknown CPU, missing memory, and dimension tampering. Focused +Windows/Linux builds, runtime smokes, and validator self-tests passed. + +### Historical schema-7 post-review convergence evidence + +After the OFD, teardown, persistent-inode, handoff, and schema-7 changes, fresh +Windows x64 and Linux x64 Release restores/builds completed with zero warnings +and zero errors. Each platform passed Unit 416/416, Contract 117/117, +Integration 302/302, Interop 75/75, and Linearizability 83/83: **993/993** with +no skips. The four additional integration cases are the load-context, +managed/native OFD, concurrent close/reopen, and shared-wrapper handoff +regressions. + +A source-fresh Linux C++ build passed 5/5 native tests. With that exact native +library installed for Python and its exact C++ agent selected, the non-stress +C#/C++/Python interoperability matrix passed 65/65, including three-owner final +cleanup with persistent `.lock`. Documentation validation and `git diff --check` +passed. Independent final source/protocol/test review found no remaining High, +Medium, or Low issue in the Linux closure or deep lock-free core. These +worktree results establish code-review closure but do not replace the clean +same-commit immutable evidence gate. + +Integration test classes are now serialized because several independent +classes each launch 8-12 subprocesses and their simultaneous execution measured +host-wide startup oversubscription against the one-second public cold-open +budget. Each scenario retains its real internal cross-process concurrency, and +the explicit two-store test preserves coverage that unrelated names remain +independent. The complete serialized Linux suite passed without increasing the +library timeout or weakening deadline checks. + +A subsequent Linux-to-Windows clean-output switch exposed one unrelated sample +test dependency: `DockerSharedMemoryLocalSampleModeRuns` invokes the sample with +`--no-build`, but the integration project did not declare that sample as a +build-only reference. The integration project now carries the same +`ReferenceOutputAssembly=false` dependency pattern already used for the broker +sample and test agents. A fresh Windows solution restore/build then completed +with 0 warnings and 0 errors and the full aggregate passed 989/989. + +### Remove classification and reclaim closure + +`TryRemove` now performs one active-lease classification scan immediately +before claiming `RemoveRequested -> Reclaiming`; the reclaimer no longer repeats +the same full lease-table scan. A fresh budget check follows classification, so +NoWait, finite, infinite, cancellation, active-lease `RemovePending`, and +post-ordering reclaim outcomes retain their documented boundaries. Checkpoint +63 deterministically covers a lease acquired after the earlier observation and +before reclaim ownership. Focused remove/reclaim/wait validation passed 111/111, +and independent review found no unresolved High or Medium issue. + +### Current pre-final convergence evidence + +The production candidate `a99a656` passed complete Windows x64 and Linux x64 +Release aggregates with no skips: Unit 416/416, Contract 117/117, Integration +298/298, Interop 75/75, and Linearizability 83/83, for **989/989** on each +platform. The later `50ea3a8` commit changes only the two evidence importers and +their self-tests; its clean Linux `-Command all` run repeated the full 989-test +Release aggregate and passed every required architecture, atomic, raw +visibility, held-lock/`strace`, 67-checkpoint crash, native, Python, Docker, +6/12-reader sample, and pack row. The optional container PID-namespace pause +row was `not-qualified` with `required: false`; the required Docker validation +passed. + +The first clean long attempt, preserved as `009-pre-final3`, stopped after its +correct 24-run benchmark because both PowerShell importers converted the +three-value midpoint `1.5` to integer `2` and compared the C# median against the +maximum. All 96 stored producer medians matched correct recomputation and every +performance gate passed; later OS stages were correctly not run after the +integrity exception. Commit `50ea3a8` uses an explicit floored midpoint in both +importers and adds non-monotonic odd/even median self-tests. Windows and Linux +validation-only suites passed, the preserved raw report revalidated across five +cultures, and independent review found no High, Medium, or Low issue. + +The fresh immutable diagnostic +`artifacts/lock-free-os-validation/009-pre-final4-linux-x64.json` then completed +with `overallStatus: pass`, start/completion clean provenance at `50ea3a8`, and +52 manifest-bound evidence files. Its exact performance summaries were: + +| Profile | Scenario | Processes | Median calls/s | Median p99 | Maximum raw stall | +|---|---|---:|---:|---:|---:| +| Legacy | acquire/release | 1 | 314,130 | 9.5 us | 489.7 us | +| Legacy | acquire/release | 8 | 317,653 | 11.8 us | 131,729.1 us | +| LockFree | acquire/release | 1 | 3,975,778 | 0.7 us | 284.2 us | +| LockFree | acquire/release | 8 | 22,070,035 | 1.4 us | 4,017.0 us | +| Legacy | publish/remove | 1 | 1,168,006 | 2.3 us | 768.1 us | +| Legacy | publish/remove | 8 | 1,177,259 | 2.3 us | 151,389.6 us | +| LockFree | publish/remove | 1 | 1,430,035 | 1.8 us | 476.7 us | +| LockFree | publish/remove | 8 | 6,545,020 | 4.6 us | 2,460.9 us | + +For both scenarios, lock-free one-process p99 beat legacy, eight-process +throughput beat legacy, eight/one lock-free p99 stayed below 3x, eight-process +p99 stayed below 10 us, and every lock-free raw stall stayed below 10 ms. This +is pre-final diagnostic evidence, not a substitute for the common-provenance +final gate. + +### Terminal clean pre-final diagnostic + +The remaining pre-final attempts were preserved rather than overwritten. In +order, they exposed and closed four release-evidence or bounded cold-start +defects: a WSL source-provenance timeout (`849a7a0` binds raw evidence to the +captured source tuple); empty strings emitted for non-executed OS fields +(`f754118` preserves JSON nulls); partially visible ready/go/done trace markers +(`5925fee` publishes each marker by same-directory atomic rename); and the +broker-key sample reusing the one-second default cold-open budget while starting +12 processes on an already loaded host (`853b10f` gives that sample operation a +single explicit ten-second budget). The marker correction passed 15 repeated +Linux `strace` gates and five complete Linux plus five Windows trace classes. +The sample correction passed 40 consecutive 12-worker Linux startups and fresh +6- and 12-worker Linux and Windows sample runs. Independent reviews of both +changes found no High, Medium, or Low finding; neither changes a layout-v2 hot +publish, acquire, release, or remove path. + +One deliberately contaminated, concurrent diagnostic left 17,733 historical +Linux rendezvous files and demonstrated a scoped Low cold-path limitation: +release-marker and owner-anchor discovery scans the flat per-distro rendezvous +directory, so extreme high-cardinality store-name churn can consume a finite +cold-open budget. Stable `.lock` and `.lifecycle` inodes cannot be routinely +deleted without breaking lock identity. This does not introduce a global lock, +does not affect an already-open store, and is outside the intended stable-name +deployment; bounded `StoreBusy` remains the specified cold-path outcome. The +final diagnostics therefore used an idle, restarted WSL tmpfs. A future +resource-protocol revision should use per-store namespaces or direct marker +resolution and add a high-cardinality cold-open regression, without changing +the lock-free key-value engine. + +After a Docker-integration-only not-qualified attempt was preserved as +`009-pre-final10`, the clean immutable diagnostic at +`artifacts/lock-free-os-validation/009-pre-final11-linux-x64.json` passed on +commit `853b10f317bcab16a4c69ead9b23b5bd6027ec7a`, tree +`69a8e453a3edc0528fab2a4a80fc151eff8820d0`, and source-manifest SHA-256 +`90F772F9FA240CE44029DF02D2EFF5D294996E69CC9184473ED83D5ECD1E1194`. +Start and completion provenance and all 33 tested-assembly rows were identical. +All 28 required rows passed; the optional Docker-pause row was correctly +not-qualified with a JSON-null execution tuple. Unit 436/436, Contract 117/117, +Integration 302/302, Interop 75/75, and Linearizability 83/83 passed, for +**1,013/1,013** across five TRX files. Native C++, Python, required Docker, +6/12-worker samples, packing, held-lock/`strace`, all 67 crash checkpoints, and +SIGSTOP recovery also passed. + +The report SHA-256 is +`28CD4C989898C481D26F77E8B1951771CFBAF1BC99ADDBC8358FBF09A66E5D27`; +its exact 52-file evidence-tree digest is +`FDA14EA9612EE0F891808FBA4348297CA766A72A4F32FBEDA541080EF8451478`; +and its raw schema-7 performance report SHA-256 is +`C621574937EA90DB110CF140641895F479E20425ACBF7B0BDFAB8BE3F2CA2CED`. +The actual release importer accepted the untouched report and recomputed the +same manifest and hashes. A separate audit independently reconstructed the +source digest from 589 blobs and found no evidence or importer finding. + +| Profile | Scenario | Processes | Median calls/s | Median p99 | Maximum raw stall | +|---|---|---:|---:|---:|---:| +| Legacy | acquire/release | 1 | 251,281 | 13.5 us | 315.5 us | +| Legacy | acquire/release | 8 | 253,850 | 15.7 us | 111,495.7 us | +| LockFree | acquire/release | 1 | 3,983,372 | 0.7 us | 4,047.6 us | +| LockFree | acquire/release | 8 | 22,008,409 | 1.5 us | 4,023.2 us | +| Legacy | publish/remove | 1 | 970,910 | 2.7 us | 4,849.8 us | +| Legacy | publish/remove | 8 | 951,879 | 2.9 us | 111,292.2 us | +| LockFree | publish/remove | 1 | 1,425,052 | 1.8 us | 1,554.0 us | +| LockFree | publish/remove | 8 | 6,492,213 | 4.6 us | 2,363.8 us | + +Acquire/release lock-free one-process p99 was 5.19% of legacy and its +eight-process throughput was 86.70x legacy; publish/remove was 66.67% and 6.82x +respectively. Lock-free eight/one-process p99 amplification was 2.14x and 2.56x. +Every contracted ratio, absolute p99, raw-stall, host-identity, affinity, +scenario-dimension, checksum, corruption, and exact-trial gate passed. This is +terminal diagnostic evidence and freezes T107/T108; it does not replace the +final PR/nightly/release and dual-platform common-provenance gate. + +### Directory handoff and delayed-helper closure + +The first full Linux tiny-operation diagnostic on commit `d834146` is preserved +at `artifacts/lock-free-os-validation/009-pre-final-linux-x64.json`. Acquire and +release passed, and the first publish/remove trial passed, but later trials +latched corruption and then recorded exactly 5,379,795,214 +`Publish.CorruptStore` results. The first traces led to +`LockFreeKeyDirectory.HelpMutation` and `TryPublishExactLocation`. Root-cause +analysis showed that delayed helpers treated legal cancellation and +same-generation location handoffs as a stable malformed store. + +The corrected protocol revalidates the canonical mutation, operation, location, +slot control, immutable directory binding, and relevant target cells as one +joint tuple before latching corruption. A stable invalid tuple requires two +identical collections plus exact no-op CAS confirmation. Canceling Insert +handoffs, first-publisher arbitration for `Unlink/Prepared`, valid alternate +locations after `Unlink/TargetSelected`, post-CAS withdrawal, old residue, and +future-generation observations now have explicit generation-fenced outcomes. +Malformed stable tuples remain fail-closed. Checkpoints 66 and 67 cover the two +new publication/revalidation windows and extend the canonical catalog to 67. + +Deterministic unit schedules cover exact, empty, valid replacement, malformed, +out-of-range, alternate-location, post-CAS, and real-reuse outcomes. The +canonical crash matrix passed 67/67 on Windows x64 and Linux x64; the final +8-process churn regression passed on both platforms. A longer pre-correction +cross-platform churn run contributed 498,742,938 public API calls with no +failure, and the final focused runs remained clean after the adjacent fixes. +Independent re-review found no High or Medium issue. It retained one +non-blocking Low observation: a crash immediately after an old-generation +zero-to-tagged location CAS may leave non-visible generation-fenced residue +until recovery or reuse cleanup; that residue cannot project bytes or overwrite +a future nonzero word. + +### Historical `844448e` pre-qualification aggregate + +On commit `844448e` before the later reclamation/benchmark delta, both Windows +x64 and Linux x64 Release +aggregates passed with zero skips: solution build 0 warnings/0 errors, Unit +415/415, Contract 117/117, Integration 295/295, Interop 75/75, and +Linearizability 83/83, for 985/985 tests per platform. Documentation validation, +`git diff --check`, and qualification `ValidateOnly` also passed on both +platforms. These are pre-qualification results and do not replace the immutable +final evidence gate. + +### Projection lifetime race closure + +The final projection re-review found and closed the raw-visibility failure in +which an exact active lease could read `DescriptorLength` successfully and then +receive an empty `DescriptorSpan`. Each property had performed a separate +preflight plus projection validation, and a legal +`Published(g) -> RemoveRequested(g)` transition between the validator's slot +control reads was treated as an invalid snapshot even though the active lease +still protected immutable metadata and bytes. + +The closed implementation now: + +- retries the bounded projection snapshot across that single legal forward + transition; +- re-proves the exact Active lease and a stable slot control before terminally + latching an impossible lifecycle or invalid mapped metadata; +- treats a copied-token release/reclaim/reuse race as benign lease expiry rather + than mapped corruption; +- keeps `ValueLease` profile-neutral and one-pass by making both engines own + invalid-token projection semantics, including legacy post-release guards; and +- appended checkpoint 65 for the metadata/control revalidation window, routed it + through the crash agent and canonical catalog, raises the PR recovery count to + 65, and added non-Participant/non-Disposal checkpoint IDs 62, 63, and 65 to the + exact suspension configuration. + +Independent reviewer executions on the resulting worktree passed: + +- 4/4 selected deterministic projection, copied-release, lifecycle-corruption, + and mapped-metadata-corruption unit cases; +- 9/9 legacy and lock-free lease contract cases; and +- 2/2 cross-process raw-visibility integration cases after the final one-pass + getter change. + +At that intermediate projection-review stage, focused implementation validation +completed the then-current 65/65 canonical checkpoint crash catalog. The later +directory closure above extends and revalidates the current catalog at 67/67. +These worktree results close the projection review with **no unresolved High or +Medium finding**; they do not claim or replace the immutable final release +evidence. + +The focused results above establish closure of the review findings; they are not +a substitute for the immutable final qualification artifacts below. + +### R2 qualification convergence and final re-review + +The first immutable candidate remains preserved. Its Linux report passed on +`73accd7`, but the same-source `009-final-pr` summary correctly failed after +1,013/1,013 tests because SC-017 configured 46 directory mutations while the +source catalog contained 50. The Linux report SHA-256 is +`6C9727CCD6C412CD9B036E2107DA389F7359281A630C120BF48919FD987668B2`; +the failed PR summary SHA-256 is +`E1B5FA0D0EF419388054B1AA310A292DB689ABBB7F33DDA5BD377166021CB79E`. +Neither is promoted as final evidence. + +R2 diagnostics subsequently rejected an obsolete atomic-test parent timeout, +ambiguous churn-test cardinality, and checkpoint 62/63 suspension boundaries. +The atomic implementation passed isolated Windows and Linux OS reports. The +qualification runner now binds churn to the one exact SC-016 method, binds each +leak assertion to its exact producing step, requires exactly one passed TRX row, +and verifies the configured top-level class plus its directly declared test +method. Its negative self-test covers role swaps, joint-mapping drift, nested +types/methods, wrong source, and wrapper/XML failure propagation. Checkpoint 62 +preserves its primary reservation outcome through cleanup. The production +reclaimer now performs checkpoint 63 before its final budget check, so a finite +remover resumed after expiry leaves a helpable `RemoveRequested` transition +rather than claiming `Reclaiming` with a stale deadline. + +The clean `009-r2-pre-final4-pr` diagnostic passed all 24 rows on exact commit +`ca200238423877044d841a3b92f93edb37385d46`, tree +`5f4454fa25bbfb67ea0e687153ef94470ca11112`, and source-manifest SHA-256 +`952FB17E4D08B5C4973A0809F0FC3541E5409B32991A07C6416709CB6C0CD5BF`. +It passed 1,014/1,014 tests, exact churn, all 50 SC-017 configurations, and all +108 suspension rows. Its summary SHA-256 is +`3D543FBF5E4C4A5C514C1C3309565F0B2575126E5B176B79E347802AC46E331A`. +Independent production, qualification-harness, and test/spec re-reviews found +no unresolved High or Medium issue. The accepted Low observations are limited +to generation-fenced, non-visible directory residue after a precisely timed +crash, which recovery/reuse can clean and which cannot project or overwrite +bytes, and the bounded Linux cold-open cost of scanning a flat rendezvous +namespace after extreme historical store-name churn, which does not touch an +open store's hot path. +These results establish convergence only. The decision remains pending until +the new final paths below pass on one immutable source state. + +### Rejected R2 final invocation + +The R2 Linux command was mistakenly started by Windows PowerShell. The preserved +schema-3 report at +`artifacts/lock-free-os-validation/009-final-r2-linux-x64.json` correctly +classified the host as Windows and returned `not-qualified`: every completed +architecture, atomic, raw, no-lock, crash, Release-test, Docker, sample, and pack +row passed, but Windows lacked the required native/Python prerequisites and +Linux-only tiny-performance, `strace`, and SIGSTOP rows were optional and +inapplicable, so the report could not satisfy the intended Linux contract. +Its SHA-256 is +`090651C119CADD7DAC2C545D04F595FD9E65F5CEFAFC1B9B79EDA009F66EAA7C`. +No production or harness defect was implicated, but the immutable R2 candidate +is rejected in full. + +The replacement command now explicitly enters an Ubuntu login shell before +running PowerShell. A Linux-x64 validation-only execution passed every +structural row with report SHA-256 +`AE16F3AED1A9E0FE5113F20AD3AB2F28AE194E5CF0717F6372BF87362A51666C`. +That validates host selection and orchestration, not release evidence. The R2 +exact-HEAD product, harness, and freeze reviews found no new High or Medium +issue; the accepted cold-open Low remains unchanged. + +### Rejected R3 final prerequisite + +The corrected Ubuntu command reached Linux x64, but the R3 report stopped at +its first executable row before restore or any store workload. `dotnet --info` +exited 1 because the package-managed SDK 10.0.109 consumed stale user-local +workload-set 10.0.102 metadata with missing manifests. The immutable report is +`artifacts/lock-free-os-validation/009-final-r3-linux-x64.json`, SHA-256 +`59419F78D559829A0E3B47AE1FAF334B5B79E83CAF8004646A1FE3687C5B86D5`. +All nine structural self-tests passed; no production row ran. This is an +environment prerequisite failure, not a product or harness finding, but it +rejects R3 in full. + +With no workload installed, `dotnet workload repair` was correctly a no-op. +`dotnet workload update` advanced the user-local manifest set to 10.0.109.1. +The repaired executable Linux architecture diagnostic then passed +`dotnet --info`, clean restore/build, and 45/45 tests; its report SHA-256 is +`799A41E8181AA299E0E0E925E3F2818935F2F1842EA94FC66C1AC6E1D6EA7F0A`. +The R4 sequence adds a non-artifact-consuming prerequisite probe before +allocating its immutable Linux path. Product code and the accepted cold-open +Low are unchanged. + +### Rejected R4 candidate and disposal test-contract correction + +R4 used clean commit `b19d982b76f2f54ebfb9f72e0f2aef85eb2632b8`, tree +`c5a94aeff5f098915f77ef9a0af8ed8e8f730571`, empty-status SHA-256 +`E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855`, +and source-manifest SHA-256 +`7033447087954594BC982FF2AD859D282527B1A2F342199F404CA9358AEAE921`. +Its Linux report passed all 28 required rows, the 1,014-test suite, native, +Python, Docker, samples, package validation, crash/trace/SIGSTOP gates, and the +exact raw performance matrix. Independent recomputation found H0/M0/L0. The +report, evidence tree, and raw-performance SHA-256 values are respectively +`1346A080DFC7B397437F26088C16C074B9BA1AE3A4E3515D2E07D2DDAA7E2BDE`, +`6DEEBF6450AA909DA6D81E1F9725F9B3CD8DFE0534AE24FE7FC2985E828A5596`, +and `2716C25672FDB332345059F83FAFDD0063E9CE8C6635B7FE198E9A4053FF6D41`. + +The same-source R4 PR summary correctly failed after 1,013/1,014 tests. Its sole +failure was the `RecoverLeases` row of +`EveryOperationAndTokenCallbackHasOnlyDocumentedDisposalRaceOutcomes`; the +summary SHA-256 is +`7A3754C670A3622C569CB5E6AFFF479C9C668D20975E31641113337213FAD177`. +The theory raced `RecoverCurrentProcessLeases: true` with a second same-process +handle's ordinary lease work even though the administrative override requires +process-wide quiescence. The reservation row carried the symmetric latent +defect. A legal recovery could therefore invalidate the second handle's lease +between acquire and projection; this was an invalid test schedule, not a store +corruption or progress failure. The same theory also dereferenced borrowed span +bytes even though racing disposal could end their public lifetime. + +The correction is confined to the disposal integration test: both concurrent +recovery cases now select `false`, and projection races assert only the safe +empty/full length outcome. Content is still verified through the unaffected +second handle and independent publication/acquisition suites. An unbound local +stress observation completed 50 repeated 15-row theory executions; it is not +promoted as durable evidence. The provenance-bound run passed 29/29 disposal- +class tests, Integration 302/302, the full solution 1,014/1,014, and independent +diff review closed with H0/M0/L0. +The clean official `009-r5-pre-final-pr` diagnostic then passed all 24 gates on +commit `527d451bd124dbbe8880fcb909b6e1bd70ad222a`, tree +`845d5f7f1ec7e816e96de6531810bf01549e8f12`, and source-manifest SHA-256 +`6988C966FD0514309635708EA5EB898F0B500D83609507EA6C30B63C55F228D7`. +Its summary and synchronization-probe SHA-256 values are +`A7C28308FCF34CF8D5703CAAAFC9039CEA839EE4AF41CBA4752B9A3455B4C8EF` +and `4ECA1C4A75DFF514FBD3AB54D6533A3DBB77DF5AD024C5A2311287301E9A8DA8`. + +R4 remains rejected because its PR tier failed and emitted no sync probe; its +nightly and release tiers were not run. The R4 Linux pass is valuable diagnostic +evidence but cannot qualify the changed R5 source. Production synchronization, +layout, public API, and the accepted Low observations remain unchanged. + +### Rejected R5 release convergence and R6 harness closure + +R5 froze clean source commit `0a1babaf2b72234a6aac3ef12121a7fee4cf594b` +after the test-only disposal correction. Its immutable Linux, PR, and nightly +artifacts passed, so they remain valuable historical evidence. The release run +is nevertheless rejected and preserved incomplete at +`artifacts/lock-free-qualification/009-final-r5-release`: its first Legacy +mixed-churn trial had not completed after more than 141 minutes because the +harness imposed 100,000,000 operations on the named-semaphore baseline. Fourteen +workers remained live and a read-only mapped sequence advanced by 27,120 in 30 +seconds, proving slow progress rather than deadlock. The remaining two Legacy +mixed trials, Legacy 100,000-frame ingest, and all LockFree rows could not fit +the six-hour whole-step deadline. R5 therefore cannot qualify the release. + +R6 preserves both profiles and every scenario. It changes completion policy at +the qualification boundary: Legacy mixed-churn and large-ingest use the same +10-second warm-up and 60-second measurement as other duration rows; LockFree +mixed-churn and large-ingest retain exact per-trial durability counts. Probe +schema 8 records `CountBoundProfiles`, `OperationTarget`, `FrameTarget`, and the +duration grace; minimum-compatible schema 8 prevents older global-target +readers from qualifying the report. Both importers reject missing, inherited, +swapped, dual, noncanonical, short-duration, or unmet evidence. + +The duration controller is an isolated-process hard boundary, not cancellation +of an in-flight store call. It arms before store creation, tracks every child +start, remains live through warm-up, measurement, result collection, child +cleanup, store disposal, and affinity disposal, and latches completion against +an absolute `Stopwatch` deadline. A late or fired deadline gives child-tree +termination 100 ms of best-effort work and then calls `Environment.FailFast` +unconditionally. Count-bound rows and fixed-work sticky-overflow do not arm this +watchdog. Five direct watchdog cases cover delayed/early dispatch, ordinary +completion, late completion, and a timer forcing `Completing -> TimedOut` while +the completing thread is blocked. Focused runtime smoke, synthetic importer +mutations, and independent adversarial review close +the R6 harness with no unresolved High or Medium finding. Production layout, +synchronization, public API, and steady-state paths are unchanged. + +## Accepted design boundaries + +These are intentional contract boundaries, not open findings: + +- The product remains a bounded shared-memory **key-value store**, not a stream, + queue, broker, or worker scheduler. External brokers distribute keys; any + publisher, worker, observer, or other process may independently access the + store. +- Layout 2.0 steady-state publish, reserve, commit, acquire, release, remove, + reclaim, and recovery paths do not acquire a named process-wide semaphore. + OS-backed serialization is confined to bounded cold lifecycle coordination. +- Layout 1.2 remains the default legacy profile. Layout 2.0 is explicit, + fail-closed, x64-only in this release, and has no in-place conversion path. +- Participant-table exhaustion remains explicit. Open returns + `ParticipantTableFull` and performs no hidden recovery or liveness scan. +- Recovery is cooperative and caller initiated; there is no hidden maintenance + thread. Ordinary callers may help published owner-free transitions without + reclaiming resources from a live owner. +- Current-process lease or reservation recovery overrides are administrative, + process-wide quiescent operations. Normal concurrent recovery uses the safe + non-override mode. +- Linux PID identity is interpreted only in its exact namespace. Once an opener + cannot prove same-namespace operation, the store irreversibly enters mixed + mode and preserves ambiguous live/registering ownership conservatively. +- Linux owner anchors are cold-lifecycle liveness evidence, not operation locks. + Cleanup deletes only well-formed, unreferenced, provably unlocked artifacts; + locked, ambiguous, malformed, and special-file entries are retained + conservatively. + +## Final evidence gate + +All of the following prelinked artifacts must exist and report success: + +1. [artifacts/lock-free-qualification/009-final-r6-pr/summary.json](../../artifacts/lock-free-qualification/009-final-r6-pr/summary.json) +2. [artifacts/lock-free-qualification/009-final-r6-pr/sync-probe.json](../../artifacts/lock-free-qualification/009-final-r6-pr/sync-probe.json) +3. [artifacts/lock-free-qualification/009-final-r6-nightly/summary.json](../../artifacts/lock-free-qualification/009-final-r6-nightly/summary.json) +4. [artifacts/lock-free-qualification/009-final-r6-nightly/sync-probe.json](../../artifacts/lock-free-qualification/009-final-r6-nightly/sync-probe.json) +5. [artifacts/lock-free-qualification/009-final-r6-release/summary.json](../../artifacts/lock-free-qualification/009-final-r6-release/summary.json) +6. [artifacts/lock-free-qualification/009-final-r6-release/sync-probe.json](../../artifacts/lock-free-qualification/009-final-r6-release/sync-probe.json) +7. [artifacts/lock-free-qualification/009-final-r6-release/os-validation.json](../../artifacts/lock-free-qualification/009-final-r6-release/os-validation.json) +8. [artifacts/lock-free-os-validation/009-final-r6-linux-x64.json](../../artifacts/lock-free-os-validation/009-final-r6-linux-x64.json) +9. [artifacts/lock-free-os-validation/009-final-r6-linux-x64.evidence/linux-tiny-performance.json](../../artifacts/lock-free-os-validation/009-final-r6-linux-x64.evidence/linux-tiny-performance.json) + +Approval requires the machine-checkable evidence to show that: + +- the PR, nightly, and release profiles completed with their required checks + passed; +- the release synchronization probe and Windows OS validation passed; +- the Linux x64 OS validation passed its required checks, including the exact + schema-8/minimum-8 one/eight-process tiny-operation raw matrix; one-process intrinsic + p99, eight-process throughput, <=3x self-amplification, <=10 us p99, and + every-lock-free-trial 10 ms stall ceilings for both scenarios; +- both exact sibling OS evidence trees, manifests, executable logs, and accepted + report/tree digests passed initial and completion validation; +- every artifact identifies the same clean commit and identical source manifest; +- the independent qualification-harness and production-code re-reviews have no + unresolved High or Medium finding; and +- the tracked tree was not edited after evidence collection. + +No final release-evidence status, commit hash, source-manifest hash, or artifact +hash is asserted in this review before those files are produced. If the nine +artifacts satisfy the conditions above, the release decision is **final +approved**. Otherwise the code and harness review remains closed, but the +feature is **not approved for release** until the failing gate is rerun on one +identical clean source state. diff --git a/specs/009-lock-free-publish-read/compatibility-results.md b/specs/009-lock-free-publish-read/compatibility-results.md new file mode 100644 index 0000000..62f7452 --- /dev/null +++ b/specs/009-lock-free-publish-read/compatibility-results.md @@ -0,0 +1,174 @@ +# Compatibility Matrix Results + +**Feature branch**: `codex/lock-free-csharp` +**Qualification time**: 2026-07-13 06:41 UTC +**Repository base commit**: `0cf7a43f9c39de1691b237a9761035339edd0964` +(`Prepare release 1.0.2`) plus the uncommitted feature-009 worktree +**Result**: PASS for the Windows x64 compatibility matrix; Linux-specific +owner-sidecar rows are NOT QUALIFIED on this host. + +## Environment + +| Item | Qualified value | +|---|---| +| Host OS | Microsoft Windows 11 Pro, version `10.0.26200`, build `26200`, 64-bit | +| Process architecture | x64 | +| Processor | 13th Gen Intel Core i9-13900, 24 cores / 32 logical processors | +| .NET SDK | `10.0.201` | +| Test host runtime | .NET `10.0.5`, x64 | +| Test adapter | xUnit VSTest Adapter `3.1.4+50e68bbb8b` | +| Current package under test | `SharedMemoryStore 2.0.0` | +| Released compatibility client | `SharedMemoryStore 1.0.2` loaded from its packed NuGet assembly in an isolated `AssemblyLoadContext` | +| Native prerequisites | C++ interop agent present and executed; Python executable and C-ABI DLL present and executed | + +PASS below means that the scenario actually executed on this host. NOT +QUALIFIED means that the test has an explicit platform guard and could not +exercise the claimed platform behavior here. Two Linux-only test methods return +early rather than reporting an xUnit skip, so the raw xUnit totals include them +as passed; this report deliberately does not count them as qualified passes. + +## Result matrix + +| Matrix lane | Executed evidence | Result | +|---|---|---| +| Current C# 2.0, layout-v1.2-only lifecycle | 12/12 legacy integration cases passed: cross-process publication, exact large value/descriptor, multi-reader leases, remove/reuse, direct reservation ingest, segmented publication, stale reservation fencing, and concurrent acquire/release/duplicate publication | PASS | +| Layout-v1.2 cross-runtime ordered pairs | 9/9 producer/consumer pairs passed for .NET, C++, and Python in both directions, including same-runtime pairs | PASS | +| Current C# 2.0, layout-v2-only lifecycle | 16/16 cases passed: publication races and bounds, 1/6/12 same/distributed-key readers, paused observer progress, 12-process lease/remove/reclaim, collision churn, and live diagnostics | PASS | +| Public/package/profile contract surface | 107/107 contract cases passed, including unchanged legacy signatures/status numbers, explicit v2 selection, layout identities, package metadata, and XML documentation | PASS | +| Existing v1.2 opened as v2 | 2/2 Windows x64 `OpenExisting`/`CreateOrOpen` header-first oversized-view cases returned `IncompatibleLayout`; existing legacy handle remained usable | PASS | +| Existing v2 opened as v1.2 | 2/2 Windows x64 `OpenExisting`/`CreateOrOpen` header-first oversized-view cases returned `IncompatibleLayout`; existing v2 handle remained usable | PASS | +| Opposite-profile `CreateNew` | 2/2 directions returned `AlreadyExists` for the same public name | PASS | +| V2 participant open/close lifecycle | 2/2 scenarios passed: capacity is consumed per handle and reusable after close; final close permits a new same-name `CreateNew` lifecycle | PASS | +| Released C# 1.0.2 against a live v2 mapping | 9/9 requested-view/open-mode combinations passed (`smaller`, `equal`, `oversized` x `CreateNew`, `OpenExisting`, `CreateOrOpen`). All attempts failed closed and the v2 payload remained readable/writable afterward | PASS | +| Default/current legacy helper against v2 | 1/1 passed: default and legacy helpers did not auto-select or reinterpret an existing v2 mapping | PASS | +| Same-name upgrade and rollback | 1/1 passed: live opposite profiles rejected one another; after all handles closed, v1.2 -> v2 -> v1.2 recreation succeeded; values did not leak across layouts and had to be republished | PASS | +| Current packed 2.0.0 consumption | Direct package-consumption script passed: clean `net10.0` project restored only from the local package, then exercised legacy publish/acquire/remove/direct-ingest/segments/recovery/disposal and explicit v2 publish/acquire/remove/profile/protocol/participant-capacity behavior. Its xUnit wrapper also passed 1/1 | PASS | +| Broker-key sample | 2/2 processes passed with 6 and 12 workers. Broker dispatch remained outside the KV store; processed/checksum counts, pending removal, missing key, and explicit zero-recovery results matched | PASS | +| C++ v1.2-only rejection of SMS2 | Actual C++ agent executed all 3 open modes, returned the required fail-closed status, never damaged payload visibility, and left a second C# v2 opener usable | PASS | +| Python v1.2-only rejection of SMS2 | Actual Python/C-ABI agent executed all 3 open modes, returned the required fail-closed status, never damaged payload visibility, and left a second C# v2 opener usable | PASS | +| Compatibility manifest | 1/1 passed: package, layout, resource protocol, C ABI, C++, and Python identities/support sets remain independently versioned | PASS | +| Linux v2 live-owner sidecar during current legacy incompatible open | Linux-only case could not execute on Windows | NOT QUALIFIED | +| Linux v2 live-owner sidecar during released-1.0.2 incompatible open | Linux-only case could not execute on Windows | NOT QUALIFIED | +| Full Linux x64 compatibility matrix | No Linux x64 runtime was used in this task | NOT QUALIFIED | + +No compatibility row failed. Across the commands below, xUnit reported 172 +passing cases. Of those, 170 actually executed their qualified scenario on +Windows x64 and two were Linux-guarded early returns. The direct package script +is additional executable evidence rather than an extra xUnit case. + +## Package and native artifact hashes + +These hashes bind the exact artifacts used or produced during this matrix. The +current package hash is from the final direct package-consumption run. + +| Artifact | Bytes | SHA-256 | +|---|---:|---| +| `artifacts/package/SharedMemoryStore.2.0.0.nupkg` | 100,313 | `C3B4E3441219FF0C5FBA63F89FD2EACC85C16F8E1DD845E334E35D572857D84D` | +| `artifacts/package/SharedMemoryStore.2.0.0.snupkg` | 44,078 | `EF78FDC0A9FAFDC593EEECA5AE5F88940026BE925D764C263D22FB0D81C0E7BE` | +| `%USERPROFILE%/.nuget/packages/sharedmemorystore/1.0.2/sharedmemorystore.1.0.2.nupkg` | 48,991 | `A3DB3D86AFD89CECC932D1A8D307A18A842A6CE7913C8C9107C4B404B8821100` | +| `artifacts/native-win/sms_cpp_interop_agent.exe` | 230,912 | `0BE45C93BE88AAF853B2981C1A8BA0F998CE73861540049DB2A166A45997574D` | +| `src/python/shared_memory_store/shared_memory_store.dll` | 1,588,736 | `88349499AA87CA6107B047C9C8ED36E39C16F74DE17AC948F615DFBBE96035BA` | + +NuGet archives contain generated ZIP metadata, so a later repack can have a +different byte hash even when compiled inputs are unchanged. Requalification +must record the newly produced package hash rather than assume this value. + +## Exact commands and counts + +All commands ran from the repository root with configuration `Release`. + +1. Pack gate -- completed successfully and produced both `.nupkg` and + `.snupkg`: + + ```powershell + dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/compatibility/current-package --no-restore + ``` + +2. Full public/package/profile contract project -- 107 passed, 0 failed: + + ```powershell + dotnet test tests/SharedMemoryStore.ContractTests/SharedMemoryStore.ContractTests.csproj -c Release --no-restore --logger "console;verbosity=normal" + ``` + +3. Same-name mixed-profile and v2 participant lifecycle -- xUnit reported 9 + passed; 8 Windows scenarios executed and one Linux-only case was not + qualified: + + ```powershell + dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --no-restore --filter "FullyQualifiedName~LockFreeProfileOpenIntegrationTests" --logger "console;verbosity=normal" + ``` + +4. Released-1.0.2, participant-capacity, upgrade/rollback, and default-profile + package matrix -- xUnit reported 13 passed; 12 Windows scenarios executed + and one Linux-only case was not qualified: + + ```powershell + dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --no-restore --filter "FullyQualifiedName~LockFreePackageIntegrationTests" --logger "console;verbosity=normal" + ``` + +5. Broker-key sample validation -- 2 passed, 0 failed: + + ```powershell + dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --no-restore --filter "FullyQualifiedName~LockFreeSampleValidationTests" --logger "console;verbosity=normal" + ``` + +6. Packed-consumer xUnit wrapper -- 1 passed, 0 failed: + + ```powershell + dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --no-restore --filter "FullyQualifiedName~PackageConsumptionIntegrationTests" --logger "console;verbosity=normal" + ``` + +7. Direct clean packed-consumer validation -- exit code 0; all printed legacy + and v2 assertions reached success: + + ```powershell + powershell -NoProfile -ExecutionPolicy Bypass -File scripts/validate-package-consumption.ps1 -Configuration Release + ``` + +8. Current C# layout-v1.2 lifecycle matrix -- 12 passed, 0 failed: + + ```powershell + $classes = @('CrossPlatformStoreIntegrationTests','PublishIntegrationTests','MultiReaderAcquireIntegrationTests','RemoveReuseIntegrationTests','ZeroCopyIngestIntegrationTests','SegmentedFrameIntegrationTests','ReservationReuseSafetyIntegrationTests','AcquireReleaseConcurrencyTests') + $filter = $classes | ForEach-Object { "FullyQualifiedName~SharedMemoryStore.IntegrationTests.$_." } + dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --no-restore --filter ($filter -join '|') --logger "console;verbosity=normal" + ``` + +9. Current C# layout-v2 lifecycle matrix -- 16 passed, 0 failed: + + ```powershell + $classes = @('LockFreePublishIntegrationTests','LockFreeMultiReaderIntegrationTests','LockFreeBroadcastLeaseIntegrationTests','LockFreeChurnIntegrationTests','LockFreeDiagnosticsIntegrationTests') + $filter = $classes | ForEach-Object { "FullyQualifiedName~SharedMemoryStore.IntegrationTests.$_." } + dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --no-restore --filter ($filter -join '|') --logger "console;verbosity=normal" + ``` + +10. Native/Python SMS2 rejection and manifest -- 3 passed, 0 failed. The C++ + and Python theory cases both executed rather than returning at their + prerequisite guard: + + ```powershell + dotnet test tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj -c Release --no-restore --filter "FullyQualifiedName~LockFreeLayoutRejectionTests" --logger "console;verbosity=normal" + ``` + +11. Layout-v1.2 .NET/C++/Python 3x3 ordered exchange matrix -- 9 passed, 0 + failed: + + ```powershell + dotnet test tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj -c Release --no-restore --filter "FullyQualifiedName~CoreExchangeMatrixTests" --logger "console;verbosity=normal" + ``` + +## Conclusions + +- The current package preserves a usable layout-v1.2 profile and requires + explicit opt-in for layout 2.0. +- Same-name mixed layouts fail closed before projecting an incompatible-sized + view; `CreateNew` retains its established `AlreadyExists` meaning. +- Released C# 1.0.2, C++ 0.1, and Python 0.1 cannot accidentally participate in + v2 data operations. Their rejection attempts leave the live v2 store usable. +- Upgrade and rollback are correctly modeled as drain, close, recreate, and + application-owned republish. No bytes cross a recreated layout boundary. +- The broker-key sample validates the intended separation: external code sends + keys to workers while SharedMemoryStore remains a bounded shared-memory KV + store. +- Windows x64 compatibility is qualified by this run. Linux x64 compatibility, + especially v1-compatible owner-sidecar preservation, still requires a native + Linux qualification run and must not be inferred from these results. diff --git a/specs/009-lock-free-publish-read/contracts/compatibility-and-rollout.md b/specs/009-lock-free-publish-read/contracts/compatibility-and-rollout.md new file mode 100644 index 0000000..e14d541 --- /dev/null +++ b/specs/009-lock-free-publish-read/contracts/compatibility-and-rollout.md @@ -0,0 +1,231 @@ +# Compatibility and Rollout Contract + +## Independent versions + +| Surface | Existing | New | +|---|---|---| +| NuGet package | 1.0.2 | target 2.0.0 | +| Mapped layout | 1.2 | 2.0 | +| Resource protocol | 1 | 2 | +| C ABI | 1.0 | unchanged; v2 unsupported | +| C++ package | 0.1.0 | unchanged; v2 unsupported | +| Python package | 0.1.0 | unchanged; v2 unsupported | + +Package, layout, resource, and C ABI versions never imply one another. The +compatibility manifest lists each distribution's create/read set explicitly. + +Layout 2.0 is not released yet, so the approved generation-tagged directory +operation/location encoding replaces the earlier draft rather than creating a +2.1 migration. The published 2.0 contract includes the `SlotCount <= 1,048,575` +limit, exact generation-tagged encodings, the versioned spill summary, and the +`PublicationIntent` field at value-slot byte offset 52, and PID-namespace +identity/mode at header offsets 264/272 plus participant offset 32. Its required- +features mask is exactly 7: bit 0 is `versioned_empty_spill_summary`, bit 1 is +`publication_intent`, and bit 2 is `pid_namespace_identity`. Any prototype built +with required-features zero, bit 0 alone, mask 3, or another superseded draft is incompatible in both directions and must +recreate its mapping; no draft-format auto-detection or in-place conversion is +supported. + +## Profile matrix + +| Requested client/profile | No mapping | Existing v1.2 | Existing v2.0 | +|---|---|---|---| +| C# 2.0 / Legacy | Create/open v1.2 per `OpenMode` | Success if dimensions match | `IncompatibleLayout` | +| C# 2.0 / LockFree | Create/open v2.0 with required-features mask 7 per `OpenMode` | `IncompatibleLayout` | Success only if dimensions and the exact required-feature set match | +| Already released C# 1.x | Create/open v1.2 | Existing behavior | Fail closed before layout data access: ordinarily `IncompatibleLayout`; `CreateNew` is `AlreadyExists`; oversized Windows views may surface existing `AccessDenied` | +| C++ 0.1 / Python 0.1 | Create/open v1.2 | Existing behavior | Deterministic incompatible result; never payload access | + +`CreateNew` still reports `AlreadyExists` when either profile already owns the +same public name. `OpenExisting` never creates a parallel mapping. Dimension or +feature mismatch within the requested major returns `IncompatibleLayout` or the +existing precise capacity/options result before unsafe access. + +An existing all-zero/unpublished header is not treated as an empty store. +Physical creation, rather than `OpenMode`, profile, dimensions, or zero magic, +is the sole initialization authority. Every profile pairing preserves the +existing bytes and reports `AlreadyExists` for `CreateNew`, `StoreBusy` for +`CreateOrOpen`, or `IncompatibleLayout` for `OpenExisting`. + +New C# 2.0 legacy-profile code uses a header-sized probe and returns +`IncompatibleLayout` for `SMS2` independently of requested dimensions. Immutable +released binaries have a safety, not exact-status, guarantee: they never return +success or access layout data. Their platform/view-size-dependent existing +non-success result is preserved and tested with the packed released version. + +## Physical resource identity + +For a public store name, v2 derives the same Windows named mapping and Linux +`.region`, `.owners`, and `.lifecycle` paths as resource-naming v1. This is +intentional fail-closed discovery. + +Resource protocol 2 changes participation: + +- one cold transaction covers physical discovery/creation, header + initialization or validation, owner publication, and participant registration + under the caller's original wait/cancellation budget; +- Windows acquires the existing named synchronization object before creating, + opening, or mapping the region and retains it through the complete cold + transaction; +- Linux acquires `.lifecycle`, reconciles markers and deletes only proven stale + resources, then opens/acquires `.lock` before mapping, owner-anchor locking, + owner-line commit, header work, and participant registration; release is + `.lock` followed by `.lifecycle`; +- failed-open mapped-resource and owner cleanup occurs only after those gates + are released, and its ordinary synchronization descriptor is disposed before + mapped-owner cleanup may re-enter `.lifecycle`; a contender rejected before + mapping publishes no owner line or release marker; +- only the attempt whose physical operation created a new region may initialize + a zero header; an opened-existing zero header has the fixed outcomes above; +- Linux owner registration/final cleanup remains under `.lifecycle`; +- every v2 Linux handle continues writing a resource-naming-v1-compatible live + `.owners` line in addition to its mapped participant record, preventing an old + opener from misclassifying/deleting a live v2 region as stale; +- every current managed Linux handle holds a private mode-`0600` + `.owners.anchor.` open-description `flock` for its mapped lifetime; + the canonical path is the exact per-store `.owners` path plus `.anchor.` and + exactly 32 lowercase hexadecimal GUID digits; + under `.lifecycle`, locked is authoritative live evidence across PID namespace + views, unlocked is stale, missing falls back to PID/start-token for C++, + Python, and older managed owners, and ambiguous/access/symlink results are + retained conservatively; +- the anchor is a managed cold-lifecycle extension: it does not change the + three-field owner line, participant layout, or interoperable `.lock`/`.lifecycle` + record locks, and it is unreachable from every v2 data operation; +- after unmapping, close/failed-open cleanup waits no more than 250 milliseconds + for `.lifecycle`; on contention or pre-commit failure it atomically publishes + one private exact-owner release marker, which a later v2 lifecycle operation + reconciles by raw exact-line removal and atomic owner-sidecar rewrite before + marker deletion; this fallback may leave the compatible owner line, anchor + pathname, or both until that later reconciliation; +- orderly close unlocks/deletes its anchor only after exact sidecar absence or + successful finalized-marker publication; if neither succeeds it retains the + lock, while process death releases it automatically and later lifecycle + cleanup removes a stale artifact only when its conservative independent probe + proves deletion safe; +- after each replacement `.owners` sidecar commit under `.lifecycle`, current C# + cleanup derives referenced tokens from canonical committed owner lines and + sweeps only canonical anchors for that store; it opens each unreferenced + candidate through a separate `O_NOFOLLOW` descriptor, verifies a regular file, + and deletes it only while a nonblocking exclusive `flock` is held; +- referenced or locked anchors, ambiguous probes, non-regular files, symbolic + links, directories, malformed names, and enumeration/open/stat/lock/delete + access errors are retained; final-owner cleanup never broad-glob deletes them; +- v1 C#, C++, and Python clients ignore these additive marker files and remain + conservative because the compatible owner line stays present until v2 + reconciliation or ordinary process-liveness cleanup; +- C# 2.0 applies the bounded owner-cleanup extension to both mapped profiles + because their Linux sidecars are shared, without changing layout-v1.2 bytes or + its ordinary per-operation locking contract; +- current C# and native Linux adapters use nonblocking `F_OFD_SETLK` on + `.lock`/`.lifecycle`, fail closed when OFD locks are unavailable, and contend + across managed assembly-load contexts and native modules in one PID; current + cleanup retains both empty lock inodes, and every close retires ordinary + synchronization before region/owner cleanup; +- released process-associated-`F_SETLK` clients remain compatible in separate + processes because traditional and OFD locks conflict; concurrent old and + current lock implementations inside one OS process are not supported because + closing a sibling descriptor can release the old implementation's lock; +- v2 releases/disposes the ordinary synchronization handle after open validation + or retains it only as a cold-path object that is unreachable from data paths; +- v2 steady-state operations never enter the Windows named mutex/semaphore or + Linux `.lock` file; +- v1.2 participants continue using resource protocol 1 for every data operation. + +The implementation must open/query an existing mapping sufficiently to read its +identity/header even when the caller-calculated length differs. A requested-size +view failure must not mask a readable incompatible major as generic +`MappingFailed`. + +## Public API compatibility + +- Existing method and enum signatures remain present. +- Existing status numeric assignments 0-22 remain unchanged. +- `SharedMemoryStoreOptions.Create(...)` and the five-dimension + `CalculateRequiredBytes(...)` remain legacy helpers. +- New profile members/helpers are additive. +- V2 participant capacity is explicit (`ParticipantRecordCount`, default 64) and + exhaustion appends `StoreOpenStatus.ParticipantTableFull` without renumbering + previous values. +- Default/manual zero enum value remains legacy. +- Layout-v1.2 tests continue exercising the original synchronization and mapped + bytes without reinterpretation. +- Public lease/reservation structs remain process-local values. Their private + representation may grow to carry v2 incarnations; because that changes runtime + size/AOT assumptions, the package uses a conservative major version and tests + supported binary/source consumption explicitly. + +## Native/Python rejection + +C++ and Python in this feature do not implement layout-v2 atomics or recovery. +Their open sequence must: + +1. discover the same physical region; +2. read enough validated header bytes to identify `SMS2`/major 2; +3. return the existing incompatible-layout public/ABI result; +4. perform no directory, slot, lease, descriptor, or payload read/write; +5. leave the mapping usable by C# v2 participants. + +Executable ordered-pair tests cover a C#-created v2 mapping opened by C++ and +Python on Windows/Linux. C ABI 1.0 remains byte-for-byte unchanged. Future v2 +native support requires an explicitly versioned ABI addition and the exact +layout/memory-order/recovery contract. + +## Upgrade paths + +There is no in-place conversion or dual writer mode. + +### Same-name cutover + +1. Stop key delivery/publication and drain application work. +2. Release leases/abort reservations and close every v1.2 participant. +3. Application owns any data migration: copy/republish required values elsewhere + before final close if needed. +4. Let normal final-owner cleanup remove the v1.2 mapping, or perform documented + operator cleanup after verifying no live owners. +5. Deploy C# 2.0 callers with explicit `StoreProfile.LockFree`/ + `CreateLockFree` and create v2 under the same public name. +6. Run health, protocol identity, collision, recovery, and short performance + checks before restoring traffic. + +### Side-by-side cutover + +Use a different **public store name**, not a hidden profile suffix. Publish data +to the new v2 store under application control, switch broker/application keys, +then drain the old v1.2 name. Each name independently fails closed. + +## Rollback + +V2 bytes cannot be reinterpreted by v1.2. To roll back: + +1. stop new v2 operations and drain/close v2 handles; +2. preserve/republish required application data through supported APIs; +3. remove the v2 mapping only after owner/liveness verification; +4. deploy legacy-profile participants and recreate v1.2 under the target name; +5. republish data and restore broker/application key delivery. + +An older client accidentally pointed at a still-live v2 name must fail +incompatible; deleting that mapping to make the old client start is never an +automatic library action. + +## Package and protocol artifacts + +Release work updates: + +- `SharedMemoryStore.csproj` package version/release notes/XML docs; +- `protocol/layout-v2.0.md` with exact offsets/fixtures; +- `protocol/resource-naming-v2.md`; +- `protocol/compatibility.json` with per-layout resource protocol support and + required-features mask 7; +- README migration/profile/progress/trust-boundary guidance; +- package-consumption tests for old legacy consumer, new legacy consumer, and + new lock-free consumer; +- C++/Python rejection tests and unchanged v1.2 interop matrix. + +## Release qualification + +A rollout artifact is not valid until Release `dotnet test`, `dotnet pack`, +layout fixtures, v1.2 regressions, v2 contracts, mixed-profile rejection, +package consumption, and the platform-specific lock-free qualification gates +pass. Performance results record hardware/OS/runtime, profile, mapping shape, +CPU affinity, warm-up, duration, trials, percentiles, throughput, fairness, and +status counts; they are not generalized to unmeasured environments. diff --git a/specs/009-lock-free-publish-read/contracts/concurrency-and-memory-ordering.md b/specs/009-lock-free-publish-read/contracts/concurrency-and-memory-ordering.md new file mode 100644 index 0000000..c0b4436 --- /dev/null +++ b/specs/009-lock-free-publish-read/contracts/concurrency-and-memory-ordering.md @@ -0,0 +1,552 @@ +# Concurrency and Memory-Ordering Contract + +## Progress definition + +Layout 2.0 is system-wide lock-free for steady-state data operations: if live +eligible callers continue taking steps, some operation completes in a finite +number of protocol transitions even when another participant stops at any data +transition. It is not wait-free and does not guarantee fairness for one caller +under adversarial same-key contention. + +No v2 success or normal-failure path for publish, reserve, segment write, +reservation projection/advance/commit/abort/dispose, acquire, lease projection/ +release/dispose, remove, reclaim, directory maintenance/help, diagnostics, or +explicit record recovery enters a named mutex, semaphore, file lock, global +writer flag, or process-owned exclusive state. + +The canonical bucket mutation word is a helpable operation descriptor, not a +lock: all information needed to finish is published before the word, every +contender helps rather than waits, and it is released before payload filling or +lease retention. + +## Atomic vocabulary + +The language-neutral operations are: + +| Operation | Required order | +|---|---| +| `LoadAcquire(word)` | Later metadata/byte reads cannot move before it | +| `StoreRelease(word, value)` | Earlier metadata/byte writes cannot move after it | +| `CompareExchange(word, expected, value)` | Sequentially consistent atomic RMW: full fence and one total order with every layout-v2 RMW | +| `Exchange(word, value)` | Sequentially consistent atomic RMW in the same total order | + +C# layout 2.0 implements RMW operations with `Interlocked` and loads/stores with +`Volatile` on aligned mapped `long` references. The acquire/remove proof depends +on full-fence, sequentially consistent ordering between lease-control and slot- +control RMWs; independent per-word acquire/release is incompatible. A stronger +implementation is allowed. A weaker implementation or ordinary mixed-width +access is not. + +## Externally observable ordering points + +| Operation | Ordering point | +|---|---| +| Explicit `TryReserve` key ownership | Slot control CAS `Initializing(g) -> Reserved(g)` with immutable `ExplicitReservation` intent, after insertion established no ordered exact-key owner | +| Explicit reservation commit visibility | Slot control CAS `Reserved(g) -> Published(g)` | +| Atomic `TryPublish`/`TryPublishSegments` success | Slot control CAS `Reserved(g) -> Published(g)` with immutable `AtomicPublication` intent; binding installation and internal Reserved are tentative | +| Acquire | Lease control CAS to `Active(r)` followed by successful directory/slot revalidation; it orders immediately before the first conflicting removal CAS when both succeed | +| Logical remove | Slot control CAS `Published(g) -> RemoveRequested(g)` | +| Lease release | Exact lease control CAS `Active(r) -> Releasing(r)` | +| Reservation abort | Slot control CAS from an owned invisible state to `Aborting(g)` | +| Safe reclamation | Slot control CAS `RemoveRequested(g) -> Reclaiming(g)` after a stable no-active-lease scan | +| Directory unlink | Exact binding CAS to zero under the canonical helpable descriptor | +| Slot reuse | Release publication of `Free(g+1)` after unlink/reset | +| Lease-record reuse | Release publication of `Free(r+1)` after relinquishment/reset | + +Participant registration is a cold-path prerequisite, not a data ordering point: +the opener writes PID/identity-kind/start fields and release-publishes an +`Active` participant record under the lifecycle lock before returning a handle. +Its index is embedded by the first slot/lease claim CAS. + +An acquire that activates a record but fails final revalidation never completes +successfully and immediately relinquishes that record. Thus its activation is an +internal attempt, not a public acquire ordering point. + +## Helpable directory mutation + +### Descriptor preparation + +The candidate/current slot contains an atomic directory-operation word encoding: + +- intent: insert or unlink; +- phase: prepared, target selected, binding changed, complete; +- target: primary lane or overflow cell index when selected; +- exact 33-bit slot generation in every nonzero phase. + +All immutable key/hash/length metadata and fixed descriptor bytes are written +before publishing insertion `Prepared`. +The mutator then CASes the canonical bucket mutation word from zero to its exact +slot binding. A stale mutation word may be cleared only after the referenced slot +generation proves it cannot describe a current operation. + +The operation generation must equal the bucket mutation binding generation and +the current slot-control generation before a helper acts. A nonzero recorded +directory location carries the same generation. This three-way generation +agreement is revalidated at each phase boundary; validation from an earlier +generation never authorizes an ordinary store or a CAS against an untagged word. + +### Helping rule + +Every caller that needs to mutate a canonical bucket first: + +1. acquire-loads its mutation word; +2. if nonzero, validates the exact slot generation and operation; +3. performs the next idempotent phase through exact full-word CAS, or clears an + exact stale descriptor/binding; +4. restarts its own lookup/mutation. + +No phase grants an unhelpable owner. Target selection is itself published by CAS +in the slot operation word before a directory-cell CAS. Therefore two helpers +cannot install one candidate binding into different cells. If another key wins +the selected empty cell, a helper CASes the unchanged target phase back to +`Prepared` and selection restarts. + +If a helper installs an old-generation binding and then discovers that its +exact operation word can no longer advance, it CAS-removes only the binding it +installed and stops. If the slot has already been reused, every old +operation/location CAS fails because generation is part of the compared word; +the helper cannot clear or complete the new lifecycle. Other callers may clear +any provably stale old-generation cell or metadata word without waiting for the +paused helper. + +Owned publication-lifecycle cancellation may race these phases. A helper +acquire-validates the immutable `PublicationIntent` after the current +generation's metadata becomes discoverable and reclassifies the exact slot +after each pause/validation window: `Aborting` or +`Reclaiming` routes to exact cancellation cleanup, a changed operation or later +generation is benign loss of authority, and only an impossible same-generation +non-cancel state is corruption. If a `BindingChanged` helper loses its +`Initializing -> Reserved` CAS, it performs this classification before +returning. Likewise, exact `Empty(binding)` observed by an overflow insertion is +benign when the same operation is canceling; it remains corruption if the exact +insert is still current in a non-cancel state because re-publishing that version +would recreate ABA. + +A directory-location publisher treats its authorization as one joint tuple: +the canonical mutation, exact operation, current location, slot control, +immutable directory binding, and selected or competing target cells. Terminal +invalid classification requires two stable acquire collections followed by +exact no-op compare/exchange confirmation of every atomic tuple member and a +fresh immutable-binding read. Any loss or movement is ordinary progress or a +budgeted retry. + +Cancellation may replace a validated Insert descriptor with `Unlink/Prepared` +before the old insert helper exact-clears its target. The delayed unlink +publisher treats target zero or a structurally valid different in-range binding +as progress and preserves the replacement; a stable malformed or out-of-range +word is corruption. Because Prepared does not name a target, independent unlink +helpers may recover different cells. The first valid location CAS wins; each +loser exact-clears only its distinct recovered old binding and follows the +winner. After `Unlink/TargetSelected`, a same-generation alternate location from +a delayed Prepared publisher is legal: helpers exact-clean the selected and +alternate old-binding witnesses plus the alternate location, while preserving +any replacement binding. A malformed alternate remains corruption. + +If an unlink publisher loses its exact operation source after its location CAS, +it withdraws only its exact old target and location. An exact committed Insert +successor or another valid replacement remains untouched. A strictly older +location is exact-cleanable residue. A future-generation location is treated as +reuse only when another member proves that the old tuple moved; a future +location enclosed by the stable exact old-generation tuple is corruption and is +preserved for diagnosis. + +Ordering is intent-specific. For `ExplicitReservation`, the exact +`Initializing -> Reserved` CAS is the public `TryReserve` ordering point and +Reserved owns the key. For `AtomicPublication`, that CAS is only an internal +prepared stage; `TryPublish`/`TryPublishSegments` order at +`Reserved -> Published`. Same-generation `Aborting`/`Reclaiming`, terminal +retirement, or a strictly newer generation before the applicable ordering point +is legal cancellation, not a duplicate-key witness. Lower generation, unknown +intent after discoverability, or any other impossible same-generation lifecycle +fails closed as `CorruptStore`. + +### Insert + +1. Claim a free slot generation and fully initialize key/length/owner metadata, + including `ExplicitReservation` or `AtomicPublication` intent. +2. Publish insertion `Prepared` and claim/help the canonical bucket descriptor. +3. Re-scan both primary buckets and logically-Present overflow for an exact + current key and classify its stable state plus intent. +4. If another exact binding is an ordered duplicate witness + (`Reserved(ExplicitReservation)`, `Published`, or `RemoveRequested`), + transition the candidate to aborting, publish terminal no-target `Rejected`, + clear the bucket descriptor, and return/recover it as duplicate. If it is + tentative (`Initializing` or `Reserved(AtomicPublication)`), help/revalidate + and retry; bounded exhaustion is `StoreBusy`, never a false `DuplicateKey`. +5. Otherwise select the first currently empty candidate lane; if none, select an + empty overflow cell in deterministic circular order. +6. Publish the target in the operation word. For an overflow target, load a + structurally valid exact summary version, revalidate the exact insert and + canonical mutation, then CAS that version to `Present(candidate)`. Revalidate + both again; an old helper never rolls this token back. +7. Only after Present publication, CAS the target cell from zero to the exact + binding; restart target selection on a conflicting CAS loss. +8. CAS-record the exact generation-tagged directory location, publish + `Reserved`, mark the generation-tagged operation complete, and CAS-clear the + exact bucket descriptor. + +Because same-key mutations share one canonical descriptor, unrelated cell +deletions cannot make two same-key candidates select different holes. Because +any caller completes the descriptor, a stopped inserter cannot block its bucket; +it may leave only the completed reservation/key/slot for explicit recovery. + +### Unlink + +1. Only `Aborting` or exclusively owned `Reclaiming` lifecycle state may prepare + unlink. +2. Claim/help the canonical descriptor. +3. Publish the recorded primary/overflow target in the operation word. +4. CAS the full exact binding to zero. A zero or different-generation cell is + treated as already unlinked only after directory validation finds no second + exact binding. +5. CAS-clear the exact generation-tagged directory location and complete the + exact generation-tagged descriptor. +6. Before releasing the mutation, capture any Present spill summary. If its + exact generation-tagged candidate, overflow location, cell, key hash, and + canonical bucket all revalidate while this mutation remains current, retain + Present without a table scan. Otherwise scan the complete overflow section + under the operation budget. A stable current entry of the same canonical + bucket retains Present. After a stable empty scan, exact operation/mutation + revalidation permits only the full-word captured `Present(X) -> Empty(X)` + CAS. Then CAS-clear the bucket mutation word. +7. Advance generation and release-publish `Free`, or publish `Retired` at + terminal generation. Ordinary slot metadata is left semantically dead and is + overwritten only by the next successful initializing owner. + +The same pre-release cleanup runs before a completed/rejected/canceled insert +releases its canonical mutation. A completed overflow insert sees its own cell +and retains Present; an insert that retried into primary or was canceled can +restore logical Empty. Budget expiry, unstable current cells, malformed tokens, +or an exact-CAS loss never manufacture Empty. They retain conservative Present +and the helpable exact mutation for a later caller. + +Reset/reuse never performs an unconditional write to a directory operation or +location that a stale helper could race. It exact-clears the old generation; +initialization of a later generation may also exact-clear recognizable residue +whose embedded generation differs from the new slot control. + +Nor does a reclaim helper clear ordinary key/hash/length/offset/binding metadata. +Multiple callers may help one `Reclaiming(g)` state; after one caller advances +control to `Free(g+1)`, a delayed helper must have no remaining ordinary write. +Free/Retired metadata is ignored, and the winner of the next +`Free(g+1) -> Initializing(g+1,p)` CAS overwrites every required lifecycle field, +including publication intent, before exposing its directory operation, +mutation, or binding. + +Readers do not consult the descriptor for a stable binding. During insertion +they may order before binding publication and return `NotFound`; during final +unlink the slot is already logically absent. + +## Reservation and publication visibility + +1. Slot `Free(g,0) -> Initializing(g,p)` CAS grants one lifecycle and atomically + carries the complete active participant token `p`; an exact participant + control recheck follows the claim. +2. Slot `DirectoryBinding`, key, lengths, fixed descriptor, offsets, and + `PublicationIntent` are written. Release publication of the exact + generation-tagged `Insert/Prepared` directory operation is the metadata-ready + marker for those ordinary writes and precedes canonical mutation and + directory-cell binding publication. A helper that acquire-loads that marker + validates intent before intent-specific action. Free/Retired and + pre-metadata Initializing—operation zero with no exact current mutation/cell + reference—and its direct unreferenced cleanup ignore stale intent bytes; a + current reference without its required marker is corruption. +3. Directory insertion publishes `Reserved`. For `ExplicitReservation`, its + exact `Initializing -> Reserved` CAS establishes key ownership and is the + public `TryReserve` ordering point. For `AtomicPublication`, Reserved is an + internal tentative stage. An installed binding that is still Initializing is + tentative for both intents. +4. Payload writes and `Advance` accounting occur only for the exact reservation + generation. `BytesAdvanced` changes monotonically with checked atomic RMW. + The successful `BytesAdvanced` compare/exchange is the `Advance` ordering + point. The operation carries one caller budget across every failed CAS: + cancellation or expiry before that CAS leaves `BytesAdvanced` unchanged, + while cancellation or expiry observed after it does not rewrite success. + `NoWait` performs one bounded probe quantum; `Infinite` retries CAS loss until + success or explicit cancellation. +5. Commit validates exact length, intent, and control, then CASes + `Reserved(g,p) -> Published(g,0)`, clearing ownership. This is the explicit + `ValueReservation.Commit` visibility point and the complete public ordering + point for `TryPublish`/`TryPublishSegments` with `AtomicPublication` intent. + +The commit release makes every preceding descriptor/payload byte visible to an +acquire that observes `Published`. An incomplete/overrun commit never performs +that CAS. Commit/abort/recovery each compare the exact generation, so only one +can leave the reservation lifecycle. + +A reservation is single-producer. Concurrent method calls through copied +`ValueReservation` structs are outside the supported contract; atomic byte +accounting prevents range overflow but does not assign disjoint write regions or +prove which bytes a caller actually filled. + +## Same-key duplicate classification + +Publish/reserve lookup must stabilize the exact directory cell, slot generation, +state, key bytes, and publication intent before returning `DuplicateKey`: + +- `Reserved(ExplicitReservation)`, `Published`, and `RemoveRequested` are + duplicate witnesses; +- `Initializing` is tentative for either intent; +- `Reserved(AtomicPublication)` is tentative because the one-call publication + has not reached Published; +- `Aborting`/`Reclaiming` is logical cancellation/cleanup and must be helped or + retried; +- unknown intent on a discoverable current lifecycle is `CorruptStore`. + +A bounded operation may return `StoreBusy` while a tentative same-key lifecycle +does not stabilize. If it later aborts, the contender may acquire the key; if it +reaches its intent-specific ordering point, the contender may return +`DuplicateKey`. `StoreFull` is independent physical pressure and may be returned +when all slots are non-reusable, including tentative slots. After an initial +same-key lookup returns absent, candidate claim precedes final directory +arbitration; a raced caller may therefore return genuine `StoreFull` before a +new same-key witness is classified. This is not a false duplicate and does not +grant a tentative lifecycle abstract key ownership. + +Allocation-scan exhaustion is not by itself a linearizable full-store result: a +concurrent reusable slot can rotate behind a sequential scanner. After bounded +help and a second claim scan, the rare capacity path uses one per-open +process-local snapshot buffer and nonblocking guard. It reads every slot control +in forward order, identifies the instant after that first collect and before the +second as a proof candidate, and reads every control again in the same order. +`StoreFull` orders at that candidate only when the second pass confirms exact +equality and both passes classify every word as structurally valid and +non-`Free`. The later confirmation callback is evidence that validates the +earlier candidate; it is not itself the physical full instant. + +The slot state machine has no reverse edge and no same-generation control ABA. +In particular, claim cleanup is +`Initializing(g,p) -> Aborting(g,0) -> Reclaiming(g,0) -> Free(g+1)` (or terminal +retirement), never `Initializing(g,p) -> Free(g)`. A free slot, any movement, or +another local proof holding the scratch guard is transient contention. The +engine applies the caller's operation-wide wait policy and repeats from fresh +same-key arbitration; `Infinite` cannot return a transient `StoreBusy`. + +## Lookup and acquire + +Directory lookup for key `K`: + +1. compute full hash and two bucket indices; +2. acquire-load every candidate binding and validate generation/hash/key bytes; +3. decode the canonical `SpillSummary`; reject malformed/reserved encodings, and + if no exact primary binding scan all overflow cells only when Present is set; +4. for a candidate, acquire-load slot control and require `Published(g)`; +5. re-read the directory cell and slot control; both must still equal the exact + observed values before lease activation begins. + +A successful lookup or maintenance classification is a cached witness from one +exact atomic directory reference word. The raw reference and its decoded slot +binding are separate values: a primary/overflow word equals its binding, while +a spill-summary reference is the complete encoded `Present(binding)` word. If a +later consumer of that witness would classify the slot as corrupt, it must not +combine an old reference observation with a newer slot lifecycle. It +acquire-reads the exact raw source word, obtains a fresh stable classification of +the separately decoded binding, then acquire-reads the same raw word again. A +source that differs from the exact raw reference on either side means +unlink/reclaim/reuse or summary replacement won the window; the operation +charges ordinary contention and restarts from a fresh lookup or maintenance +retry. Corruption is permitted only when the same exact reference word encloses +the repeated invalid slot snapshot. This is joint validation, not a new atomic +multi-word primitive. Directory-location publishers extend that source proof to +the complete canonical/operation/location/control/binding/target tuple and use +the no-op confirmation rule above before terminal invalid classification. + +Every slot control accepted by directory logic must also satisfy the complete +wire shape. `Initializing` and `Reserved` carry a configured structurally valid +participant token. `Free`, `Published`, `RemoveRequested`, `Aborting`, +`Reclaiming`, and `Retired` carry no participant; `Retired` additionally carries +the terminal generation. A malformed owner/state/generation combination is +invalid even if its generation matches the directory binding, and only a +structurally valid newer control is stale rather than corrupt. + +Exact stored-key equality is chunked and probes the same operation-wide budget; +deadline or cancellation propagates `StoreBusy` or `OperationCanceled` rather +than being converted to a false `NotFound`. The null-key stale-cell classifier +does not scan key bytes. + +Before releasing a canonical mutation while its spill summary is Present, +cleanup first validates the summary's exact binding/location/cell/hash tuple. +That validation treats the complete encoded summary as the source word and its +embedded binding separately under the source/slot/source rule above. +An exact current witness retains Present without an overflow-table scan. If the +candidate is stale or absent, a complete bounded scan either proves Empty or +finds another exact current same-canonical witness Y. While the same mutation is +still current, cleanup may full-word-CAS `Present(X)` to `Present(Y)`. A raw +Present(Y) recurrence cannot authorize a delayed clearer: its prerequisite +stable-empty scan could not have succeeded while the same nonwrapping Y binding +was current. A stale setter can at worst leave another conservative Present. +Only an exact `Present(X) -> Empty(X)` CAS after stable-empty scan is permitted; +no Empty identity is reused. + +Acquire then: + +1. CAS `Free(r,0)` to `Claiming(r,p)`, atomically carrying the complete active + participant token, then revalidate that exact participant control; +2. write the exact slot binding; +3. release-CAS `Claiming(r) -> Active(r)`; +4. acquire-revalidate the exact directory cell and `Published(g)` slot; +5. on success return the lease; otherwise CAS out of `Active`, recycle the + record, and return `NotFound`/retry/`StoreBusy` as the observed ordering + allows, or `CorruptStore` only when an unchanged exact reference word encloses + a repeated malformed slot classification. + +It does not read/project descriptor or payload before final success. A claiming +record is not reclamation authority; a later active record cannot return after +removal because final revalidation fails. + +Lease allocation-scan exhaustion is only a capacity candidate because a `Free` +record can rotate behind a sequential scanner. The rare slow path owns one +eager `long[LeaseRecordCount]` process-local snapshot and nonblocking guard per +open handle. It reads every lease control in record order, identifies the +instant after the first collect and before the second as the proof candidate, +then repeats the collect in the same order. `LeaseTableFull` orders at that +candidate only when both passes are structurally valid and all non-`Free`, and +the second controls exactly equal the first. `Claiming`/`Active` controls require +a structurally valid configured participant token; `Free`, `Releasing`, +`Recovering`, and `Retired` require participant zero; incarnation and terminal +retirement shapes must also be valid. Malformed controls fail `CorruptStore`. +Lease controls advance incarnation or retire instead of restoring an old Free +word, so equality cannot hide ABA. A free record, movement, or another local +proof holding the guard is transient contention: `NoWait` returns `StoreBusy`, +while finite/infinite callers retry under the original operation-wide budget. +The buffer and guard are neither mapped nor cross-process synchronization. + +## Remove, release, and reclaim + +Remove CASes `Published(g) -> RemoveRequested(g)` before scanning leases. New +acquires then fail their published-state validation. It scans lease records by: + +```text +control1 = LoadAcquire(record.Control) +if control1 is Active: + binding = record.SlotBinding + control2 = LoadAcquire(record.Control) + count only if control1 == control2 and binding == removed binding +``` + +If any exact active record exists, remove returns `RemovePending`. If the caller +bound expires after logical removal but before its fixed lease-table scan and +classification finish, it also returns conservative `RemovePending`; a later +remove/release/helper retries classification. Otherwise it returns `Success` +after attempting cooperative reclaim. Both statuses describe logical absence; +neither promises physical unlink/reuse completed synchronously. The acquire/ +remove race is safe because activation and logical removal are SC RMWs in one +total order: + +- activation before remove is visible to the subsequent scan, unless release + already ended protection; +- activation after remove cannot pass final published-state revalidation. + +Release CASes only its exact record incarnation from `Active(r,p)` to +`Releasing(r,0)`, ending public +projection lifetime. It resets/recycles that record and may help reclaim an +exact `RemoveRequested` slot. Concurrent remove, release, retrying remove, +allocation pressure, and recovery may all attempt reclamation; one +`RemoveRequested -> Reclaiming` CAS wins. + +## Projection rules + +Reservation and lease property access validates local handle state and exact +token incarnation without a named/global lock. A successful lease may project a +`Published` or `RemoveRequested` generation because removal preserves existing +leases. A reservation may project writable bytes only in its exact `Reserved` +generation and within the unadvanced announced range. + +Callers must not use a returned span/memory beyond its documented token lifetime. +The protocol prevents safe API projection after lifetime; it cannot revoke an +unsafe pointer or already copied span held by incorrect application code. + +## Retry, deadline, and cancellation + +CAS loss proves another participant changed shared state and is therefore +progress. Operations restart from a validation boundary, call `SpinWait`, and +sample deadline/cancellation at bounded intervals. + +Before that workflow's public ordering point, a caller relinquishes owner-controlled slot and +lease claims before returning `StoreBusy`/`OperationCanceled`. An unbound slot +can transition directly to unowned `Aborting`; a bound slot publishes a complete +helpable unlink descriptor. The returning process need not win another bucket +descriptor, so adversarial later mutations cannot force it to retain participant +ownership past the caller bound. An `Aborting` lifecycle is helpable maintenance, +not a leaked reservation. Thus an atomic convenience publication may abort from +Reserved, while a successfully returned explicit reservation may not be +reclassified as a failed `TryReserve`. After a public ordering point, +cancellation does not rewrite the result. Logical remove returns conservative `RemovePending` if its +post-ordering scan cannot finish within the bound; the key remains absent. All +bounded public-operation tests allow the selected limit plus 250 ms for this +finite handoff/cleanup. + +Normal recovery may run concurrently but never changes an owner-controlled slot +whose exact participant remains live Active. The current-process reservation +override is valid only after process-wide reserve/publish/token/writable-view +quiescence, and participant Closing/Recovering is itself a claim-closed +quiescent handoff. Therefore supported recovery cannot race a live owner across +either intent's public ordering point. An override invoked concurrently with a +live writer is outside the public outcome contract; exact generation fencing +must still protect later lifecycles. + +## Participant lifecycle + +The cold lifecycle lock protects only selection/initialization of participant +records and prevents an incomplete opener from exposing a token. Normal data +claims use the locally cached 28-bit index+incarnation token. They perform an +acquire recheck of its participant control after a successful first claim CAS, +but never acquire the cold lock or mutate the participant registry. + +Disposal sets local entry closed, changes its exact `Active` participant record +to `Closing` before resource cleanup, and boundedly scans/help-cleans exact +captured slot and lease controls carrying its token. A stable exact +`Closing`/`Recovering` is an unconditional recovery handoff even while its PID is +live; only `Registering`/`Active` requires liveness classification. +Record-local explicit recovery classifies PID/start identity before clearing an +exact reference. A safely stale participant may enter `Recovering` only for a +final zero-reference retirement pass. `Closing`/`Recovering` makes a post-claim +participant recheck fail, so no legitimate new owner-controlled claim persists. +Only a final reference-free scan permits CAS to PID-free, universally helpable +`Reclaiming`; helpers exact-CAS that control to the next Free incarnation or +Retired record without ordinary identity-field writes. Free/Retired identity +fields are semantically dead, and the next exclusive Registering owner +overwrites all of them before publishing Active, so a delayed helper cannot +erase a reused incarnation. + +## Diagnostics and counters + +Correctness never depends on a diagnostic counter. Hot counters are striped by +local handle/participant or updated outside the critical visibility transition +to avoid introducing a global cache-line bottleneck. Scans use the same stable +double-read patterns but do not claim records solely for snapshot consistency. + +## Disposal ordering + +One process-local atomic lifetime word rejects new operations once disposing and +counts operations already entered. Disposal waits only for that handle's entered +calls, publishes exact `Active -> Closing`, then uses one finite post-ownership +cleanup allowance to release/abort exact ownership attributable to its +participant record and attempt reference-free retirement before unmapping. If +the allowance expires, another recovery caller may finish the still claim-closed +participant without waiting for the disposer. Other +handles and mappings continue. A local thread paused after entry may delay +disposal of that handle, but cannot prevent system-wide store progress. + +## Required model checks + +Before completing the full API, a finite-state/deterministic scheduler must cover +every pause between the atomic steps for: + +- same-key insert descriptors with unrelated lane insertion/removal; +- primary spill and overflow publication; +- insertion help/owner recovery; +- commit/acquire; +- acquire/remove; +- release/reclaim/unlink; +- abort versus commit/recovery; +- stale descriptor/binding/lease after generation reuse; +- a helper paused after each operation/location validation and resumed after + completion, slot reclaim, and reuse; +- local disposal/operation. +- participant open/exhaustion/closing/recovery and a crash immediately after the + first slot/lease claim CAS. + +Any stable-key false miss, two successful current generations, live-owner +recovery, stale-token mutation, or unhelpable descriptor is a convergence failure, +not an allowed outcome. diff --git a/specs/009-lock-free-publish-read/contracts/layout-v2.md b/specs/009-lock-free-publish-read/contracts/layout-v2.md new file mode 100644 index 0000000..2c531f9 --- /dev/null +++ b/specs/009-lock-free-publish-read/contracts/layout-v2.md @@ -0,0 +1,517 @@ +# Mapped Layout 2.0 Contract + +## Identity + +| Contract | Value | +|---|---| +| Magic | `SMS2` (`0x32534D53`, little-endian) | +| Layout | major 2, minor 0 | +| Resource protocol | 2 | +| Endianness | little-endian only | +| Architecture | x64 only in layout-2.0 C# support; other architectures reject before mapping mutation | +| Atomic width | aligned 64-bit only | +| Minimum slot generation | 1 | +| Slot-count range | `1..1,048,575` (`2^20-1`) | +| Maximum slot index | `1,048,574` (zero based) | +| Required features | bit 0 (`0x1`): `versioned_empty_spill_summary`; bit 1 (`0x2`): `publication_intent`; bit 2 (`0x4`): `pid_namespace_identity` | + +An opener validates identity, required feature bits, complete header bounds, and +requested dimensions before obtaining any descriptor/payload pointer. Unknown +major versions or required features are incompatible. A higher minor version is +compatible only when its required feature set is fully understood and every +section used by this participant has a recognized length/stride. + +The current pre-release 2.0 contract requires bits 0, 1, and 2, a mask of `7`, +exactly. A prior v2 binary that supports required-features zero rejects bit 0 as +unknown; a bit-0-only draft rejects bit 1, and a mask-3 draft rejects bit 2. The +current binary rejects every older shape because their spill-summary, +publication-intent, or process-identity semantics cannot be interpreted safely. +These compatibility fences change neither the layout nor resource-protocol +version. + +## Canonical section order + +```text +0 +| StoreHeaderV2 (64-byte aligned length) +| ParticipantRecordV2[] (64-byte stride) +| PrimaryDirectoryBucket[] (128-byte stride) +| OverflowBinding[SlotCount] (8-byte cells) +| LeaseRecordV2[] (64-byte stride) +| ValueSlotMetadataV2[] (128-byte stride) +| KeyStorage[SlotCount] (aligned fixed stride) +| DescriptorStorage[SlotCount] (aligned fixed stride) +| PayloadStorage[SlotCount] (aligned fixed stride) +` end <= TotalBytes +``` + +All section starts are at least 8-byte aligned. Header, participant records, +primary buckets, lease records, and slot metadata begin on 64-byte boundaries. +Padding bytes are zero at initialization and ignored by readers. + +## Sizing + +Let: + +```text +PrimaryLaneCount = NextPowerOfTwo(max(32, checked(SlotCount * 4))) +LanesPerBucket = 8 +BucketCount = PrimaryLaneCount / LanesPerBucket +OverflowCount = SlotCount +ParticipantCount = configured ParticipantRecordCount (default 64, max 2^20-1) +ParticipantIndexBits = ceil(log2(ParticipantCount + 1)) +ParticipantGenerationBits = 28 - ParticipantIndexBits (minimum 8) +ParticipantIndexMask = (1 << ParticipantIndexBits) - 1 +ParticipantGenerationMask = (1 << ParticipantGenerationBits) - 1 +ParticipantToken = (participantGeneration << ParticipantIndexBits) | (recordIndex + 1) +KeyStride = Align8(max(1, MaxKeyBytes)) +DescriptorStride = Align8(max(1, MaxDescriptorBytes)) +PayloadStride = Align8(max(1, MaxValueBytes)) +``` + +`SlotCount` is in `1..2^20-1`, so `PrimaryLaneCount` is at most `2^22`. +Creation/sizing rejects an out-of-range lock-free slot count before mapping +creation or open. The legacy profile retains its own validation contract. + +Bucket mixing maps every key to two buckets and one canonical home bucket. The +layout calculator uses checked 64-bit arithmetic for every product/addition and +aligns each section. `CalculateRequiredBytes(StoreProfile.LockFree, ...)` and +`CreateLockFree(...)` use this one canonical calculation. An existing mapping's +header values, offsets, lengths, and strides must exactly match the requested +dimensions. + +## Atomic record rules + +1. Atomic words are naturally 8-byte aligned both by record stride and field + offset. +2. A shared atomic word is never read or written through a 32-bit field. +3. A participant uses `Interlocked` for read-modify-write and `Volatile` acquire/ + release access according to the memory-ordering contract. +4. Non-atomic metadata is mutable only while an atomic state grants exclusive + initialization/reclamation ownership; it is immutable while published or + protected by a lease. +5. No on-memory field contains a pointer, `nint`, `size_t`, managed object ID, + OS handle, enum with unspecified width, or native structure. +6. Every remaining reserved/padding field is zero on creation and ignored on + open unless a future required feature assigns it. Slot offset 52 is assigned + by required-feature bit 1; header offsets 264/272 and participant offset 32 are + assigned by bit 2 and are not padding. + +## Binding codec + +Every directory and mutation word uses this unsigned bit pattern through a +signed 64-bit atomic storage location: + +```text +bits 0..30: slot index plus one (1..2^31-1) +bits 31..63: slot generation (1..2^33-1) +all zero: empty +``` + +Decoding zero, slot zero/out-of-range, generation zero, or a binding whose slot +control generation differs is invalid/stale. Exact CAS always compares the full +64 bits. A slot reaching the terminal generation is retired and never wraps. + +## Spill-summary codec + +The first atomic word of each canonical primary bucket is a versioned negative +cache encoded independently from `IndexBinding` while preserving the same exact +slot generation: + +```text +bits 0..19: candidate slot index plus one (1..2^20-1) +bits 20..52: candidate slot generation (1..2^33-1) +bit 53: Present (1 = overflow scan required, 0 = confirmed empty) +bits 54..63: reserved zero +all zero: initial EmptyNone +``` + +`Present(X)` and `Empty(X)` carry the same exact candidate identity. A completed +empty scan clears with one full-word `Present(X) -> Empty(X)` CAS and never +restores zero. Candidate identities never repeat because slot generations never +wrap, so a delayed setter's old expected Empty token and a delayed clearer's old +expected Present token cannot match a later summary version. Index zero, +generation zero, a slot index greater than or equal to the current header's +configured `SlotCount`, or a reserved bit is corruption; the token is retained +and the mapping fails closed. + +## Store header requirements + +The header contains, in fixed-width fields: + +- identity/version/header length/resource protocol; +- required and optional feature masks; +- total bytes and random nonzero Store ID; +- atomic store control; +- all public dimensions; +- participant-record count/offset/length/stride; +- primary bucket/lane/overflow dimensions; +- every section offset, length, and stride; +- PID-namespace identity/mode at exact byte offsets 264/272; +- bounded diagnostic counter offsets/counts; +- header checksum only if a future required feature defines its atomic update + behavior (not required by 2.0). + +`StoreHeaderV2` is encoded/validated by explicit constants and offsets. Managed +`Marshal.SizeOf` is verified by tests but is not the language-neutral authority. +On Linux `PidNamespaceId` is the positive numeric token parsed from +`/proc/self/ns/pid`; on Windows it is zero. Creation writes it and atomic mode +`Enabled=1` (or `Mixed=2` when unproven) before `Ready`. A different or +unproven Linux opener release-publishes irreversible Mixed before its first +Registering CAS and then retains ordinary KV access. + +Store control values are `Initializing=1`, `Ready=2`, `Corrupt=3`, and +`Unsupported=4`. Initialization release-publishes `Ready`. A path that has +stabilized and revalidated persistent mapped structural corruption full-word- +CASes `Ready` to `Corrupt`; the state is terminal. Each public mapped-data +operation acquire-loads the control before projection or mutation. A corrupt +mapping rejects later opens as incompatible. Caller-owned malformed inputs, +ordinary capacity/contended/canceled outcomes, and legal raced observations do +not change this word. These are mapped atomics, not a lock or OS primitive. + +## Primary directory bucket + +One 128-byte bucket contains: + +```text +offset 0: SpillSummary (atomic versioned Present/Empty u64) +offset 8: Mutation (atomic binding or zero) +offset 16: Lane[8] (eight atomic bindings) +offset 80: reserved zero padding through byte 127 +``` + +The mutation binding refers to a slot whose generation-tagged +`DirectoryOperation` identifies insert or unlink and phase. It is publishable +only after all descriptor fields are initialized. A helper may act only when the +binding, operation, and slot control generations agree. A helper that finds a +stale generation may CAS-clear only that exact mutation word. + +An exact current insert helper CAS-publishes `Present(candidate)` before any +overflow-cell CAS and revalidates both the operation and canonical mutation +afterward. Before releasing a completed mutation that may have touched the +summary, a helper scans overflow for a stable current binding whose hash maps to +that canonical bucket. A current witness retains Present. Only an empty full +scan followed by exact operation/mutation revalidation permits the captured +Present token to become its matching Empty token. Cancellation, deadline, +instability, or a CAS loss retains conservative Present and leaves the mutation +helpable. + +## Overflow directory + +The overflow section contains `SlotCount` atomic binding cells. It has no +sentinel other than zero, no tombstones, and no header count required for +correctness. A directory binding may exist in exactly one primary or overflow +cell. Diagnostics derive occupancy by bounded scan. + +Primary lanes, overflow cells, and versioned Present spill summaries are exact +atomic directory reference words. Classification keeps the captured raw word +separate from its decoded slot binding: a lane/cell raw word equals the binding, +while a summary raw word is the complete encoded `Present(binding)` value. A +would-be corrupt binding classification is reportable only after the same raw +source word is acquire-read unchanged around a repeated stable, fully +shape-validated snapshot of the decoded slot binding. Source movement requires +a budgeted lookup or maintenance retry instead. This source/slot/source rule is +joint validation, not a mapped multi-word atomic or a new shared owner. + +## Participant record + +One 64-byte record is claimed only under the cold lifecycle lock and contains: + +```text +offset 0: Control (atomic u64; includes PID) +offset 8: IdentityKind (i32) +offset 12: reserved zero (i32) +offset 16: ProcessStartValue (i64) +offset 24: OpenSequence (i64) +offset 32: PidNamespaceId (u64) +offset 40: reserved zero through byte 63 +``` + +Participant control encoding is: + +```text +bits 0..2: state (Free=0, Registering=1, Active=2, Closing=3, + Recovering=4, Reclaiming=5, Retired=6; 7 invalid) +bits 3..30: participant incarnation (high unused bits zero per layout codec) +bits 31..62: positive PID while Registering/Active/Closing/Recovering +bit 63: reserved zero +``` + +Under the cold lock, `Free -> Registering` atomically publishes PID and +incarnation before other identity fields are written. The exclusive claimant +writes the exact admitted PID namespace before an opener release-publishes +`Active` and before any data control may reference it. A stable Active snapshot +jointly fences this field with control and compares it with the current +namespace before PID/start lookup. Registering presence-only classification +uses the header identity only while mode is Enabled; Mixed makes it Unsupported +because its per-record ordinary fields may still be mixed old/new values. A +recovery reader snapshots control before acquire-loading mode, and the opener +publishes Mixed before its claim. The 28-bit hot token packs +record index plus one in the layout's `ParticipantIndexBits` and the same +incarnation in the remaining bits. The record is not reused until a reference +scan proves no exact token remains. + +The encoded index-plus-one must be in `1..ParticipantCount`; zero is an invalid +token. Participant generation starts at one and may not exceed +`ParticipantGenerationMask` even though the participant control reserves a +28-bit physical field. Bits above the configured generation mask in both control +and token are zero. Closing/recovery of the terminal configured generation +publishes `Retired`; it never increments to zero or uses the larger physical +field range. + +Identity-kind assignments are `Unknown=0`, `WindowsProcessCreationFileTime=1`, +and `LinuxProcStartTicks=2`. Unknown identity permits normal access but explicit +stale-owner recovery remains conservative/unsupported. Values 3 and above are +unknown required semantics for recovery and must not be guessed. + +## Slot metadata + +The 128-byte slot metadata record has these exact offsets: + +```text +offset 0: Control (atomic u64; state + generation + participant token) +offset 8: DirectoryBinding (u64; immutable exact binding for this lifecycle) +offset 16: DirectoryLocation (atomic u64) +offset 24: DirectoryOperation (atomic u64) +offset 32: KeyHash (u64) +offset 40: KeyLength (i32) +offset 44: DescriptorLength (i32) +offset 48: ValueLength (i32) +offset 52: PublicationIntent (i32) +offset 56: BytesAdvanced (atomic i64) +offset 64: CommitSequence (i64) +offset 72: KeyOffset (i64) +offset 80: DescriptorOffset (i64) +offset 88: PayloadOffset (i64) +offset 96: reserved zero through byte 127 +``` + +Slot control encoding is: + +```text +bits 0..2: Free=0, Initializing=1, Reserved=2, Published=3, + RemoveRequested=4, Aborting=5, Reclaiming=6, Retired=7 +bits 3..35: nonzero 33-bit slot generation +bits 36..63: complete 28-bit participant token in Initializing/Reserved; + zero in Free/Published/RemoveRequested/Aborting/Reclaiming/Retired +``` + +The participant token in `Initializing`/`Reserved` must be structurally valid +for the header's configured `ParticipantRecordCount`: index-plus-one is in +range, incarnation is nonzero and within its configured token bits, and no +unused bit is set. Every slot generation is nonzero and bounded. `Retired` is +valid only at the terminal slot generation. Any other state/generation/owner +shape is corruption; in particular, an owned unowned-state control is not a +current binding, and only a structurally valid strictly newer control may make +an older directory binding stale. + +Publication-intent encoding is: + +```text +0: None +1: ExplicitReservation (public TryReserve / ValueReservation workflow) +2: AtomicPublication (public TryPublish / TryPublishSegments workflow) +3..2^31-1 and negative values: invalid +``` + +`PublicationIntent` is ordinary owner-written metadata. The successful +`Free -> Initializing` claimant writes it with the slot's ordinary +`DirectoryBinding`, key/length/offset metadata, and descriptor before +release-publishing the current-generation `Insert/Prepared` +`DirectoryOperation`. That exact nonzero operation word is the metadata-ready +marker; its later phase/operation-intent changes and eventual clear remain exact- +generation CAS operations. Canonical-bucket mutation and directory-cell binding +publication MUST follow the marker. Intent is immutable for the rest of that +slot generation. + +`Pre-metadata Initializing` means exact current `Initializing` with a zero +`DirectoryOperation` and no current-generation canonical mutation or directory +cell referencing the slot. `None` or stale/unknown intent bytes are ignored in +that state and while `Free`/`Retired`. If stale-owner recovery changes that +unmarked claim directly to unreferenced `Aborting`/`Reclaiming`, cleanup also +does not interpret the stale ordinary intent bytes. A current-generation +mutation/cell without the preceding valid operation marker is structural +corruption, not an alternative publication path. While the marker or any valid +later current-generation reference exists, only intent values 1 and 2 are +valid; an unknown intent is `CorruptStore`. Reclaim does not zero the field. The +next exclusive claimant overwrites it before publishing the next lifecycle's +marker. + +The intent selects public ordering without changing slot states. For +`ExplicitReservation`, `Initializing -> Reserved` is the public `TryReserve` +ordering point and `Reserved` is a duplicate-key witness. For +`AtomicPublication`, `Reserved` is an internal tentative stage; +`Reserved -> Published` is the public convenience-publication ordering point, +and only `Published`/`RemoveRequested` is its terminal duplicate-key witness. +An intent-aware lookup may help/revalidate a tentative lifecycle and may return +`StoreBusy` on bounded exhaustion, but it must not report `DuplicateKey` solely +from `Initializing` or `Reserved(AtomicPublication)`. + +This intent classification does not give duplicate detection priority over +physical allocation in a race. After an initial same-key lookup returns absent, +the candidate claim precedes final directory arbitration and may return +`StoreFull` when every slot is non-Free, including when tentative lifecycles +occupy the remaining capacity. + +The public result is certified without adding a layout field. One local +`Int64[SlotCount]` buffer per open handle records a first forward collect of slot +controls; a second forward collect must be exactly equal, structurally valid, +and entirely non-`Free`. The full instant is the candidate point between the +collects, confirmed by the completed second pass. Controls never roll back +within one generation, so equality cannot conceal ABA. The buffer and its +nonblocking local guard are process memory, not mapped protocol state, and no +shared counter, lock word, named primitive, or OS synchronization participates. +An equal malformed control fails `CorruptStore`; it never contributes occupancy +evidence. A free/moving control or local guard conflict is contention governed +by the caller's operation-wide wait policy, not `StoreFull`. + +Directory-location encoding is: + +```text +all zero: None +bits 0..1: kind (Primary=1, Overflow=2; 0 only for None; 3 invalid) +bits 2..23: zero-based absolute cell index within the selected section +bits 24..56: exact nonzero 33-bit slot generation +bits 57..63: reserved zero +``` + +The index must be within `PrimaryLaneCount` or `OverflowCount` for its kind. A +nonzero location is valid only when its generation equals the directory binding +and current slot lifecycle generation. + +Directory-operation encoding is: + +```text +bits 0..1: intent (None=0, Insert=1, Unlink=2; 3 invalid) +bits 2..4: phase (None=0, Prepared=1, TargetSelected=2, + BindingChanged=3, Rejected=4, Complete=5; 6..7 invalid) +bits 5..6: target kind (None=0, Primary=1, Overflow=2; 3 invalid) +bits 7..28: zero-based target cell index +bits 29..61: exact nonzero 33-bit slot generation +bits 62..63: reserved zero +``` + +The only legal zero word is `intent=None, phase=None, target=None`. `Prepared` +and `Rejected` require target kind/index zero but retain the exact nonzero slot +generation; Rejected is the terminal +same-key/canceled insert outcome with no binding installed. `TargetSelected` +and `BindingChanged` require a section-valid target. Insert `Complete` also +retains that target. Unlink `Complete` may use target None/index zero when a +generation-matching binding was already absent and the helper therefore had no +cell to clear. Insert and Unlink otherwise use the same phases; +`BindingChanged` means binding installed or exact binding cleared according to +intent. Helpers CAS the full operation word, never one subfield. Every nonzero +phase must carry the same generation as the mutation binding and current +lifecycle before the helper performs a write. + +Every write derived from an operation is either an exact CAS on a +generation-tagged operation/location word or an exact CAS on a generation-tagged +directory binding. If a target-cell CAS succeeds but the old operation can no +longer advance, the helper rolls back only that exact old binding. A stale +helper may therefore leave recognizable old-generation residue for another +caller to clear, but it cannot overwrite or unlink a later lifecycle. + +Directory-location publication adds no wire state. Its validation tuple is the +canonical mutation word, exact operation word, current location word, slot +control, immutable directory binding, and selected or competing target cells. +A terminal invalid classification requires two identical acquire collections of +that tuple, exact no-op compare/exchanges confirming every atomic member, and a +fresh immutable-binding read; any lost comparison is progress and requires +retry rather than corruption. + +When cancellation hands an Insert to `Unlink/Prepared`, an empty target or a +structurally valid different in-range binding is legal target-loss progress; +malformed or out-of-range target state is corruption. The first valid +`Unlink/Prepared` location publisher wins, and a loser exact-clears only its +distinct recovered old binding. `Unlink/TargetSelected` may observe a valid +same-generation alternate location and exact-cleans both old-binding witnesses +and that alternate location while preserving empty cells and valid replacement +bindings. If the source is lost after a location CAS, the publisher withdraws +only its exact old target and location; it never removes a committed Insert +successor or another valid replacement. + +Strictly older location residue is exact-cleanable. A future location is benign +reuse only when another member of the old tuple has moved; a future location +inside the confirmed exact old-generation tuple is corruption and is preserved +for diagnosis. These rules change validation and helping only; all offsets and +operation/location encodings remain unchanged. + +`protocol/layout-v2.0.md` and fixtures reproduce these authoritative offsets and +encodings; tests reject drift before implementation proceeds. + +## Lease record + +The 64-byte lease record contains: + +```text +offset 0: Control (atomic u64; state + record incarnation + participant token) +offset 8: SlotBinding (u64) +offset 16: AcquireSequence (i64) +offset 24: reserved zero through byte 63 +``` + +Lease control encoding is: + +```text +bits 0..2: Free=0, Claiming=1, Active=2, Releasing=3, + Recovering=4, Retired=5 +bits 3..35: nonzero 33-bit lease-record incarnation +bits 36..63: complete 28-bit participant token in Claiming/Active; + zero in Free/Releasing/Recovering/Retired +``` + +Record incarnation starts at one, advances before reuse, and retires the record +instead of wrapping. + +`Claiming`/`Active` require a structurally valid participant token for the +configured participant table. `Free`/`Releasing`/`Recovering`/`Retired` require +participant zero; `Retired` is valid only at terminal record incarnation, and +state values 6-7 are invalid. Any invalid state/incarnation/owner/token shape is +`CorruptStore`. + +An exhausted lease scan is only a capacity candidate because reusable records +can rotate behind a sequential scanner. Each open handle may eagerly own one +private `Int64[LeaseRecordCount]` snapshot protected by a nonblocking local +guard. The proof reads every control in record order, records the candidate +instant between passes, and repeats the collect in the same order. Only two +structurally valid, entirely non-`Free`, exactly equal collects confirm that +candidate and permit `LeaseTableFull`. Every reuse advances incarnation or +retires, so equality cannot hide ABA. A malformed control is `CorruptStore`; a +free/moving control or local guard conflict follows the operation-wide wait +policy as contention. The buffer and guard are private process state and add no +mapped field, shared counter, named primitive, or OS synchronization. + +## Byte-storage rules + +- Keys are exact opaque nonempty bytes. Key equality always checks length and + every byte after hash/binding filtering. +- Descriptor and payload lengths may be zero. +- Descriptor and payload bytes are immutable after publication. +- Unused bytes in a fixed stride have no semantic value and are never projected + beyond the stored length. +- Ordinary slot metadata is semantically dead while its control is Free or + Retired. Reclaim helpers do not zero it; the next successful Initializing + claimant overwrites every required lifecycle field before directory + publication, preventing a delayed reclaim helper from erasing a reused slot. +- Aborted/reclaimed bytes need not be securely erased; trusted mapped writers + and data remanence policy remain outside this feature. Metadata lengths and + generation fencing make those bytes inaccessible through safe APIs. + +## Initialization and corruption + +The cold attempt that physically creates the region zeroes/initializes every +atomic word, slot generation, participant and lease-record incarnation, fixed +offset, and Store ID before publishing `Ready`. Physical creation disposition, +not open mode or observed zero bytes, is the sole initialization authority. An +opener of an existing zero header never writes it: `CreateNew` reports +`AlreadyExists`, `CreateOrOpen` reports `StoreBusy`, and `OpenExisting` reports +`IncompatibleLayout`. Platform abandonment and stale-resource cleanup may make +a later attempt the physical creator, but opening an extant unpublished region +does not. See `protocol/resource-naming-v2.md` for the ordered cold transaction. + +Impossible bounds, overlapping sections, unsupported features, misalignment, +invalid/reserved operation or location bits, generation mismatches, or +current-generation bindings outside their legal sections are +reported as incompatible/corrupt before unsafe projection. Normal stale +transitional state is recovered/helped under the concurrency contract rather +than globally marking the store corrupt. diff --git a/specs/009-lock-free-publish-read/contracts/public-api.md b/specs/009-lock-free-publish-read/contracts/public-api.md new file mode 100644 index 0000000..6bf905c --- /dev/null +++ b/specs/009-lock-free-publish-read/contracts/public-api.md @@ -0,0 +1,325 @@ +# Public API Contract + +## Compatibility principle + +Layout 2.0 adds an explicitly selected concurrency profile; it does not add a +second key-value abstraction. Existing v1.2 callers retain their exact helper +method signatures, numeric enum assignments, operation names, overload shapes, +and default behavior. + +## Profile selection + +```csharp +namespace SharedMemoryStore; + +public enum StoreProfile +{ + Legacy = 0, + LockFree = 1 +} + +public sealed class SharedMemoryStoreOptions +{ + public StoreProfile Profile { get; init; } = StoreProfile.Legacy; + public int ParticipantRecordCount { get; init; } = 64; + + // All existing properties remain. + + // Existing signature and layout-v1.2 result remain unchanged. + public static long CalculateRequiredBytes( + int slotCount, + int maxValueBytes, + int maxDescriptorBytes, + int maxKeyBytes, + int leaseRecordCount); + + // New profile-aware helper; profile is first to avoid overload ambiguity. + public static long CalculateRequiredBytes( + StoreProfile profile, + int slotCount, + int maxValueBytes, + int maxDescriptorBytes, + int maxKeyBytes, + int leaseRecordCount, + int participantRecordCount = 64); + + // Existing Create(...) signature remains byte-for-byte unchanged and Legacy. + + public static SharedMemoryStoreOptions CreateLockFree( + string name, + int slotCount, + int maxValueBytes, + int maxDescriptorBytes, + int maxKeyBytes, + int leaseRecordCount, + int participantRecordCount = 64, + OpenMode openMode = OpenMode.CreateOrOpen, + bool enableLeaseRecovery = false); +} +``` + +Manually constructed options select layout 2.0 only when `Profile` is explicitly +`LockFree`. The existing `Create(...)` helper always sets `Legacy`. Neither open +path auto-detects and changes the requested profile. + +`ParticipantRecordCount` is ignored by the legacy layout calculator/engine and +is a required positive v2 capacity no greater than 1,048,575. One open v2 store +handle consumes one participant record; the default supports 64 simultaneous +handles. + +`SlotCount` remains a required positive capacity for both profiles. The +lock-free profile additionally limits it to 1,048,575 so the exact slot +generation can be carried by every portable single-word directory helper +reference. `CreateLockFree`, profile-aware sizing, manual v2 option validation, +and v2 header validation all enforce the same range before payload projection. +The legacy helper and engine retain their existing upper-bound behavior. + +## Opened protocol identity + +```csharp +public readonly record struct StoreProtocolInfo( + StoreProfile Profile, + int LayoutMajorVersion, + int LayoutMinorVersion, + int ResourceProtocolVersion, + ulong RequiredFeatures, + ulong OptionalFeatures); + +public sealed class MemoryStore : IDisposable +{ + public StoreProfile Profile { get; } + public StoreProtocolInfo ProtocolInfo { get; } +} +``` + +These properties are immutable for a handle and safe without shared locking. +For the current pre-release layout 2.0 shape, `RequiredFeatures` is `7`: bit 0 +is the versioned spill summary, bit 1 is intent-aware publication ordering, and +bit 2 is PID-namespace-safe recovery identity. Required-features-zero, +bit-0-only, and mask-3 draft mappings return +`StoreOpenStatus.IncompatibleLayout`; those draft clients likewise reject the +current mask before payload projection. + +## Store operations + +The following existing workflows remain the public v2 surface: + +- `MemoryStore.TryCreateOrOpen` +- `TryPublish` and `TryPublishSegments` +- `TryReserve`, `ValueReservation.GetSpan`, `DangerousGetMemory`, `Advance`, + `Commit`, `Abort`, and `Dispose` +- `TryAcquire`, zero-copy `ValueLease` length/span properties, `Release`, and + `Dispose` +- `TryRemove` +- explicit lease and reservation recovery +- diagnostics snapshot +- `StoreWaitOptions` overloads +- store disposal + +No queue, dequeue, claim, acknowledgement, work assignment, subscriber, stream, +or broker API is added. + +## Status contract + +All existing `StoreOpenStatus` and `StoreStatus` numeric assignments remain +unchanged. Layout 2.0 uses them as follows: + +| Condition | Result | +|---|---| +| Exact key owned by `Reserved(ExplicitReservation)`, `Published`, or `RemoveRequested` | `DuplicateKey` | +| No published current generation | `NotFound` | +| Exact confirmed all-non-Free double collect of the value-slot controls | `StoreFull` | +| Exact confirmed all-non-Free double collect of structurally valid lease controls | `LeaseTableFull` | +| Removal won but matching active leases remain, or bounded post-removal classification/reclaim work is incomplete | `RemovePending` | +| Caller retry/deadline budget exhausted | `StoreBusy` | +| Cancellation observed before ordering point | `OperationCanceled` | +| Stale/wrong lease record incarnation | `InvalidLease` or `LeaseAlreadyReleased` according to observable history | +| Stale/wrong reservation generation | `InvalidReservation` or `ReservationAlreadyCompleted` according to observable history | +| Requested profile differs from existing mapping | `StoreOpenStatus.IncompatibleLayout` | +| `CreateNew` finds an existing unpublished zero header | `StoreOpenStatus.AlreadyExists`; existing bytes remain unchanged | +| `CreateOrOpen` finds an existing unpublished zero header whose initialization ownership cannot be proven | `StoreOpenStatus.StoreBusy`; existing bytes remain unchanged | +| `OpenExisting` finds an existing unpublished zero header | `StoreOpenStatus.IncompatibleLayout`; existing bytes remain unchanged | +| Cold coordination cannot be entered within the caller's budget | `StoreOpenStatus.StoreBusy` or `StoreOpenStatus.OperationCanceled` according to the observed bound/cancellation | +| No free v2 participant record for a new handle | `StoreOpenStatus.ParticipantTableFull` (appended numeric value 11) | +| Existing v2 store control is `Corrupt` | `StoreOpenStatus.IncompatibleLayout` | + +The cold create/open budget begins before lifecycle coordination and covers +physical discovery or creation, mapping, header work, and handle registration. +Only the attempt that physically created the region may initialize it. No +successful handle is returned until that ownership has been transferred and the +ordered cold gates have been released. + +There is no ordinary `IndexFull`: layout 2.0 overflow capacity equals +`SlotCount`. Failure to place a binding while a value slot is owned is an +inconsistent/corrupt state after bounded revalidation, not a smaller documented +capacity. + +Persistent mapped structural corruption is store-wide. The detecting path +exact-CASes the shared store control from `Ready` to `Corrupt`; every later +operation on any handle returns `CorruptStore` before a new mapped projection or +mutation, and a later open fails with `IncompatibleLayout`. An already borrowed +span cannot be revoked. Invalid caller-owned inputs and documented concurrent +or token-history outcomes remain local results and never poison the store. + +`Initializing` and `Reserved(AtomicPublication)` are mapped states that remain +tentative to the public operation, not `DuplicateKey` witnesses. A same-key caller helps/revalidates them and may +return `StoreBusy` only after exhausting its bounded local budget. If the +tentative lifecycle aborts, it retries key ownership; if it reaches its +intent-specific ordering point, the caller observes the corresponding +`DuplicateKey`. `StoreFull` remains a physical capacity result: a tentative +non-Free slot is not reusable and can contribute to full-store pressure. After +an initial absent-key lookup, a raced insertion may return `StoreFull` at its +candidate claim before final same-key arbitration; the API does not give +`DuplicateKey` precedence over genuine physical exhaustion in that race. +An exhausted scan is provisional. `StoreFull` is exposed only after two +same-order, structurally valid, all-non-Free control snapshots match exactly; +the ordering point is the confirmed candidate instant between those collects. +A free/moving slot or local proof-buffer contention follows the wait policy and +may yield `StoreBusy`, but never a false capacity result. + +Lease-record scan exhaustion is also provisional. `LeaseTableFull` is exposed +only after two same-order, structurally valid, all-non-Free lease-control +snapshots match exactly; malformed state/incarnation/owner/token shapes return +`CorruptStore`. A free/moving record or local lease-proof-buffer contention +follows the same wait policy and cannot become a false capacity result. + +## Wait semantics + +`StoreWaitOptions` retains its existing public shape. + +- Legacy profile: bounds the existing shared synchronization acquisition. +- Lock-free profile: bounds local retry, revalidation, cooperative help, and + backoff. +- It does not wait for a broker message, key arrival, removal, free value slot, + or free lease record. +- `NoWait` performs the minimum safe attempt and returns a normal lifecycle, + capacity, or `StoreBusy` result. +- A transient capacity proof restarts from fresh same-key arbitration for finite + and infinite callers. `Infinite` does not return transient `StoreBusy`; it + continues until a confirmed result, another normal outcome, or cancellation. +- Cancellation cannot undo a successful operation after its documented ordering + point. +- Before an ordering point, cancellation/expiry relinquishes owner-controlled + slot/lease claims. It may publish an unowned helpable abort/unlink descriptor + before returning; this is not a leaked reservation and any caller can finish + physical cleanup. + +Ordering is workflow-specific. `TryReserve` orders at +`Initializing -> Reserved`; `TryPublish` and `TryPublishSegments` order only at +`Reserved -> Published`; a later explicit `ValueReservation.Commit` also orders +value visibility at `Reserved -> Published`. + +## Atomic convenience publication + +`TryPublish` and `TryPublishSegments` are one-call atomic convenience +publications. Their internal slot carries `AtomicPublication` intent. Both +`Initializing` and `Reserved` remain tentative to the public call, and failures +before the exact `Reserved -> Published` CAS leave no newly published key. +Another caller must not turn either tentative state into `DuplicateKey` merely +because a directory binding is physically present. Once Published wins, the +public call has ordered and cancellation cannot rewrite its result. + +## Reservation lifetime + +A successful reservation owns one exact key/slot generation and returns +store-owned writable memory. Descriptor bytes are fixed before success. The +caller must account for exactly the announced payload length before commit. + +The slot carries `ExplicitReservation` intent. Its exact +`Initializing -> Reserved` CAS is the `TryReserve` ordering point. A physically +discoverable `Initializing` binding is tentative and helpable, not by itself a +`DuplicateKey` witness; exact Reserved is an ordered explicit reservation and +does block the same key. + +Supported recovery never cancels a resource owned by a live `Active` +participant. Normal recovery preserves it. Current-process reservation recovery +is a test/controlled-shutdown override that requires process-wide writer and +writable-view quiescence; racing that override with `TryReserve`, convenience +publication, token activity, or attached store-handle disposal is outside the +public result contract. Once a +reservation owner is safely stale or has published a quiescent +Closing/Recovering handoff, recovery may invalidate its token and reclaim its +slot. + +The reservation is an exclusive single-producer token. Copying it for ordinary +value passing is allowed, but concurrent `GetSpan`, `DangerousGetMemory`, +`Advance`, `Commit`, `Abort`, or `Dispose` calls through copies are unsupported. +The library prevents out-of-bounds/later-generation mutation; it does not assign +disjoint ranges to concurrent producers or verify that bytes were written before +the caller accounts for them. + +- Safe spans are valid only until the next reservation lifecycle action on that + token and never after commit, abort, recovery, token disposal, or store + disposal. +- `DangerousGetMemory` is retained-capable but has the same logical lifetime; + using it later is explicitly unsafe. +- Commit is atomic visibility, not a payload copy. +- Disposing a still-current reservation performs best-effort abort. +- A copied stale token cannot commit, advance, abort, or expose writable memory + after generation reuse. + +## Lease lifetime + +A successful acquire returns one shared read lease over immutable descriptor and +payload bytes in mapped memory. Several processes may lease the same generation. + +- Projection requires no named/global lock. +- Spans are valid until that exact lease is released/recovered or its local store + handle is disposed. +- Release is exactly once for a record incarnation. Copied/stale tokens cannot + release a later record reuse. +- Logical removal prevents new leases but existing valid leases retain their + bytes until release/recovery. +- Lease tokens are process-local API values, not blittable, serializable, or + transferable shared-memory records. + +## Diagnostics additions + +The existing `DiagnosticsSnapshot` remains available and gains additive v2 +properties for: + +- current profile/protocol; +- initializing/reserved/reclaiming/retired slot counts; +- claiming/recovering lease counts; +- primary directory occupancy and buckets with logically-Present versioned + spill summaries; +- overflow occupancy/scans; +- CAS retries, helped transitions, contention-budget exhaustion; +- stale token and owner-classification outcomes. + +Legacy tombstone/compaction properties remain numerically meaningful for v1.2 +and report zero/not-applicable for v2. A snapshot is bounded and moment-in-time, +not transactionally exact. + +## Disposal + +Disposal is local to one `MemoryStore` handle. It atomically rejects new local +operations, waits for already-entered local calls before unmapping, completes +that participant record's safely owned reservations/leases, proves no shared +control references its participant index, advances/frees the participant record, +and invalidates its tokens/views. It does not set a mapping-wide disposing state +or wait for other handles. + +`TryRemove` success means the key is logically absent and its bounded lease scan +completed without a protecting active lease. `RemovePending` means the key is +logically absent but either a protecting lease was observed or the caller bound +ended before classification/reclaim completed. Either path may leave physical +unlink/slot reuse to cooperative helping; callers may retry remove and must use +capacity outcomes rather than infer that a particular slot was synchronously +recycled. + +## Allocation contract + +After store initialization and path warm-up, successful and expected-failure v2 +publish/reserve/commit, acquire/project/release, remove, and retry paths allocate +0 managed heap bytes per operation. Public tokens remain value types. Test hooks, +diagnostic snapshots, recovery reports, and open/dispose are not claimed as hot +zero-allocation paths unless separately documented. + +Opening a lock-free handle eagerly allocates private `long[SlotCount]` and +`long[LeaseRecordCount]` capacity-proof buffers: approximately eight bytes per +configured value slot or lease record. The lease buffer is about 1 KiB for 128 +records, 64 KiB for 8,192, and 8 MiB for 1,048,576. Each is reused for the +handle's lifetime behind its own nonblocking local guard. They are neither +shared-memory capacity nor operation allocations and never serialize another +process/handle. diff --git a/specs/009-lock-free-publish-read/contracts/recovery.md b/specs/009-lock-free-publish-read/contracts/recovery.md new file mode 100644 index 0000000..17d5836 --- /dev/null +++ b/specs/009-lock-free-publish-read/contracts/recovery.md @@ -0,0 +1,312 @@ +# Recovery Contract + +## Scope + +Recovery restores bounded resources abandoned by a terminated participant. It is +explicit, caller controlled, and record local. It is not required for healthy +steady-state progress and never acquires authority over the whole mapping. + +Two mechanisms are distinct: + +- **Helping** completes an already published idempotent transitional operation + (`DirectoryOperation`, `Aborting`, `Reclaiming`, `Releasing`) without deciding + that an owner is dead. +- **Recovery** changes an owner-controlled `Initializing`, `Reserved`, `Claiming`, + or `Active` lifecycle only after classifying its exact participant incarnation, + unless that exact participant has already published `Closing`/`Recovering`. + +Any data caller may help. Explicit recovery performs owner classification only +for owner-controlled `Registering`/`Active` participants. Exact stable +`Closing`/`Recovering` is instead the participant owner's durable quiescent +handoff and requires no OS liveness decision. + +## Participant identity + +Every owner-controlled slot/lease state atomically stores: + +```text +participant token = (participant record incarnation, participant record index + 1) +``` + +That active participant record stores PID, identity kind, process-start value, +exact PID-namespace identity, and a nonwrapping participant-record incarnation. +The mapping Store ID, slot +generation, and lease-record incarnation further bind the token. PID alone is +never sufficient. + +The platform adapter classifies: + +| Result | Meaning | +|---|---| +| Live | PID exists and start identity matches | +| Stale | PID is absent or exists with a different start identity, with sufficient platform evidence | +| CurrentHandleAllowed | Owner is the calling process/handle and the explicit test/controlled-shutdown option permits recovery | +| Unsupported | Platform, namespace, permission, or identity source cannot make a safe determination | +| Inconsistent | Stored identity/state is structurally invalid | + +Unsupported, live, and inconsistent classifications preserve owner-controlled +state. A store/container deployment must share a process identity/liveness view +sufficient for recovery; otherwise normal access remains supported but recovery +reports unsupported. + +Header offsets 264/272 store the creator namespace and atomic recovery mode. +Linux derives the positive ID from `/proc/self/ns/pid`; Windows stores zero. A +different or unproven opener release-publishes monotonic Mixed before its first +Registering CAS and then retains ordinary KV access. Recovery snapshots the +participant control before acquire-loading mode. In Mixed, Registering is always +Unsupported because its per-record ordinary fields can be mixed; Active may be +classified only when its stable per-record namespace exactly matches the +caller's current namespace. The namespace gate precedes PID/start lookup. +Closing/Recovering are already claim-closed and remain helpable in either mode. + +## Participant recovery + +Participant records are initialized only under the cold lifecycle lock and +become referenceable only after `Active` publication. Lease and reservation +recovery continue to recover only their documented resource type: they double- +read the exact participant control/identity, classify namespace/PID/start, and CAS the +individual lease/slot control. A stale participant record remains `Active` and +therefore unreusable while any resource type still references it. + +After record-local recovery, an internal retirement pass definitely classifies +only `Registering`/`Active` participant candidates. It exact-CASes each stale +`Active(incarnation, pid)` candidate to claim-closed +`Recovering(incarnation, pid)` before taking an absence proof; `Closing` and +already-`Recovering` candidates are admitted from their exact captured controls +without process classification because they are already claim-closed. Slot and lease claims +require an exact `Active` post-claim recheck, so any claim ordered before that +fence is visible to the following fresh scan and no valid claim can become usable +after it. The pass then scans every value slot and lease record once, and only an +exact candidate token/control absent from that complete scan may CAS to unowned +`Reclaiming` with PID zero and exact-CAS advance to `Free(next incarnation)` or +`Retired`. Stale `Registering` may retire directly because it has never published +a referenceable Active token. Free/Retired identity fields are semantically dead; +the next exclusive Registering owner overwrites them before Active publication, +preventing a delayed helper from erasing a reused incarnation. Current-process +test/shutdown override does not retire a live `Active` participant record. If retirement +is interrupted, another recovery caller helps `Recovering` or `Reclaiming`; a new +opener cannot reuse it prematurely. + +## Lease recovery + +`TryRecoverLeases` preserves its current options/report surface and scans a +bounded lease table without a global data lock. + +### Current-process override precondition + +`RecoverCurrentProcessLeases: false` is the normal recovery mode and remains +safe concurrently with lease acquisition, projection, use, and release. A live +current-process participant is preserved in that mode. + +`RecoverCurrentProcessLeases: true` is an administrative override for tests and +controlled shutdown. Before selecting it, the caller MUST establish +process-wide quiescence for every store handle attached to this mapping: no +current-process thread may be acquiring a lease, projecting `ValueSpan` or +`DescriptorSpan`, consuming a previously borrowed span, or releasing a lease. +That quiescence MUST remain in force until `TryRecoverLeases` returns. Outstanding +abandoned Active lease records may remain; they are the override's intended +targets. + +The library does not add a process-local acquisition/recovery gate to enforce +this administrative precondition, because every hot-path lease operation would +otherwise pay for an exceptional shutdown/test policy. Concurrent invocation of +the override with current-process lease activity is outside the supported +contract. Exact record-incarnation fencing still prevents an old token from +mutating a later incarnation, but callers MUST NOT infer that an acquire racing +the override will return a still-valid lease. The precondition is process-wide, +not handle-local, because one recovery scan can target Active records created by +any handle in the calling process. + +For each record: + +1. acquire-load control; skip `Free`/retired records; +2. decode the participant token already present in `Claiming`/`Active`, then + double-read the corresponding participant record around its identity fields; +3. help `Releasing`/already-owned `Recovering` phases without liveness judgment; +4. classify the stable referenced participant, except that exact stable + participant `Closing`/`Recovering` is an unconditional handoff for both + `Claiming` and `Active` regardless of whether that process is still live; +5. for a safely stale `Active(r)`, or a current-process `Active(r)` explicitly + admitted by the test/shutdown override, CAS exactly to `Recovering(r)`; +6. for a safely stale `Claiming(r)`, CAS exactly to its recovery/reset phase. A + live current-process `Claiming(r)` remains active/not-eligible even when the + override is set, unless exact participant `Closing` or published + `Recovering`/`Reclaiming` state proves the claimant is quiescent; +7. make ordinary fields semantically dead by advancing record incarnation and + publishing `Free(r+1)` or retired; +8. if the exact protected slot is `RemoveRequested`, attempt cooperative reclaim. + +The conservative `Claiming` exception is required because its owner may still +have ordinary `SlotBinding`/`AcquireSequence` initialization writes in flight. +Recycling that record solely on a same-process override could let a delayed write +corrupt a later incarnation. Stale-process classification is sufficient because +that writer is gone; participant closing/recovery is sufficient because it +publishes the required quiescence or helpable handoff. + +Recovery helpers do not zero `SlotBinding` or `AcquireSequence`: a helper paused +before such an ordinary write could resume after another helper has published +`Free` and a new claimant has reused the record. Free/retired readers ignore +those fields, and the next exclusive claimant overwrites both before `Active` +publication. + +An `Active` CAS to `Recovering` is the recovered lease's release point. A live +release winning first makes the recovery CAS fail; recovery reclassifies the new +state and never decrements or clears it again. A recovered/copied lease token +fails its record-incarnation check and cannot release a later lease. + +Report meanings remain: + +- `ScannedRecordCount`: records examined; +- `RecoveredLeaseCount`: exact stale lease/claim records relinquished; +- `ActiveLeaseCount`: live/not-eligible records preserved; +- `UnsupportedLeaseCount`: owner safety could not be established; +- `FailedRecoveryCount`: inconsistent or repeatedly changing records not safely + mutated within the caller bound. + +## Reservation recovery + +`TryRecoverReservations` preserves its current options/report surface and scans +value slots. + +### Current-process reservation override precondition + +`RecoverCurrentProcessReservations: false` is normal recovery and is safe +concurrently with healthy publication activity. It preserves every +owner-controlled `Initializing` or `Reserved` lifecycle whose exact participant +is still live Active, regardless of `PublicationIntent`. + +`RecoverCurrentProcessReservations: true` is an administrative override for +tests and controlled shutdown. Before selecting it, the caller MUST establish +process-wide quiescence across every handle attached to the mapping: no thread +may be executing `TryReserve`, `TryPublish`, `TryPublishSegments`, reservation +projection, `Advance`, `Commit`, `Abort`, or reservation disposal, or using a +previously borrowed writable span/memory, or disposing a `MemoryStore` handle +attached to the mapping. Quiescence MUST remain until recovery returns. +Concurrent use of this override with current-process writer activity is outside +the supported result contract. The library does not add a hot-path process-wide +gate to enforce this exceptional policy. + +Even with the override, a live pre-metadata `Initializing` claimant is not +recycled merely from stale/`None` intent bytes: its ordinary intent/key/length +writes may still be in flight. It becomes eligible only through stale-process +classification or exact participant `Closing`/`Recovering` handoff. A stable +`Reserved` lifecycle may be recovered by the override after the required +quiescence. + +For each slot: + +1. acquire-load exact control, decode its participant token, and stabilize the + referenced participant identity; +2. ignore `Published`, `RemoveRequested`, `Free`, and `Retired` as reservation + recovery targets; +3. help existing `Aborting`/`Reclaiming` and directory descriptors; +4. classify stable owner-controlled `Initializing(g)` or `Reserved(g)`. + `Pre-metadata Initializing` is exact current Initializing with a zero + directory-operation word and no exact current canonical mutation/directory + cell; a current reference without its required `Insert/Prepared` marker is + corruption. Once that metadata-ready marker or a valid later reference is + acquire-observed, validate + `PublicationIntent=ExplicitReservation|AtomicPublication`; unknown intent is + corruption, while Free/Retired and pre-metadata Initializing ignore stale + ordinary intent bytes. Direct unreferenced cleanup after recovery of such a + claim also ignores those bytes; +5. for safely stale ownership, CAS exactly to `Aborting(g)`; +6. use the same generation-tagged abort/unlink helper as ordinary abort and + reclamation; help/claim the canonical descriptor and clear only the exact + generation-matching binding/location/operation if installed; +7. advance generation and publish `Free(g+1)` or `Retired`; leave ordinary + metadata semantically dead for the next exclusive Initializing claimant to + overwrite. + +Recovery never implements a second unlink protocol and never unconditionally +zeros directory or ordinary slot metadata. A publisher or recovery helper that +resumes after its exact lifecycle was recovered fails generation revalidation +and has no ordinary cleanup write capable of altering a later slot generation. + +Recovery never copies or publishes incomplete bytes. An +`ExplicitReservation` is publicly ordered at `Initializing -> Reserved`; an +`AtomicPublication` remains tentative in Reserved and is publicly ordered only +at `Reserved -> Published`. Either owner-controlled lifecycle may be reclaimed +only after supported stale/quiescent classification. If commit wins +`Reserved -> Published` before the recovery CAS, recovery preserves that +committed value for either intent and reports the race as non-recovered/active +according to the stable state. A former reservation token fails +generation/owner checks after recovery. + +Report meanings remain: + +- `ScannedSlotCount`: slots examined; +- `RecoveredReservationCount`: exact stale invisible lifecycles reclaimed; +- `ActiveReservationCount`: live/not-eligible reservations preserved; +- `UnsupportedReservationCount`: safe owner classification unavailable; +- `FailedRecoveryCount`: inconsistent/repeatedly changing state not safely + changed within the bound. + +## Removal/reclamation after a dead lease + +Recovery does not clear a value slot merely because it recovered one lease. It +attempts `RemoveRequested -> Reclaiming` only after a fresh stable scan finds no +other exact active lease record. Any concurrent acquire active before logical +removal is included; any activation after logical removal cannot return success. + +## Handle disposal + +Normal `MemoryStore.Dispose` knows its own participant record and does not need OS +liveness classification. After closing local operation entry, it CASes that +record `Active -> Closing` before any resource cleanup. It then spends one fresh, +finite 250 ms post-ownership allowance scanning and best-effort aborting/releasing +only exact captured controls carrying its participant token, helps transitions, +and attempts a bounded exact zero-reference retirement before unmapping. If the +allowance expires, the participant remains claim-closed `Closing`; any unrelated +explicit recovery caller may recover its resources and retire it without waiting +for or classifying the still-live disposer. If another thread still owns +a reservation/lease token through that facade, subsequent safe token actions +return disposed/invalid. Other handles in the same process remain live because +their participant indices differ. + +## Bounds and outcomes + +Recovery observes `StoreWaitOptions` as a bound on scan retries, helping, OS +classification, and cancellation. It never returns while holding a mutation +descriptor it claimed; it completes or leaves a fully helpable published +descriptor. Expected results use existing statuses/reports: + +- success with zero or more recoveries; +- operation canceled; +- local contention budget exhausted (`StoreBusy`) with report counts collected + so far and no false recovered count; +- unsupported classification represented in report counts; +- corruption/failed counts for unsafe structural state. + +Participant-table exhaustion is an open outcome, not a recovery mutation: +`StoreOpenStatus.ParticipantTableFull` rejects only the new handle. Callers may +explicitly recover stale participants from an already-open handle and retry +open; the library does not run a hidden recovery pass. If every participant +record is occupied and no handle remains open, a new open deterministically +returns `ParticipantTableFull`; it does not classify or reclaim owners while +opening. Deployments that must preserve an existing mapping across total process +loss therefore provision participant headroom and keep a recovery-capable handle +available, or explicitly recreate the fixed-capacity store. + +## Safety properties + +1. Recovery mutation compares Store ID, complete participant token and + PID/start identity, slot/record index, and exact incarnation/state as + applicable. +2. No recovery action can change `Published` back to an invisible state. +3. No live or unsupported `Registering`/`Active` owner is reclaimed without the + explicit current-process test/shutdown override already present in public + options and its process-wide quiescence precondition. Exact stable + `Closing`/`Recovering` is owner-published authority and does not require that + override or an OS liveness result. +4. A recovery winner invalidates every old token before storage/record reuse. +5. Recovery of one owner cannot stop ordinary operations on unrelated keys. +6. Capacity is considered restored only when a fill-to-capacity test can reuse + every safely recoverable slot/record, not merely when a diagnostic counter + increments. +7. A process stopping immediately after `Free -> Initializing` or + `Free -> Claiming` remains classifiable because that CAS already contains its + active participant index. +8. Recovery never interprets tentative `Reserved(AtomicPublication)` as a + completed public publish and never interprets unknown intent as either public + workflow. diff --git a/specs/009-lock-free-publish-read/contracts/validation-and-performance.md b/specs/009-lock-free-publish-read/contracts/validation-and-performance.md new file mode 100644 index 0000000..afaedc4 --- /dev/null +++ b/specs/009-lock-free-publish-read/contracts/validation-and-performance.md @@ -0,0 +1,410 @@ +# Validation and Performance Contract + +## Evidence layers + +No single stress run proves this protocol. Release evidence combines all five +layers below. + +### 1. Deterministic transition schedules + +Production atomic transitions expose stable internal checkpoint IDs through a +zero-cost generic/static instrumentation seam. The production no-op type must +inline away; tests use a controlled scheduler. A coverage test fails when a new +transition checkpoint lacks before/after ordering-point, pause, crash, and race +classification. The cross-process agent uses an internal friend-only factory to +instantiate the same generic protocol core with checkpoint instrumentation; +ordinary public construction always selects the inlined no-op specialization. + +Required families include: + +- same-key insertion/help with unrelated primary-lane mutation; +- primary-to-overflow spill and exact unlink; +- commit/acquire and incomplete commit; +- acquire/remove and 12-reader pending removal; +- release/reclaim and exactly one reuse; +- abort/commit/recovery; +- recovery/live release; +- stale descriptor/binding/reservation/lease after reuse; +- a helper paused after every operation/location validation and resumed only + after another participant completes the mutation, reclaims the slot, and + reuses it for a later generation; +- generation and record-incarnation retirement; +- disposal/operation and retained borrowed-memory lifetime. +- participant registration/exhaustion/close/recovery and a crash immediately + after the first participant-bearing slot/lease claim CAS; +- cancellation/deadline immediately before and after every public ordering point, + including verification that no owner-controlled claim remains on a canceled + pre-ordering path; +- intent publication immediately before directory discoverability, including + rejection of every unknown intent on a current discoverable lifecycle and + tolerance of stale bytes in `Free`, `Retired`, and pre-metadata + `Initializing`; +- explicit reservation immediately before and after + `Initializing -> Reserved(ExplicitReservation)`, and atomic convenience + publication immediately before and after both its tentative + `Initializing -> Reserved(AtomicPublication)` transition and its public + `Reserved -> Published` ordering point; +- same-key contenders proving that `Reserved(ExplicitReservation)`, + `Published`, and `RemoveRequested` can justify `DuplicateKey`, while + `Initializing` and `Reserved(AtomicPublication)` require help/revalidation; +- physical capacity exhaustion while the final slot is held in tentative + `Initializing` or `Reserved(AtomicPublication)`, followed by exact capacity + restoration after abort/reclaim; +- normal reservation recovery preserving every live Active owner, plus the + administrative current-process override only under the documented + process-wide writer/writable-view quiescence precondition; +- Insert cancellation handed to `Unlink/Prepared`, with a delayed location + publisher classifying target states that are exact old binding, empty, valid + in-range replacement, malformed, and mapping-out-of-range; +- two `Unlink/Prepared` helpers recovering distinct same-generation cells, with + the first valid location publication winning and the loser clearing only its + exact recovered binding, plus `Unlink/TargetSelected` observing each of exact, + empty, valid replacement, malformed, and out-of-range target state at a + same-generation alternate location; +- source loss immediately after a location CAS, both before terminal Unlink and + after a committed Insert successor, proving exact old target/location cleanup + without successor loss; and +- older location residue, true future-generation reuse, and a future location + enclosed by an otherwise exact stable old-generation tuple, including a + forced no-op-confirmation race that must retry rather than report corruption. + +Every pair is scheduled immediately before and after each ordering point. Only +the outcome sets in the feature spec are accepted. + +### 2. Bounded linearizability checker + +Generate small histories with 2-4 actors, 1-2 keys, and 6-12 calls. Record call +intervals, inputs, status, returned generation/bytes, and logical token IDs. An +offline backtracking checker enumerates sequential orders that preserve real-time +precedence against a simple reference model containing absent/reserved/published/ +removed values, shared leases, fixed slot/lease capacity, commit/abort, removal, +and stale tokens. The model distinguishes `Reserved(ExplicitReservation)` from +the non-public tentative claim inside an atomic convenience publication: +`TryReserve` orders at explicit `Reserved`, whereas `TryPublish` and +`TryPublishSegments` remain one abstract call that orders at `Published`. +Duplicate-key transitions use only the intent-aware witnesses listed above. + +`StoreFull` is validated as a physical-capacity outcome, not invented as key +ownership for a tentative publication. Production history capture therefore +records a semantic proof candidate after the first all-non-Free collect and a +separate confirmation only after the complete second collect matches. A strict +history accepts `StoreFull` only with its own distinct confirmed proof satisfying +`operation entry < candidate < confirmation < operation return`. The candidate, +not the later confirmation/checkpoint callback, is the physical ordering point. +An unconfirmed candidate, a candidate outside the call interval, or a delayed +claim/free timestamp cannot justify capacity. Exact Claim/Free/Retire events +remain lifecycle-integrity coverage only; the checker does not reconstruct a +possibly false simultaneous-full interval from delayed callbacks. A history +without the required confirmed proof cannot use an internal lifecycle to excuse +`StoreFull`. + +`LeaseTableFull` uses the same strict evidence rule with a distinct lease-proof +kind and the configured `LeaseRecordCount`: one candidate plus a later +confirmation must satisfy +`operation entry < candidate < confirmation < operation return`. The strict +checker rejects missing, out-of-window, wrong-capacity, or reused lease proofs. +Production-backed histories cover stable lease exhaustion and a release moving +between collects; the latter rejects the candidate and an Infinite caller +retries to success. After confirmation, acquire revalidates the exact directory +binding and Published generation so the proof and target existence share a +valid ordering instant. + +Deterministic schedules include stable full tables, a reusable slot moving +between collects, same-handle proof-guard contention, progress through a second +handle, malformed words in either pass, cancellation/NoWait/finite/Infinite +policy outcomes, and a failed pre-metadata claim proving the original control +word never reappears. Lease schedules additionally cover every malformed +state/incarnation/owner/token shape, target removal before the capacity +candidate, record movement, and the per-handle proof guard. Warmed full and +unstable-proof loops are included in the exact zero-allocation gate. + +Runs use reproducible seeds and minimize/print a failing history. Independent-key +histories are partitioned only when global capacity cannot couple them. The raw +memory-order path does not log through shared atomic counters that could add +fences and conceal a defect. + +### 3. Cross-process checkpoint and crash agent + +A C# test agent opens either profile, executes one requested operation, prints a +machine-readable checkpoint, and waits for controller input. The controller may +continue or kill it and then proves healthy same/unrelated-key outcomes, +recovery, fill-to-capacity restoration, and stale-token rejection. + +Required platform modes: + +- portable Windows/Linux cooperative checkpoint pause; +- `Process.Kill(entireProcessTree: true)` crash; +- Linux `SIGSTOP`/`SIGCONT` smoke; +- Linux `docker pause`/resume/kill where container prerequisites are available. + +External Windows thread suspension is optional diagnostic coverage, not a gate, +because a deterministic in-protocol checkpoint is safer and reproducible. + +### 4. Raw Release memory-order tests + +Independent processes run without scheduling/logging hooks. A producer fills +sequence, complement, key identity, generation, and deterministic full-payload +patterns before commit. Readers verify every byte; removers reuse slots +aggressively. Any partial, torn, stale, or mixed pattern fails immediately. + +Layout tests assert every atomic word's exact 8-byte alignment, one-width access, +publication order, and nonreuse while a lookup/lease can reference a generation. +They also assert required-features mask 7, `PublicationIntent` at exact slot +offset 52, PID-namespace identity/mode at header offsets 264/272 and participant +offset 32, their state/ordering rules, and fail-closed rejection between current, +required-features-zero, bit-0-only, and mask-3 draft mappings before payload projection. +The primitive suite includes a cross-process two-word Dekker test mirroring +lease activation versus logical removal: after each side performs its SC +`Interlocked` RMW, both sides observing the other's old value is forbidden. +Windows/Linux x64 are mandatory. A Linux ARM64 weekly/release job is required +before a later compatible protocol advertises ARM64; current v2 open tests require +non-x64 to return `UnsupportedPlatform`. + +### 5. Stress, allocation, tracing, and benchmarks + +Correctness stress uses unique generation patterns, reproducible PRNG seeds, +status histograms, early/late latency windows, and an actual fill-to-capacity +check after churn/recovery. + +Exact allocation gates run on a dedicated warmed thread, settle GC, sample +`GC.GetAllocatedBytesForCurrentThread`, execute at least 1,000,000 complete +cycles, and perform no lambda, assertion, formatting, or result collection in +the measured region. BenchmarkDotNet `MemoryDiagnoser` is supporting evidence, +not the exact-zero oracle. + +## Absence of the operation lock + +Use three independent checks: + +1. Inject a counting/throwing `ISharedStoreSynchronization`; reset after open and + assert zero calls for every v2 steady-state success/normal-failure path. +2. Hold the legacy Windows named synchronization object or Linux `.lock` byte + range indefinitely using the existing foreign-lock harness; warmed v2 data + operations must continue. +3. On Linux, trace a marked warmed measurement interval with + `strace -ff -yy -e fcntl,flock` and assert no store `.lock` `F_OFD_SETLK`, + `F_OFD_SETLKW`, `F_SETLK`, or `F_SETLKW` call. + +Windows ETW wait tracing is useful supplemental evidence but not a deterministic +release gate. Cold create/open/final-cleanup activity is excluded from the +steady-state interval and reported separately. + +## Benchmark methodology + +The multi-process harness emits JSON and records: + +- repository commit, package/layout/resource versions; +- OS build, architecture, .NET runtime/GC mode; +- CPU model, logical/physical count, assigned CPU set/affinity, memory; +- store dimensions and profile; +- exact key/payload/descriptor distribution and collision construction; +- process roles, lease duration, churn pattern; +- three trials: duration-bound rows use a 10-second minimum warm-up and + 60-second measurement, while count-bound rows must reach their exact target; +- the median reported trial; +- aggregate/per-process throughput, fairness, p50/p95/p99, maximum sampled + latency, producer allocation counts, whether any copy counter is actually + instrumented, structural copy-path evidence, and every status count. + +The qualification orchestrator supplies the probe's repository commit and +working-tree state as one validated pair from its independently captured start +provenance. The probe captures that pair, host metadata, and assembly hashes +once before workload execution; standalone runs use bounded, fail-closed Git +discovery when the pair is absent. Partial or malformed overrides are rejected. +Qualification revalidates the reported pair against clean start/completion +provenance, the normalized source manifest, and fresh tested-assembly hashes. + +Do not oversubscribe the machine. Give each participant one logical processor +where possible. If required participants cannot be assigned, report the workload +as not qualified rather than treating an underprovisioned result as a product +failure. + +## Required workload matrix + +| Workload | Participants | Pattern | Required evidence | +|---|---|---|---| +| Tiny operation | 1, 2, 4, 8, 12 processes | 8-byte rotating keys, 1-byte values, publish/remove and acquire/release | ops/s, p50/p95/p99, Windows 4x/80% target; Linux one-process intrinsic p99, eight-process throughput, <=3x self-amplification, <=10 us p99, and no >10 ms sampled stall | +| Same-key broadcast | 1, 2, 4, 6, 8, 12 readers | one 256-byte value, full checksum, immediate release | 6-reader >=4x and 12-reader >=7x single-reader throughput | +| Distributed keys | 1, 2, 4, 6, 8, 12 readers | 256 uniform stable keys, 256-byte values | 6-reader >=4.5x and 12-reader >=8x, zero false misses/checksum errors | +| Broker-directed primary | one zero-copy producer, 1 or 12 assigned readers, one observer | 1.3 MB frames, 16-byte descriptors, 256 rotating keys; test pipe sends keys only | 12-reader publication rate >=80% of one-reader, end-to-end latency, `ProducerStoreOperationAllocatedBytes == 0`, and structural direct-reservation-write/borrowed-lease-read evidence; the retained non-instrumented `FullPayloadCopies` field is not treated as a measured zero | +| Mixed churn | 12 readers, 2 publisher/removers | >=256 live collision-heavy keys, 10,000,000 cycles, including keys sharing one canonical bucket | correctness/capacity, zero stale-helper mutation, late missing/publish p99 <=2x early p99 | +| Participant suspension | distributed and churn roles | pause 30 s at every checkpoint | same healthy set >=90% own baseline on suitable keys/capacity | +| Large ingest | one producer, 1 and 12 readers | 100,000 direct 1.3 MB frames | exact bytes, allocation/copy evidence, throughput | + +The test broker is a lightweight test-only pipe protocol. It sends committed +keys and expected generations and receives application-level acknowledgements; +none of that state enters the production package or mapping. + +## Test tiers + +The qualification runner records three independent seeded counts. Production +race repetitions (`SMS_PRODUCTION_RACE_REPETITIONS`) execute the real mapped +`MemoryStore` action pair and are the only count credited to SC-011. Production +generated histories (`SMS_PRODUCTION_HISTORY_COUNT`) capture 6-12 real calls +from 2-4 actors for the reference checker and failure minimizer. Reference-model +checker invocations (`SMS_CHECKER_HISTORY_REPETITIONS`) exercise model/order +coverage only and are never reported as production race executions. + +### Pull request gate + +- build/analyzers and all existing fast unit/contract tests; +- deterministic schedules for the minimal lifecycle and directory descriptor; +- bounded linearizability smoke with fixed seeds; +- exact layout/alignment/API/profile/compatibility fixtures, including required + mask 7, publication intent at slot offset 52, and PID-namespace identity/mode; +- checkpoint pause/crash smoke for every transition family; +- raw visibility smoke; +- participant default/sizing/exhaustion/first-claim/reuse contracts; +- zero-allocation one-million-cycle gates; +- hostile held-legacy-lock test; +- Release package creation and package-consumption smoke. + +### Nightly Windows and Linux + +- randomized/minimized linearizability histories; +- 10,000,000-cycle churn; +- complete checkpoint/pause/crash matrix; +- Linux syscall trace; +- short multi-process performance matrix; +- disposal, collision/overflow, capacity, retirement, and recovery stress; +- Docker pause/recovery where supported. + +### Weekly/release qualification + +- complete three-trial workload matrix: duration-bound rows measure 60 seconds + and count-bound rows reach their machine-recorded profile-aware targets; +- at least 100,000,000 mixed operations in every lock-free mixed-churn trial; +- 100,000 direct 1.3 MB frames in every lock-free large-ingest trial (at least + 130 GB written per trial); +- 10,000 injected reservation/lease-owner termination cases; +- all 1,000,000-repetition race-family stress requirements after finite + deterministic schedules; +- legacy/new package, same-name incompatibility, upgrade/rollback, native/Python + rejection, full tests, and pack; +- Linux ARM64 memory-order run when that target is available/advertised. + +Windows x64 and Linux x64 qualification are distinct mandatory result sets. Each +runs the primitive mapped-atomic litmus and the raw full publish/acquire/remove/ +reuse pattern test with production checkpoints/logging disabled. Missing access +to one platform is recorded as **not qualified**, never as pass. The same raw +test fails on any torn, partial, stale, or mixed generation and does not use +shared diagnostic counters in its measured path. + +The probe controller enforces a per-trial deadline for every duration-bound row: +warm-up plus measurement plus the configured cleanup grace. At timeout it gives +tracked child-tree termination a bounded 100 ms best-effort budget, then +unconditionally fail-fast terminates the isolated probe. Count-bound rows remain +under the qualification step deadline and emit periodic progress heartbeats. Schema-v8 evidence records +`CountBoundProfiles`, `OperationTarget`, and `FrameTarget`; both targets may +never be positive on one row. + +## Qualification evidence contract + +The release gate is the schema-v4 output of +`scripts/run-lock-free-qualification.ps1`, not the exit status of any single +test or benchmark command. It explicitly restores the solution, runs the full +Release suite with TRX evidence, and then runs configured focused stresses. A +configured repetition/case count is credited only when its exact family +completion marker or TRX row count is present. The three zero-owner/leak claims +are mapped to named passing tests that inspect final diagnostics and prove full +slot/lease/participant capacity; copying their labels from configuration is not +evidence. + +Every performance report must contain exactly one row for each required +profile/scenario/process-count/trial tuple and exactly one summary row per tuple. +Unexpected, missing, duplicate, smoke-only, oversubscribed, affinity-incomplete, +wrong-duration, wrong-frame-target, or wrong-operation-target rows cannot satisfy +the gate. Correctness counters are checked before threshold calculations. + +The Linux-x64 `-Command all` report contains one required +`linux-tiny-performance` row. It runs schema-8 SyncProbe mode `sync` for exactly +Legacy and LockFree, `acquire-release` and `publish-remove`, process counts 1 and +8, 10-second warm-up, 60-second measurement, and three trials. All 24 raw rows +must be qualification measurements with zero failures, no oversubscription, +zero operation/frame targets, the recorded LockFree count-bound policy (which is +inapplicable to these time-based scenarios), complete unique per-row affinity, +internally consistent operation/status/worker +counters, and reproducible 8-row medians. Every raw row has at least two recorded +store operations per completed cycle and exactly one successful operation pair +per cycle (`Acquire`/`Release` or `Publish`/`Remove`); checksum-mismatch and +corruption-reason histogram rows are forbidden. For each scenario, lock-free +one-process median p99 divided by legacy one-process p99 is at most 1.0; +lock-free eight-process median throughput divided by legacy eight-process +throughput is at least 1.0; lock-free eight-process median p99 divided by its +own one-process median p99 is at most 3.0 and is at most 10 microseconds +absolute. Every individual lock-free raw row at either process count—not only +its median maximum—has `MaxMicroseconds <= 10000`. The same OS row remains +visible as optional/not-qualified on Windows and executes no workload there. + +Affinity CPU identifiers are unique native identifiers in `[0,63]` that the +probe reports as successfully applied, matching its current 64-bit native +affinity mask. They are not required to be less than the process-visible +logical-processor count because constrained Linux CPU sets can expose sparse +identifiers (for example, CPUs 8 through 15 with a visible count of eight). + +The release summary consumes two schema-v3 +`scripts/validate-lock-free-os.ps1 -Command all -Configuration Release` +reports: one `windows-x64` and one `linux-x64`. Both must pass every row marked +`required`, and both must match the release runner's repository commit and +normalized source-manifest SHA-256. Platform-inapplicable or explicitly optional +rows remain visible with `required: false`; missing required tools or platform +evidence produces **not qualified**. + +For each OS report, the release runner derives the exact sibling `.evidence` +root from the report path. Manifest paths must be unique, normalized repository- +relative paths contained by that root; no report, root, component, or descendant +may be a reparse point. The manifest and recursive actual file set must be +identical, every length/SHA-256 must match, and every executable result's stdout +and stderr path/hash must bind one manifested file. Every passing executed row, +including `clean`, must retain its command and bound logs; only structural +`self-test-*` rows and optional not-qualified platform rows are exempt from that +executable-row requirement. An exempt row encodes `command`, `stdout`, `stderr`, +and both stream digests as JSON `null`; empty strings, whitespace, partial +command/log tuples, and missing nullable fields are invalid rather than aliases +for absent evidence. The Linux raw performance JSON is validated again by the +release runner, including provenance and tested-assembly hashes. Accepted +OS report hashes and canonical evidence-tree digests are recorded in the +schema-4 release summary and revalidated before completion. + +Both scripts use new evidence paths and refuse to overwrite an existing result. +They record commit/tree/status/source, script/config/solution, runtime/toolchain, +stdout/stderr, and artifact hashes. Exit code 0 is pass, 1 is failure, and 2 is +not-qualified. `-ValidateOnly` performs structure/configuration checks without +workloads and emits `overallStatus: validation-only`; it is never qualification +evidence. See `release-qualification.md` for the current result set and preserved +historical failures. + +## Convergence gates + +Implementation stops and raises a design flag when a gate fails with the same +underlying invariant after two evidence-driven protocol corrections, or +immediately when correction would require a forbidden global owner, unavailable +128-bit atomic, weakened capacity/correctness contract, native runtime shim, or +new material public semantic choice. + +1. **Atomic layout**: no unaligned/mixed-width atomic or partially initialized + published reference; mapped Interlocked litmus passes Windows/Linux x64. +2. **Minimal lifecycle**: one-key reserve/commit/acquire/remove/release/reuse + passes every controlled schedule and the reference model. +3. **Directory**: same-key help/spill/unlink creates no duplicate current binding, + stable-key false miss, early capacity loss, unhelpable descriptor, or + old-generation helper mutation after slot reuse. Every nonzero operation and + location matches the exact mutation/slot generation. A spill summary is + Present before its cell, changes only by exact version CAS, reaches versioned + Empty only after a stable full scan, and cannot be cleared or resurrected by + delayed helpers after later generations. Directory-location handoff schedules + pass joint-tuple/no-op confirmation, first-publisher arbitration, alternate + location cleanup, and post-CAS source-loss rollback without false corruption + or removal of a committed successor or valid replacement. +4. **Reclamation**: pause/crash cannot reclaim live ownership, mutate through a + stale token, leak safely recoverable capacity, or block unrelated progress. +5. **Platform**: raw Release visibility passes and steady-state code never + touches the OS operation lock. +6. **Performance**: short profiling shows unrelated-key scaling and no material + uncontended regression after hot cache-line/dispatch causes are corrected. +7. **Release/code review**: long stress, compatibility, package, or independent + concurrency review has no unresolved invariant violation. + +Threshold failure caused only by an unqualified/undersized machine is reported +as missing qualification evidence, not silently passed. A correctness failure is +never averaged away by throughput. diff --git a/specs/009-lock-free-publish-read/data-model.md b/specs/009-lock-free-publish-read/data-model.md new file mode 100644 index 0000000..1d92e7e --- /dev/null +++ b/specs/009-lock-free-publish-read/data-model.md @@ -0,0 +1,599 @@ +# Data Model: Lock-Free Shared-Memory Key-Value Store + +## Model Boundary + +Layout 2.0 is a fixed-capacity, little-endian shared-memory protocol. It contains +no managed references, native pointers, process-local handles, queues, worker +assignments, or broker state. Every offset is relative to the mapping base and +every independently changing shared control is an aligned 64-bit word. + +The public key-value model remains: + +```text +opaque key -> at most one current immutable value generation + | + +-> zero or more shared read leases +``` + +## Store Header V2 + +The header is immutable after `Initializing -> Ready` except for the store +control's irreversible `Ready -> Corrupt` transition and explicitly identified +atomic counters/diagnostic hints. + +| Field | Width | Meaning | +|---|---:|---| +| Magic | 32 | `SMS2` little-endian identity | +| Layout major/minor | 16 + 16 | `2.0` | +| Header length | 32 | Exact bytes covered by this header version | +| Resource protocol | 32 | `2` | +| Required feature bits | 64 | Features every opener must understand | +| Optional feature bits | 64 | Ignorable/additive features | +| Total bytes | 64 | Exact mapped capacity | +| Store ID | 64 | Random nonzero mapping incarnation | +| Store control | 64 atomic | Store state and initialization incarnation | +| Slot count | 32 | Configured bounded value generations; `1..1,048,575` in layout 2.0 | +| Lease-record count | 32 | Configured simultaneous lease capacity | +| Participant-record count | 32 | Configured open-handle capacity; default 64, maximum 1,048,575 | +| Max key/descriptor/value bytes | 32 each | Public input bounds | +| Participant index/generation bits | 32 each | Layout-derived split totaling 28 token bits | +| Bucket/lane dimensions | 32 each | Primary directory shape | +| Section offsets/lengths/strides | 64 each | Bounds-checked relative locations | +| PID namespace ID | 64 | Creator's Linux `/proc/self/ns/pid` numeric token; zero on Windows | +| PID namespace mode | 64 atomic | Monotonic recovery mode: `Enabled=1`, `Mixed=2` | +| Diagnostic counters | 64 atomic each | Approximate monotonic event totals | + +Header validation checks magic/version/features before any payload projection, +then validates every multiplication, alignment, monotonic section boundary, +stride, count, and exact option match with checked arithmetic. + +Layout 2.0 currently requires feature mask `7`: bit 0 is the versioned-empty +exact-generation spill summary, bit 1 assigns per-slot `PublicationIntent`, and +bit 2 assigns the exact store/participant Linux PID-namespace identity. A +required-features-zero, bit-0-only, or mask-3 draft mapping is incompatible, +because those shapes cannot express the current publication and recovery +ordering. + +The creator writes header byte offset 264 and the initial mode before `Ready`. +A Linux opener whose current namespace is different or unproven release- +publishes `Mixed` at offset 272 before its first `Registering` CAS, then proceeds +with ordinary KV access. The downgrade never returns to Enabled. Windows stores +zero and begins Enabled. + +Store states: + +```text +Zero -> Initializing -> Ready + | | + +-----> Corrupt <-+ + \-----> Unsupported +``` + +Only cold create/open participates in the legacy-compatible named lifecycle +lock. A steady-state operation that proves persistent mapped structural +corruption full-word-CASes `Ready` to `Corrupt`; this is an atomic shared-memory +fail-closed signal, not OS synchronization, and it never returns to `Ready`. +Ordinary success, caller-input failure, contention, cancellation, and legal +concurrent lifecycle observations never change the store state. + +### Cold-open initialization authority + +Physical creation disposition is transient process-local state, not a mapped +header field. Each cold attempt records either `CreatedNew` or +`OpenedExisting`. Only `CreatedNew` authorizes `Zero -> Initializing` and the +initial writes that follow it; an open mode, requested profile or dimensions, +and observed zero bytes are not ownership evidence. + +An `OpenedExisting` zero header is never modified. `CreateNew` reports +`AlreadyExists`, `CreateOrOpen` reports `StoreBusy`, and `OpenExisting` reports +`IncompatibleLayout`. The cold transaction retains its ordered platform gates +through initialization or validation and participant registration, then either +transfers mapped-resource ownership exactly once or releases those gates before +failed-open owner cleanup. One caller wait/cancellation budget covers the whole +transaction. + +## Participant Registry V2 + +One 64-byte participant record represents one open `MemoryStore` handle. + +| Field | Width | Meaning | +|---|---:|---| +| Control | 64 atomic | 3-bit state + 28-bit incarnation + 32-bit PID; top bit reserved zero | +| Identity kind | 32 | Windows creation time, Linux proc start ticks, or unsupported | +| Process-start value | 64 | Platform value interpreted only with identity kind | +| Open sequence | 64 | Diagnostic identity, not hot-path ordering | +| PID namespace ID | 64 | Exact admitted Linux namespace token; zero on Windows | +| Reserved | remaining bytes | Zero in layout 2.0 | + +Participant state encodings are `Free=0`, `Registering=1`, `Active=2`, +`Closing=3`, `Recovering=4`, `Reclaiming=5`, and `Retired=6`; value 7 is invalid. +Open selects and initializes a record while holding the allowed cold lifecycle +lock, then release-publishes `Active` before returning the handle. An incomplete +participant record cannot be referenced by a data control and is safely cleared +only after the cold lock is reacquired following creator termination. + +Registration writes the per-record PID namespace before `Active`. A stable +Active identity snapshot jointly includes control, PID/start, open sequence, and +that namespace value; classification compares it with the caller's current +namespace before PID/start observation. While `Registering`, ordinary record +fields may still be a previous incarnation's mixture. Presence-only +classification therefore uses the creator header namespace only while mode is +Enabled and is Unsupported in Mixed, never trusting the partial per-record +field. A recovery reader snapshots participant control and only then acquire- +loads mode: any cross-namespace opener release-publishes Mixed before its claim, +so a reader that sees that claim also preserves it. Closing and Recovering are +already claim-closed and remain helpable in either mode. + +The hot-path participant token is 28 bits. Its low +`ceil(log2(ParticipantRecordCount + 1))` bits encode record index plus one and its +remaining bits encode participant incarnation. With the default 64 records this +is 7 index bits plus 21 incarnation bits; at the maximum 1,048,575 records at +least 8 incarnation bits remain. Normal operations cache the complete token. A +participant record cannot return to `Free(next incarnation)` until local +disposal or explicit stale-owner recovery has prevented new claims and a bounded +full reference scan proves no slot, lease, or directory operation contains that +token. It retires before the configured token incarnation wraps. + +Participant transitions are: + +```text +Free(g,pid=0) -> Registering(g,pid) -> Active(g,pid) +Active -> Closing (normal disposal after local operation entry closes) +Active -> Recovering (safely stale owner, for final zero-reference retirement) +Closing/Recovering -> Reclaiming(g,pid=0) (stable zero-reference scan) +Reclaiming -> Free(g+1,pid=0) or Retired (universally helpable metadata clear) +``` + +`Closing` is release-published only after the facade gate has stopped and +drained local calls, and before disposal begins mapped-resource cleanup. +`Closing` and `Recovering` are therefore exact, claim-closed owner handoffs: a +recovery caller does not classify their PID/start identity and may recover an +exact referenced slot/lease even while that process remains live. Retirement +still requires a fresh bounded exact-token absence scan and a full-control CAS. + +`Reclaiming` carries no live PID or ownership token. Any cold opener or recovery +caller may exact-CAS its control to the next Free/Retired control. It performs no +ordinary identity-field writes: those fields are semantically dead while Free or +Retired, and the next exclusive Registering owner overwrites every identity field +before publishing Active. This prevents a delayed reclaim helper from erasing a +later participant incarnation. Diagnostics counts Reclaiming separately from +usable Free records. + +## Directory Binding + +A directory cell is one aligned atomic 64-bit word. The fixed codec is: + +```text +0 = Empty +(slotGeneration[33 bits] << 31) | (slotIndex + 1)[31 bits] = Bound +``` + +The slot index portion is never zero. A bound cell is valid only while: + +1. its decoded slot is in range; +2. the slot control carries the same nonzero generation; +3. the slot state owns a key lifecycle (`Initializing`, `Reserved`, `Published`, + `RemoveRequested`, `Aborting`, or `Reclaiming` as applicable); +4. once metadata is discoverable, `PublicationIntent` is a known nonzero value; + and +5. the slot's exact stored key matches the caller key. + +Stale or impossible bindings are never followed into payload storage. A normal +operation may help clear a binding only when the exact slot generation proves it +obsolete; otherwise it reports corruption/retry according to the contract. + +## Primary Key Directory + +The primary directory is an array of buckets. Each bucket contains: + +| Field | Width | Meaning | +|---|---:|---| +| SpillSummary | 64 atomic | Versioned Present/Empty token carrying exact insertion slot and generation | +| Mutation | 64 atomic | Empty or exact binding of a helpable directory operation | +| Lane 0..7 | 64 atomic each | Empty or exact directory binding | + +Two independently mixed bucket indices are derived from the full 64-bit key +hash. Candidate lanes are scanned in one deterministic order shared by every +participant. Total primary lanes are approximately four times `SlotCount`, for +no more than approximately 25% primary load when `SlotCount` keys exist. + +`SpillSummary` is a versioned negative cache, not a count. Bits 0..19 carry +slot-index-plus-one, bits 20..52 carry the 33-bit slot generation, bit 53 is +Present, and bits 54..63 are zero. Raw zero is only the initial empty state. +Every nonzero candidate index must also be below this mapping's configured +`SlotCount`; a codec-shaped but mapping-out-of-range identity is corruption and +cannot act as an Empty negative cache. +Every overflow attempt CASes an exact prior summary to `Present(candidate)` +before its cell CAS. A stable empty full scan under the exact canonical mutation +CASes `Present(X)` to `Empty(X)`, preserving X's identity rather than restoring +zero. Lookup skips overflow for initial or versioned Empty. False-positive scans +are allowed on interrupted cleanup; a false negative for a current or +publishable overflow binding is forbidden. + +Cleanup first revalidates the summary's exact candidate binding, overflow +location, cell, hash, and canonical bucket. That stable witness retains Present +without a table scan. A stale, absent, or unstable candidate falls back to the +complete bounded overflow scan before Empty is permitted. If that scan finds a +different exact current spill witness Y, the still-current canonical mutation +may full-word-CAS `Present(X)` to `Present(Y)`. This prevents a removed newest +spill from making every later primary mutation rescan. Reusing Present(Y) is +safe only while Y's exact nonwrapping binding is current: an older clearer +could not have completed its required stable-empty scan while that same Y was +present. Empty identities are never reintroduced. + +`Mutation` is not an owner lock. Before claiming it, a mutator writes a complete +insert/unlink descriptor into the referenced slot. A caller that sees a nonzero +word validates the binding and idempotently completes or clears that operation. +The word covers only key-claim/final-unlink for one canonical home bucket and is +released before payload filling or lease retention. + +## Overflow Directory + +The overflow directory contains exactly `SlotCount` aligned binding cells. +Cells are scanned in a deterministic hash-derived circular order and return +directly to zero on exact removal. It has no tombstones, links, resize, compact, +or epoch switch. + +Capacity invariant: + +```text +live directory bindings <= owned non-Free value slots <= SlotCount +``` + +When a new publisher already owns one slot and needs overflow, at most +`SlotCount - 1` cells can contain other live bindings. Consequently at least one +cell can accept the binding. A full scan that finds no valid placement is an +inconsistent state, not an ordinary lower index capacity. + +## Value Slot V2 + +Each value slot has fixed metadata and disjoint fixed-stride key, descriptor, +and payload storage. + +| Field | Width | Mutation rule | +|---|---:|---| +| Control | 64 atomic | 3-bit state + 33-bit generation + 28-bit participant token | +| Directory binding | 64 | Exact binding installed for this lifecycle | +| Directory location | 64 atomic | None or generation-tagged primary/overflow cell used by exact unlink | +| Directory operation | 64 atomic | None or generation-tagged insert/unlink phase prepared before bucket descriptor publication | +| Key hash | 64 | Immutable for owned lifecycle | +| Key/descriptor/value length | 32 each | Immutable after initialization | +| Publication intent | 32 | `None=0`, `ExplicitReservation=1`, or `AtomicPublication=2`; immutable after metadata publication | +| Bytes advanced | 64 atomic | Monotonic within reservation length | +| Commit sequence | 64 | Diagnostic/order identity, not a cross-key transaction | +| Key/descriptor/payload offsets | 64 each | Layout-derived and bounds checked | + +Slot control encoding is fixed: + +```text +bits 0..2: state +bits 3..35: nonzero 33-bit slot generation +bits 36..63: complete participant token, or zero in unowned states +``` + +The first `Free(g, participant=0) -> Initializing(g, participant=p)` CAS already +identifies a published active participant record. The binding codec carries the +same generation in 33 bits plus a positive 31-bit slot index. Retirement occurs +before either representation wraps. + +The exclusive claimant overwrites `PublicationIntent` and every other ordinary +lifecycle field before release-publishing the exact current-generation +`Insert/Prepared` directory operation. That operation is the metadata-ready +marker and precedes canonical mutation and directory-cell publication. +Free/Retired and a pre-metadata Initializing lifecycle—operation zero with no +current-generation mutation/cell reference—ignore stale, `None`, or unknown +bytes at offset 52; direct unreferenced cleanup after recovery of that claim does +the same. A current reference without its required operation marker is +corruption. While the marker or a valid current reference exists, only +`ExplicitReservation` and `AtomicPublication` are valid; unknown intent fails +closed as `CorruptStore`. Reclaim does not zero this ordinary field, so a delayed +helper has no write that can erase a later generation. + +Public workflow and ordering are intent-specific: + +| Public workflow | Intent | Public ordering point | Pre-`Published` duplicate witness | +|---|---|---|---| +| `TryReserve` returning `ValueReservation` | `ExplicitReservation` | `Initializing -> Reserved` | exact `Reserved` | +| `TryPublish` / `TryPublishSegments` | `AtomicPublication` | `Reserved -> Published` | none; `Initializing` and `Reserved` remain tentative | + +`Published` and `RemoveRequested` are duplicate witnesses for either intent. +An exact tentative binding remains physical/helpable, but a same-key contender +must help/revalidate it and may return `StoreBusy` only after bounded retry +exhaustion; it cannot return `DuplicateKey` solely from that tentative state. +This key-ownership rule does not redefine capacity: every non-Free slot is +physically unavailable, so `StoreFull` may observe tentative Initializing or +Reserved lifecycles. After an initial absent-key lookup, a raced insertion may +return `StoreFull` at candidate claim before its final same-key arbitration; a +real physical-capacity result does not imply or require a duplicate-key witness. + +Scan exhaustion alone is not that witness. Each open handle owns an eager local +`long[SlotCount]` scratch snapshot and a nonblocking local guard used only on the +rare full candidate path. It collects every control in slot order, records the +instant between collects as a candidate, and repeats the collect in the same +order. Only structurally valid, all-non-Free, exactly equal collects confirm the +candidate and return `StoreFull`. A guard conflict, `Free`, or movement leaves +the candidate unconfirmed and is retried according to `StoreWaitOptions`. +Structural validity covers the complete control word: generation is nonzero and +bounded; `Initializing`/`Reserved` carry a structurally valid token for a +configured participant record; `Free`/`Published`/`RemoveRequested`/`Aborting`/ +`Reclaiming`/`Retired` carry token zero; and `Retired` uses the terminal +generation. Any other shape is `CorruptStore`, not occupancy evidence, even if +the malformed word is identical in both collects. + +This proof relies on a strictly forward slot lifecycle. In particular, a failed +pre-metadata claim never restores `Free(g)` after publishing +`Initializing(g,p)`; it hands off through `Aborting(g,0)` and +`Reclaiming(g,0)` before reaching `Free(g+1)` or terminal `Retired(g)`. Thus an +exact control value cannot disappear and reappear between the two reads. The +scratch array costs approximately `8 * SlotCount` bytes per process/open handle +(about 8 MiB at `2^20 - 1` slots), is not mapped/shared state, and introduces no +per-operation allocation or cross-process owner. + +The lock-free `SlotCount` ceiling is `2^20 - 1`, so the primary directory has at +most `2^22` lanes and every primary/overflow target needs at most 22 bits. +`DirectoryLocation` stores target kind (2 bits), target index (22), and exact +slot generation (33); seven high bits are reserved. `DirectoryOperation` stores +intent (2), phase (3), target kind (2), target index (22), and exact slot +generation (33); two high bits are reserved. All-zero means no location or no +operation. Every nonzero word has a nonzero generation matching the lifecycle. + +Operations are None, Insert, or Unlink and progress `Prepared -> +TargetSelected -> BindingChanged -> Complete` with a section-bounded target, or +`Prepared -> Rejected` when an insert loses duplicate/cancellation arbitration +before installing a binding. Helpers compare/exchange complete words. They may +act only when operation/location generation matches both the bucket mutation +binding and slot control. Consequently a helper paused across slot reclaim/reuse +cannot match a later lifecycle, even when intent/phase/target values repeat. + +Generation dominance is asymmetric. A helper for generation `G` may clear its +exact `G` word or a strictly older residue after revalidating its exact binding +and operation. A generation greater than `G` proves benign reuse only when the +old canonical tuple has moved; a future location enclosed by an otherwise +stable exact `G` tuple is malformed and is preserved while corruption is +reported. Because all-zero remains the unversioned empty value, a delayed +`0 -> tagged(G)` CAS can briefly install only its own older residue; +postvalidation withdraws that exact word and a current later generation may +exact-clear it. No older helper may clear or replace a later generation's +nonzero word. + +Cancellation dominance is also asymmetric. Once exact slot control changes +from owned `Initializing`/`Reserved` to unowned `Aborting` or `Reclaiming`, an +insert helper that revalidates that state switches to the cancellation path; it +does not classify the state as corrupt or try to make the slot `Reserved`. +Phase validation is only a snapshot: a helper that loses the race after +validation re-reads the exact operation and slot generation before reporting a +failure. An overflow `Empty(binding)` created by exact cancellation is a legal +terminal version for that insertion and prevents an older setter from +re-publishing `Present(binding)`. + +Location publication is validated as one joint tuple: canonical mutation, +operation, location, slot control, immutable directory binding, and selected or +competing target cells. A repeated invalid tuple is terminal only after stable +double collection plus exact no-op atomic confirmation; movement restarts or +ends the stale helper. During canceling `Insert -> Unlink/Prepared` handoff, a +cleared target or structurally valid replacement is progress, while a stable +malformed or out-of-range target is corruption. The first valid Prepared unlink +location wins; a loser exact-clears only its distinct old binding. A later +TargetSelected unlink may exact-clean a same-generation alternate location and +old-binding cells while preserving replacements. Post-CAS source loss withdraws +only the publisher's exact old target/location and cannot erase a committed +Insert successor. + +For `ExplicitReservation`, the exact +`Initializing(g,p) -> Reserved(g,p)` CAS is the public reserve ordering point. +Losing that CAS to legal cancellation keeps the lifecycle tentative and has no +abstract key-ownership effect. For `AtomicPublication`, the same CAS is only an +internal prepared stage; its public convenience operation orders at +`Reserved(g,p) -> Published(g,0)`. Lower-generation, unknown discoverable +intent, or impossible same-generation observations remain `CorruptStore`. + +Supported recovery does not cancel resources owned by a live `Active` +participant. Normal recovery preserves them; current-process reservation +recovery requires process-wide writer quiescence, and exact +`Closing`/`Recovering` is already a quiescent owner handoff. Thus supported +recovery can reclaim an ordered explicit reservation or a tentative atomic +publication only after the owner is stale or quiescent. Racing the +current-process administrative override with a live reserve/publish call is +outside the public result contract; generation fencing still prevents mutation +of a later lifecycle. + +### Slot states + +Slot state encodings are `Free=0`, `Initializing=1`, `Reserved=2`, +`Published=3`, `RemoveRequested=4`, `Aborting=5`, `Reclaiming=6`, and +`Retired=7`. + +| State | Key discoverable? | Acquirable? | Writable? | Reusable? | +|---|---|---|---|---| +| Free | No | No | No | Yes | +| Initializing | Physically after binding install, for helping only; not a public duplicate witness | No | Owner only | No | +| Reserved | Yes; duplicate only for `ExplicitReservation`, tentative for `AtomicPublication` | No | Owner only | No | +| Published | Yes | Yes | No | No | +| RemoveRequested | Yes, duplicate detection | No new leases | No | No | +| Aborting | Yes until exact binding clear | No | No | No | +| Reclaiming | Yes until exact binding clear | No | No | No | +| Retired | No | No | No | Never | + +### Slot transitions and ordering points + +```text +Free(g,0) --claim CAS-------------> Initializing(g,p,intent) +Initializing(g,p,ExplicitReservation) --reserve CAS--> Reserved(g,p) [TryReserve point] +Initializing(g,p,AtomicPublication) --prepare CAS-----> Reserved(g,p) [still tentative] +Reserved(g,p) --commit CAS-------> Published(g,0) [commit point; TryPublish/TryPublishSegments point] +Published(g,0) --remove CAS------> RemoveRequested(g,0) [logical-removal point] +Initializing/Reserved(g,p) --abort CAS--> Aborting(g,0) +RemoveRequested(g,0) --no leases CAS---> Reclaiming(g,0) +Aborting/Reclaiming --unlink/advance---> Free(g+1,0) or Retired(g,0) +``` + +Simple and segmented publication use `AtomicPublication`, may go from +`Initializing` through the same prepared `Reserved` state, and order only at +the commit CAS within their one public call. Explicit direct ingest uses +`ExplicitReservation`, returns after `Reserved`, and later `Commit` orders +value visibility at the same commit CAS. No payload or descriptor byte changes +after `Published` becomes observable. + +Helpable abort/reclaim finalization exact-clears only generation-tagged +operation/location words and directory bindings. It does not zero ordinary slot +metadata: a delayed helper may resume after another helper publishes Free and a +new owner reuses the slot. Free/Retired metadata is ignored, and the next +successful `Free -> Initializing` claimant overwrites every lifecycle metadata +field under exclusive generation ownership before publishing a binding. + +`ValueReservation` is an exclusive single-producer lifecycle token. It may be +copied as a C# value for ordinary passing, but concurrent `GetSpan`, `Advance`, +`Commit`, or `Abort` calls on copies are unsupported; the library guarantees +bounds and lifecycle fencing, not coordination of overlapping producer writes. + +## Lease Record V2 + +Each record independently represents one protecting read lease. + +| Field | Width | Meaning | +|---|---:|---| +| Control | 64 atomic | 3-bit state + 33-bit record incarnation + 28-bit participant token | +| Slot binding | 64 | Exact slot/generation protected | +| Acquire sequence | 64 | Diagnostic identity | + +Lease states: + +Lease state encodings are `Free=0`, `Claiming=1`, `Active=2`, `Releasing=3`, +`Recovering=4`, and `Retired=5`; values 6-7 are invalid in layout 2.0. + +```text +Free(r,0) -> Claiming(r,p) -> Active(r,p) -> Releasing(r,0) -> Free(r+1,0) + | | + `--------------+-> Recovering(r,0) -> Free(r+1,0) +``` + +- The first claim CAS already embeds an active participant token. `Claiming` is + therefore recoverable even when later target fields are incomplete, but it is + not reclamation authority. An acquire publishes `Active` and then revalidates + the directory and slot. +- `Active` matching the exact slot binding protects that generation. +- The `Active -> Releasing` CAS is the release ordering point. A public token + carries record index and incarnation, so a copied stale token cannot act on a + later lease in the same record. +- A crash in `Releasing` is helpable because protection ended at its CAS. +- Recovery may claim only an `Active` record whose exact participant incarnation + is safely classified stale. + +An exhausted allocation scan is not a simultaneous capacity witness because a +`Free` record can rotate behind it. Each open handle therefore owns an eager +local `long[LeaseRecordCount]` snapshot and nonblocking guard for the rare proof +path. It collects controls twice in record order. Only two structurally valid, +all-non-Free, exactly equal collects confirm `LeaseTableFull` at the candidate +instant between them. `Claiming`/`Active` require a structurally valid configured +participant token; every unowned state requires token zero, and retirement is +valid only at the terminal incarnation. Malformed controls are corruption. Every +reuse advances incarnation (or retires), so an exact control cannot disappear +and recur between collects. Free/change/guard conflict is contention under the +operation-wide wait policy. The snapshot costs `8 * LeaseRecordCount` private +bytes per handle and introduces neither per-operation allocation nor shared/OS +synchronization. + +## Participant Incarnation + +Participant identity is the stable lookup: + +```text +(mapping Store ID, participant record index, participant record incarnation) + -> (PID, identity kind, process-start value) +``` + +The record incarnation distinguishes later handles reusing one record. The +hot slot/lease control carries the complete compact index+incarnation token, and +the no-reuse-until-no-references rule supplies a second fence. Process start distinguishes PID reuse. Slot generation and lease- +record incarnation complete individual token identity. + +## Public Token Handles + +The facade stores engine-neutral private handles. + +```text +ReservationHandle: + Store ID, participant index/incarnation, slot index/generation, payload length + +LeaseHandle: + Store ID, participant index/incarnation, slot index/generation, + lease-record index/incarnation +``` + +Every token action first checks the local handle lifetime, then the relevant +shared control incarnation. Stale tokens can return only documented invalid or +already-completed outcomes and never mutate a current lifecycle. + +## Recovery Decision + +A recovery attempt is a caller-controlled classification plus exact CAS: + +| Classification | Mutation | +|---|---| +| Live owner | None; count/report active | +| Safely stale owner | Claim exact state/incarnation, finish abort/release/reclaim | +| Unsupported/unknown | None; count/report unsupported | +| Inconsistent identity/state | None unless exact help rule proves obsolete; report failed/corrupt | + +Recovery scans are moment-in-time and may race live actions. Classification does +not grant global authority. Every mutation rechecks the exact control word it +classified. + +## Diagnostics Snapshot + +Diagnostics aggregate bounded scans and local atomic counters: + +- free, initializing/reserved, published, remove-requested, reclaiming, retired + slots; +- active/claiming/recovering/free lease records; +- active/closing/recovering/free/retired participant records and open failures; +- primary occupancy, buckets with spill, overflow occupancy, maximum observed + candidate scan, CAS retries, and contention-budget exhaustion; +- publish/acquire/remove/release/recovery status counts; +- invalid/stale token attempts and recovery classifications. + +Counts may reflect different instants. Diagnostics never change ownership merely +to obtain an exact snapshot and never make a data operation wait for the scan. + +## Entity Relationships and Invariants + +```text +Store(1) + |-- ParticipantRecord(ParticipantRecordCount) --> owner liveness identity + |-- PrimaryDirectory(1) --binding--> ValueSlot(0..SlotCount) + |-- OverflowDirectory(1) --binding-> ValueSlot(0..SlotCount) + |-- ValueSlot(SlotCount) --owns-----> Key + Descriptor + Payload + `-- LeaseRecord(LeaseRecordCount) --> exact ValueSlot generation +``` + +Global invariants: + +1. One exact key has at most one live directory binding. +2. One directory binding names exactly one slot generation; stale bindings never + project bytes. +3. Every nonzero directory operation/location names the same generation as its + owning slot lifecycle; a stale helper can clear or roll back only the exact + old tagged word/binding and cannot mutate a later lifecycle. +4. A value becomes acquirable only at its commit CAS after all immutable bytes. +5. A successful lease has one active record before returning and revalidated the + exact published binding after activation. +6. A slot is reusable only after logical removal/abort, zero matching active + leases, exact binding clear, exact old-generation helper-word cleanup, and + generation advance; ordinary stale metadata is ignored until the next + exclusive initializing owner overwrites it. +7. No single slot, lease record, directory lane, recovery caller, or process owns + authority required for operations on all other suitable keys. +8. Every owner-controlled slot/lease state contains a nonzero participant token + matching an `Active` participant control; that record remains unreused until + every exact token reference is gone. +9. A cached directory reference and a slot snapshot form one valid observation + only when the exact raw source word is unchanged on both sides of stable + classification of its separately decoded slot binding. Primary/overflow + words equal the binding; a spill-summary source is the complete encoded + `Present(binding)` word. Source movement orders a fresh lookup or maintenance + retry; an unchanged exact reference word enclosing a malformed control orders + fail-closed corruption. +10. Directory-reachable slot controls always obey their full wire shape: + owner-controlled states carry a structurally valid configured participant + token, unowned states carry zero, generations are nonzero and bounded, and + Retired is terminal. diff --git a/specs/009-lock-free-publish-read/linux-owner-release-marker-results.md b/specs/009-lock-free-publish-read/linux-owner-release-marker-results.md new file mode 100644 index 0000000..16d6fab --- /dev/null +++ b/specs/009-lock-free-publish-read/linux-owner-release-marker-results.md @@ -0,0 +1,102 @@ +# Linux Owner-Release Marker Qualification + +## Current cold-open transaction supersession (2026-07-14) + +The original runs below established bounded release-marker fallback after a +mapped handle existed. A later structural review found that failed-open safety +also requires cross-process coordination before any mapping: otherwise an older +client can expose a zero region before entering its lifecycle lock and a newer +opener can mistake those bytes for initialization authority. + +The current implementation acquires `.lifecycle` and then `.lock` before +mapping, retains both through physical-disposition-aware header work and +participant registration, and releases them before any failed-open mapped-owner +cleanup. Only the physical creator initializes a zero header. The former test +named "Public failed open after mapping uses the bounded marker path" has been +replaced by `OpenBlockedBeforeMappingDoesNotPublishOwnerOrReleaseMarker`; the +current five-test Linux owner-release-marker suite passed 5/5. The expanded +profile/open matrix and exact-once cleanup tests also passed as part of the full +295/295 Linux Integration aggregate. + +Each admitted mapped handle locks a private regular-file owner anchor before +committing its sidecar line. Cleanup opens candidate anchors independently with +`O_NOFOLLOW`, requires regular-file metadata and an unlocked proof, and treats +unavailable or ambiguous `statx` evidence conservatively. Tests cover crashes +before sidecar commit, durable release-marker reconciliation, malformed and +special-file artifacts, and the blocked-before-mapping window, which publishes +neither an owner line nor a marker. A 12-process cold-open fan-out is bounded by +the caller's original finite wait budget. These worktree results are diagnostic; +the immutable final Linux and release evidence paths remain the qualification +authority. + +The historical results below remain useful evidence for marker durability, +mode, reconciliation, and bounded close behavior; their failed-open ordering is +not the current protocol. + +**Date**: 2026-07-13 +**Historical result**: PASS on Linux x64 for the pre-FR-056 protocol + +## Environment + +- Host: Windows x64 with Docker Engine 29.3.1, Linux containers +- Image: `mcr.microsoft.com/dotnet/sdk:10.0` +- Image ID: `sha256:4207e009b0b8c470b08db499ab86c33fcf29a2bd2849ab44c251ff1c560a0ecf` +- Repository digest: `mcr.microsoft.com/dotnet/sdk@sha256:ea8bde36c11b6e7eec2656d0e59101d4462f6bd630730f2c8201ed0572b295d5` +- SDK: 10.0.301 +- Test runtime: .NET 10.0.9, x64 +- Isolation: repository bind-mounted read-only; source copied to an ephemeral + 2 GiB container tmpfs, so Linux build outputs did not modify the host tree + +The installed WSL SDK was not qualified because its workload-manifest resolver +failed with `MSB4242` before project evaluation. The Docker result below is the +Linux-x64 qualification evidence. + +## Command + +```powershell +$repo=(Get-Location).Path +docker run --rm --mount "type=bind,source=$repo,target=/src,readonly" --tmpfs /work:exec,size=2g -w /work mcr.microsoft.com/dotnet/sdk:10.0 bash -lc 'tar -C /src --exclude=.git --exclude=bin --exclude=obj --exclude=artifacts -cf - . | tar -C /work -xf - && dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --filter FullyQualifiedName~LinuxOwnerReleaseMarkerIntegrationTests --logger "console;verbosity=normal"' +``` + +## Results + +The initial historical three-test run passed 3/3: + +| Test | Duration | Result | +|---|---:|---| +| Concurrent blocked releases publish distinct private markers without loss | 292 ms | PASS | +| Final-owner marker permits `CreateNew` while the releasing process remains alive | 256 ms | PASS | +| Held `.lifecycle` bounds dispose, preserves sibling use, and removes only the exact ghost | 268 ms | PASS | + +After adding malformed-finalized-marker fail-closed coverage, the historical run +at that checkpoint passed 4/4 in 1.5517 seconds. That pre-FR-056 suite then added +the old failed-after-mapping path and passed 5/5 in 2.7571 seconds: + +| Test | Duration | Result | +|---|---:|---| +| Concurrent blocked releases publish distinct private markers without loss | 323 ms | PASS | +| Final-owner marker permits `CreateNew` while the releasing process remains alive | 261 ms | PASS | +| Public failed open after mapping uses the bounded marker path (historical; superseded) | 873 ms | PASS | +| Malformed finalized marker rejects open and remains present | 1 ms | PASS | +| Held `.lifecycle` bounds dispose, preserves sibling use, and removes only the exact ghost | 259 ms | PASS | + +All marker files observed by the tests had Unix mode `0600`. The ordinary +`.lock` remained independently acquirable while `.lifecycle` was held. The +eight concurrent disposals completed in one bounded interval and produced eight +distinct exact-owner markers; the next opener reconciled all of them without a +lost release. + +A neighboring Linux regression run filtered to +`LinuxOwnerReleaseMarkerIntegrationTests`, +`LockFreeProfileOpenIntegrationTests`, and +`MultiStoreLifecycleIntegrationTests` passed 17/17 with no skips in one second. + +## Historical conclusion + +This checkpoint established that Linux close/open-failure region teardown no +longer had an infinite lifecycle-lock wait. Its mapped-after-failure ordering is +superseded by the current blocked-before-mapping cold transaction above. The +durability conclusion remains applicable: a permitted close fallback publishes +a replayable exact-owner marker through same-directory temporary-file rename, +and reconciliation orders raw owner-line removal plus atomic sidecar rewrite +before marker deletion. diff --git a/specs/009-lock-free-publish-read/plan.md b/specs/009-lock-free-publish-read/plan.md new file mode 100644 index 0000000..37efe53 --- /dev/null +++ b/specs/009-lock-free-publish-read/plan.md @@ -0,0 +1,402 @@ +# Implementation Plan: Lock-Free Shared-Memory Key-Value Store + +**Branch**: `codex/lock-free-csharp` | **Date**: 2026-07-12 | **Spec**: [spec.md](spec.md) + +**Input**: Feature specification from `specs/009-lock-free-publish-read/spec.md` + +## Summary + +Add an explicit C# lock-free profile to the existing bounded shared-memory +key-value store while retaining layout-v1.2 as the default legacy profile. The +new mapped layout 2.0 uses only naturally aligned 64-bit atomic control words, +a configurable participant registry (64 handles by default), a fast fixed-bucket +key directory with a capacity-preserving bounded overflow directory, +generation-fenced value slots, generation-tagged directory operation/location +words, per-slot publication intent that separates explicit reservation ordering +from atomic convenience publication, incarnation-fenced lease records, and +cooperative per-record recovery. Persistent mapped structural corruption is +published once through the header's terminal `Ready -> Corrupt` atomic latch; +all later mapped-data operations fail closed without an OS lock. +The lock-free profile caps `SlotCount` at 1,048,575 so every directory helper +state fits in one portable 64-bit atomic word. A public `MemoryStore` facade preserves the +recognizable key-value, reservation, zero-copy lease, removal, recovery, wait, +diagnostics, and disposal workflows. Named cross-process synchronization is +retained only for cold mapping initialization and compatibility validation; no +v2 steady-state data operation acquires it. + +## Technical Context + +**Language/Version**: C# 14 on .NET 10; mapped protocol documented with +fixed-width, language-neutral little-endian fields. + +**Primary Dependencies**: .NET base class library and existing Windows/Linux OS +adapters only; the Linux adapter uses libc `open`, `flock`, and `statx` for +cold-lifecycle owner anchors. xUnit and BenchmarkDotNet remain test/benchmark +dependencies. Interoperable `.lock`/`.lifecycle` coordination uses direct libc +`F_OFD_SETLK` open-file-description locks with a per-wrapper non-reentrant local +arbiter. OFD locks provide the missing same-PID boundary across assembly-load +contexts and native modules without a managed process-wide registry. Unsupported +kernels/filesystems fail closed. No new runtime package or broker dependency. + +**Storage**: Fixed-capacity named shared memory. Existing mapped layout 1.2 +remains unchanged. New mapped layout 2.0 (`SMS2`) contains an explicit header, +participant records, fixed CAS directory buckets, bounded overflow cells, +value-slot metadata and key/descriptor/payload storage, and lease records. The same public store name +resolves to the same physical mapping so a profile mismatch fails closed rather +than opening a parallel empty store. + +**Testing**: Existing xUnit unit, contract, integration, interop, package, and +Docker suites; deterministic atomic-transition schedules; a bounded +linearizability checker; cross-process checkpoint/pause/crash agents; raw +Release memory-order litmus tests; zero-allocation loops; BenchmarkDotNet; and a +multi-process benchmark/OS-lock tracing harness. Linux lifecycle tests kill or +pause owners around anchor creation, sidecar commit, release-marker publication, +and cleanup; special-file, symlink, malformed, locked, and orphan artifacts are +also exercised. + +**Target Platform**: Windows x64 and Linux x64, including same-host Linux +containers. Layout 2.0 initially rejects non-x64 processes even when they are +64-bit/little-endian. Linux ARM64 is a weekly/release memory-order qualification +target before a later compatible feature/version advertises that architecture. +The Linux anchor adapter treats an unavailable/blocked `statx` call as +ambiguous-live evidence, so metadata uncertainty cannot authorize deletion. + +**Project Type**: Reusable NuGet library in the existing multi-language +monorepo. Layout 2.0 is implemented by C# first; current C++ and Python clients +remain layout-v1.2-only and reject v2. + +**Performance Goals**: Meet SC-001 through SC-018, including 0 B/op warmed data +paths; on Windows at least 4x legacy aggregate throughput and 80% lower p99 for +the eight-process tiny-operation workload; on Linux no uncontended p99 or +eight-process throughput regression, at most 3x lock-free one-to-eight-process +p99 amplification, at most 10 microseconds eight-process p99, and no raw +lock-free trial stall above 10 ms in the exact three-by-60-second one/eight- +process acquire/release and publish/remove matrix; scale the broker-directed 1.3 MB workload from +6 to 12 readers without a store-wide lock; and retain early/late churn p99 +within 2x. + +**Constraints**: Preserve existing public status numeric values and all legacy +method signatures; no in-place v1.2 conversion; no 128-bit atomic requirement; +no named/global exclusive owner on v2 data paths; no hidden maintenance thread; +no mandatory payload copy; bounded retries/cancellation; fixed value, lease, and +participant capacity; lock-free `SlotCount` in `1..1,048,575`; every persistent +directory helper reference fenced by the exact 33-bit slot generation; trusted +same-host writers; explicit recovery only; and layout-v2 required feature mask +`7` (versioned spill summary, publication intent at slot offset 52, and exact +Linux PID-namespace identity at header offset 264/participant offset 32). +Named/file locking remains permitted only for bounded cold create/open/close +coordination. One cold-open scope owns those gates and the mapped resources +until initialization or validation and participant registration finish. On +Windows the named gate precedes mapping. On Linux `.lifecycle` precedes +reconciliation and stale data deletion, `.lock` precedes mapping, and release +occurs in reverse order. After held gates are released, ordinary synchronization +is disposed before failed-open or handle-close region cleanup can re-enter +`.lifecycle`. Current implementations retain the empty `.lock` and `.lifecycle` +inodes as stable rendezvous resources. Only the physical creator initializes an +unpublished header; an opener never treats zero bytes or `OpenMode` as creation +authority. Each C# Linux lock wrapper owns an OFD-lock descriptor and local +non-reentrant gate. The C++ adapter retains one shared `FileState`/descriptor +and timed mutex per canonical path inside one module; descriptors from different +modules or managed load contexts contend in the kernel. Unlock failure +retires/closes the affected descriptor before reopening its local gate. Each +managed Linux handle additionally holds +a private regular-file `flock` anchor after mapping and before its owner-sidecar +line is committed. The anchor is liveness evidence, never a data-operation lock; +cleanup removes only a canonical unreferenced anchor proven regular and unlocked +through a separately opened `O_NOFOLLOW` descriptor. Locked, nonregular, +malformed, inaccessible, or otherwise ambiguous artifacts are retained. +Every v2 operation also acquire-loads the aligned store control before a new +mapped projection or mutation. This is one ordinary shared-memory atomic read, +not a process-held critical section. Only revalidated persistent mapped +structural corruption can full-word-CAS `Ready` to terminal `Corrupt`; caller +input, capacity, contention, cancellation, and legal lifecycle races cannot +poison the mapping. + +**Scale/Scope**: One producer and 6-12 broker-directed workers are the primary +benchmark, but the contract supports multiple independent publishers, readers, +removers, observers, diagnostics callers, and store handles. V2 defaults to 64 +simultaneously open handles and permits explicit participant sizing. Validation includes +up to 100 million lifecycle operations, 100,000 1.3 MB direct ingests, and 10,000 +recovery cases in release/nightly tiers. The v2 value-slot count is configurable +from 1 through 1,048,575; larger capacities require a later layout version. + +## Constitution Check + +*GATE: Passed before Phase 0 research and re-checked after Phase 1 design.* + +- Library and package first: PASS. `MemoryStore` remains a general-purpose + key-value library facade; broker delivery and worker selection are test/sample + concerns only. +- Stable contracts and semantic versioning: PASS. Existing signatures, enum + assignments, resource discovery, and layout 1.2 remain supported. Layout 2.0, + resource protocol 2, package major-version impact, and rollback are explicit + and independently tested. +- Test-driven production quality: PASS. Tests precede each engine phase and + cover atomic layout, public contracts, deterministic races, linearizability, + crash recovery, disposal, allocation, performance, package consumption, and + platform behavior. +- .NET 10 baseline and portable core: PASS. C#/.NET 10 is first, while every + mapped field, state transition, ordering point, owner identity, and memory + order is documented without managed-object identity. Platform liveness and + mappings stay behind adapters. The non-portable Linux `flock`/`statx` owner + anchor is isolated in `Interop`, is absent from mapped layout and hot data + paths, preserves PID/start fallback for native/Python/older owners, and fails + conservative on unsupported or ambiguous artifacts. +- Minimal, observable, dependency-conscious design: PASS. No runtime dependency, + global mutable configuration, console output, broker, or hidden worker is + added. Maintenance is bounded and cooperatively helped by callers; diagnostics + are snapshot-based and caller controlled. + +The post-design re-check remains PASS. The directory overflow reserve preserves +the configured value-slot capacity even for exact hash collisions, so no hidden +index capacity or index-full status is introduced. Participant records are an +explicit open-handle capacity with a distinct appended open status. They are +claimed only on the cold path; a first slot/lease claim performs one acquire +validation of its own cache-line-isolated record but no hot-path registry RMW or +cross-participant cache-line contention. The approved v2 capacity ceiling makes +the primary/overflow target fit in 22 bits, leaving 33 bits in both directory +operation and location words for the exact slot generation. This closes the +stale-helper ABA window without a non-portable 128-bit atomic or a blocking +quiescence scheme. The same slot ceiling permits a one-word versioned +spill-summary codec (20-bit index, 33-bit generation, Present, reserved bits) +whose required-feature bit fences the earlier pre-release Boolean hint. Exact +Present/Empty CAS transitions restore fast missing lookups after churn without +allowing a delayed setter or clearer to manufacture a false negative. +The second required-feature bit assigns slot offset 52 to immutable publication +intent, so helpers and duplicate classification cannot confuse an explicit +reservation with the internal reserved stage of atomic convenience publication. +The third required-feature bit assigns exact Linux PID-namespace identities to +the store header and participant records, so a PID is never classified through +a different namespace view. Older drafts and current mask-7 clients reject one +another before payload projection. + +## Architecture and Dependency Direction + +```text +Public MemoryStore / ValueReservation / ValueLease + | + +--> legacy layout-1.2 engine --> existing synchronized protocol + | + `--> lock-free layout-2.0 engine + |--> terminal mapped-corruption control + |--> atomic directory + slot + lease protocols + |--> participant registry/recovery classification + `--> memory-map and cold-lifecycle platform adapters + |--> cold-open scope + creation disposition + |--> Windows named gate -> mapping + `--> Linux lifecycle -> lock -> mapping/owner + +Tests/benchmarks --> public facade + internal deterministic checkpoint seam +Protocol docs ----> every language implementation (C# v2 now; C++/Python reject) +``` + +`MemoryStore` owns the local mapped-memory lifetime and dispatches to exactly +one concrete engine. Public reservation and lease structs continue to call back +through that facade, but carry profile-neutral lifecycle and record incarnation +tokens. The lock-free engine depends on fixed-width protocol primitives; those +primitives do not depend on diagnostics, test orchestration, or public wrappers. +Platform-specific process-start/liveness classification remains isolated. A +`SharedStoreOpenScope` retains the ordered platform gates across mapping, +header work, and participant registration and reports a local +`RegionOpenDisposition`; engines may initialize only `CreatedNew`. Successful +construction transfers the region and synchronization resource exactly once, +while failed construction releases the inner and outer gates before disposing +resources whose owner cleanup can re-enter lifecycle coordination. Facade +construction occurs only after those gates are released. +On Linux, a handle acquires its per-owner anchor while the per-store lifecycle +gate is held, commits the exact `PID:start:token` sidecar line, and keeps the +anchor locked until its mapped view is gone and the line is absent or covered by +a durable release marker. A later lifecycle action reconciles markers and, after +an atomic sidecar replacement, sweeps only canonical unreferenced anchors that a +separate regular-file descriptor proves unlocked. This closes the crash window +between anchor creation and sidecar commit without introducing a store-wide hot +owner or interpreting foreign PID namespaces as local liveness. + +## Project Structure + +### Documentation (this feature) + +```text +specs/009-lock-free-publish-read/ +|-- spec.md +|-- plan.md +|-- research.md +|-- data-model.md +|-- quickstart.md +|-- contracts/ +| |-- public-api.md +| |-- layout-v2.md +| |-- concurrency-and-memory-ordering.md +| |-- recovery.md +| |-- compatibility-and-rollout.md +| `-- validation-and-performance.md +|-- checklists/ +| `-- requirements.md +`-- tasks.md +``` + +### Source Code (repository root) + +```text +protocol/ +|-- README.md +|-- layout-v1.2.md +|-- layout-v2.0.md +|-- resource-naming-v1.md +|-- resource-naming-v2.md +`-- compatibility.json +src/SharedMemoryStore/ +|-- MemoryStore.cs # Stable public facade +|-- SharedMemoryStoreOptions.cs # StoreProfile and v2 helpers +|-- ValueLease.cs +|-- Ingest/ValueReservation.cs +|-- Engines/ +| |-- IStoreEngine.cs +| `-- LegacyV12/ +| `-- LegacyV12StoreEngine.cs # Extracted, behavior-equivalent v1.2 path +|-- LockFree/ +| |-- LockFreeStoreEngine.cs +| |-- AtomicControlWord.cs +| |-- IndexBinding.cs +| |-- LockFreeKeyDirectory.cs +| |-- LockFreeSlotTable.cs +| |-- LockFreeLeaseRegistry.cs +| |-- LockFreeParticipantRegistry.cs +| |-- LockFreeRecovery.cs +| |-- ParticipantIncarnation.cs +| `-- LockFreeDiagnostics.cs +|-- LayoutV2/ +| |-- LayoutV2Constants.cs +| |-- StoreLayoutV2.cs +| `-- SharedRecordsV2.cs +|-- Lifecycle/ +| `-- StoreLifecycleGate.cs # Nonblocking operation entry +`-- Interop/ + |-- SharedStorePlatform.cs # Platform cold-open entry + |-- SharedStoreOpenScope.cs # Gate/resource transaction owner + |-- RegionOpenDisposition.cs # Created-new versus opened-existing proof + |-- WindowsSharedMemoryRegion.cs + `-- LinuxSharedMemoryRegion.cs +tests/ +|-- SharedMemoryStore.UnitTests/ # State machine, layout, retry, rollover +|-- SharedMemoryStore.ContractTests/ # API, status, package/profile contracts +|-- SharedMemoryStore.IntegrationTests/ # In-process/cross-process races and crash +|-- SharedMemoryStore.InteropTests/ # v1.2 native rejection of v2 +|-- SharedMemoryStore.LockFreeAgent/ # Deterministic checkpoint participant +`-- SharedMemoryStore.LinearizabilityTests/ +benchmarks/SharedMemoryStore.Benchmarks/ # Single-process allocation/latency +benchmarks/SharedMemoryStore.SyncProbe/ # Multi-process JSON benchmark/tracing tool +samples/LockFreeBrokerKeys/ # Test broker sends keys; store remains KV +``` + +**Structure Decision**: Preserve public source locations and isolate volatile +layout-2.0 algorithms under `LockFree`/`LayoutV2`. Extract the current v1.2 body +behind the facade without changing its observable behavior. Keep protocol +documents at repository root because all language distributions must understand +which layouts they may open. Test-only scheduling and broker orchestration live +outside the package. + +## Delivery Strategy + +1. Freeze legacy behavior with characterization and public API snapshots. +2. Add profile/participant sizing, layout-v2 records and participant registry, + atomic codecs, and cross-process aligned-atomic litmus tests before exposing + data operations. Establish the cold-open transaction first: acquire platform + coordination before mapping, carry physical creation disposition through + header work, initialize only a newly created region, register the handle + before release, and clean up failed opens only after ordered gate release. + Slot/lease claim controls atomically embed a participant token; no post-claim + identity window is permitted. +3. Implement reserve/commit/acquire/remove/release/reuse for one key with + deterministic checkpoints and a reference-model checker. Every helper phase, + directory-location publication, and cleanup uses a generation-tagged exact + CAS so an older helper can only install/clear its own stale reference and can + never match a reused lifecycle. Generation-mismatch cleanup is directional: + an older helper preserves every future-generation word, while a current + helper may exact-clear strictly older residue after fresh ownership + validation. Publications from all-zero are postvalidated and rolled back + only by comparison with the exact value just published. Insert helpers also + reclassify the exact slot after validation windows: a concurrent transition + to `Aborting`/`Reclaiming` routes to cancellation cleanup, while a changed + operation or generation ends the stale helper without a corruption result. + The C# directory-location publisher proves terminal invalid state from two + stable forward/reverse collections of the canonical mutation, operation, + location, control, immutable binding, and target tuple, followed by exact + no-op CAS confirmation. Cancellation target loss accepts empty or valid + replacement progress; `Unlink/Prepared` uses first-location-wins arbitration; + `Unlink/TargetSelected` cleans same-generation alternate locations; and + post-location-CAS source loss withdraws only exact old target/location state. + Older residue remains exact-cleanable, while a future location is reuse only + after another old-tuple member moves and is corruption inside a confirmed old + tuple. This is a helping/validation change with no wire-encoding change. + The exclusive claimant writes immutable `PublicationIntent` before + release-publishing the current-generation `Insert/Prepared` metadata-ready + marker, which precedes canonical mutation and directory-cell discoverability. + `Reserved(ExplicitReservation)` is an ordered key + owner; `Reserved(AtomicPublication)` remains tentative until commit. +4. Add the primary directory and spill-safe overflow, then multi-key concurrency, + simple/segmented publish, direct ingest, diagnostics, and local disposal. +5. Add exact owner-incarnation recovery and cross-process crash/pause coverage. + On Linux, publish PID-namespace identities before participation, isolate + `flock`/`statx` anchors in the cold-lifecycle adapter, unmap before releasing + liveness, reconcile durable close markers, and repair only canonical unlocked + pre-sidecar orphans while retaining every ambiguous artifact. +6. Thread the shared store-control capability through every structural validator + and projection predicate. Verify cross-handle `Ready -> Corrupt` propagation, + terminal reopen rejection, nonthrowing cleanup, and non-poisoning caller + failures with corruption injection tests. +7. Complete compatibility rejection, docs/sample/package changes, allocation + gates, performance matrix, and release qualification. Linux `-Command all` + owns an explicit raw tiny-operation matrix; the release runner independently + validates it and the exact non-reparse OS evidence trees, then revalidates + accepted report/tree digests at completion. Keep Legacy mixed-churn and + large-ingest rows as fixed-duration comparisons, apply the 100,000,000- + operation and 100,000-frame durability targets to lock-free rows only, and + bind that completion policy in schema-v8 per-run evidence with controller- + enforced duration deadlines. Arm each atomic monotonic-deadline watchdog + before store setup; reject overdue completion even if timer dispatch is + delayed. On timeout, give tracked child-tree termination a bounded 100 ms + budget and then unconditionally fail-fast the isolated probe process rather + than unwinding an in-flight infinite store operation. + +Each stage must keep the minimal lifecycle linearizable. Repeated failure of the +atomic, directory, reclamation, platform, or performance convergence gates in +`contracts/validation-and-performance.md` stops implementation for design review. + +## Semantic Version and Deployment + +- Mapped layout identity: new major `2.0` with magic `SMS2`. +- Resource protocol: version 2; physical region/lifecycle discovery names remain + the same so incompatible opens fail closed. +- NuGet: target `2.0.0` because public token representation and concurrency/wait + semantics expand even though legacy source workflows and enum values remain. +- Default profile: `StoreProfile.Legacy`, preserving compiled/default v1.2 use. +- Adoption: callers explicitly create/open v2 with `CreateLockFree` or `Profile`. +- Default v2 participant capacity: 64 open handles; exhaustion returns the + appended `StoreOpenStatus.ParticipantTableFull` without disturbing live handles. +- No conversion: drain/close and recreate/republish under the same name, or use a + new public store name for side-by-side rollout. Rollback likewise recreates a + v1.2 mapping and republishes application-owned data. + +## Complexity Tracking + +No constitution violations are planned. The second internal engine, mapped +layout, fault agent, and linearizability test project are required by the +explicit compatibility and lock-free correctness goals; none is a runtime +service or application-specific integration. + +## Phase 0 Research Summary + +See [research.md](research.md). The capacity-preserving directory design resolves +the material key-index fork without weakening configured slot capacity or +requiring a global rebuild. The approved generation-tagged descriptor redesign +resolves the observed helper/reuse ABA failure by trading an explicit v2 +`SlotCount` maximum for a portable single-word proof. Platform atomic behavior +remains an implementation gate proven by executable Windows/Linux tests. + +## Phase 1 Design Summary + +See [data-model.md](data-model.md), the contracts under [contracts](contracts/), +and [quickstart.md](quickstart.md). All public ordering points, atomic widths, +state transitions, ownership lifetimes, compatibility identities, validation +tiers, and non-convergence conditions are specified before task generation. diff --git a/specs/009-lock-free-publish-read/qualification-config.json b/specs/009-lock-free-publish-read/qualification-config.json new file mode 100644 index 0000000..c79b0d6 --- /dev/null +++ b/specs/009-lock-free-publish-read/qualification-config.json @@ -0,0 +1,169 @@ +{ + "schemaVersion": 5, + "seed": 1592590848, + "tiers": { + "pr": { + "stepTimeoutSeconds": 900, + "checkerHistoryRepetitionsPerFamily": 1024, + "productionHistoryCountPerFamily": 2, + "productionRaceRepetitionsPerFamily": 1024, + "churnCycles": 10000, + "recoveryCases": 67, + "performanceMode": "all", + "performanceWarmupSeconds": 2, + "performanceDurationSeconds": 3, + "performanceDurationBoundGraceSeconds": 60, + "performanceTrials": 1, + "mixedOperations": 100000, + "largeFrames": 100, + "largeFrameBytes": 1363148, + "suspensionBaselineSeconds": 1, + "suspensionPauseSeconds": 1, + "suspensionWarmupSeconds": 1, + "directoryGenerationStressRepetitions": 50 + }, + "nightly": { + "stepTimeoutSeconds": 7200, + "checkerHistoryRepetitionsPerFamily": 10000, + "productionHistoryCountPerFamily": 8, + "productionRaceRepetitionsPerFamily": 1000000, + "churnCycles": 10000000, + "recoveryCases": 10000, + "performanceMode": "all", + "performanceWarmupSeconds": 2, + "performanceDurationSeconds": 10, + "performanceDurationBoundGraceSeconds": 60, + "performanceTrials": 1, + "mixedOperations": 10000000, + "largeFrames": 10000, + "largeFrameBytes": 1363148, + "suspensionBaselineSeconds": 3, + "suspensionPauseSeconds": 5, + "suspensionWarmupSeconds": 2, + "directoryGenerationStressRepetitions": 24000 + }, + "release": { + "stepTimeoutSeconds": 21600, + "checkerHistoryRepetitionsPerFamily": 10000, + "productionHistoryCountPerFamily": 16, + "productionRaceRepetitionsPerFamily": 1000000, + "churnCycles": 100000000, + "recoveryCases": 10000, + "performanceMode": "full", + "performanceWarmupSeconds": 10, + "performanceDurationSeconds": 60, + "performanceDurationBoundGraceSeconds": 60, + "performanceTrials": 3, + "mixedOperations": 100000000, + "largeFrames": 100000, + "largeFrameBytes": 1363148, + "suspensionBaselineSeconds": 10, + "suspensionPauseSeconds": 30, + "suspensionWarmupSeconds": 10, + "directoryGenerationStressRepetitions": 1000000 + } + }, + "boundedOperationSlackMilliseconds": 250, + "suspensionMinimumHealthyThroughputRatio": 0.9, + "requiredLeakAssertions": [ + { + "id": "slot-owner-count=0", + "evidenceStep": "churn", + "testNameContains": "LockFreeChurnIntegrationTests.CollisionHeavyMultiProcessRemoveReuseRestoresCapacityAndKeepsLateLatencyBounded", + "assertedState": "ActiveReservationCount, InitializingSlotCount, ReservedSlotCount, ReclaimingSlotCount, and every directory occupancy are zero; FreeSlotCount equals SlotCount" + }, + { + "id": "lease-owner-count=0", + "evidenceStep": "churn", + "testNameContains": "LockFreeChurnIntegrationTests.CollisionHeavyMultiProcessRemoveReuseRestoresCapacityAndKeepsLateLatencyBounded", + "assertedState": "ActiveLeaseCount is zero and FreeSlotCount equals SlotCount after the final lifecycle" + }, + { + "id": "unreferenced-stale-participant-count=0", + "evidenceStep": "recovery", + "testNameContains": "LockFreeCrashRecoveryIntegrationTests.EveryCanonicalCheckpointCanBeKilledRecoveredAndFilledToCapacity", + "assertedState": "every terminated checkpoint participant is recovered and the complete participant, slot, and lease capacity can be reclaimed" + } + ], + "completionEvidence": { + "referenceModelFamilies": [ + "publish-publish", + "publish-reserve", + "reserve-reserve", + "commit-acquire", + "acquire-remove", + "release-reclaim", + "recovery-live-action", + "disposal-operation", + "participant-capacity", + "value-capacity", + "lease-capacity", + "cancellation", + "stale-token" + ], + "productionHistoryFamilies": [ + "publish-publish", + "publish-reserve", + "reserve-reserve", + "commit-acquire", + "acquire-remove", + "release-reclaim", + "recovery-live-lease", + "disposal-operation" + ], + "productionRaceFamilies": [ + "publish-publish", + "publish-reserve", + "reserve-reserve", + "commit-acquire", + "acquire-remove", + "release-reclaim", + "recovery-live-lease", + "disposal-operation" + ] + }, + "performanceMatrix": { + "profiles": ["Legacy", "LockFree"], + "countBoundProfiles": ["LockFree"], + "shortScenarios": { + "acquire-release": [1, 2, 4, 8, 12], + "publish-remove": [1, 2, 4, 8, 12], + "same-key-read": [1, 2, 4, 6, 8, 12], + "distributed-key-read": [1, 2, 4, 6, 8, 12] + }, + "releaseOnlyScenarios": { + "broker-directed": [1, 12], + "mixed-churn": [12], + "large-ingest": [1, 12], + "sticky-overflow-miss": [1] + }, + "lockFreeOnlyScenarios": ["sticky-overflow-miss"] + }, + "linuxTinyPerformance": { + "mode": "sync", + "profiles": ["Legacy", "LockFree"], + "scenarios": ["acquire-release", "publish-remove"], + "processCounts": [1, 8], + "syncKeysPerWorker": 2, + "syncMaximumWorkerCount": 12, + "syncCanonicalBucketCount": 16, + "syncKeyCatalogSha256": "9A7E93EB1382F2665155971C64C10D4C29039916CD9E314DB72B9906549656D2", + "syncKeyCanonicalBucketAssignments": [ + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, + 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11 + ], + "minimumThroughputRatio": 1.0, + "maximumUncontendedP99Ratio": 1.0, + "maximumScaleP99Ratio": 3.0, + "maximumP99Microseconds": 10.0, + "maximumStallMicroseconds": 10000 + }, + "suspensionCheckpointIds": [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 29, 30, 31, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 46, 47, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 62, 63, 65, 66, 67 + ], + "platforms": ["windows-x64", "linux-x64"] +} diff --git a/specs/009-lock-free-publish-read/quickstart.md b/specs/009-lock-free-publish-read/quickstart.md new file mode 100644 index 0000000..8453a4c --- /dev/null +++ b/specs/009-lock-free-publish-read/quickstart.md @@ -0,0 +1,344 @@ +# Quickstart: Lock-Free Key-Value Store with Broker-Directed Workers + +This example uses an application-owned broker to send keys to workers. The +shared-memory package stores values by key; it does not choose workers, enqueue +keys, or acknowledge work. + +The API below is the layout-2.0 surface. All participants must use the +same name and dimensions. + +## 1. Create explicit lock-free options + +```csharp +using SharedMemoryStore; + +const string StoreName = "frames-v2"; + +var options = SharedMemoryStoreOptions.CreateLockFree( + name: StoreName, + slotCount: 256, + maxValueBytes: 1_300_000, + maxDescriptorBytes: 16, + maxKeyBytes: 32, + leaseRecordCount: 128, + openMode: OpenMode.CreateOrOpen, + enableLeaseRecovery: true); + +var open = MemoryStore.TryCreateOrOpen(options, out var store); +if (open != StoreOpenStatus.Success || store is null) +{ + throw new InvalidOperationException($"Cannot open {StoreName}: {open}"); +} + +using (store) +{ + Console.WriteLine($"Layout {store.ProtocolInfo.LayoutMajorVersion}." + + $"{store.ProtocolInfo.LayoutMinorVersion}, {store.Profile}"); + // Run the process role shown below. +} +``` + +`StoreProfile.Legacy` remains the default for existing options/helpers. Lock-free +layout 2.0 is never selected implicitly. `CreateLockFree` defaults to 64 +participant records—one per simultaneously open `MemoryStore` handle. Set +`participantRecordCount` explicitly when the deployment needs more; a full table +rejects only the new open with `ParticipantTableFull`. Recovery is explicit and +runs through an already-open handle; opening does not silently classify stale +participants. Provision headroom (and keep a recovery-capable handle available) +when an existing mapping must survive broad process churn. + +Layout 2.0 accepts `slotCount` values from 1 through 1,048,575. The explicit +ceiling keeps every helpable directory reference generation-fenced in one +portable atomic word; larger stores require a future layout version. + +## 2. Producer: write directly into mapped storage + +The producer owns frame creation and publishes the key to its broker only after +the store commit succeeds. + +```csharp +static StoreStatus PublishFrame( + MemoryStore store, + ReadOnlySpan key, + ReadOnlySpan descriptor, + Stream frameSource, + int frameLength, + IApplicationKeyBroker broker) +{ + var status = store.TryReserve(key, frameLength, descriptor, out var reservation); + if (status != StoreStatus.Success) + { + return status; + } + + using (reservation) + { + while (reservation.RemainingBytes != 0) + { + var destination = reservation.GetSpan(); + var read = frameSource.Read(destination); + if (read == 0) + { + return StoreStatus.ReservationIncomplete; + } + + status = reservation.Advance(read); + if (status != StoreStatus.Success) + { + return status; + } + } + + status = reservation.Commit(); + if (status != StoreStatus.Success) + { + return status; + } + } + + // Application concern: choose a worker and deliver/track the key. + broker.PublishKey(key); + return StoreStatus.Success; +} +``` + +There is no producer-owned full-frame buffer and no library copy after the +source fills the reservation. Partial bytes remain invisible until commit. +One reservation belongs to one producer; do not concurrently write/advance +through copied reservation structs. In lock-free v2, this explicit reservation +orders at `Reserved(ExplicitReservation)` and then becomes visible at commit. +The `TryPublish` and `TryPublishSegments` convenience APIs instead use an +internal `Reserved(AtomicPublication)` stage that remains tentative until the +one public call reaches `Published`; a same-key contender cannot use that +tentative stage alone as `DuplicateKey`. Both kinds of non-Free slot still +consume physical capacity and may contribute to `StoreFull`. + +## 3. Six to twelve workers: acquire the assigned key + +Each worker receives a key from the application broker, independently opens the +same store, and reads the immutable shared bytes. + +```csharp +static void ProcessAssignedKey( + MemoryStore store, + ReadOnlySpan key, + IApplicationKeyBroker broker, + int workerId) +{ + var status = store.TryAcquire(key, out var lease); + if (status == StoreStatus.NotFound) + { + broker.ReportMissing(workerId, key); // Application retry/dead-letter policy. + return; + } + + if (status != StoreStatus.Success) + { + broker.ReportStoreFailure(workerId, key, status); + return; + } + + using (lease) + { + DecodeFrame(lease.DescriptorSpan, lease.ValueSpan); + } + + broker.Acknowledge(workerId, key); // Not stored in SharedMemoryStore. +} +``` + +A copied key message is not a store lease. A worker must acquire and release its +own lease. Missing/removed keys are normal key-value outcomes, not broker state. + +## 4. Independent observer: read the same key concurrently + +The observer is not a worker and has no exclusive claim. It may lease the exact +same value while a worker holds it. + +```csharp +static bool TryObserve(MemoryStore store, ReadOnlySpan key, out ulong checksum) +{ + checksum = 0; + if (store.TryAcquire(key, out var lease) != StoreStatus.Success) + { + return false; + } + + using (lease) + { + checksum = ComputeChecksum(lease.ValueSpan); + return true; + } +} +``` + +Pausing this observer retains only its lease record and that value generation. +Other readers of the same key and operations on other keys continue. + +## 5. Remove only after application-level processing policy allows it + +The application—not the store—decides when assigned work and observation permit +cleanup. + +```csharp +var remove = store.TryRemove(key); +switch (remove) +{ + case StoreStatus.Success: + // Logically absent with no protecting lease observed. Physical slot + // reuse may already be complete or may finish cooperatively. + break; + + case StoreStatus.RemovePending: + // Logically absent now. Existing leases or bounded post-removal work may + // remain; a final release or later remove/help will reclaim exactly once. + break; + + case StoreStatus.NotFound: + // Already absent. + break; + + default: + HandleStoreStatus(remove); + break; +} +``` + +After logical removal, a new acquire does not succeed. A lease established +before removal continues projecting the exact immutable bytes. Republishing the +same key remains duplicate until that generation is safely reclaimed. + +## 6. Handle bounded pressure explicitly + +```csharp +var wait = new StoreWaitOptions(TimeSpan.FromMilliseconds(25), cancellationToken); +var status = store.TryAcquire(key, wait, out var lease); + +// Distinct application decisions: +// NotFound -> broker/application key is no longer current +// LeaseTableFull -> configured read-protection capacity is exhausted +// StoreBusy -> this caller exhausted its local retry/contention budget +// OperationCanceled -> caller canceled before acquisition ordered +``` + +V2 wait options do not wait for a key or capacity to appear and never acquire a +global store-operation lock. + +## 7. Recover a terminated participant explicitly + +Run recovery from an authorized control path after the application has evidence +that a participant terminated. Unknown/live owners are preserved. + +```csharp +var leaseStatus = store.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: false), + out var leaseReport); + +var reservationStatus = store.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: false), + out var reservationReport); + +Console.WriteLine( + $"lease={leaseStatus}, recovered={leaseReport.RecoveredLeaseCount}, " + + $"active={leaseReport.ActiveLeaseCount}, unsupported={leaseReport.UnsupportedLeaseCount}"); + +Console.WriteLine( + $"reservation={reservationStatus}, " + + $"recovered={reservationReport.RecoveredReservationCount}, " + + $"active={reservationReport.ActiveReservationCount}, " + + $"unsupported={reservationReport.UnsupportedReservationCount}"); +``` + +Recovery restores only safely stale slots/records. It is not needed to unblock +unrelated healthy operations and does not run in a hidden thread or during +opening. If every participant record is occupied and no handle remains open, +opening returns `ParticipantTableFull`; recreate the store or use deployment +coordination appropriate to that total-process-loss scenario. + +Keep `RecoverCurrentProcessLeases` false during normal operation; that mode is +safe while readers on any process acquire, project, use, and release leases. The +true value is an administrative test/controlled-shutdown override. Before using +it, stop new current-process acquisitions and drain all lease projection, +borrowed-span use, and release activity across **every** handle in the process +that is attached to this mapping. Keep those activities quiescent until recovery +returns. Merely draining the handle that calls recovery is insufficient, and the +library intentionally adds no hot-path gate to enforce this shutdown policy. + +Keep `RecoverCurrentProcessReservations` false during normal operation as well; +normal recovery preserves an exact live Active reservation/publication owner. +Before selecting the true administrative override, quiesce `TryReserve`, +`TryPublish`, `TryPublishSegments`, reservation projection and borrowed writable +memory, `Advance`, `Commit`, `Abort`, and reservation disposal across every +current-process handle attached to this mapping; do not dispose those store +handles concurrently either. Maintain that quiescence until recovery returns. +Racing the override with current-process publication activity is outside the +supported result contract. + +## 8. Inspect pressure without pausing data paths + +```csharp +if (store.TryGetDiagnostics(out var snapshot) == StoreStatus.Success) +{ + ExportGauge("sms.free_slots", snapshot.FreeSlotCount); + ExportGauge("sms.active_leases", snapshot.ActiveLeaseCount); + ExportGauge("sms.pending_removals", snapshot.PendingRemovalCount); + // V2 additive fields expose spill, retries, helping, and recovery pressure. +} +``` + +Snapshots may combine observations from nearby instants. They are safe during +live operations and do not make publishers/readers wait for a consistent global +snapshot. + +## 9. Upgrade and rollback safely + +- Do not point legacy and lock-free participants at the same live mapping. The + library rejects the incompatible profile. +- For same-name cutover, drain/close all legacy handles, recreate layout 2.0, + and republish application-owned data. +- For side-by-side deployment, use an explicitly different public store name and + switch application/broker keys under application control. +- Rollback recreates v1.2; it never reinterprets v2 bytes. + +## Application-owned broker sketch + +```csharp +public interface IApplicationKeyBroker +{ + void PublishKey(ReadOnlySpan key); + void Acknowledge(int workerId, ReadOnlySpan key); + void ReportMissing(int workerId, ReadOnlySpan key); + void ReportStoreFailure(int workerId, ReadOnlySpan key, StoreStatus status); +} +``` + +This interface is illustrative sample code and is not part of the +SharedMemoryStore package. + +## Validated package sample + +The repository sample can use either the source project (the default) or the +packed `SharedMemoryStore` 2.0.0 package. The package-consumer path is selected +with `-p:UsePackedSharedMemoryStore=true`; restore it from the directory +containing `SharedMemoryStore.2.0.0.nupkg`, then run the same built sample: + +```powershell +dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/quickstart-package +dotnet restore samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj ` + -p:UsePackedSharedMemoryStore=true ` + --source artifacts/quickstart-package +dotnet build samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj -c Release ` + --no-restore -p:UsePackedSharedMemoryStore=true +dotnet run --project samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj -c Release ` + --no-build -p:UsePackedSharedMemoryStore=true -- --workers 6 --frames 12 +dotnet run --project samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj -c Release ` + --no-build -p:UsePackedSharedMemoryStore=true -- --workers 12 --frames 24 +``` + +On 2026-07-13, this end-to-end package path built with zero warnings and zero +errors on Windows x64/.NET 10. Both runs completed successfully: 12/12 frames +with 6 workers and 24/24 frames with 12 workers. In each run the worker and +independent-observer checksums matched, lease-protected removal returned +`RemovePending`, the bounded missing-key read returned `NotFound`, diagnostics +reported layout 2.0, and explicit lease/reservation recovery reported no stale +records. diff --git a/specs/009-lock-free-publish-read/recovery-results.md b/specs/009-lock-free-publish-read/recovery-results.md new file mode 100644 index 0000000..52b9bbd --- /dev/null +++ b/specs/009-lock-free-publish-read/recovery-results.md @@ -0,0 +1,125 @@ +# Lock-Free Checkpoint Crash-Recovery Evidence + +## Directory publication/revalidation extension (2026-07-14) + +The append-only catalog now contains 67 checkpoints. Checkpoint 66 pauses after +an Empty-location source tuple has been revalidated and before the location CAS; +checkpoint 67 pauses after location publication and before source +revalidation. They make cancellation handoff, first-publisher arbitration, +post-CAS withdrawal, alternate-location cleanup, and generation-reuse behavior +crash-observable through the production callback. + +Focused Release runs of the complete catalog on commit `844448e` passed 67/67 on Windows +x64 and 67/67 on Linux x64 (WSL2). The corresponding final 8-process fixed-key +churn regression passed on both platforms, and the full Release aggregate on +each platform passed 985/985 tests with zero failures or skips. These historical +snapshot runs establish implementation/review closure; the immutable final qualification +workflow remains the release-evidence authority. + +## Projection and recovery-window extension (2026-07-14, historical) + +At this intermediate stage the append-only catalog contained 65 checkpoints. +IDs 62-65 covered canceled +insert cleanup, the no-active-lease reclaim proof, participant generation +advance, and the lease-projection metadata/control revalidation window. A +focused Release run of that complete catalog passed 65/65 on Windows +x64. The earlier Linux x64 run below covered the then-current 61-entry catalog; +the current 67-entry result above supersedes that former Linux gap. + +## PID-namespace checkpoint extension (2026-07-14) + +At the time of this extension the append-only catalog contained 61 checkpoints. +Checkpoint 61 pauses after +the per-record PID-namespace write and before Active publication. Focused +Release runs of the complete 61-case crash catalog passed 61/61 on Windows x64 +and 61/61 on Linux x64 (WSL2). The Linux namespace-focused unit set passed +55/55 and the focused profile/open integration set passed 11/11. These focused +runs verify routing and crash recovery; final tier artifacts remain recorded by +the qualification workflow. + +**Date**: 2026-07-13 +**Branch**: `codex/lock-free-csharp` +**Evidence commit base**: `0cf7a43` plus the feature worktree +**Host**: Microsoft Windows 10.0.26200, x64 (`win-x64`) +**SDK/runtime**: .NET SDK 10.0.201, .NET runtime 10.0.5, x64 + +## Release command + +```powershell +dotnet build tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --no-restore +dotnet test tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj -c Release --no-build --filter "FullyQualifiedName~LockFreeCrashRecoveryIntegrationTests" --logger "console;verbosity=minimal" +``` + +Result: + +- build: PASS, 0 warnings and 0 errors; +- checkpoint crash matrix: PASS, 31/31 tests, 0 skipped, approximately 3 seconds test duration; +- repeat stability check: PASS, 3 consecutive runs and 93/93 checkpoint cases; +- process termination mode: `Process.Kill(entireProcessTree: true)` at the exact instrumented protocol checkpoint. + +## Original 31-entry matrix coverage (2026-07-13 snapshot) + +At this historical checkpoint, +`LockFreeCrashRecoveryIntegrationTests.CanonicalCheckpoints` enumerated the +production `LockFreeCheckpointCatalog.Entries` collection directly. Therefore a +new catalog entry automatically adds a required crash case instead of relying on +a duplicated test list. + +| Checkpoint family | Catalog entries exercised | +|---|---:| +| Publish | 2 | +| Reserve | 2 | +| Commit | 2 | +| Abort | 2 | +| Acquire | 2 | +| Project | 2 | +| Release | 2 | +| Remove | 2 | +| Reclaim | 3 | +| Directory | 4 | +| Diagnostics | 2 | +| Recovery | 2 | +| Disposal | 2 | +| Participant | 2 | +| **Total** | **31** | + +For every entry, the child opened the real friend-instrumented layout-v2 engine, +emitted one JSON checkpoint signal, and blocked inside the production callback. +While it was stopped, the controller published, acquired, checked, released, +and removed an unrelated key. The controller then killed the child, ran explicit +lease and reservation recovery with current-process recovery disabled, and +required all recovery passes to converge without a failed-recovery count. + +## Safety and capacity evidence + +- **Live-owner preservation**: the controller retained a live lease during every + child pause and recovery. Recovery continued to report at least one active + lease, and the controller verified the original bytes before releasing it. +- **Killed-token fencing**: outside participant-registration checkpoints, the + child captured an independent lease handle before entering its target + operation. After kill and recovery, the controller reconstructed that + adversarial internal token and proved it was invalid and could not release a + later lease-record incarnation. +- **Local stale-copy fencing**: a copied controller lease was released, the same + table was reused, and the copied token could not release the replacement. +- **Value capacity restoration**: after cleanup, all 4 configured slots accepted + publications and the fifth returned `StoreFull`. +- **Lease capacity restoration**: all 4 configured lease records could be active + simultaneously and the fifth acquire returned `LeaseTableFull`. +- **Participant capacity restoration**: with the controller handle open, 7 more + handles opened successfully for the configured capacity of 8 and the ninth + returned `ParticipantTableFull`. This passed after crashes at both + `ParticipantAfterActivePublication` and `DisposalAfterParticipantRelease`, + covering stale active participants that had no remaining slot/lease reference. +- **No store-wide failure**: no case returned `CorruptStore`, leaked recoverable + slot/lease capacity, invalidated the live owner, or prevented unrelated-key + progress. + +## Original platform qualification boundary (2026-07-13) + +This historical run qualified the portable checkpoint pause and process-kill recovery mode +on the current Windows x64 host. Linux x64 `SIGSTOP`/`SIGCONT` and Docker +pause/resume/kill were not available in this Windows run and remain **not +qualified** by this historical section; the later Linux current-catalog result +is recorded above, while final platform qualification remains machine-derived +from the immutable release evidence. diff --git a/specs/009-lock-free-publish-read/release-qualification.md b/specs/009-lock-free-publish-read/release-qualification.md new file mode 100644 index 0000000..23cfb20 --- /dev/null +++ b/specs/009-lock-free-publish-read/release-qualification.md @@ -0,0 +1,317 @@ +# Lock-Free Release Qualification + +## Frozen one-shot qualification contract + +This tracked document is frozen before the final runs. It does not narratively +declare a result and must not be edited after a run to turn a failure into a +pass. The release is **QUALIFIED if and only if** every predicate below is true +in the linked raw JSON; otherwise it is **NOT QUALIFIED**. The raw JSON below is +the authoritative final record, even when it reports `failed`, +`not-qualified`, or is absent. + +| Tier or platform | Authoritative raw evidence | +|---|---| +| PR | [`009-final-r6-pr/summary.json`](../../artifacts/lock-free-qualification/009-final-r6-pr/summary.json) and [`009-final-r6-pr/sync-probe.json`](../../artifacts/lock-free-qualification/009-final-r6-pr/sync-probe.json) | +| Nightly | [`009-final-r6-nightly/summary.json`](../../artifacts/lock-free-qualification/009-final-r6-nightly/summary.json) and [`009-final-r6-nightly/sync-probe.json`](../../artifacts/lock-free-qualification/009-final-r6-nightly/sync-probe.json) | +| Release | [`009-final-r6-release/summary.json`](../../artifacts/lock-free-qualification/009-final-r6-release/summary.json), [`009-final-r6-release/sync-probe.json`](../../artifacts/lock-free-qualification/009-final-r6-release/sync-probe.json), and the runner-created Windows [`009-final-r6-release/os-validation.json`](../../artifacts/lock-free-qualification/009-final-r6-release/os-validation.json) | +| Linux x64 | [`009-final-r6-linux-x64.json`](../../artifacts/lock-free-os-validation/009-final-r6-linux-x64.json) and its required raw [`linux-tiny-performance.json`](../../artifacts/lock-free-os-validation/009-final-r6-linux-x64.evidence/linux-tiny-performance.json) | + +The predicate is: + +1. The three summaries have schema 4, `validationOnly: false`, their exact + `pr`, `nightly`, and `release` tiers, and `overallStatus: passed`; every + required result is passed and neither skips performance nor OS validation. +2. Every summary has `provenance.workingTreeState: clean`; its start and + completion `commit`, `headTree`, `workingTreeState`, `statusSha256`, and + `sourceManifestSha256` are equal; its tested-assembly manifests are equal; + and its `completion-integrity` result is passed. +3. All three summaries identify one identical clean commit, tree, status hash, + and source-manifest hash. Their evidence manifests reproduce the hashes of + the linked `sync-probe.json` files and all subsidiary logs/TRX files. +4. The release summary and both OS reports identify that same provenance. The + Windows and Linux reports have schema 3, `validationOnly: false`, + `qualifiedArchitecture: true`, `overallStatus: pass`, and every required + row passed. The release summary's `dual-platform-os-evidence` result is + passed. Optional rows may be `not-qualified` only when `required: false`. + Each OS report's manifest is the exact file set below its sibling `.evidence` + directory: paths are unique, normalized, in-root and non-reparse; every + length/hash and executable-row stdout/stderr binding matches the file on + disk. The release summary records each accepted OS report hash/tree digest + and revalidates both at completion. +5. The release `sync-probe.json` has executable schema 8, exact configured + rows/trials/counts, matching source and tested-assembly provenance, zero + correctness failures, and every threshold accepted by the runner. The + independent review in `code-review.md` has no unresolved High or Medium + finding for the same committed source. The Linux OS report additionally has + one required `linux-tiny-performance` row and the Windows OS report has the + same row as optional/not-qualified. The Linux row binds schema-8 raw JSON for + exactly Legacy/LockFree x acquire-release/publish-remove x process-counts 1 + and 8 x three 60-second trials after 10 seconds of warm-up, with complete affinity, + unique native CPU IDs in `[0,63]`, zero failures, at least two operations and + exactly one successful operation pair per completed cycle, no checksum or + corruption evidence, and reproducible summaries. For each scenario, + one-process lock-free/legacy p99 is at most 1.0, eight-process lock-free/ + legacy throughput is at least 1.0, lock-free eight/one-process p99 is at most + 3.0, lock-free eight-process p99 is at most 10 microseconds, and every raw + lock-free `MaxMicroseconds` is at most 10,000. + +If any predicate is false, any command exits nonzero, or any artifact path +already exists before its one shot, the freeze is invalid. Preserve the failed +artifacts, revise the implementation or this pre-run contract as appropriate, +commit a new clean tree, choose new immutable paths, and rerun the complete +sequence. Never repair or copy JSON by hand and never edit tracked evidence +bookkeeping after the qualifying run. + +## Exact execution sequence + +Run from the clean commit. The ignored `artifacts/` tree retains authoritative +raw evidence without changing repository provenance. + +```powershell +# Linux x64 prerequisite probe; it creates no final artifact on failure +wsl -d Ubuntu -- bash -lc 'set -e; dotnet --info >/dev/null; dotnet workload list >/dev/null; command -v git >/dev/null; command -v pwsh >/dev/null; command -v cmake >/dev/null; command -v python >/dev/null; command -v docker >/dev/null; command -v strace >/dev/null; docker version --format "{{.Server.Version}}" >/dev/null' +if ($LASTEXITCODE -ne 0) { throw 'Linux prerequisite probe failed.' } + +# Linux x64, explicitly inside Ubuntu on the same clean commit +wsl -d Ubuntu -- bash -lc 'cd /mnt/c/Users/rantr/source/repos/SharedMemoryStore && pwsh -NoProfile -File ./scripts/validate-lock-free-os.ps1 -Command all -Configuration Release -OutputPath artifacts/lock-free-os-validation/009-final-r6-linux-x64.json' + +# Windows x64, on that same clean commit +pwsh ./scripts/run-lock-free-qualification.ps1 -Tier pr ` + -EvidenceRunId 009-final-r6-pr +pwsh ./scripts/run-lock-free-qualification.ps1 -Tier nightly ` + -EvidenceRunId 009-final-r6-nightly +pwsh ./scripts/run-lock-free-qualification.ps1 -Tier release ` + -EvidenceRunId 009-final-r6-release ` + -AdditionalOsEvidence artifacts/lock-free-os-validation/009-final-r6-linux-x64.json +``` + +The configured release run keeps every Legacy and LockFree matrix row. Legacy +mixed-churn and large-ingest are bounded 10-second-warm-up/60-second comparison +trials; every lock-free mixed-churn trial is count-bound at 100,000,000 +operations and every lock-free large-ingest trial at 100,000 direct 1.3 MB +frames. Schema 8 records and validates each effective target. The remaining +release contract is +1,000,000 production repetitions per SC-011 family, 1,000,000 directory- +generation repetitions, 10,000 recovery cases, three trials, and a 30-second +suspension gate. +The Linux `-Command all` run also executes the independently validated one/eight- +process tiny-operation matrix described above and preserves its raw JSON inside the +Linux OS evidence tree. +Exit code `0` is necessary but not sufficient: the JSON predicate above is the +final authority. Exit code `1` is failure. Exit code `2`, `validation-only`, a +missing prerequisite, a skipped gate, or an unsupported environment is +not-qualified. + +## Success-criterion evidence map + +Every row below is conditional: **PASS** means the named machine gate is passed +in the final release JSON with the exact clean provenance above. No prose can +override a failing or missing field. + +| Criterion | Final machine-derived gate | +|---|---| +| SC-001 | Every release LockFree `sync-probe` mixed-churn row records `OperationTarget: 100000000`, reaches that target with zero correctness failures, and `raw-visibility`, `churn`, and `owner-leak-assertions` are passed; Legacy rows record zero targets and at least 60 measured seconds. | +| SC-002 | Release `sync-probe` same-key-read 6/1 and 12/1 median throughput assertions are at least 4x and 7x. | +| SC-003 | Release `sync-probe` distributed-key-read 6/1 and 12/1 median throughput assertions are at least 4.5x and 8x. | +| SC-004 | Release `sync-probe` broker-directed 12-reader publication rate is at least 80% of its one-reader rate. | +| SC-005 | Release `participant-suspension` is `passed` with `qualification: sc005-qualified`, every configured checkpoint, 30-second pauses, and healthy-throughput ratio at least 0.9. | +| SC-006 | Release `sync-probe` 8-process Windows tiny-operation throughput/p99 assertions pass; Linux `linux-tiny-performance` binds the exact one/eight-process three-trial raw matrix and passes uncontended p99, eight-process throughput, <=3x self-amplification, <=10 us absolute p99, and every-raw-trial 10 ms maximum-stall gates for both scenarios; `dual-platform-os-evidence` and completion OS-tree revalidation pass. | +| SC-007 | Release `wait-policy` includes the no-operation-lock proof; Windows `no-lock-held` and Linux `no-lock-held` plus required `no-lock-linux-strace` OS rows pass. | +| SC-008 | Release `sync-probe` exact allocation scope reports `ProducerStoreOperationAllocatedBytes == 0` and the full-suite allocation tests pass for warmed publish/reserve/commit and acquire/project/release paths. | +| SC-009 | Every release LockFree large-ingest row records `FrameTarget: 100000`, reaches it, reports zero producer store-operation allocation, and carries `structural-direct-reservation-write-and-borrowed-lease-read` copy evidence with zero correctness failures; Legacy comparison rows record zero targets and at least 60 measured seconds. | +| SC-010 | Release `recovery` proves exactly 10,000 cases and full capacity, `owner-leak-assertions` passes, and both OS reports' required crash/checkpoint rows pass. | +| SC-011 | Release `production-race-stress` has exactly one valid marker for each of eight families with 1,000,000 production races; `production-generated-histories` and `reference-model-histories` exact family/count gates also pass. | +| SC-012 | Release `wait-policy` TRX passes the full wait/cancellation matrix with `completionAllowanceMilliseconds=250`; all three `owner-leak-assertions` mappings pass. | +| SC-013 | Release `full-test-suite`, `unit-contract`, `contract`, `package-consumption`, build, and both OS reports' required `release-tests`, native, Python, Docker, and pack rows pass. | +| SC-014 | Release full-suite sample validation passes and each OS report's required `sample-6` and `sample-12` rows passes. | +| SC-015 | The release full-suite TRX passes the barrier-controlled 12-process lease/removal/reclamation integration test with zero non-passed tests. | +| SC-016 | Release `churn` proves the configured lifecycle count and final capacity; every mixed-churn trial has zero correctness failures and late/early p99 at most 2.0. | +| SC-017 | Release `directory-generation-stress` is `passed` with `qualification: sc017-qualified-count-and-correctness` and exactly 1,000,000 repetitions across all configured mutation checkpoints. | +| SC-018 | All three release sticky-overflow-miss trials meet the exact 4,096-slot/10,000-cycle/16,384-sample shape, observe real spill and cleanup, drain occupancy, add zero late scans, have zero failures, and report late/early p99 at most 2.0. | + +## Diagnostic and rejected-candidate lineage + +Historical development runs include Windows release tests, bounded SC-011 +production races and generated histories, native/Python checks, and focused +Linux tests. Their counts belong to the source snapshots that produced them and +are intentionally not restated as current-tree totals. They are useful +regression evidence only: they are not at the final immutable paths, do not +jointly prove the final clean source manifest, and satisfy none of the +conditional release rows above. + +## Preserved historical failure + +The failed PR attempt at +`artifacts/lock-free-qualification/20260713T173242Z-pr/` remains intentionally +preserved. Its `summary.json` SHA-256 is +`436C4777B057472A448F0F925C3A16A778834977182ACF88C75E84C6B1F607E1`. +Its ordinary eight-process publish/remove workload returned `CorruptStore`. +That artifact predates exact-reference revalidation and is failure evidence, +not evidence for the final tree. + +## First final candidate and R2 convergence lineage + +The first immutable final candidate is also preserved rather than rewritten. +Its Linux report, `artifacts/lock-free-os-validation/009-final-linux-x64.json`, +passed all required rows on commit `73accd7b33730027fed8da54d99d72239eeb1d59`, +tree `2d0deebeb5da5b9be578297fbaf7965079d3b63f`, and source-manifest SHA-256 +`780D563964615DB8C5BF234D257F074737D1223EC98660E4FDA77D022C8235EF`. +The report SHA-256 is +`6C9727CCD6C412CD9B036E2107DA389F7359281A630C120BF48919FD987668B2`; +its raw performance report SHA-256 is +`EA05742E2CEC2F2A3032C18EADE888C0DDD026A8C08358AD1477B57BFC966CC7`. +The matching `009-final-pr` attempt then passed the complete 1,013-test suite +but correctly failed SC-017 because its generated configuration covered 46 +directory mutations while the source catalog contained 50. Its summary SHA-256 +is `E1B5FA0D0EF419388054B1AA310A292DB689ABBB7F33DDA5BD377166021CB79E`. +That failure rejects the entire first candidate despite the Linux pass. + +The preserved R2 diagnostics then closed four distinct gaps. The +`009-r2-pre-final-pr` attempt exposed an obsolete 45-second parent timeout in a +raw mapped-memory atomic proof, not a store failure; isolated Windows and Linux +atomic reports passed with SHA-256 +`B39F7970C718C8A4EDF384AFD3A196D29B8C4F8B91FF131C31DFF67910E74FD7` +and `51CC15DDB526B330801F4815BD44A330B22B6EBEF0E211898A09A293D742742E`. +The rejected parent-run summary SHA-256 is +`FC0949A5D7B56EFC7A5E707A3B8ED50C41738C2CD90A94264A88AA8E7BDF21E6`. +One separate full-solution observation found a test-only 50-millisecond setup +budget expiring before an instrumented reservation checkpoint under parallel +load. That observation emitted no named immutable diagnostic artifact or hash; +focused repetition closed it with a two-second budget and 2.25-second pause and +is deliberately recorded only as unbound test history. +`009-r2-pre-final2-pr` rejected two churn result rows where the contract requires +the one exact SC-016 method; its summary SHA-256 is +`3483EE66ED1DDA3E71864CC16D73C5569D195BF3C74AF800CDDCBD1CF044E0ED`. +`009-r2-pre-final3-pr` then passed 104/108 checkpoint/workload rows and isolated +checkpoint 62 oracle handling and checkpoint 63 finite-budget ownership in both +suspension workloads. Its summary SHA-256 is +`48F632E637B2999AFD877FCCA094322D6C9CA029903E495CF32D80C43A9FC66A`. +The former was corrected in the crash command; the latter required a production +reclaimer change so an expired remover cannot claim `Reclaiming` after +suspension and leave the key unhelpable. + +Finally, the clean diagnostic `009-r2-pre-final4-pr` passed all 24 qualification +rows on commit `ca200238423877044d841a3b92f93edb37385d46`, tree +`5f4454fa25bbfb67ea0e687153ef94470ca11112`, and source-manifest SHA-256 +`952FB17E4D08B5C4973A0809F0FC3541E5409B32991A07C6416709CB6C0CD5BF`. +It passed 1,014/1,014 tests, the single exact 10,000-cycle churn test, all 50 +SC-017 configurations, and 108/108 suspension rows across 54 checkpoints and +two workloads. Its summary, synchronization probe, and suspension SHA-256 values +are respectively +`3D543FBF5E4C4A5C514C1C3309565F0B2575126E5B176B79E347802AC46E331A`, +`9788BF7222BFE8B77E4512EECFEE9788BD1D898763AABBA0417F16D2AF8DE3EC`, +and `8BC2FD8F2BBFA288EE67610E43155186E5A445443059586BF1AEC34F8C7DAA35`. +All R2 artifacts named in this section are diagnostic lineage only. They do not +replace the frozen common-provenance final sequence linked above. + +## Rejected R2 final invocation + +The second candidate is preserved at +`artifacts/lock-free-os-validation/009-final-r2-linux-x64.json`. It is schema 3 +`not-qualified` with report SHA-256 +`090651C119CADD7DAC2C545D04F595FD9E65F5CEFAFC1B9B79EDA009F66EAA7C` +and clean provenance for commit `00c0dda2f3412bdba0faac487cc5ab5596ced7fb`, +tree `a960691095f5a555aa95869eeb233d561231d64d`, and source-manifest SHA-256 +`92BAA2E3DF0BE60BFB3FF314821A98ECAE3F3BCA730CA36BC008E46D3C03D981`. +The command was mistakenly launched by Windows PowerShell, so the report +correctly identified a Windows host: completed architecture, atomic, raw, +no-lock, crash, Release-test, Docker, sample, and pack rows passed, while the +required native and Python rows rejected the missing Windows `cmake` and the +Linux-only tiny-performance/`strace`/SIGSTOP rows were optional and not +applicable. The report therefore cannot satisfy the intended Linux contract. +This is an operator-platform failure, not a product or completed-test failure, +and it invalidates the entire R2 candidate. + +Before the R3 freeze, the explicit Ubuntu login-shell command above passed +validation-only orchestration as Linux x64 with every structural self-test +passed. That diagnostic report is +`artifacts/lock-free-os-validation/009-r3-linux-invocation-validate.json`, SHA-256 +`AE16F3AED1A9E0FE5113F20AD3AB2F28AE194E5CF0717F6372BF87362A51666C`. +It proves correct platform selection and orchestration only; it is not final +qualification evidence. + +## Rejected R3 final prerequisite + +The third candidate is preserved at +`artifacts/lock-free-os-validation/009-final-r3-linux-x64.json`. It is an +executable Linux-x64 schema-3 `fail` report with SHA-256 +`59419F78D559829A0E3B47AE1FAF334B5B79E83CAF8004646A1FE3687C5B86D5` +and exact clean provenance for commit +`4a972ce095fd79b76ffaa77778c4f6d1a7011921`, tree +`3d7a22b9c12dbb0e20fb85b65b75df9e2fb3be87`, and source-manifest SHA-256 +`117C476DD55EA6148C16D15707528B2503ECCAB75FF9D31FF4D6DC0BBC6B4795`. +All nine structural self-tests passed, then the first executable row, +`dotnet --info`, exited 1 before restore or any store workload. Ubuntu's healthy +package-managed SDK 10.0.109 was reading a stale user-local workload set +10.0.102 with missing manifests. `dotnet workload repair` correctly reported no +installed workloads and made no change; `dotnet workload update` advanced the +empty-workload manifest set to 10.0.109.1, after which both `dotnet --info` and +`dotnet workload list` exited 0. + +The repaired environment then passed the executable Linux-x64 architecture +diagnostic at +`artifacts/lock-free-os-validation/009-r4-preflight-linux-x64.json`: clean +`dotnet-info`, restore, build, and 45/45 architecture tests. Its report SHA-256 +is `799A41E8181AA299E0E0E925E3F2818935F2F1842EA94FC66C1AC6E1D6EA7F0A`. +This closes the environment failure but remains diagnostic only. The R3 path is +immutable and rejected. + +## Rejected R4 candidate and disposal test-contract correction + +The R4 Linux report is preserved at +`artifacts/lock-free-os-validation/009-final-r4-linux-x64.json`. It passed on +clean commit `b19d982b76f2f54ebfb9f72e0f2aef85eb2632b8`, tree +`c5a94aeff5f098915f77ef9a0af8ed8e8f730571`, empty-status SHA-256 +`E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855`, +and source-manifest SHA-256 +`7033447087954594BC982FF2AD859D282527B1A2F342199F404CA9358AEAE921`. +The schema-3 report passed all 28 required rows, the complete 1,014-test suite, +and the exact Linux tiny-operation matrix; its optional Docker PID-namespace +pause row was `not-qualified`. The report, evidence-tree, and raw-performance +SHA-256 values are respectively +`1346A080DFC7B397437F26088C16C074B9BA1AE3A4E3515D2E07D2DDAA7E2BDE`, +`6DEEBF6450AA909DA6D81E1F9725F9B3CD8DFE0534AE24FE7FC2985E828A5596`, +and `2716C25672FDB332345059F83FAFDD0063E9CE8C6635B7FE198E9A4053FF6D41`. + +The same-source `artifacts/lock-free-qualification/009-final-r4-pr/summary.json` +is schema 4 `failed`. It passed 1,013/1,014 tests—Contract 117/117, +Linearizability 83/83, Interop 75/75, Unit 437/437, and Integration 301/302—then +correctly rejected the sole failure, +`LockFreeDisposalIntegrationTests.EveryOperationAndTokenCallbackHasOnlyDocumentedDisposalRaceOutcomes(operation: RecoverLeases)`. +Its summary SHA-256 is +`7A3754C670A3622C569CB5E6AFFF479C9C668D20975E31641113337213FAD177`. +The failed PR emitted no `sync-probe.json`; R4 nightly and release were not run. + +The theory had selected `RecoverCurrentProcessLeases: true` while a second +current-process handle performed ordinary lease work. That administrative +override explicitly requires process-wide quiescence, so reclaiming the newly +acquired lease was permitted and the test schedule was outside contract. Its +reservation sibling carried the same latent defect. The test-only correction +uses both normal concurrent-safe recovery modes and stops dereferencing borrowed +projection bytes because racing disposal may invalidate their lifetime. Production +code, layout, lock freedom, and the public recovery contract are unchanged. +Focused repetition, the disposal class, Integration, the full solution, and an +independent H0/M0/L0 review passed. The clean provenance-bound +`009-r5-pre-final-pr` diagnostic then passed all 24 gates and 1,014/1,014 tests +on commit `527d451bd124dbbe8880fcb909b6e1bd70ad222a`. + +R4 is rejected in full despite its Linux pass. R5 subsequently passed its +Linux, PR, and nightly gates on one clean source, but its release synchronization +probe exposed a harness convergence defect. The global count target was applied +to each Legacy mixed-churn trial before the qualifying LockFree trials and would +also have applied the 100,000-frame target to Legacy large ingest. During the +first Legacy mixed trial, all workers remained responsive and a read-only mapped +sequence sample advanced by 27,120 in 30 seconds, proving slow serialized +progress rather than deadlock; after more than 141 minutes, two identical Legacy +trials plus the remaining matrix could not fit the six-hour whole-step deadline. +The incomplete `009-final-r5-release` directory is preserved and R5 is rejected +as a final sequence. + +R6 makes the already feature-specific acceptance policy explicit: Legacy rows +remain same-machine duration comparisons; count targets apply only to LockFree. +Schema 8 records the selected count-bound profiles and per-run targets, config +schema 5 records the duration cleanup grace, duration rows have a controller- +enforced deadline, and both importers reject target/timing tampering. Only the +complete R6 sequence above can qualify; no R4 or R5 artifact can be reused to +satisfy a final predicate because R6 must bind both platforms and every tier to +one identical source manifest. diff --git a/specs/009-lock-free-publish-read/research.md b/specs/009-lock-free-publish-read/research.md new file mode 100644 index 0000000..723dd50 --- /dev/null +++ b/specs/009-lock-free-publish-read/research.md @@ -0,0 +1,607 @@ +# Phase 0 Research: Lock-Free Shared-Memory Key-Value Store + +## Decision 1: Preserve one public store facade with an explicit profile + +**Decision**: Keep `MemoryStore`, `ValueReservation`, and `ValueLease` as the +recognizable public workflow. Add `StoreProfile` with numeric zero equal to the +legacy v1.2 profile and an init-only `SharedMemoryStoreOptions.Profile` defaulting +to that value. Preserve the exact existing `Create(...)` and +`CalculateRequiredBytes(...)` signatures and their v1.2 meaning. Add +`CreateLockFree(...)` and a profile-aware sizing overload. V2 adds configurable +`ParticipantRecordCount` with a default of 64 and appends +`StoreOpenStatus.ParticipantTableFull`. Internally dispatch to one concrete +legacy or v2 engine. + +**Rationale**: Existing source and compiled consumers continue requesting the +same layout and using the same tokens. An explicit profile prevents silent +auto-upgrade and makes deployment intent reviewable. The facade avoids a second +parallel public store API and keeps zero-copy lifetimes anchored to one mapped +handle. + +**Alternatives rejected**: + +- A separate `LockFreeMemoryStore` duplicates every operation and token and + invites semantic drift. +- Changing existing helper signatures by appending an optional profile breaks + already compiled call sites because the CLR method signature changes. +- Automatically opening whichever layout exists makes creation, rollback, and + accidental mixed deployments ambiguous. + +## Decision 2: Use a true layout-v2 and fail closed on the same name + +**Decision**: Define mapped layout `2.0`, magic `SMS2`, and resource protocol 2. +Retain the same physical mapping/region and cold lifecycle resource names for a +given public store name. Use the existing named lock only while creating a zero +mapping and validating its header. Never enter it from a v2 steady-state data +operation. A current requested profile/header mismatch returns +`StoreOpenStatus.IncompatibleLayout` before payload projection or mutation. +Already released clients that cannot map enough bytes to inspect an unknown +header still fail closed through their existing mapping/open failure status. + +**Rationale**: Layout 1.2 assumes serialization while v2 relies on atomic state +machines. Treating the redesign as 1.3 would understate incompatibility. Keeping +one physical discovery name ensures an old process cannot silently create a +parallel empty legacy store while v2 data exists. Cold serialization also closes +the simultaneous v1/v2 `CreateOrOpen` zero-header race. + +**Alternatives rejected**: + +- Reinterpreting v1.2 in place permits incompatible synchronization protocols to + mutate the same bytes. +- Profile-suffixed physical names hide an incompatible deployment as two + independent stores. +- A new initialization-only lock unknown to old clients does not serialize an + old creator racing a new creator. + +## Decision 3: Build the protocol from aligned 64-bit atomics + +**Decision**: Every independently changing shared control value is a naturally +8-byte-aligned signed `long` accessed atomically through `Interlocked` and +`Volatile`. No shared control word is ever accessed atomically at mixed widths. +Metadata is written only while its containing control word grants an exclusive +initializing state, then published with a release/full-fence transition. Readers +perform acquire reads and validate the binding/control pair before using +metadata. The initial v2 protocol accepts x64 Windows/Linux processes only and +does not require 128-bit CAS or `MemoryBarrierProcessWide`. Other 64-bit +architectures return `UnsupportedPlatform` until separately qualified. Every v2 +RMW is sequentially consistent/full-fence in one cross-word order; the acquire/ +remove handshake does not rely on independent release/acquire ordering alone. + +**Rationale**: .NET exposes efficient 64-bit compare/exchange and acquire/release +reads/writes on the supported baseline. Mapped views share coherent physical +memory across same-host processes. Keeping every observable transition in one +word makes crash points explicit and prevents torn multi-field ownership. +Executable litmus tests remain mandatory because public managed documentation +does not provide a standalone cross-process formal memory-model guarantee. + +**Sources**: + +- [.NET memory-mapped files](https://learn.microsoft.com/en-us/dotnet/standard/io/memory-mapped-files) +- [`Interlocked.CompareExchange`](https://learn.microsoft.com/en-us/dotnet/api/system.threading.interlocked.compareexchange) +- [`Volatile.Read`/`Write`](https://learn.microsoft.com/en-us/dotnet/api/system.threading.volatile) +- [Win32 `InterlockedCompareExchange64`](https://learn.microsoft.com/en-us/windows/win32/api/winnt/nf-winnt-interlockedcompareexchange64) +- [Win32 file mapping](https://learn.microsoft.com/en-us/windows/win32/memory/file-mapping) + +**Alternatives rejected**: + +- A 128-bit binding/state CAS is not portably exposed by the .NET 10 baseline. +- Process-wide memory barriers on every operation are unnecessarily expensive + when all participants obey the documented atomic protocol. +- Managed references, pointers, object identities, or platform-sized integers + cannot form an interoperable mapped contract. + +## Decision 4: Use fixed CAS buckets plus a capacity-preserving spill directory + +**Decision**: Store keys in generation-owned slot key storage. The primary key +directory has two deterministic buckets per hash and eight aligned 64-bit +binding lanes per bucket. The total primary cell count is approximately four +times `SlotCount`, keeping maximum primary load near 25 percent. Each canonical +home bucket also has a versioned `SpillSummary` word and one helpable mutation +descriptor word. A separate overflow directory contains exactly `SlotCount` +binding cells. The versioned-empty summary codec is a required layout-2.0 +feature, so earlier pre-release v2 clients that interpreted offset zero as a +monotonic Boolean reject these mappings in both directions. + +A binding uses 31 bits for `slotIndex + 1` and 33 bits for a nonzero slot +generation; zero means empty. A slot retires before generation wrap. + +Layout 2.0 caps `SlotCount` at `2^20 - 1` (1,048,575). Therefore the canonical +primary directory has at most `2^22` lanes and every primary or overflow target +index fits in 22 bits. `DirectoryOperation` packs intent (2 bits), phase (3), +target kind (2), target index (22), and the exact slot generation (33), leaving +two reserved high bits. `DirectoryLocation` packs target kind (2), target index +(22), and the same generation (33), leaving seven reserved high bits. Every +nonzero operation/location belongs to exactly one slot generation. + +Before installing or finally unlinking a key, a mutator publishes a descriptor +in its already initialized slot and CASes the canonical bucket's mutation word +from zero to that exact binding. Any contender that finds the word occupied +finishes the described idempotent operation and restarts; it never waits for the +owner. This localized helping closes the race in which same-key inserters could +observe different holes while unrelated deletions occur. The descriptor is +released before payload filling, commit, acquisition, or lease retention, so a +paused publisher retains only its key/slot rather than a bucket progress gate. + +A helper accepts an operation or recorded location only when its embedded +generation equals both the bucket mutation binding and the slot control +generation. Every phase transition and location publication/clear is an exact +full-word CAS. If a helper resumes after the slot was reclaimed and reused, its +CAS cannot match the newer operation/location. A stale target-cell CAS is safe +because the cell binding also carries the old generation; failure to advance +the exact old operation triggers exact rollback, and any observer may clear the +provably stale binding. No ordinary store from an old helper may overwrite a +new lifecycle. + +While owning the descriptor, publication scans candidate lanes in deterministic +order. If all candidates are occupied, it full-word-CASes the bucket summary +from its exact prior version to `Present(candidate)` before publishing into the +overflow directory. The summary packs 20 bits of slot-index-plus-one, the full +33-bit nonwrapping slot generation, one Present bit, and ten zero reserved bits. +Spill insertion scans cells in one deterministic hash-derived circular order +and rechecks exact keys through their referenced slots. Lookup scans overflow +only for a structurally valid Present summary. + +Before releasing a completed insert/cancel/unlink mutation, a helper captures +the exact Present token and performs a budgeted stable scan for any current +overflow binding with that canonical bucket. After an empty full scan and exact +operation/mutation revalidation, it CASes only `Present(X) -> Empty(X)`. It does +not return to initial zero: the retained exact identity makes every summary +version unique, so a delayed setter expecting an older empty value and a delayed +clearer expecting an older Present value both fail after later churn. A current +overflow witness retains Present; deadline, unstable, or malformed observations +retain the conservative token and the helpable mutation. Malformed codec or +reserved bits fail closed as corruption. + +The overflow directory preserves the capacity contract: while a publisher owns +one reserved value slot, at most `SlotCount - 1` other live bindings exist, so at +least one of its `SlotCount` cells is empty if primary placement is unavailable. +Deletion returns cells directly to empty because lookup scans a fixed candidate +set or the complete logically-present overflow directory; no probe chain or +tombstone must be preserved. A conservative Present token may survive a bounded +or interrupted cleanup, but any later exact mutation can re-run the scan. Normal +infinite-budget churn converges to versioned Empty and restores O(1) primary-only +missing lookups. No rebuild, compaction, epoch flip, or stalled maintenance owner +exists. + +**Rationale**: The common path is a short fixed scan with stable deletion and no +global maintenance. The bounded fallback handles adversarial exact hash +collisions without reporting directory-full before value capacity is exhausted. +Pathological spill lookup is deliberately O(`SlotCount`) but remains bounded, +observable, and lock-free. The per-home descriptor serializes only short +directory mutations and is lock-free because its complete operation is fully +described before publication and any caller can finish it. This is preferable to +silently reducing configured capacity or introducing a fragile general +reclamation scheme. The explicit 1,048,575-slot ceiling is substantially above +the practical mapped capacity for large values and purchases a deterministic +33-bit generation fence in every helper-visible word. It is preferable to a +platform-specific 128-bit CAS, crash-sensitive hazard registration, or a paused +helper blocking slot reuse. + +**Alternatives rejected**: + +- Set-associative buckets without full-capacity overflow can saturate before + value slots and would require a new public capacity outcome. +- Lock-free open addressing with permanent tombstones eventually degrades all + missing lookups under churn; clearing/moving tombstones safely requires extra + coordination. +- Cooperative whole-table migration needs quiescence before table reuse; a + paused reader can pin an epoch and eventually stop maintenance progress. +- Chained nodes tied to reusable value slots require safe unlinking/reclamation + and can form stale links or ABA cycles. +- Leaving generation only in the bucket mutation binding allows a helper to + validate that binding, pause across completion/reclaim/reuse, and then act on + untagged operation/location words whose bit pattern may repeat. +- Shared hazard/refcount registration either introduces a crash-recovery leak or + permits a stopped helper to delay reclamation, contradicting the progress + requirement. + +## Decision 5: Fence slots with one generation/state control word + +**Decision**: Each slot has an atomic control word containing 3 state bits, 33 +slot-generation bits, and a 28-bit participant token. The participant token +packs a nonzero record index and that record's incarnation using layout-derived +bit widths. The binding codec reserves 31 bits for `slotIndex + 1` and the same +33-bit generation. A `Free` control has participant token zero. The first +`Free -> Initializing` CAS already embeds the opener's active participant token, +so a crash before any later metadata write remains classifiable. A slot advances +generation before returning to `Free`; terminal generation publishes `Retired` +instead of wrapping. + +The former reserved `int32` at slot offset 52 is `PublicationIntent`: +`None=0`, `ExplicitReservation=1`, and `AtomicPublication=2`. Required-feature +bit 1 (`publication_intent`) makes that interpretation mandatory; together with +the versioned spill-summary bit, the current required mask is `3`. The intent is +ordinary metadata written under exclusive Initializing ownership before any +directory-cell binding publication and immutable for that generation. The exact +current-generation `Insert/Prepared` directory operation is release-published as +the metadata-ready marker before canonical mutation or directory-cell +publication. Pre-metadata Initializing means operation zero with no exact +current mutation/cell reference. Unknown intent on a discoverable current +lifecycle, or a current mutation/cell without its required marker, is +corruption; stale bytes are ignored in Free/Retired and pre-metadata +Initializing. + +The core state flow is: + +```text +Free(0) -> Initializing(p) -> Reserved(p) -> Published(0) -> RemoveRequested(0) + | | | + `-> Aborting <-' `-> Reclaiming -> Free(next generation) + +Commit clears the participant token because published values are unowned; +abort/recovery clears it while publishing a fully helpable state. Any recoverable +owner-controlled state may be changed only after participant classification. A +slot at generation limit becomes Retired instead of Free. +``` + +Key, lengths, offsets, intent, and the fixed descriptor bytes are prepared +before publishing the directory mutation descriptor. Binding installation +makes a lifecycle physically discoverable for helping, but ordering depends on +intent. An explicit direct reservation establishes key ownership and becomes a +duplicate witness at the exact `Initializing -> Reserved` CAS. Simple and +segmented convenience publication use `AtomicPublication`; their Initializing +and Reserved stages remain tentative, and the public operation orders only at +the `Reserved -> Published` commit CAS. A contender may exhaust its bounded +budget as `StoreBusy`, but cannot report `DuplicateKey` solely from a tentative +intent/state. Both workflows remain invisible to acquisition until Published. +Logical removal is the +`Published -> RemoveRequested` CAS. Reclamation owns the slot with +`RemoveRequested -> Reclaiming`, clears the exact directory binding and +generation-tagged directory metadata with exact CAS, advances generation, and +publishes `Free`. Reclaim helpers do not zero ordinary slot metadata: two +helpers may both observe `Reclaiming`, and a delayed loser must have no ordinary +write capable of erasing a reused generation. Metadata is semantically ignored +while Free/Retired and is completely overwritten only after the successful next +`Free(g+1) -> Initializing(g+1,p)` claim grants exclusive ownership. A later +lifecycle treats any older tagged helper residue as stale and may exact-clear +it; it never interprets that residue as its own operation or location. + +A lookup or maintenance result is only a cached exact-reference-word witness. +Reclamation may clear that word and its operation descriptor before advancing +the slot generation, so a later consumer must not combine the old source +observation with the newer slot snapshot and call the transitional pair corrupt. +On a would-be corruption, the chosen rule is source/slot/source joint +revalidation: acquire-read the exact raw reference word, take a fresh stable and +fully shape-validated snapshot of its separately decoded slot binding, and +acquire-read the same raw word again. A primary/overflow source word equals the +binding; a spill-summary source is the complete encoded `Present(binding)` word. +A changed source restarts a budgeted lookup or maintenance retry; only an +unchanged exact reference word enclosing a repeated invalid slot shape fails +closed. This preserves corruption detection without adding a multi-word atomic, +shared epoch, or cross-process synchronization owner. + +An owned reservation may publish unowned `Aborting` while another process is +between insert-helper validations. That transition is legal protocol progress, +not corruption. The helper re-reads the exact operation/control identity before +classifying a failed `Reserved` publication, switches to cancellation cleanup +for `Aborting`/`Reclaiming`, and treats an exact versioned Empty spill token from +that cancellation as terminal. Generation-tagged comparisons still fence a +helper that resumes only after reclaim and reuse. + +The same rule is required outside the helper loop. For explicit reservation, +recovery/cancellation before `Initializing -> Reserved` leaves no abstract key +owner; after that CAS the reserve has ordered. For atomic convenience +publication, Reserved is still tentative and recovery may discard it until the +Published CAS wins. Supported recovery never cancels a live Active owner: +normal recovery preserves it, and the current-process override requires +process-wide writer quiescence. A concurrent override and live writer is +outside the result contract, while exact generation fencing remains mandatory. +Lower-generation, unknown discoverable intent, or impossible same-generation +states remain `CorruptStore`, preserving fail-closed detection. + +**Rationale**: The control word fences every stale token, while the immutable +intent separates two public policies that intentionally reuse the same internal +state chain. This avoids treating an internal Reserved stage of atomic publish +as a completed public operation or weakening explicit zero-copy reservation +semantics. Transitional states retain only one slot/key and are locally +helpable. Generation retirement is deterministic and safer than an ABA-producing +wrap. + +**Alternatives rejected**: + +- Separate atomic state and generation fields admit torn identity observations. +- Clearing the key binding at logical removal would allow republish while old + leases still retain the generation, changing current duplicate-key behavior. +- Reusing a slot before clearing its exact old binding permits stale lookup and + deletion to affect the new lifecycle. +- Inferring public ordering from `Reserved` without an intent conflates an + explicit reservation with the internal reserve phase of atomic convenience + publication and permits false `DuplicateKey` results. + +## Decision 6: Make lease records the reclamation authority + +**Decision**: Do not use a decrementing slot reader count as the sole authority. +Each lease record has a 64-bit control word containing 3 state bits, 33 record- +incarnation bits, and the same 28-bit participant token, plus the exact slot +binding. Acquisition CASes `Free(r, participant=0)` directly to +`Claiming(r, participant)`, so recovery knows the owner even if the process stops +at that instruction. It then fills the target binding, publishes `Active`, and +revalidates both directory binding and slot `Published` state. If revalidation +fails it relinquishes the record and does not expose payload bytes. + +Removal first changes the slot to `RemoveRequested`, then scans for stable active +records matching the exact binding. A claiming acquisition that becomes active +after removal must fail its post-validation; an acquisition active before +removal is visible to the subsequent scan. Release is one CAS out of `Active`. +Remove, final release, later remove calls, allocation pressure, and explicit +recovery may all attempt the same idempotent reclaim CAS. Exactly one succeeds. + +**Rationale**: Record state makes crash recovery idempotent. A process cannot die +between an unrecorded reader-count increment and a recoverable lease. Several +readers of one value remain independent and a paused lease pins only its record +and value generation. + +**Alternatives rejected**: + +- A shared `UsageCount` increment/decrement has unrecoverable crash windows and + cannot distinguish stale record reuse for the same slot generation. +- Per-key reader locks violate progress and the no-exclusive-owner requirement. +- Copying payload bytes removes the lease problem but violates zero-copy scope. + +## Decision 7: Register recoverable participant incarnations before data claims + +**Decision**: Layout 2.0 contains a fixed participant-record table configured at +creation (64 records by default, maximum 1,048,575). One open `MemoryStore` +handle consumes one record. The 28-bit participant token uses +`ceil(log2(ParticipantRecordCount + 1))` low bits for record index plus one and +the remaining bits for record incarnation. The default therefore has 7 index +bits and 21 incarnation bits; the maximum table still has at least 8 incarnation +bits. Records retire rather than repeat a token. + +Under the allowed cold lifecycle lock, open CASes a free participant control to +`Registering`, atomically including PID and record incarnation, writes explicit +platform identity kind/process-start value and the exact Linux PID-namespace +token, then release-publishes `Active` before the public handle can execute a +data operation. The creator first writes its namespace token and an Enabled/ +Mixed recovery mode into the store header before `Ready`. A different or +unproven Linux opener release-publishes the irreversible Mixed mode before its +first Registering CAS and then continues ordinary KV access. Windows uses zero. + +Slot and lease controls atomically embed the complete participant token in their +first claim CAS and revalidate the exact participant control after claiming. +Normal data operations carry that locally cached token and do not mutate the +participant record. A participant record is not reused +until local disposal or explicit stale-owner recovery has closed entry and a +complete bounded scan proves no slot, lease record, or directory descriptor +references its token. Its incarnation advances within the configured token codec +before reuse and retires instead of wrapping. + +Normal handle disposal stops local entry, waits for already-entered local calls, +changes its participant record to `Closing`, relinquishes exact referenced +ownership, proves no references remain, publishes unowned/helpable `Reclaiming`, +then advances/free-publishes or retires the record and unmaps only afterward. +Stale-owner recovery may similarly use `Recovering -> Reclaiming` after PID/ +namespace/start classification and a zero-reference scan. A stable Active +snapshot includes the per-record namespace and compares it before any PID/start +lookup. A partial Registering record instead uses the creator header namespace +for presence-only classification only while mode is Enabled; Mixed makes that +classification Unsupported because ordinary fields may still contain an earlier +incarnation's bytes. Closing/Recovering handoffs remain helpable. Another +handle in the same or another process uses a different record. Unknown liveness +preserves the active participant and reports unsupported. + +If the table is full, open returns the appended +`StoreOpenStatus.ParticipantTableFull`; already-open handles and their data paths +are unaffected. + +**Rationale**: PID alone is reusable and namespace-relative, and writing identity +after a slot/lease CAS creates an unrecoverable crash window. A store-global +namespace mode closes the crash immediately after the Registering CAS: a cross- +namespace opener release-publishes Mixed before that CAS, and recovery acquire- +loads mode after snapshotting control. Cold registration gives the first hot CAS +a compact recoverable token without adding registry cache-line traffic to normal +operations. Delaying participant-record reuse until all references disappear +makes the compact embedded token safe without fitting a full process identity in +every claim word. + +**Alternatives rejected**: + +- Post-claim owner metadata cannot distinguish a dead claimant from a live + process paused immediately after CAS. +- Packing a probabilistic PID/start/nonce hash into 64 bits weakens the exact + incarnation guarantee and still leaves slot generation/state requirements. +- A native 128-bit atomic owner+generation word adds a runtime dependency and is + unavailable on the baseline. +- Automatic background recovery violates caller control and could misclassify + live owners. + +## Decision 8: Treat waits as bounded local retry/backoff + +**Decision**: Preserve `StoreWaitOptions` source shape. In v2 it bounds CAS retry, +state revalidation, helping, and backoff; it does not wait for a global lock or +turn the store into a key-arrival/capacity notification service. Cancellation +wins if observed before the operation's linearization point. `StoreBusy` means +the local contention budget expired. `StoreFull`, `LeaseTableFull`, +`DuplicateKey`, `RemovePending`, and `NotFound` retain distinct meanings. +`StoreFull` is determined by physical slot reuse, not abstract key ownership: +every non-Free slot, including tentative atomic-publication and cleanup states, +consumes capacity. After an initial absent-key lookup, candidate claim precedes +final same-key arbitration, so genuine `StoreFull` may precede `DuplicateKey` in +a race when no candidate is reusable. Because sequential scans can miss a free +slot rotating behind the scanner, scan exhaustion is provisional. After one +helping/reprobe pass, the allocator uses a process-local nonblocking guard and +eager `long[SlotCount]` scratch buffer to perform two same-order control-word +collects. Two structurally valid, all-non-Free, exactly equal collects confirm +the full instant between them. A malformed slot state/generation/owner/token +shape is `CorruptStore`, even when the malformed word is equal across passes. +Free/change/guard conflict is ordinary contention: `NoWait` terminates +`StoreBusy`, while finite and infinite callers retry under their operation-wide +budget. Lease-record scan exhaustion uses the +same exact-proof rule with an eager per-open `long[LeaseRecordCount]` buffer and +nonblocking local guard. Two same-order, structurally valid, all-non-Free, +exactly equal lease-control collects confirm `LeaseTableFull` at the candidate +instant between them. Free/change/guard conflict is contention, malformed +state/incarnation/owner/participant-token shape is `CorruptStore`, and monotonic +lease incarnation advance prevents control ABA. + +The proof requires no exact control ABA. Failed pre-metadata claims therefore +follow the existing helpable forward path to `Aborting`/`Reclaiming` and advance +generation; they never restore the original same-generation `Free` word. The +buffer is private to one open handle, costs eight bytes per slot (about 8 MiB at +the layout ceiling), and causes neither per-operation allocation nor mapped, +named, or OS synchronization. + +Use `SpinWait`/bounded yield on the hot retry path and check time/cancellation at +bounded intervals so fast successes avoid repeated clock or token overhead. A +caller that has not crossed its public ordering point must relinquish its +owner-controlled claim before returning. It may hand a slot to the unowned, +fully helpable `Aborting` state when physical unlink cannot finish inside the +bound; this is not a leaked reservation. Once a public ordering point has +occurred, cancellation does not rewrite the normal result. `TryRemove` reports +logical absence and returns conservative `RemovePending` when its post-removal +lease classification cannot finish within the bound; physical reclaim may +finish cooperatively after `Success`/`RemovePending`. + +**Rationale**: This preserves deterministic bounded calls and existing status +numbers while removing the shared semaphore meaning from v2. Capacity and key +state are external/application concerns unless a future explicit API adds +notifications. + +## Decision 9: Use a lock-free local lifetime gate + +**Decision**: Replace per-operation monitor entry with one atomic local lifetime +word holding a dispose flag and active-operation count. Operation entry CASes the +count only while open; exit decrements it. Disposal sets the flag and waits only +for operations already entered through that local handle before unmapping. +Borrowed views remain invalid after release/abort/recovery/handle disposal as +documented. + +**Rationale**: No managed monitor is paid on each v2 operation, and a paused +operation cannot affect other handles or processes. Waiting during explicit +local disposal is allowed because mapped memory cannot be safely unmapped while +that handle is executing. + +## Decision 10: Keep native clients v1.2-only and version independently + +**Decision**: C++ and Python remain layout-v1.2/resource-protocol-1 participants +in this feature and must return an incompatible-layout result for v2 before +payload access. Update compatibility metadata and executable mixed-version +tests, but do not change C ABI 1.0. Target NuGet 2.0.0 independently of mapped +layout 2.0. + +**Rationale**: A partial native implementation of atomic ordering or recovery is +more dangerous than explicit rejection. The package major communicates expanded +public token representation and wait semantics even while the legacy facade and +enum numeric assignments remain recognizable. + +## Decision 11: Validate safety before throughput + +**Decision**: Use five layers: deterministic transition schedules; small-history +linearizability checking; cross-process checkpoint/pause/crash tests; raw Release +memory-order and generation-pattern litmus tests; and allocation/performance/OS +tracing. Keep PR, nightly, and release workloads separate as defined in +`contracts/validation-and-performance.md`. + +**Rationale**: Random stress alone rarely hits the critical instruction window, +while deterministic hooks can accidentally add fences that hide memory-order +bugs. Both instrumented and raw paths are required. The full matrix writes over +130 GB and runs for hours, so portable PR validation must not pretend to be the +release qualification. + +**Non-convergence rule**: Stop rather than weaken the feature if aligned mapped +atomics fail on Windows/Linux, the one-key lifecycle cannot pass controlled +schedules, directory collisions create duplicate current generations, recovery +can reclaim a live owner or leak a dead one repeatedly, a steady-state path +touches the OS operation lock, or short profiling shows no unrelated-key scaling +after two evidence-driven correction cycles. + +## Decision 12: Separate intrinsic latency from parallel scaling on Linux + +**Decision**: Qualify both acquire/release and publish/remove at one and eight +processes with three independent 60-second Release trials after a 10-second +warmup. The lock-free profile must meet all of these independently recomputed +conditions: + +- at one process, median p99 latency is no greater than the matching legacy + median p99; +- at eight processes, median throughput is no lower than the matching legacy + median throughput; +- lock-free eight-process median p99 is at most three times its matching + one-process median p99 and is also at most 10 microseconds; +- every lock-free raw trial at both process counts has an unevictable observed + maximum across its sampled candidates of at most 10 milliseconds. + +The executable benchmark uses two deterministic keys per worker, a fixed +canonical bucket assignment, separate early and late Algorithm-R reservoirs, +and an unevictable running maximum. Qualification binds the raw evidence to the +exact Linux host, architecture, process count, CPU model, and clean commit, then +recomputes all eight scenario/profile/process-count summary rows from the raw +trials. Both PowerShell importers compute the midpoint with explicit floor +semantics and self-test distinct odd and even value sets so a three-trial +median cannot inherit PowerShell's round-to-nearest integer conversion. + +**Evidence**: The original probe accidentally allowed fixed-key collisions and +could lose an early maximum when its latency reservoir replaced that sample. +After correcting both defects, eight-process lock-free operation retained a +large throughput advantage and sub-10-microsecond p99, while the legacy Linux +file lock produced a deceptively low p99 by serializing incumbents even though +its raw stalls reached tens of milliseconds. Reducing the lease table from 64 +records to one, changing the slot hash, and changing the slot probe stride did +not materially improve lock-free eight-process p99, so each experiment was +reverted. + +**Rationale**: A serialized implementation is not a valid parallel-latency +oracle. The one-process comparison checks intrinsic operation cost, the +eight-process throughput comparison checks useful scaling, the lock-free +one-to-eight ratio limits contention growth, and the absolute p99 and raw-maximum +limits protect tail latency without rewarding convoying. This preserves a hard +performance contract while keeping correctness, host binding, and raw-tail +requirements independent of any aggregate score. + +## Decision 13: Use OFD locks and stable Linux rendezvous inodes + +**Decision**: Current C# and C++ Linux adapters issue nonblocking +`F_OFD_SETLK` open-file-description locks for byte `[0,1)` on both `.lock` and +`.lifecycle`. Each C# wrapper owns its descriptor and a per-wrapper +non-reentrant local gate. C++ wrappers in one module retain the existing shared +per-path `FileState`/descriptor and timed mutex; distinct native modules, +managed assembly-load contexts, and other open descriptions contend in the +kernel. `EINVAL`, `ENOSYS`, and unsupported-filesystem outcomes fail closed as +Unsupported; there is no fallback to process-associated locking. + +Current cleanup retains the empty mode-`0600` `.lock` inode, matching the +already-persistent `.lifecycle` inode. Successful, failed, initialized, and +uninitialized teardown releases held gates and disposes ordinary +synchronization before mapped-region/owner cleanup can enter `.lifecycle`. +Unlock failure closes/retires the descriptor before its local gate is reopened. +These rules affect only cold coordination and legacy v1.2 operations; no v2 +steady-state key-value path consults a registry or enters an OS lock. + +**Rationale**: The earlier shared-descriptor registry fixed sibling close within +one loaded C# assembly, but its static state was per `AssemblyLoadContext`, not +per OS process. Independently loaded packages and an in-process native adapter +could still use different traditional `F_SETLK` descriptors; same-PID calls did +not contend, and closing either descriptor could release every traditional lock +for that inode. OFD locks bind ownership to each open description, conflict even +inside one PID, and are not released by closing an unrelated descriptor. They +also conflict with traditional record locks used by released clients in other +processes. A persistent pathname removes the independent unlink/recreate inode +split, while descriptor-before-region teardown remains defense in depth for an +older participant that still deletes `.lock` after the last owner. + +Concurrent use of an older process-associated-lock implementation and a current +OFD implementation in one OS process is explicitly unsupported: closing any new +contender descriptor can release the older implementation's process-associated +lock. Cross-process old/new compatibility and same-process current/current +managed/native compatibility remain supported. + +**Evidence**: Linux regressions cover `.lock` and `.lifecycle`, same-thread and +timed local contenders, foreign-process exclusion, two copies of the current +assembly in separate collectible load contexts, and a same-PID native-style OFD +descriptor in both ownership directions. A concurrent final-close/reopen test +holds the persistent pathname lock, proves the reopened legacy store's own hot +operation returns `StoreBusy`, and proves foreign exclusion until release. +Failed-open ordering records synchronization disposal before region-owner +cleanup. The current C++ adapter uses the same command and closes its lock before +region owner cleanup. + +**Alternatives rejected**: + +- A managed static registry cannot provide process scope across load contexts or + native modules. +- Traditional `F_SETLK` plus per-module mutexes cannot enforce same-PID mutual + exclusion and retains the sibling-close hazard. +- Reentrant local monitors permit accidental same-thread unlock of a live gate. +- Switching to `flock` would not conflict with released record-lock clients. +- Teardown ordering alone cannot eliminate every unlink/recreate window; stable + rendezvous inodes make pathname identity independent of close timing. diff --git a/specs/009-lock-free-publish-read/sc011-production-race-results.md b/specs/009-lock-free-publish-read/sc011-production-race-results.md new file mode 100644 index 0000000..199d90b --- /dev/null +++ b/specs/009-lock-free-publish-read/sc011-production-race-results.md @@ -0,0 +1,95 @@ +# SC-011 Production Race Evidence + +**Root seed**: `1592590848` +**Authoritative command**: `scripts/run-lock-free-qualification.ps1` + +## Evidence boundary + +SC-011 counts only invocations of the real mapped `MemoryStore` action pair. A +reference-model permutation is checker coverage, not a production race. A +generated history is linearizability evidence, not a high-count production +race. The qualification runner records all three counts independently and +rejects missing, duplicate, renamed, or wrong-seed completion markers. + +The required production matrix has eight families: + +| Family marker | Contracted action pair | +|---|---| +| `publish-publish` | atomic publication / atomic publication | +| `publish-reserve` | atomic publication / explicit reservation | +| `reserve-reserve` | explicit reservation / explicit reservation | +| `commit-acquire` | reservation commit / acquire | +| `acquire-remove` | acquire / remove | +| `release-reclaim` | final lease release / same-key reclaim and reuse | +| `recovery-live-lease` | normal recovery with overrides disabled / live lease release | +| `disposal-operation` | local handle disposal / acquire on that exact handle | + +Each non-disposal family reuses persistent actor threads and a two-actor start +barrier, but executes both production calls once per repetition. The disposal +family keeps only the mapping and an independent keeper handle persistent. It +opens a fresh participant handle and races exactly one `Dispose` call with one +operation for every repetition. Therefore `completed=N`, `disposeCalls=N`, and +`freshHandles=N`; operations are not credited multiple times against one +long-lived disposal interval. The keeper verifies after every race that another +handle can still read the exact payload. + +Normal recovery uses `RecoverCurrentProcessLeases: false`. A live exact owner +must never be recovered, failed, or classified unsupported; its concurrent +release remains `Success`. The current-process administrative override is +intentionally absent because racing that override with live lease use is outside +the supported contract. + +## Qualification command + +The nightly and release tiers set the count to at least one million per family: + +```powershell +pwsh -NoProfile -File scripts/run-lock-free-qualification.ps1 ` + -Tier release ` + -OutputDirectory artifacts/lock-free-qualification-release ` + -AdditionalOsEvidence artifacts/lock-free-os-validation/linux-x64-final.json +``` + +The generated run directory contains immutable stdout for the +`production-race-stress` step plus `summary.json`. The runner verifies one exact +derived-seed marker for every family and records a total of 8,000,000 real +production races when the configured count is 1,000,000. + +## Convergence stress + +A pre-release Windows x64 convergence run on 2026-07-13 executed 100,000 real +races in every family (800,000 total) and passed in 44.3 seconds. The disposal +family executed 100,000 fresh opens and 100,000 `Dispose` calls, reaching both +documented orderings (`operationWins=49,358`, `disposalWins=50,642`). This run is +diagnostic; only a passing runner-owned million-per-family artifact is credited +to the release gate. + +The earlier six-family artifact is withdrawn. It omitted both reservation +arbitration families, used the current-process recovery override concurrently +with live lease activity, and credited one disposal interval as one million +races. None of those counts is used by the current runner. + +## Production-captured histories + +The production-history tier independently captures all eight families. Each +history contains 6-12 real calls from 2-4 actors, retains invocation/entry/return/ +response envelopes and real reservation or lease tokens, and is checked by the +reference model. A failure is minimized and reported with its root and derived +family seed. The reference-only tier includes the same two reservation +arbitration families and models normal same-participant recovery as preservation, +not administrative override recovery. + +## Regressions exposed during convergence + +The production harness previously found two release/reclaim defects that a +model-only permutation count did not expose: + +1. A second directory lookup could observe final reclamation and leak the + internal absent result as `NotFound` from same-key publication. +2. Capacity could become reusable between the first claim and helper scan, but + publication did not re-probe after successful helping and returned transient + `StoreFull`. + +The engine now treats second-lookup absence as permission to continue and +re-probes capacity after successful reclaim helping. The fixed-seed production +stress keeps `NotFound` and unjustified `StoreFull` outside the family oracle. diff --git a/specs/009-lock-free-publish-read/spec.md b/specs/009-lock-free-publish-read/spec.md new file mode 100644 index 0000000..95640b1 --- /dev/null +++ b/specs/009-lock-free-publish-read/spec.md @@ -0,0 +1,946 @@ +# Feature Specification: Lock-Free Shared-Memory Key-Value Store + +**Feature Branch**: `codex/lock-free-csharp` + +**Created**: 2026-07-12 + +**Status**: Draft + +**Input**: User description: "Keep SharedMemoryStore a key-value store over +shared memory. Producer-consumer processing is only one use case, and an +external message broker load-balances work by sending keys to workers. Workers +and other processes need independent access to stored data. Create a C#-first +lock-free implementation so one process cannot lock the store and harm the +performance or progress of all other processes." + +## Product Scope + +This feature preserves SharedMemoryStore as a bounded, named, general-purpose +key-value store. Opaque byte keys address immutable values in shared memory. +Any authorized process can publish, acquire, release, or remove data according +to the existing key-value lifecycle; the library does not assign work to +processes and does not track message acknowledgement. + +In the primary producer-worker use case, a message broker outside this library +sends keys to 6-12 workers. A worker acquires the value for its assigned key and +reads it without copying. Other processes may independently acquire the same key +or unrelated keys for monitoring, enrichment, diagnostics, or other application +workflows. Multiple valid read leases for one published value remain supported. + +The feature replaces store-wide steady-state serialization with lock-free +progress. Lock-free means that suspending or terminating any participant during +a steady-state data operation cannot prevent all other eligible participants +from completing operations while relevant data or capacity exists. It does not +promise that every individual operation is wait-free or immune to starvation +under adversarial same-key contention. + +Existing layout-v1.2 mappings and their public behavior remain a supported +compatibility profile. The new lock-free shared-memory contract has a distinct +compatibility identity so old and new participants can never mutate one mapping +under incompatible synchronization assumptions. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Read Values Concurrently by Key (Priority: P1) + +Many processes independently receive or choose keys and acquire the matching +shared value. Workers may receive keys from an external broker, while unrelated +processes may read the same values or different values for their own purposes. +No reader is required to coordinate through the store with other readers. + +**Why this priority**: Key-addressed zero-copy reads are the library's core +value. The store must remain useful beyond one producer-consumer topology, and a +slow or paused reader must not serialize all other readers. + +**Independent Test**: Publish a known set of keyed values, then run 6 and 12 +worker processes plus independent observer processes. Send keys to workers from +a test broker, let observers acquire overlapping keys, and verify exact bytes, +concurrent lease validity, progress, allocation behavior, and the absence of a +store-wide operation lock. + +**Acceptance Scenarios**: + +1. **Given** one published key, **When** several processes acquire it + concurrently, **Then** every successful caller receives a valid lease over + the same immutable value generation. +2. **Given** an external broker sends different keys to different workers, + **When** workers acquire those keys, **Then** the store returns values by + exact key and does not participate in worker selection, delivery, or + acknowledgement. +3. **Given** workers are reading assigned values, **When** an unrelated process + acquires one of the same keys, **Then** that additional lease is allowed and + does not invalidate worker leases. +4. **Given** one reader pauses while holding a lease, **When** other processes + acquire the same or unrelated published keys, **Then** they continue making + progress and observe complete immutable values. +5. **Given** no current value exists for a key, **When** a process acquires it, + **Then** it receives the documented not-found outcome rather than a false + contention or corruption result. + +--- + +### User Story 2 - Publish Directly Into Shared Memory (Priority: P1) + +A producer reserves bounded capacity for an opaque key, descriptor, and complete +payload length, fills store-owned memory directly, and commits the value without +first creating an intermediate full-payload buffer. Other producers may publish +unrelated keys, and readers continue acquiring already committed values while a +reservation is being filled. + +**Why this priority**: The primary high-throughput use case depends on zero-copy +ingest, but publication cannot achieve that goal if it holds a global lock that +delays every reader and unrelated writer. + +**Independent Test**: Run direct frame ingestion concurrently with readers and +with publication of unrelated keys. Verify exact-byte commit, invisibility +before commit, duplicate-key behavior, zero-copy access, progress on unrelated +keys, and no per-value steady-state allocation. + +**Acceptance Scenarios**: + +1. **Given** free capacity and a valid new key, **When** a producer reserves, + fills, accounts for exactly the announced payload length, and commits, **Then** + one complete immutable value becomes visible atomically under that key. +2. **Given** a reservation is still being filled, **When** any process acquires + its key, **Then** the reservation and all partial payload bytes remain + invisible. +3. **Given** a producer is paused while filling one reservation, **When** other + processes read committed keys or publish unrelated keys, **Then** those + operations remain able to complete while capacity permits. +4. **Given** two producers race to publish the same absent key, **When** their + publications contend, **Then** no more than one value generation becomes + current and every caller receives a deterministic documented result. +5. **Given** the announced payload length is not exactly accounted for, **When** + commit is attempted, **Then** the value remains invisible and no partial or + overrun bytes become published. + +--- + +### User Story 3 - Remove and Reuse Values Without Stalling the Store (Priority: P1) + +Any authorized process can logically remove a key. New acquisitions stop seeing +the removed generation, existing leases remain readable, and storage is reused +only after the final protecting lease is released. Removal or reclamation of one +key does not globally pause operations on other keys. + +**Why this priority**: A key-value store needs safe churn. Lock-free publication +and lookup are insufficient if remove, final release, reuse, or index maintenance +can still stop every process. + +**Independent Test**: Continuously publish, acquire, remove, release, and reuse a +large rotating key set from independent processes. Pause participants at every +observable lifecycle stage and verify linearizable same-key outcomes, immutable +existing leases, stale-token rejection, unrelated-key progress, and bounded +capacity behavior. + +**Acceptance Scenarios**: + +1. **Given** a published value with no active lease, **When** removal succeeds, + **Then** new acquisitions return not found and its capacity becomes safely + reusable. +2. **Given** one or more active leases, **When** removal wins, **Then** new + acquisitions cannot lease that generation, existing leases keep reading the + exact bytes they acquired, and reclamation waits only for those leases. +3. **Given** acquire and remove race for the same generation, **When** both + finish, **Then** either acquisition established a valid protecting lease + before logical removal or removal won and acquisition did not expose the + value. +4. **Given** a removed key is awaiting final release, **When** processes operate + on unrelated keys, **Then** they continue without waiting for the retained + lease. +5. **Given** a stale reservation or lease token survives storage reuse, **When** + it attempts to commit, abort, project, release, or remove, **Then** it cannot + affect the current generation. +6. **Given** a process pauses while helping publish or remove a key and the + original value slot is later reclaimed and reused, **When** the paused process + resumes, **Then** its older directory work cannot alter, unlink, or complete + the newer value generation. + +--- + +### User Story 4 - Survive Participant Pauses and Failures (Priority: P2) + +The store remains available when a publisher, reader, remover, or diagnostic +caller is paused or terminated. An authorized caller can explicitly recover +eligible abandoned reservations and leases without treating a live owner as +dead or allowing a former owner to modify reused state. + +**Why this priority**: The principal reason for removing the global lock is to +prevent one unhealthy process from harming every healthy process. Crash recovery +must restore capacity without recreating global blocking or corrupting active +data. + +**Independent Test**: Suspend and terminate processes during reservation, +commit, lookup, lease registration, removal, release, diagnostics, and recovery. +Continue operations from healthy processes, then perform explicit recovery and +verify progress, owner classification, capacity restoration, and stale-owner +fencing. + +**Acceptance Scenarios**: + +1. **Given** a process is suspended at any steady-state operation transition, + **When** other live processes operate on suitable keys and capacity, **Then** + at least one eligible operation continues to complete without the suspended + process resuming. +2. **Given** a publisher terminates with an incomplete reservation, **When** an + authorized caller confirms it is recoverable, **Then** partial bytes never + become visible and recoverable capacity is restored. +3. **Given** a reader terminates with a lease, **When** authorized recovery + confirms that exact owner and lease incarnation are stale, **Then** the lease + is released safely and cannot later release a reused generation. +4. **Given** owner liveness is unknown or still live, **When** recovery is + attempted, **Then** the store preserves the protected state and reports the + documented unsupported or active-owner outcome. +5. **Given** a formerly stalled owner resumes after its reservation or lease was + recovered, **When** it uses an old token, **Then** the operation is rejected + without mutating current state. + +--- + +### User Story 5 - Operate and Upgrade the Store Safely (Priority: P3) + +An operator can understand capacity, contention, active reservations, leases, +pending removals, recovery, and key-index health while live operations continue. +Existing users retain the documented public API and can distinguish legacy and +lock-free mappings during deployment or rollback. + +**Why this priority**: Lock-free concurrency is production-ready only when its +pressure and recovery behavior are observable and incompatible participants are +fenced before accessing memory. + +**Independent Test**: Exercise mixed operations, pressure, participant churn, +diagnostics, legacy opening, lock-free opening, incompatible opening, packaging, +and rollback. Verify diagnostic bounds, public statuses, compatibility outcomes, +and unchanged key-value semantics. + +**Acceptance Scenarios**: + +1. **Given** a live store under mixed load, **When** diagnostics are requested, + **Then** operators can distinguish capacity exhaustion, local contention, + retained leases, pending removals, abandoned state, recovery, and index + pressure without globally pausing data operations. +2. **Given** an existing layout-v1.2 mapping, **When** an upgraded client opens + it through the supported compatibility profile, **Then** current key-value, + lease, and result-status behavior remains available. +3. **Given** a client does not support the lock-free mapping contract, **When** + it attempts to open that mapping, **Then** it fails closed with a documented + non-success compatibility/open outcome before projecting payload memory. +4. **Given** a service uses an external broker to distribute keys, **When** it + adopts the lock-free store, **Then** broker topology and acknowledgement + semantics remain outside the store and require no migration into the library. +5. **Given** a live managed Linux owner whose PID is hidden from another + process by a different PID-namespace view, **When** that process performs a + cold lifecycle operation, **Then** the live mapping is retained without + adding synchronization to any key-value operation. +6. **Given** a physical region whose header is still unpublished and zero, + **When** any same-profile or opposite-profile client attempts to open it, + **Then** the opener leaves every byte unchanged and returns `AlreadyExists` + for `CreateNew`, `StoreBusy` for `CreateOrOpen`, or `IncompatibleLayout` for + `OpenExisting`. + +### Edge Cases + +- A key is empty, oversized, hash-colliding, concurrently inserted, concurrently + removed, or repeatedly reused across many generations. +- Two or more processes publish the same absent key while other processes search + through the same collision chain. +- A spill-summary setter or clearer pauses after exact validation while helpers + finish that lifecycle, preserve a versioned Empty identity, and later publish + a different overflow generation for the same canonical bucket. +- A producer reserves a key and pauses before writing, during writing, after all + bytes are written, or immediately before or after commit visibility. +- A producer writes fewer or more bytes than announced, commits twice, aborts + after commit, or retains writable memory after its lifetime. +- A zero-length value or descriptor is published and acquired. +- Acquisition races publication, removal, final lease release, recovery, index + maintenance, store disposal, and generation rollover. +- Many processes repeatedly acquire one hot key while other processes access a + broad key set. +- One process retains a lease indefinitely; only that value generation's + reclamation and bounded capacity are affected. +- Multiple handles in one process and handles in different processes operate on + the same and different keys concurrently. +- Capacity is one, all slots are occupied, all lease-tracking capacity is used, + or capacity is retained entirely by removed generations with active leases. +- The configured participant table is full, one process owns several store + handles, a participant terminates immediately after claiming shared state, or + a participant record approaches its incarnation limit. +- High churn creates index pressure while missing-key lookup and new publication + continue. +- A process is suspended after reserving shared state but before returning its + public token. +- A publisher, reader, remover, diagnostic caller, or recovery caller terminates + at each observable state transition. +- Owner process identifiers are reused or liveness cannot be established. +- A managed Linux owner's PID is not visible to another process while both see + the same shared-memory resources; the owner exits normally or is terminated. +- A stale reservation, lease, recovery, or removal token acts after generation, + incarnation, sequence, or storage-position reuse. +- A participant pauses after observing an in-progress directory mutation, other + participants finish it and reuse the value slot repeatedly, and the paused + participant later resumes with observations from the older generation. +- Lifecycle or identity counters approach their supported numeric limits. +- A bounded operation is canceled or expires at the same instant its target + state becomes available. +- A store handle is disposed while another local operation is active, while + other process handles remain live, or while borrowed memory is retained. +- A creator pauses after exposing a physical region but before publishing its + header, including an older client that mapped before entering cold + coordination; another opener must not infer initialization ownership from + open mode, dimensions, profile, or zero bytes. +- Diagnostics observe counters and gauges while they change concurrently and + therefore may be moment-in-time rather than transactionally exact. +- A broker delivers a missing, removed, duplicate, or delayed key; the store + answers only the current key-value lookup and does not acknowledge the message. +- An untrusted process modifies mapped bytes; protection from malicious mapped + writers remains outside the trust boundary. + +## Concurrency Outcome Contract + +The following outcome sets make same-key and lifecycle races testable. A caller +may additionally receive its documented input-validation, disposed, +incompatible, access-denied, unsupported-platform, or corruption outcome when +that condition genuinely applies. Steady-state `StoreStatus.StoreBusy` is +allowed only when the caller's bounded local retry budget is exhausted; it does +not mean another process owns a global store lock. Cold +`StoreOpenStatus.StoreBusy` may additionally report cold-gate contention or an +existing unpublished region whose initialization ownership cannot be proven. + +| Concurrent actions | Allowed observable outcome | +|---|---| +| Two atomic convenience publications of the same absent key | At most one publication returns `Success`, ordered at its exact `Reserved(AtomicPublication) -> Published` transition. Another caller returns `DuplicateKey` only after observing a current `Published`/`RemoveRequested` duplicate witness, `StoreFull` if no physical candidate slot is reusable, or `StoreBusy` if its bounded retry budget expires while the winner remains tentative. `Initializing` and `Reserved(AtomicPublication)` alone never justify `DuplicateKey`. Two current generations are never visible. | +| Two explicit reservations of the same absent key | At most one reserve returns `Success`, ordered at its exact `Initializing -> Reserved(ExplicitReservation)` transition. Another caller returns `DuplicateKey` after observing that exact-key explicit-reservation witness, `StoreFull` if no physical candidate slot is reusable before duplicate arbitration, or `StoreBusy` if its bounded retry budget expires before it can determine the winner. | +| Atomic convenience publication and explicit reservation of the same absent key | `Reserved(ExplicitReservation)` is an ordered duplicate-key witness. `Reserved(AtomicPublication)` is tentative: a contender MUST help/revalidate and may return `DuplicateKey` only after the atomic publication becomes `Published`, may proceed if that tentative lifecycle aborts, may return physical `StoreFull` before acquiring a candidate slot, or may return `StoreBusy` after bounded retry exhaustion. | +| Commit and acquire of the reservation's key | If acquire orders first, it returns `NotFound`; if commit orders first, acquire returns `Success` with the complete committed generation. Acquire may return `StoreBusy` only after bounded retry exhaustion. Partial bytes are never returned. | +| Acquire and remove of one published generation | If acquire orders first, it returns `Success` with a protecting lease and removal returns `RemovePending`. If logical removal orders first, acquire returns `NotFound` and removal returns `Success` or `RemovePending` according to established leases or bounded post-removal classification/reclaim work. Either caller may return `StoreBusy` only before logical removal after bounded retry exhaustion. | +| Final lease release and reclamation | The valid final release returns `Success` and causes exactly one safe reclamation. A duplicate or stale release returns `LeaseAlreadyReleased` or `InvalidLease` and cannot reclaim a later generation. | +| Remove and republish of the same key | Publication returns `DuplicateKey` while the published or pending-removal generation still owns the key. Publication may return `Success` only after that lifecycle is safely reclaimed and the new publication wins ownership. | +| Normal recovery and a live reservation/lease action | With the applicable current-process overrides disabled, recovery remains safe concurrently with normal resource activity and preserves every resource whose exact participant is live Active. An explicit reserve orders at `Initializing -> Reserved(ExplicitReservation)`; an atomic convenience publication orders at `Reserved(AtomicPublication) -> Published`. Normal recovery neither invalidates those live workflows nor undoes a committed value or releases a later incarnation. Stale-process and exact `Closing`/`Recovering` ownership remain independently recoverable. | +| Current-process lease-recovery override and current-process lease activity | `RecoverCurrentProcessLeases: true` is an administrative test/controlled-shutdown override, not a concurrent lease operation. Before invoking it, the caller MUST quiesce all current-process lease acquisition, projection, borrowed-span use, and release across every handle attached to the mapping, and MUST maintain that quiescence until recovery returns. Concurrent use of this override with that activity is outside the supported contract; no process-local hot-path gate is added to enforce the precondition. | +| Current-process reservation-recovery override and current-process publication activity | `RecoverCurrentProcessReservations: true` is an administrative test/controlled-shutdown override, not a concurrent publication operation. Before invoking it, the caller MUST quiesce reservation and publication creation, writable projection/use, progress, commit, abort, reservation disposal, and store-handle disposal across every handle attached to the mapping, and MUST maintain that quiescence until recovery returns. Concurrent use of this override with that activity is outside the supported contract; no process-local hot-path gate is added to enforce the precondition. | +| Cancellation/deadline and target-state availability | If the operation orders before cancellation/deadline observation, it returns its normal result. Otherwise it returns `OperationCanceled` or `StoreBusy` and leaves no newly owned reservation or lease. | +| Local disposal and an operation on that handle | If the operation orders first, it returns its normal result and disposal then invalidates local borrowed access. If disposal orders first, the operation returns `StoreDisposed`. Other process handles remain unaffected, and neither path exposes synchronization or mapped-memory exceptions. | +| Index maintenance and lookup of a stable published key | Lookup returns `Success` with the exact key/value or `StoreBusy` after bounded retry exhaustion. It does not return `NotFound` solely because unrelated maintenance is moving through an intermediate state. | + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The library MUST remain a bounded named key-value store in which + opaque byte keys address immutable shared-memory values. +- **FR-002**: The library MUST NOT assign keys to workers, load-balance work, + track message delivery, acknowledge broker messages, or impose exclusive work + claims as part of the key-value contract. +- **FR-003**: Keys MUST retain exact byte equality, non-empty validation, + configured size limits, and deterministic collision handling. +- **FR-004**: At most one published current value generation MAY exist for one + exact key at a time. +- **FR-005**: Processes MUST be able to publish unrelated keys concurrently, + subject only to bounded capacity and conflicts involving shared internal state. +- **FR-006**: Concurrent publication of the same absent key MUST make at most one + generation current and MUST return deterministic documented outcomes to every + participant. +- **FR-007**: The existing simple and segmented publication workflows MUST + preserve their documented byte, descriptor, duplicate-key, and visibility + semantics. Each is one atomic convenience publication whose public success + orders only at the exact `Reserved(AtomicPublication) -> Published` + transition; its preceding `Initializing` and `Reserved` states remain + tentative internal work. +- **FR-008**: A producer MUST be able to reserve bounded capacity for a key, + fixed descriptor, and complete non-negative payload length before visibility. + This explicit-reservation workflow MUST carry + `PublicationIntent=ExplicitReservation` and public reservation MUST order only + at the exact `Initializing -> Reserved(ExplicitReservation)` transition. An + `Initializing` lifecycle is tentative and MAY be physically discoverable for + helping, but MUST NOT alone establish reserved-key ownership or justify + `DuplicateKey`. +- **FR-009**: A reservation MUST expose store-owned writable payload memory so + the producer can fill it without an intermediate full-payload buffer. +- **FR-010**: Commit MUST succeed only after exactly the announced payload length + has been explicitly accounted for; incomplete or overrun commit attempts MUST + remain invisible and return deterministic outcomes. +- **FR-011**: Commit MUST make the complete key, descriptor, payload, and value + generation visible atomically; readers MUST observe either no value or one + complete immutable generation. Publication of either intent orders at the + exact transition to `Published`; for an explicit reservation this is the + later commit operation, while for an atomic convenience workflow it is the + sole public operation's success point. +- **FR-012**: Aborting or disposing an incomplete reservation MUST keep it + invisible and restore safely reusable capacity. +- **FR-013**: Writable and read-only mapped-memory views MUST be borrowed for a + precise documented lifetime. Public safe access MUST stop projecting a view + after its lifetime; explicitly unsafe escaped references remain outside the + library's protection boundary. +- **FR-014**: Any authorized process MUST be able to acquire the current value + for an exact key without obtaining exclusive processing ownership. +- **FR-015**: Multiple threads and processes MUST be able to hold simultaneous + valid read leases for the same published value generation. +- **FR-016**: A successful lease MUST expose the exact immutable payload and + descriptor directly from shared memory without a mandatory payload copy. +- **FR-017**: A lease MUST protect its exact value generation from storage reuse + until released or safely recovered. +- **FR-018**: Projecting or inspecting an already valid lease MUST NOT require a + store-wide exclusive operation lock for each property or memory view. +- **FR-019**: Releasing or disposing a valid lease MUST relinquish only that + lease incarnation and MUST NOT affect other leases for the same generation. +- **FR-020**: Successful logical removal MUST prevent new leases for the removed + generation at one observable point while preserving every lease established + before that point. +- **FR-021**: Reclamation and reuse MUST occur only after logical removal and the + release or safe recovery of every protecting lease for that generation. +- **FR-022**: Acquire/remove and release/reclaim races MUST produce documented + outcomes consistent with one ordering of their observable completion points. +- **FR-023**: A key awaiting reclamation MUST retain the existing duplicate-key + behavior until its lifecycle permits documented reuse. +- **FR-024**: A stable published key MUST NOT spuriously return not found because + an unrelated key is being published, removed, reclaimed, or maintained. +- **FR-025**: Suspending or terminating any participant at any steady-state + publish, reserve, commit, abort, acquire, project, release, remove, reclaim, or + index-maintenance transition MUST NOT prevent another live eligible + participant from completing an operation while suitable keys or capacity + exist. +- **FR-026**: Steady-state progress MUST NOT depend on process-owned or globally + exclusive synchronization state. +- **FR-027**: Every steady-state success and normal-failure path for simple and + segmented publication, reservation access and progress, commit, abort, + reservation disposal, acquisition, lease projection, release, lease disposal, + removal, final reclamation, and index maintenance MUST NOT acquire a named + cross-process lock or any other globally exclusive operation owner. +- **FR-028**: Every successful publication, commit, acquisition, logical removal, + and release MUST have a single documented externally observable ordering + point for same-key race analysis. +- **FR-029**: An individual operation that cannot complete within the caller's + selected contention or wait bound MUST return a deterministic busy, canceled, + timeout, capacity, or lifecycle outcome without reporting a false success or + corrupting state. +- **FR-030**: Optional waiting for a key-state, capacity, or recovery transition + MAY use bounded notification, but successful fast-path operations MUST NOT pay + a mandatory globally exclusive synchronization acquisition. +- **FR-031**: The store MUST remain fixed-capacity and MUST return distinguishable + outcomes for value-slot exhaustion, lease-tracking exhaustion, duplicate key, + local contention exhaustion, pending reclamation, and lock-free-profile + participant/open-handle tracking exhaustion. `StoreFull` is a physical + capacity outcome: every non-`Free` slot, including tentative `Initializing` + and `Reserved(AtomicPublication)`, is unavailable until safely reclaimed. A + sequential allocation scan is only a candidate because a reusable slot can + move behind it. Public `StoreFull` therefore requires two same-order collects + of every structurally valid slot control, with every control non-`Free` and + the second collect exactly equal to the first. Slot controls MUST progress + monotonically for a generation—failed claims use + `Initializing -> Aborting -> Reclaiming -> Free(next generation)` and MUST NOT + roll back to the same `Free` word—so exact equality cannot hide ABA. The + structural check MUST require a nonzero bounded generation, a structurally + valid configured participant token in `Initializing`/`Reserved`, participant + zero in `Free`/`Published`/`RemoveRequested`/`Aborting`/`Reclaiming`/`Retired`, + and the terminal generation in `Retired`. An invalid state/generation/owner + shape MUST return `CorruptStore`, even when two malformed words compare equal. + The confirmed ordering point is the candidate instant after the first collect + and before the second. A free slot, changed control, or concurrent use of the + process-local proof buffer is contention, not capacity; `NoWait` returns + `StoreBusy`, while finite/infinite calls retry under their existing budget. + A publication/reservation operation whose initial same-key lookup was absent MAY + therefore return `StoreFull` at its later candidate claim before final + arbitration with a raced same-key lifecycle; duplicate-key status has no + precedence over genuine physical exhaustion in that race. Lease allocation + scan exhaustion is likewise provisional because a reusable lease record can + rotate behind the scanner. Public `LeaseTableFull` requires two same-order + collects of every lease control, with both passes structurally valid, + non-`Free`, and exactly equal. Owner-controlled states MUST carry a + structurally valid configured participant token; unowned states MUST have no + participant, and invalid state/incarnation/owner shapes MUST return + `CorruptStore`. Lease incarnation advance or terminal retirement prevents + control-word ABA between collects. A free or changed lease control, or + concurrent use of that handle's lease-proof buffer, is contention: + `NoWait` returns `StoreBusy`, while finite/infinite calls retry under the + operation-wide budget. Only the confirmed candidate instant between the + collects may order `LeaseTableFull`. +- **FR-032**: A stalled or failed reservation owner MAY retain only its key, one + reservation lifecycle, and its configured slot capacity; a stalled or failed + lease owner MAY retain only its lease record and protected value generation. + Neither MAY retain global authority required for unrelated operations or + additional readers of an already published value to progress. +- **FR-033**: Explicit reservation recovery MUST discard incomplete bytes, + preserve committed values, restore only safely recoverable capacity, and fence + recovered tokens from later mutation. Recovery MUST reclaim stranded bounded + resources rather than restore global steady-state progress. Normal recovery + MUST preserve an owner-controlled lifecycle whose exact participant is live + Active. Current-process reservation override MUST require documented + process-wide writer and writable-view quiescence; racing that override with + current-process publication activity is outside the supported result contract. +- **FR-034**: Explicit lease recovery MUST distinguish live, stale, unsupported, + and inconsistent owners before relinquishing a lease and MUST reject a stale + release after recovery or reuse. +- **FR-035**: Owner and token identity MUST distinguish later incarnations even + when a process identifier, participant record, lease record, key, or storage + position is reused. The first atomic claim of a value slot or lease record MUST + already identify a recoverable participant incarnation; ownership identity + MUST NOT depend on a later non-atomic write by the claimant. +- **FR-036**: The library MUST NOT rely on hidden background workers for + publication, lookup, reclamation, index maintenance, recovery, diagnostics, + retries, or cleanup. +- **FR-037**: Steady-state publish/reserve/commit and acquire/project/release + paths, including expected duplicate, missing, full, and local-contention + outcomes, MUST avoid per-operation runtime heap allocation after + initialization and warm-up. Each open lock-free handle MAY eagerly reserve one + process-local `Int64[SlotCount]` StoreFull proof buffer and one + `Int64[LeaseRecordCount]` LeaseTableFull proof buffer (approximately eight + bytes per configured record: 1 KiB for 128 lease records, 64 KiB for 8,192, + and 8 MiB for 1,048,576); the proof paths MUST perform no per-operation + allocation and MUST place no proof counter or buffer in shared memory. +- **FR-038**: Diagnostics MUST expose capacity, active reservations, published + values, active leases, pending removals, reclamation, local contention, + recovery, invalid-token outcomes, and key-index health. +- **FR-039**: Diagnostics MAY be moment-in-time rather than transactionally + exact, but requesting them MUST be safe during live operations and MUST NOT + impose a global exclusive data-path pause or require data operations to wait + for the diagnostic caller. +- **FR-040**: Disposing one local handle MUST invalidate its borrowed views and + tokens without corrupting or globally blocking other attached handles. +- **FR-041**: Disposal races with every public operation MUST produce only + documented normal or disposed outcomes and MUST NOT expose invalid mapped + memory or synchronization exceptions. +- **FR-042**: Existing key-addressed `TryPublish`, reservation, + `TryPublishSegments`, `TryAcquire`, lease release, `TryRemove`, recovery, + diagnostics, wait-policy, and disposal semantics MUST remain publicly + recognizable and behaviorally compatible in the lock-free profile. +- **FR-043**: Unsupported clients MUST fail closed on the lock-free mapped + representation before projecting payload memory or mutating shared state. + Current clients that can validate the header MUST report the documented + incompatible-layout outcome; already released clients MAY surface an existing + non-success mapping/open outcome when their requested view prevents header + validation. +- **FR-044**: Documentation and samples MUST show one zero-copy producer, 6-12 + broker-directed workers, simultaneous non-worker readers, removal with active + leases, bounded pressure, disposal, and explicit recovery while keeping broker + responsibilities outside the store. +- **FR-045**: The trusted same-host process boundary MUST remain explicit; the + feature MUST NOT claim cross-host access, persistence, or protection from + malicious processes with mapped-memory write access. +- **FR-046**: Explicit recovery MAY coordinate ownership classification and the + exact records being recovered, but healthy steady-state data operations MUST + NOT acquire or wait for a global recovery lock. +- **FR-047**: Every durable in-progress key-directory mutation and every durable + directory-location reference in the lock-free profile MUST identify the exact + value generation it belongs to. Work that resumes after the referenced slot + has been reclaimed or reused MUST be unable to match, alter, unlink, or + complete the newer generation. Generation-mismatch cleanup MUST be + directional: generation `G` MAY exact-CAS its own tagged word or a strictly + older residue, but MUST preserve every observed word tagged with a generation + greater than `G` because that word belongs to a reused lifecycle. Publication + from an unversioned empty word MUST be postvalidated and compensating cleanup + MUST compare only against the publisher's exact generation-tagged value. +- **FR-048**: The lock-free profile MUST support between 1 and 1,048,575 value + slots. Creation requests outside that range MUST fail validation before a + mapping is created or opened. This limit applies only to the lock-free + profile and MUST NOT silently change the legacy profile's validation contract. +- **FR-049**: Overflow lookup in the lock-free profile MUST use a one-word + exact-generation versioned spill summary. An exact current insert MUST publish + Present before its overflow-cell CAS. A helper MAY publish logical Empty only + by exact full-word CAS after a stable empty bounded scan and exact canonical + mutation revalidation, and MUST preserve a nonrepeating version identity so a + delayed setter or clearer cannot ABA through empty. Budget, instability, or + malformed state MUST NOT create a false-negative overflow decision. A stable + full scan that finds a different exact current spill witness MAY full-word-CAS + Present to that witness under the same revalidated canonical mutation; a + Present identity may recur only while its exact nonwrapping binding remains + current, which cannot revive an older stable-empty clear authorization. +- **FR-050**: A legal reservation transition from `Initializing` or `Reserved` + to the unowned `Aborting`/`Reclaiming` cleanup lifecycle MAY race any insert + helper phase. After every validation window, a helper MUST distinguish this + cancellation state from structural corruption and either complete exact + cancellation cleanup or stop benignly. In particular, a delayed + `BindingChanged` helper MUST NOT report corruption merely because its exact + `Initializing` slot can no longer become `Reserved`, and an exact versioned + `Empty(binding)` published by cancellation MUST suppress an older overflow + setter without being reclassified as malformed state. A helper that resumes + after validating `Insert` MAY exact-clear that insert's target after + cancellation has handed the same canonical mutation to `Unlink/Prepared`. + A delayed unlink location publisher that revalidates the canceling + `Aborting`/`Reclaiming` lifecycle MUST treat an empty target or a structurally + valid different in-range binding as legal progress and MUST preserve the + replacement; a stable malformed or mapping-out-of-range target remains + `CorruptStore`. A helper that resumes + after cancellation, reclaim, or reuse MUST remain generation-fenced and MUST + NOT alter the later lifecycle. Losing `Initializing -> Reserved` to exact + `Aborting`/`Reclaiming`, terminal retirement, or a strictly later generation + means the tentative reservation was legally canceled and returns + `InvalidReservation` after helping cleanup; a lower generation or an + impossible same-generation state remains fail-closed `CorruptStore`. Winning + `Initializing -> Reserved(ExplicitReservation)` is the public explicit-reserve + ordering point. The same transition for `AtomicPublication` remains tentative + for the outer operation. Supported normal recovery preserves the live owner; + an administrative current-process override is permitted only after the + quiescence required by FR-033. +- **FR-051**: Every claimed lock-free value-slot lifecycle MUST carry one + immutable 32-bit `PublicationIntent` ordinary-metadata value at byte offset + 52: `None=0`, `ExplicitReservation=1`, or `AtomicPublication=2`. The exclusive + `Initializing` owner MUST write a nonzero known intent with the key, lengths, + descriptor, and other ordinary lifecycle metadata before release-publishing + the exact current-generation `Insert/Prepared` directory operation. That word + MUST be the metadata-ready marker and MUST precede canonical mutation and + directory-cell binding publication. Reclaim MAY leave the ordinary bytes + stale and the next exclusive claimant MUST overwrite them. `Free`, `Retired`, + and pre-metadata `Initializing`—operation zero with no exact current mutation + or directory-cell reference—MUST ignore stale/`None` intent; + direct unreferenced cleanup of such a safely recovered claim MUST remain + possible without interpreting those ordinary bytes; + every current discoverable lifecycle with an unknown intent MUST fail closed + as `CorruptStore`; a current mutation/cell reference without its required + operation marker is also corruption. `Reserved(ExplicitReservation)`, `Published`, and + `RemoveRequested` are duplicate-key witnesses; `Initializing` and + `Reserved(AtomicPublication)` are not. +- **FR-052**: A directory lookup or maintenance reference MUST be treated as a + cached exact-reference-word witness, not as permanent ownership of the decoded + slot binding. For a primary lane or overflow cell the exact source word is the + binding itself; for a versioned `Present(binding)` spill summary the exact + source word is the complete encoded summary and the referenced binding is + decoded separately. If later slot classification would return `CorruptStore`, + the implementation MUST acquire-read that exact source word, take a fresh + stable snapshot of the separately decoded slot binding, and acquire-read the + same source word again. If either source read no longer equals the exact raw + reference word, unlink/reclaim/reuse or summary replacement has overtaken the + cached observation and the caller MUST perform a budgeted fresh lookup or + maintenance retry instead of reporting corruption. Only an unchanged exact + reference word around a repeated invalid slot snapshot may fail closed. + Directory-location publication MUST apply the same rule to one joint tuple: + canonical mutation, exact operation, current location, slot control, + immutable directory binding, and every selected or competing target cell. + Before returning `CorruptStore`, two stable acquire collects MUST be followed + by exact no-op compare/exchange confirmation of the atomic tuple members and + a fresh immutable-binding read; any loss is progress or retry, not corruption. + For `Unlink/Prepared`, the first valid location publication wins arbitration; + a losing helper MUST exact-clear only its distinct recovered old binding and + preserve an empty or structurally valid replacement target. If + `Unlink/TargetSelected` later finds another structurally valid + same-generation location, it MUST exact-clean both old-binding witnesses and + the alternate location while preserving any replacement it does not own. + After a location CAS, loss of the exact unlink source MUST withdraw that + helper's exact old target and location; an exact committed `Insert` successor + or other valid replacement MUST remain untouched. A structurally valid older + location is exact-cleanable residue. A future-generation location is benign + reuse only when another member of the old tuple proves movement; if the exact + old-generation tuple remains stable around that future location, the shape is + corruption and the future word is preserved for diagnosis. Every + slot control used to classify a live directory reference MUST also have a + valid generation, state, and owner shape: + `Initializing`/`Reserved` require a structurally valid configured participant + token; `Free`, `Published`, `RemoveRequested`, `Aborting`, `Reclaiming`, and + `Retired` require participant zero; and `Retired` requires the terminal + generation. Only a structurally valid strictly newer control may classify an + old binding as stale. This revalidation MUST add no shared counter, lock, or OS + synchronization. +- **FR-053**: Layout v2 MUST store the creator's exact Linux PID-namespace + numeric identity in the header and the registering process's identity in each + participant record; Windows values MUST be zero. The header MUST also contain + an aligned atomic recovery mode that begins `Enabled` only when the creator's + identity is proven. A different or unproven Linux opener MUST release-publish + irreversible `Mixed` before its first `Registering` CAS and MUST retain + ordinary KV access. Recovery MUST snapshot participant control before acquire- + loading the mode. In Mixed, a partial `Registering` identity MUST be + Unsupported and preserved; a stable Active identity MAY be classified only + after its per-record namespace exactly matches the caller's current namespace. + Namespace mismatch or inability to prove the current namespace MUST become + Unsupported before PID/start lookup. Closing/Recovering handoffs remain + helpable. Header offsets 264/272 and participant offset 32 MUST be compatibility- + fenced by required-feature bit 2 without changing existing offsets or strides. +- **FR-054**: Every current C# Linux mapped handle MUST derive a private + mode-`0600` owner-liveness anchor from the unchanged GUID third field of its + three-field `.owners` line, acquire an exclusive open-description `flock` + before publishing that line, and hold it for the mapped-view lifetime. Under + `.lifecycle`, a separately opened probe MUST classify a contended anchor as + live regardless of PID visibility, an acquirable anchor as stale, a missing + anchor by the existing PID/start-token fallback for older C#, C++, and Python, + and access/symlink/directory/other ambiguity conservatively as live. A + same-process registry MUST make local classification explicit. Cleanup MUST + commit exact owner absence before deleting a stale anchor; orderly close MUST + unmap before unlocking and MUST unlock only after exact sidecar absence or + finalized exact-owner release-marker publication, while process death MUST + release the lock automatically. The anchor mechanism MUST NOT change mapped + layout, the owner-line format, the interoperable record locks, or any hot + key-value path. +- **FR-055**: When a v2 operation proves persistent mapped structural + corruption after the required exact-reference and race revalidation, it MUST + atomically and irreversibly transition the shared store control from `Ready` + to `Corrupt`. Every later public operation on every attached handle MUST + acquire-check that control before projecting or mutating mapped data and + return `CorruptStore`; opening the corrupt mapping MUST fail closed as + `IncompatibleLayout`. Already borrowed spans cannot be revoked, but a later + projection through an issued token MUST fail. The latch MUST NOT be set for + malformed caller-owned input, documented token history, capacity, + contention, cancellation, disposal, or a legal concurrent state change. + Cleanup that discovers corruption MUST latch it without throwing and MUST + preserve the malformed record. This per-operation check and the transition + MUST use only mapped 64-bit atomics and MUST add no OS synchronization, + process-held lock, or shared hot-path counter. +- **FR-056**: A cold create/open attempt MUST acquire the platform's required + ordered lifecycle coordination before creating, opening, or mapping the + physical region and MUST retain that coordination through header + initialization or validation and handle registration. Only the attempt that + proves it physically created the region MAY initialize an unpublished + header; `OpenMode`, requested profile or dimensions, and observed zero bytes + MUST NOT confer that authority. An already-existing zero header MUST remain + byte-for-byte unchanged and return `AlreadyExists` for `CreateNew`, + `StoreBusy` for `CreateOrOpen`, or `IncompatibleLayout` for `OpenExisting`. + The caller's original wait and cancellation budget MUST cover the complete + cold transaction. Ordered gates MUST be released before failed-open mapped + resource cleanup that may re-enter outer lifecycle coordination, and no store + handle may escape until resource ownership has been transferred exactly once. +- **FR-057**: Current C# and native Linux participants MUST use nonblocking + open-file-description record locks on byte `[0,1)` for `.lock` and + `.lifecycle`, fail closed as Unsupported when OFD locking is unavailable, and + preserve exclusion across separately loaded managed assemblies and native + modules in the same PID. Current cleanup MUST retain the empty mode-`0600` + `.lock` inode as a stable rendezvous, and every successful, failed, initialized, + or uninitialized teardown MUST retire ordinary synchronization after held gates + are released but before mapped-region/owner cleanup can enter `.lifecycle`. + Released traditional-record-lock participants MUST remain compatible across + processes; concurrently mixing such an older implementation with a current + implementation inside one OS process is explicitly unsupported because old + process-associated locks can be released by closing any sibling descriptor. + These rules MUST add no lock acquisition to a v2 steady-state key-value path. + +### Library Contract & Compatibility *(mandatory)* + +- **LC-001**: The initial production-ready implementation MUST focus on the + current C#/.NET package, including its public surface, tests, documentation, + diagnostics, benchmarks, samples, and packaging. +- **LC-002**: The new lock-free mapped representation MUST use a compatibility + identity distinct from layout v1.2; incompatible participants MUST fail before + accessing payload memory. +- **LC-003**: Layout-v1.2 mappings MUST remain usable through their supported + compatibility profile; no in-place reinterpretation or mixed synchronization + participation is permitted. +- **LC-004**: The public key-value model MUST remain unchanged: keys identify + values, acquisitions are shared read leases, and the store does not become a + queue, stream, broker, dispatcher, or exclusive-claim work pool. +- **LC-005**: Existing public status numeric assignments MUST remain stable; + new contention, compatibility, incarnation, or recovery outcomes MUST be + appended or separated without renumbering the legacy contract. +- **LC-006**: Public contracts MUST define key equality, duplicate publication, + reservation lifetime, exact-byte commit, value visibility, lease lifetime, + logical removal, reclamation, token incarnation, recovery, waiting, disposal, + and same-key race outcomes. +- **LC-007**: Public documentation MUST define system-wide lock-free progress + and explicitly state that the feature does not guarantee wait-free completion + or equal progress for every caller under sustained same-key contention. +- **LC-008**: Public documentation MUST separate empty/missing, duplicate, + capacity, pending removal, local contention, timeout, cancellation, disposed, + incompatible, unsupported recovery, and corruption outcomes. +- **LC-009**: Resource ownership documentation MUST state who owns each cold + create/open transaction and its physical-initialization authority, store + handle, reservation, borrowed writable view, read lease, recovery decision, + and mapped-memory lifetime, including successful ownership transfer and + failed-open cleanup ordering. +- **LC-010**: C++ and Python implementation of the new layout is outside this + feature, but the shared-memory visibility, ordering, identity, progress, and + recovery contracts MUST be documented without relying on managed-object + identity. +- **LC-011**: The implementation plan MUST classify the NuGet semantic-version + impact separately from the new mapped-layout compatibility version and include + deployment and rollback guidance. +- **LC-012**: The core package MUST retain its minimal runtime dependency model + and MUST NOT require a message broker, dispatcher, scheduler, service host, + logging framework, or hidden worker. +- **LC-013**: Package-consumption and compatibility tests MUST cover legacy-only, + lock-free-only, incompatible mixed-version, upgrade, and rollback scenarios. +- **LC-014**: Performance documentation MUST distinguish uncontended latency, + same-key contention, unrelated-key concurrency, capacity pressure, recovery, + and large-payload zero-copy measurements. +- **LC-015**: The layout-v2 contract MUST publish the generation-fenced + directory-reference representation and the 1,048,575-slot maximum as + cross-runtime compatibility rules. Clients that do not implement the exact + representation MUST reject the mapping rather than infer or reinterpret it. +- **LC-016**: Layout-v2 mappings using the versioned spill-summary codec, + publication-intent metadata, and PID-namespace recovery identity MUST carry + required-feature bits 0, 1, and 2, respectively, and the exact required- + features mask MUST be 7. Earlier required-features-zero, bit-0-only, and + mask-3 v2 binaries and current binaries MUST + reject one another before payload projection; this pre-release compatibility + fence MUST be reflected in executable constants, fixtures, and protocol + documentation without changing the 2.0 topology. + +### Key Entities *(include if feature involves data)* + +- **Shared-Memory Key-Value Store**: One bounded named mapping containing the + key index, value generations, reservations, leases, and reusable storage. +- **Key**: An opaque non-empty byte sequence compared by exact bytes and used + solely to address the current visible value. +- **Publication Reservation**: Temporary ownership of bounded store capacity for + one key, fixed descriptor, and announced payload length before visibility. +- **Publication Intent**: Immutable per-lifecycle ordinary metadata that + distinguishes an explicitly returned reservation from the tentative internal + reservation used by one atomic convenience publication. +- **Published Value Generation**: One complete immutable descriptor and payload + currently addressable by a key and protected from reuse by its lifecycle + identity and active leases. +- **Read Lease**: A shared, zero-copy, read-only protection token for one exact + published value generation; several leases may coexist. +- **Pending Removal**: A logically absent value generation retained only until + its protecting leases are released or safely recovered. +- **Participant Incarnation**: Identity that distinguishes a particular owner + and token lifetime from later reuse of process identifiers or shared records. +- **Recovery Decision**: A caller-controlled classification and mutation that + restores only safely abandoned reservation or lease state. +- **Diagnostics Snapshot**: Bounded consumer-requested measurements of store + capacity, lifecycle state, contention, recovery, and index health. +- **External Key Dispatcher**: An application-owned broker or coordinator that + may send keys to workers but has no ownership role inside the store contract. + +## Success Criteria *(mandatory)* + +### Benchmark Workload Matrix + +Relative performance criteria use the same machine, operating system, runtime, +store capacities, process placement, and configured profile-aware completion +policy for the legacy and lock-free profiles. Throughput scenarios run three trials; duration-bound rows +use at least 10 seconds of warm-up and 60 seconds of measurement, while +count-bound rows must reach their exact target. Reported results use the median +trial. Participants receive one logical processor each +where the host permits it, without oversubscribing the host, and the report +records processor allocation, OS, runtime, payload size, key distribution, +lease duration, churn pattern, and final statuses. + +| Workload | Participants | Data and access pattern | Primary measurements | +|---|---|---|---| +| Tiny operation baseline | 1, 2, 4, 8, and 12 independent processes | 8-byte keys, 1-byte payloads, distinct rotating keys; publish/remove and acquire/release cycles with immediate release | Aggregate operations/second and p50/p95/p99 latency | +| Same-key broadcast reads | 1, 2, 4, 6, 8, and 12 readers | One stable key, 256-byte payload, full-payload checksum, immediate release | Aggregate acquire/read/release throughput, p99, and checksum equality | +| Distributed-key reads | 1, 2, 4, 6, 8, and 12 readers | 256 stable keys selected uniformly, 256-byte payloads, full-payload checksum, immediate release | Scaling, p99, misses, and checksum equality | +| Primary broker-directed workload | One zero-copy producer, 1 or 12 broker-directed readers, and one observer | 1.3 MB frames, 16-byte descriptors, 256 rotating keys; broker sends each committed key to one reader, observer samples overlapping keys, cleanup removes only after assigned processing | Publication rate, end-to-end key availability, reader throughput, allocations, copies, and status outcomes | +| Mixed lifecycle churn | 12 readers plus 2 publisher/remover processes | At least 256 live keys, collision-heavy key set, random publish/acquire/release/remove/reuse operations for 10,000,000 cycles | Per-key correctness, throughput stability, early/late p99, and capacity recovery | +| Participant suspension | Same as distributed-key and mixed-churn workloads | Suspend one participant for 30 seconds at each observable steady-state transition | Healthy-process throughput, affected keys/capacity, and progress | +| Large-payload ingest | One producer plus 1 and 12 readers | 100,000 frames of 1.3 MB each using direct reservation, exact-byte commit, broker-directed acquire, and safe removal | Producer-owned allocation, library copies, throughput, and payload checksums | + +Both profiles remain present in the mixed-churn and large-ingest matrix. Legacy +rows are duration-bound comparison measurements using the same 10-second warm-up +and 60-second measured window as ordinary throughput rows. The lock-free rows +are count-bound durability qualifications: every mixed-churn trial executes at +least 100,000,000 operations and every large-ingest trial executes 100,000 +frames. The report records the count-bound profile set plus each row's effective +operation/frame target; validation rejects inherited, swapped, dual, missing, +or unmet targets and independently proves the full duration of comparison rows. + +### Measurable Outcomes + +- **SC-001**: A lock-free 100,000,000-operation mixed run with concurrent publication, + reservation, commit, acquire, projection, release, remove, and reuse across at + least 12 reader processes and additional publisher/remover processes completes + with zero partial payloads, mixed generations, false successful removals, + stale-token mutations, access violations, deadlocks, or global livelocks. +- **SC-002**: In the same-key broadcast-read workload, 6 reader processes + achieve at least 4 times one-reader acquire/read/release throughput and 12 + readers achieve at least 7 times one-reader throughput. +- **SC-003**: In the distributed-key workload, 6 reader processes achieve at + least 4.5 times one-reader throughput and 12 readers achieve at least 8 times + one-reader throughput. +- **SC-004**: In the primary broker-directed workload, the zero-copy producer + with 12 readers sustains at least 80% of its publication rate measured with + one reader. +- **SC-005**: In each participant-suspension workload, pausing one participant + for 30 seconds leaves the same set of healthy processes operating on unrelated + suitable keys at least 90% of their own pre-suspension aggregate throughput + while capacity permits; the paused participant is excluded from both sides of + the comparison. +- **SC-006**: In the 8-process tiny-operation workload, + the Windows lock-free profile delivers at least 4 times layout-v1.2 aggregate + throughput and reduces p99 acquire/release and publish/remove latency by at + least 80%. On Linux, for both acquire/release and publish/remove, lock-free + one-process p99 is no greater than layout-v1.2 one-process p99, lock-free + 8-process aggregate throughput is no lower than layout v1.2, and lock-free + 8-process p99 is at most 3 times its own one-process p99 and at most 10 + microseconds absolute. Every sampled lock-free trial at both process counts + has maximum latency at most 10 milliseconds. This separates intrinsic + latency and contention amplification from the legacy file-lock incumbent's + serialized completion distribution. +- **SC-007**: Instrumented steady-state validation records zero dependency on a + process-owned or globally exclusive synchronization owner for successful + publication, lookup, lease, removal, reclamation, and index-maintenance paths. +- **SC-008**: After initialization and warm-up in the primary and tiny-operation + workloads, publication/reservation/commit and acquire/projection/release each + report 0 bytes of runtime heap allocation per operation across at least + 1,000,000 operations. +- **SC-009**: In each lock-free large-payload ingest trial, all 100,000 direct publications + complete without a producer-owned full-payload allocation, without a + library-level full-payload copy after the producer fills reserved memory, and + without any reader payload copy required by the library. +- **SC-010**: Across 10,000 injected reservation-owner and lease-owner + termination cycles, explicit recovery exposes zero partial publications, + reclaims zero live ownership, accepts zero stale token actions, and restores + all capacity classified as safely recoverable. +- **SC-011**: Exhaustive controlled race tests for atomic-publish/atomic-publish, + atomic-publish/explicit-reserve, explicit-reserve/explicit-reserve, + commit/acquire, acquire/remove, release/reclaim, supported recovery/live + activity, and disposal/operation produce only the documented outcome sets + across at least 1,000,000 repetitions per race family. +- **SC-012**: Every bounded contention, capacity, recovery, cancellation, and + wait test returns within the caller's selected limit plus 250 milliseconds and + leaves no leaked reservation or lease ownership. +- **SC-013**: Existing layout-v1.2 contract tests, lock-free profile contract and + integration tests, package-consumption tests, full release tests, and package + creation all pass in the release configuration. +- **SC-014**: A consumer can follow the documented sample to publish directly, + distribute keys through its own broker, read from 6-12 workers and an + additional observer process, remove values safely, and recover a terminated + participant without reading implementation source. +- **SC-015**: In a barrier-controlled test, 12 processes simultaneously lease + one key and observe the same checksum; removal becomes pending, all 12 views + remain valid, no new lease succeeds after removal, and exactly one safe + reclamation occurs after the final release. +- **SC-016**: The 10,000,000-cycle mixed lifecycle churn workload + completes without corruption or leaked safely recoverable capacity, and its + late-run missing-key and publication p99 latency remains within 2 times its + early-run p99 latency on the same environment. +- **SC-017**: In controlled collision-heavy tests, a participant is paused at + every directory-mutation transition while other participants complete the + operation, reclaim the referenced slot, and reuse it for a later generation. + Across at least 1,000,000 repetitions, the resumed participant performs zero + mutations against the later generation and all capacity remains recoverable. +- **SC-018**: In three trials of at least 10,000 complete collision-heavy spill + churn cycles, diagnostics observe a real spill and nonzero real cleanup scans, + then report logical spill count zero and overflow occupancy zero before the + late missing-key window. That late window adds zero overflow scans, has zero + correctness failures, and its p99 latency is at most 2 times the corresponding + early missing-key p99 on the same environment. + +## Assumptions + +- SharedMemoryStore remains a key-value store. Load balancing, key delivery, + retries, acknowledgements, and worker scheduling belong to an external broker + or application layer. +- Multiple processes may acquire the same key concurrently. The store does not + infer exclusive processing ownership from a read lease. +- The primary measured workload has one zero-copy producer and 6-12 workers, but + the public store remains valid for additional readers, publishers, removers, + diagnostic callers, and non-broker use cases. +- Values are immutable after publication. Updating a key continues to use the + documented remove-and-republish lifecycle; atomic in-place mutation and an + additive atomic-replace API are outside this feature. +- The store provides per-key operation semantics but no cross-key transaction, + atomic batch, or total ordering guarantee. +- Capacity remains fixed at store creation. Lock-free progress does not imply + unlimited storage, guaranteed success, or immunity from capacity retained by + valid leases. +- Layout 2.0 has a configurable participant-record capacity, defaulting to 64 + open store handles. Each handle consumes one record so its identity is present + in the first atomic slot/lease claim; exhaustion rejects a new open without + affecting already-open handles or steady-state data progress. +- Layout 2.0 intentionally caps value-slot capacity at 1,048,575 so every + persistent directory intent and location can carry enough value-generation + identity to reject stale helpers after slot reuse using the portable atomic + contract. Larger lock-free stores require a future mapped-layout version + rather than weakening the generation fence. +- Lock-free is a system-wide progress guarantee, not a wait-free guarantee for + each caller. Same-key contention may cause bounded retries or a documented + contention outcome. +- Store creation, opening, and final cleanup are cold paths and may use bounded + lifecycle coordination. Explicit recovery may coordinate owner classification + and the records it changes, but neither lifecycle nor recovery coordination + may become an operation lock required by healthy steady-state data access. +- No hidden background worker performs maintenance, reclamation, recovery, + diagnostics, retries, or notification on behalf of callers. +- The deployment boundary remains trusted same-host processes. Cross-host + behavior, persistence guarantees, and protection from malicious mapped-memory + writers are outside scope. +- The first implementation targets the C#/.NET package on its currently + supported Windows, Linux, and same-host container environments. +- C++ and Python support for the new mapped contract is deferred. Those clients + must reject it until they implement the documented protocol. diff --git a/specs/009-lock-free-publish-read/tasks.md b/specs/009-lock-free-publish-read/tasks.md new file mode 100644 index 0000000..0e61e40 --- /dev/null +++ b/specs/009-lock-free-publish-read/tasks.md @@ -0,0 +1,358 @@ +# Tasks: Lock-Free Shared-Memory Key-Value Store + +**Input**: Design documents from `specs/009-lock-free-publish-read/` + +**Prerequisites**: `plan.md`, `spec.md`, `research.md`, `data-model.md`, +`contracts/`, and `quickstart.md` + +**Tests**: Required. For every behavior phase, create/run the listed failing +test before implementing the corresponding production task. + +**Organization**: Tasks are grouped by user story. User Story 2 precedes User +Story 1 because the read story's independent test needs a real committed v2 +value; both remain P1 and are validated independently. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: May run in parallel with adjacent tasks because files and dependencies + do not overlap. +- **[US#]**: Maps to the numbered user story in `spec.md`. +- Every task names the primary file(s) it changes or validates. + +## Phase 1: Setup and Baseline + +**Purpose**: Preserve the known-good legacy baseline and add test-only project +shells without changing runtime behavior. + +- [X] T001 Run `dotnet test SharedMemoryStore.slnx -c Release` and `dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release`; record commands, environment, pass/fail counts, and package API snapshot in `specs/009-lock-free-publish-read/baseline.md` +- [X] T002 Create a compiling test-only executable stub in `tests/SharedMemoryStore.LockFreeAgent/SharedMemoryStore.LockFreeAgent.csproj` and `tests/SharedMemoryStore.LockFreeAgent/Program.cs`, add it to `SharedMemoryStore.slnx`, and add a build/copy project reference from `tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj` so project-scoped integration runs can launch it +- [X] T003 Create `tests/SharedMemoryStore.LinearizabilityTests/SharedMemoryStore.LinearizabilityTests.csproj` and add it to `SharedMemoryStore.slnx` with xUnit/project references matching existing test projects +- [X] T004 Create a reproducible tracked `benchmarks/SharedMemoryStore.SyncProbe/SharedMemoryStore.SyncProbe.csproj` and `benchmarks/SharedMemoryStore.SyncProbe/Program.cs` scaffold from the documented baseline: controller/worker subprocesses, acquire-release and publish-remove modes, 1/2/4/8 processes, three trials, bounded latency samples, JSON environment/results, p50/p95/p99/max, fairness, status failures, and no runtime package dependency; add it to `SharedMemoryStore.slnx` (the ignored local artifact may be consulted but is not an input contract) + +**Checkpoint**: Existing tests/package pass and new empty tooling projects build. + +--- + +## Phase 2: Foundational Profile, Facade, Layout, and Atomic Gate + +**Purpose**: Establish compatibility-preserving dispatch and prove the platform +can support the atomic protocol before implementing user stories. + +**CRITICAL**: No lock-free user-story implementation begins until the mapped +atomic litmus passes on the current supported platform and the legacy suite still +passes after facade extraction. + +### Failing tests first + +- [X] T005 [P] Add failing API/profile/signature/numeric-assignment tests for `StoreProfile`, `CreateLockFree`, profile-aware sizing, `ParticipantRecordCount` default 64/validation, appended `ParticipantTableFull=11`, `ProtocolInfo`, and unchanged legacy members in `tests/SharedMemoryStore.ContractTests/LockFreeProfileApiContractTests.cs` +- [X] T006 [P] Add failing facade-routing and opaque token-incarnation tests using a fake engine in `tests/SharedMemoryStore.UnitTests/MemoryStoreFacadeTests.cs` +- [X] T007 [P] Add failing layout-2.0 participant/slot/lease control encodings, lock-free `SlotCount` range `1..1,048,575`, exact 22-bit target plus 33-bit generation encodings/reserved-bit rejection for `DirectoryLocation` and `DirectoryOperation`, exact participant index/generation masks and configured terminal retirement, participant section/64-byte stride, all offsets, 8-byte atomic alignment, binding codec, sequentially consistent RMW requirement, x64 gate, and checked-overflow tests in `tests/SharedMemoryStore.ContractTests/LockFreeLayoutContractTests.cs` +- [X] T008 [P] Add failing local lifetime-gate race/allocation tests that pause entry/exit/dispose transitions in `tests/SharedMemoryStore.UnitTests/LockFreeLifecycleGateTests.cs` +- [X] T009 [P] Add failing same-name v1.2/v2 header-first incompatible-size tests, participant open/close/reuse/exhaustion behavior, and Linux v1-compatible live owner-sidecar tests in `tests/SharedMemoryStore.IntegrationTests/LockFreeProfileOpenIntegrationTests.cs` +- [X] T010 Add failing cross-process aligned mapped `Interlocked`/`Volatile` publication/CAS tests, a two-word acquire/remove Dekker litmus that forbids both participants observing the old value after SC RMW, and non-x64 rejection in `tests/SharedMemoryStore.IntegrationTests/MappedAtomicLitmusIntegrationTests.cs` and `tests/SharedMemoryStore.LockFreeAgent/Program.cs` + +### Foundational implementation + +- [X] T011 Implement `StoreProfile`, `ParticipantRecordCount`, additive profile/sizing helpers, appended `ParticipantTableFull`, unchanged legacy signatures/status numbers, and `StoreProtocolInfo` in `src/SharedMemoryStore/SharedMemoryStoreOptions.cs`, `src/SharedMemoryStore/StoreStatus.cs`, and `src/SharedMemoryStore/StoreProtocolInfo.cs` until T005 passes +- [X] T012 Implement engine-neutral `LeaseHandle`, `ReservationHandle`, metrics, and synchronous span-safe `IStoreEngine` abstractions in `src/SharedMemoryStore/Engines/StoreTokenHandles.cs`, `src/SharedMemoryStore/Engines/EngineMetrics.cs`, and `src/SharedMemoryStore/Engines/IStoreEngine.cs` +- [X] T013 Extract current layout-v1.2 behavior into `src/SharedMemoryStore/Engines/LegacyV12/LegacyV12StoreEngine.cs`, implement `src/SharedMemoryStore/Engines/StoreEngineFactory.cs`, and refactor `src/SharedMemoryStore/MemoryStore.cs`, `src/SharedMemoryStore/ValueLease.cs`, and `src/SharedMemoryStore/Ingest/ValueReservation.cs` into the stable facade until T006 and all existing legacy tests pass +- [X] T014 Replace monitor-based operation entry with a CAS/ref-count local gate while preserving disposal ordering in `src/SharedMemoryStore/Lifecycle/StoreLifecycleGate.cs` until T008 and existing disposal tests pass +- [X] T015 [P] Implement the lock-free `SlotCount` ceiling and layout-v2 participant/slot/lease constants/codecs, exact participant index/generation masks with retirement at the configured hot-token maximum, generation-tagged 22-bit-target directory location/operation encodings with reserved-bit validation, explicit records, checked calculator, binding codec, SC RMW wrappers, and offset assertions in `src/SharedMemoryStore/LayoutV2/LayoutV2Constants.cs`, `src/SharedMemoryStore/LayoutV2/SharedRecordsV2.cs`, `src/SharedMemoryStore/LayoutV2/StoreLayoutV2.cs`, `src/SharedMemoryStore/Options/SharedMemoryStoreOptionsValidator.cs`, `src/SharedMemoryStore/LockFree/ParticipantToken.cs`, `src/SharedMemoryStore/LockFree/IndexBinding.cs`, `src/SharedMemoryStore/LockFree/DirectoryLocation.cs`, `src/SharedMemoryStore/LockFree/DirectoryOperation.cs`, and `src/SharedMemoryStore/LockFree/AtomicControlWord.cs` until T007 passes +- [X] T016 Refactor existing-region discovery to probe actual header identity before requested-size projection, preserve Linux live owner records, retain the legacy-compatible named lock only for cold initialization, and dispatch profile discovery in `src/SharedMemoryStore/Interop/SharedStorePlatform.cs`, `src/SharedMemoryStore/Interop/WindowsSharedMemoryRegion.cs`, and `src/SharedMemoryStore/Engines/StoreEngineFactory.cs` +- [X] T017 Implement layout-v2 cold header/section initialization, participant `Free/Registering/Active/Closing/Reclaiming` allocation, empty-handle close/reuse/exhaustion, x64 validation, and open cleanup in `src/SharedMemoryStore/LockFree/LockFreeParticipantRegistry.cs` and `src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs` until T009 passes +- [X] T018 Complete the raw two-process atomic litmus commands in `tests/SharedMemoryStore.LockFreeAgent/Program.cs`, run T010 in Release on the current development OS, and record architecture/runtime results in `specs/009-lock-free-publish-read/atomic-litmus-results.md`; stop local implementation if aligned mapped atomics fail, while retaining the distinct dual-platform release gate +- [X] T019 Run all legacy unit/contract/integration/interop tests and package consumption after extraction; record the green compatibility checkpoint in `specs/009-lock-free-publish-read/baseline.md` + +**Checkpoint**: Legacy is unchanged, v2 opens safely, mapped atomics pass, and the +facade/layout foundation is ready. + +--- + +## Phase 3: User Story 2 - Publish Directly Into Shared Memory (Priority: P1) + +**Goal**: Reserve one exact key/slot, write directly into mapped memory, and make +complete bytes visible at one commit CAS without a global operation owner. + +**Independent Test**: Concurrent simple, segmented, and direct reservations +publish exact bytes; unfinished bytes stay invisible; same-key races have one +winner; a paused publisher does not stop unrelated publication. + +### Failing tests first + +- [X] T020 [P] [US2] Add failing exhaustive finite-state binding/control/generation-tagged directory-operation/location tests for every insert-help pause, including pause after validation followed by descriptor completion, reclaim, slot reuse, stale target-cell installation, exact rollback, and no later-generation mutation; retain bidirectional checkpoint completeness and publish/publish reference histories in `tests/SharedMemoryStore.UnitTests/LockFreeCheckpointCoverageTests.cs`, `tests/SharedMemoryStore.UnitTests/LockFreeDirectoryStateTests.cs`, `tests/SharedMemoryStore.LinearizabilityTests/ReferenceStoreModel.cs`, `tests/SharedMemoryStore.LinearizabilityTests/LinearizabilityChecker.cs`, `tests/SharedMemoryStore.LinearizabilityTests/CheckerSelfTests.cs`, and `tests/SharedMemoryStore.LinearizabilityTests/PublicationHistoryTests.cs` +- [X] T021 [P] [US2] Add failing exact-collision tests for two-choice primary lanes, spill-summary gating, full `SlotCount` overflow admission, generation-tagged exact unlink, and stale-helper residue cleanup without later-generation damage in `tests/SharedMemoryStore.UnitTests/LockFreeDirectoryCollisionTests.cs` +- [X] T022 [P] [US2] Add failing first-claim participant-token, reservation generation, exclusive single-producer, exact-advance, commit/abort/recovery, terminal retirement, writable lifetime, and cancellation/deadline immediately before/after binding and commit tests in `tests/SharedMemoryStore.UnitTests/LockFreeReservationStateTests.cs` +- [X] T023 [P] [US2] Add failing public simple/segmented/reservation visibility, empty/oversized/zero-length boundaries, status, cancellation cleanup, and zero operation-synchronizer-call tests for the lock-free profile in `tests/SharedMemoryStore.ContractTests/LockFreePublishContractTests.cs` +- [X] T024 [US2] Add failing concurrent same-key/unrelated-key publisher, paused insertion-helper, and bounded cancellation/deadline ownership-cleanup integration tests in `tests/SharedMemoryStore.IntegrationTests/LockFreePublishIntegrationTests.cs` + +### Implementation + +- [X] T025 [P] [US2] Implement a generic/static no-op production checkpoint specialization, friend-only instrumented engine factory for the cross-process agent, canonical checkpoint catalog, and scheduler in `src/SharedMemoryStore/LockFree/LockFreeCheckpoint.cs`, `src/SharedMemoryStore/Properties/AssemblyInfo.cs`, and `tests/SharedMemoryStore.UnitTests/TestSupport/ControlledLockFreeScheduler.cs` until the T020 completeness test proves every checkpoint has before/after/pause/crash/race classifications +- [X] T026 [US2] Implement participant-bearing generation-fenced free-slot claim with exact participant revalidation, stale generation-tagged directory-residue cleanup, complete ordinary metadata overwrite only after exclusive `Free -> Initializing` claim, payload accounting, commit/abort ownership clearing, cancellation handoff, and retirement without delayed-helper ordinary cleanup stores in `src/SharedMemoryStore/LockFree/LockFreeSlotTable.cs` until T022 and the revised T020 pass +- [X] T027 [US2] Implement primary bucket/overflow lookup and a single helpable generation-tagged insert/unlink/abort protocol: require operation/location/binding/control generation agreement, use exact full-word CAS for every phase/location write, roll back an old target binding when its operation cannot advance, treat stale residue as helpable rather than later-generation corruption, preserve exact-key/spill/capacity invariants, and expose the common abort/unlink helper in `src/SharedMemoryStore/LockFree/LockFreeKeyDirectory.cs` until T020-T021 pass +- [X] T028 [US2] Implement sparse lazily warmed per-slot writable memory-manager pages so span-only/read-only handles do not allocate one managed object per configured slot, retain zero-allocation reuse after warm-up, and enforce reservation view lifetime in `src/SharedMemoryStore/LockFree/LockFreeReservationMemory.cs` +- [X] T029 [US2] Implement v2 `TryReserve`, simple publish, segmented publish, exclusive reservation projection/advance/commit/abort/dispose, stale-publisher generation revalidation, the shared exact abort/unlink helper (no direct descriptor zeroing), reservation-aware participant Closing cleanup/zero-reference proof, pre-ordering cancellation cleanup/helpable handoff, post-ordering normal outcomes, and facade routing without the operation synchronizer in `src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs` until T023-T024 and revised T020 pass +- [X] T030 [US2] Add and pass warmed zero-allocation reservation/abort, duplicate, invalid/incomplete, and bounded successful commits up to configured capacity in `tests/SharedMemoryStore.UnitTests/LockFreePublishAllocationTests.cs`; defer the one-million complete publish/remove reuse cycle to T051 + +**Checkpoint**: User Story 2 is independently usable; publication has one key +owner, zero-copy commit visibility, bounded retries, and unrelated-key progress. + +--- + +## Phase 4: User Story 1 - Read Values Concurrently by Key (Priority: P1) + +**Goal**: Let many independent processes acquire the same or different keys and +project immutable mapped bytes through incarnation-fenced shared leases. + +**Independent Test**: Publish stable values, then 6/12 workers and an observer +lease overlapping keys, verify exact bytes/checksums, and pause one reader +without stopping other readers. + +### Failing tests first + +- [X] T031 [P] [US1] Add failing participant-bearing lease first-claim/revalidation, activate/release ownership clearing, reuse/incarnation, and cancellation/deadline immediately before/after activation tests in `tests/SharedMemoryStore.UnitTests/LockFreeLeaseRegistryTests.cs` +- [X] T032 [P] [US1] Add failing lookup double-validation, commit/acquire, missing-key, hash-collision, stale-binding, bounded cleanup schedules, and minimal acquire histories in `tests/SharedMemoryStore.UnitTests/LockFreeAcquireStateTests.cs` and `tests/SharedMemoryStore.LinearizabilityTests/AcquireHistoryTests.cs` +- [X] T033 [P] [US1] Add failing public lease projection/lifetime/status/profile/cancellation and zero operation-synchronizer-call contract tests in `tests/SharedMemoryStore.ContractTests/LockFreeLeaseContractTests.cs` +- [X] T034 [US1] Add failing 1/6/12-reader same-key and distributed-key cross-process scenarios plus paused observer in `tests/SharedMemoryStore.IntegrationTests/LockFreeMultiReaderIntegrationTests.cs` and corresponding agent commands in `tests/SharedMemoryStore.LockFreeAgent/Program.cs` + +### Implementation + +- [X] T035 [US1] Implement participant-token-fenced global lease-record claim with exact participant revalidation, activation, stable scan, ownership-clearing exact release/recovery handoff, recycle, cancellation cleanup, and retirement in `src/SharedMemoryStore/LockFree/LockFreeLeaseRegistry.cs` until T031 passes +- [X] T036 [US1] Implement double-validated primary/marked-overflow exact-key lookup and stale-binding help in `src/SharedMemoryStore/LockFree/LockFreeKeyDirectory.cs` until T032 passes +- [X] T037 [US1] Implement v2 `TryAcquire`, post-activation directory/slot/participant revalidation, zero-copy descriptor/payload projection, release/dispose, lease-aware participant Closing cleanup/zero-reference proof, bounded cancellation cleanup, and facade routing without the operation synchronizer in `src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs` until T033 passes +- [X] T038 [US1] Complete acquire/release/checksum/pause commands in `tests/SharedMemoryStore.LockFreeAgent/Program.cs` and pass T034 without using a production broker or global lock +- [X] T039 [US1] Add and pass the barrier-controlled 12-process same-key lease/checksum setup portion in `tests/SharedMemoryStore.IntegrationTests/LockFreeBroadcastLeaseIntegrationTests.cs` +- [X] T040 [US1] Add and pass a one-million-cycle warmed zero-allocation acquire/project/release and expected-miss/full gate in `tests/SharedMemoryStore.UnitTests/LockFreeAcquireAllocationTests.cs` +- [X] T041 [US1] Extend `benchmarks/SharedMemoryStore.SyncProbe/Program.cs` with legacy/v2 selection, affinity-if-available, 1/6/12 same-key and distributed-key modes, status histograms, and stable JSON, then run the short reader scaling smoke and record `specs/009-lock-free-publish-read/benchmark-results/smoke-readers.json` + +**Checkpoint**: User Story 1 is independently usable by broker-directed workers +and unrelated readers; one reader never owns progress for the store or key. + +--- + +## Phase 5: User Story 3 - Remove and Reuse Without Stalling (Priority: P1) + +**Goal**: Logically remove at one CAS, preserve existing leases, reject new +leases, and reclaim exactly once after the last exact lease. + +**Independent Test**: Race acquire/remove/release/reclaim/republish over colliding +and unrelated keys, pause every transition, and verify exact old lease bytes and +safe generation reuse. + +### Failing tests first + +- [X] T042 [P] [US3] Add failing SC-ordered acquire/logical-remove deterministic schedules and minimal histories, including activation and cancellation/deadline immediately before/after removal plus conservative `RemovePending` on post-removal scan expiry, in `tests/SharedMemoryStore.UnitTests/LockFreeRemoveStateTests.cs` and `tests/SharedMemoryStore.LinearizabilityTests/RemoveHistoryTests.cs` +- [X] T043 [P] [US3] Add failing release/reclaim, cancellation handoff, generation-tagged exact-once unlink-help, helper pause-after-validation through reclaim/reuse, stale operation/location/binding residue cleanup, stale release/remove, and republish-after-reuse tests in `tests/SharedMemoryStore.UnitTests/LockFreeReclamationTests.cs` +- [X] T044 [P] [US3] Add failing public logical-success/conservative `RemovePending` for active leases or bounded classification/cooperative-physical-reclaim/duplicate-until-reclaim/cancellation and zero operation-synchronizer-call contracts in `tests/SharedMemoryStore.ContractTests/LockFreeRemoveContractTests.cs` +- [X] T045 [US3] Extend the failing 12-process barrier scenario through remove, rejected new acquire, final release, and one reclamation in `tests/SharedMemoryStore.IntegrationTests/LockFreeBroadcastLeaseIntegrationTests.cs` +- [X] T046 [US3] Complete the failing collision-heavy multi-process remove/reuse and early/late missing/publication latency test in `tests/SharedMemoryStore.IntegrationTests/LockFreeChurnIntegrationTests.cs`, including disjoint keys sharing one canonical bucket and asserting zero `CorruptStore`, false miss, later-generation mutation, or leaked capacity + +### Implementation + +- [X] T047 [US3] Implement SC-ordered logical `Published -> RemoveRequested`, stable exact lease scan, `Success` only after completed no-active classification, conservative `RemovePending` for active leases or post-ordering bound expiry, and facade routing without the operation synchronizer in `src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs` until T042 and T044 pass +- [X] T048 [US3] Implement cooperative exact-once reclaim through the common generation-tagged unlink/abort helper, exact operation/location clearing without unconditional stores, no ordinary metadata zeroing by delayed helpers, generation advance/retirement, stale-residue handling, and release/allocation-pressure helping in `src/SharedMemoryStore/LockFree/LockFreeReclaimer.cs` until revised T043 passes +- [X] T049 [US3] Integrate release-triggered and retrying-remove reclamation in `src/SharedMemoryStore/LockFree/LockFreeLeaseRegistry.cs` and `src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs` until T045 passes +- [X] T050 [US3] Complete churn/remove/reuse agent commands and pass the bounded integration workload in `tests/SharedMemoryStore.LockFreeAgent/Program.cs` and `tests/SharedMemoryStore.IntegrationTests/LockFreeChurnIntegrationTests.cs` +- [X] T051 [US3] Add and pass one-million complete warmed publish/reserve/commit/acquire/project/release/remove/reuse zero-allocation cycles plus exact capacity-restoration gates in `tests/SharedMemoryStore.UnitTests/LockFreeRemoveReuseAllocationTests.cs` + +**Checkpoint**: User Story 3 safely churns all configured capacity with existing +leases intact and no global index maintenance. + +--- + +## Phase 6: User Story 4 - Survive Participant Pauses and Failures (Priority: P2) + +**Goal**: Healthy processes progress past stopped participants; explicit exact- +incarnation recovery restores only safely abandoned slots/records. + +**Independent Test**: Pause/kill agents at every checkpoint, continue healthy +operations, recover, fill to capacity, and replay stale tokens after reuse. + +### Failing tests first + +- [X] T052 [P] [US4] Add failing participant Registering/Active/Closing/Recovering/Reclaiming/reuse/retirement, PID-reuse/process-start identity, complete token, first-claim crash, table exhaustion, diagnostics, helping, and unsupported classification tests in `tests/SharedMemoryStore.UnitTests/LockFreeParticipantRegistryTests.cs` +- [X] T053 [P] [US4] Add failing reservation recovery versus commit/abort/helper and cancellation/deadline schedules, participant retirement, and report-count tests in `tests/SharedMemoryStore.UnitTests/LockFreeReservationRecoveryTests.cs` +- [X] T054 [P] [US4] Add failing lease recovery versus live release/reclaim/record reuse and cancellation/deadline schedules, participant retirement, and report-count tests in `tests/SharedMemoryStore.UnitTests/LockFreeLeaseRecoveryTests.cs` +- [X] T055 [US4] Add failing checkpoint pause/kill/recover/fill/stale-token scenarios by consuming every entry in the canonical checkpoint catalog in `tests/SharedMemoryStore.IntegrationTests/LockFreeCrashRecoveryIntegrationTests.cs` and `tests/SharedMemoryStore.LockFreeAgent/Program.cs` +- [X] T056 [P] [US4] Add failing local handle disposal versus every operation/token callback while a second handle progresses in `tests/SharedMemoryStore.IntegrationTests/LockFreeDisposalIntegrationTests.cs` + +### Implementation + +- [X] T057 [US4] Complete participant control/token creation, PID/identity-kind/start classification, conservative liveness, zero-reference retirement, and stale Registering handling in `src/SharedMemoryStore/LockFree/LockFreeParticipantRegistry.cs`, `src/SharedMemoryStore/LockFree/ParticipantIncarnation.cs`, and `src/SharedMemoryStore/Leasing/LeaseOwnerClassifier.cs` until T052 passes +- [X] T058 [US4] Implement participant-token exact-CAS reservation recovery/help/reporting and bounded cancellation handoff in `src/SharedMemoryStore/LockFree/LockFreeRecovery.cs` until T053 passes +- [X] T059 [US4] Implement participant-token exact-CAS lease claim/active recovery, record-incarnation fencing, reclaim help, reporting, and bounded cancellation handoff in `src/SharedMemoryStore/LockFree/LockFreeRecovery.cs` until T054 passes +- [X] T060 [US4] Complete and harden participant `Active -> Closing -> Reclaiming`, all-resource token cleanup, stable zero-reference proof, record reuse/retirement helping, and local dispose ordering in `src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs`, `src/SharedMemoryStore/LockFree/LockFreeParticipantRegistry.cs`, and `src/SharedMemoryStore/MemoryStore.cs` until T056 passes +- [X] T061 [US4] Complete every entry in the then-current agent checkpoint/crash catalog, run T055 in Release, and record capacity/live-owner/stale-token evidence in `specs/009-lock-free-publish-read/recovery-results.md`; later catalog additions remain owned by their convergence task + +**Checkpoint**: User Story 4 proves stopped owners retain only local bounded +resources and recovery cannot reclaim live/current ownership. + +--- + +## Phase 7: User Story 5 - Operate and Upgrade Safely (Priority: P3) + +**Goal**: Expose bounded diagnostics, reject incompatible participants, preserve +legacy/package behavior, and document explicit rollout/rollback. + +**Independent Test**: Exercise mixed load and diagnostics while opening legacy, +v2, old native/Python, package, upgrade, and rollback scenarios. + +### Failing tests first + +- [X] T062 [P] [US5] Add failing additive diagnostics/profile/participant occupancy/exhaustion/spill/retry/help/recovery snapshot contracts in `tests/SharedMemoryStore.ContractTests/LockFreeDiagnosticsContractTests.cs` +- [X] T063 [P] [US5] Add failing live diagnostics versus data-operation progress tests in `tests/SharedMemoryStore.IntegrationTests/LockFreeDiagnosticsIntegrationTests.cs` +- [X] T064 [P] [US5] Add failing compatibility-manifest and updated C++/Python v1.2-only v2 fail-closed rejection-before-payload tests in `tests/SharedMemoryStore.InteropTests/LockFreeLayoutRejectionTests.cs` +- [X] T065 [P] [US5] Add failing packed C# 1.0.2 smaller/equal/oversized/all-open-mode fail-closed mapping tests, new default-legacy/explicit-v2/header-first incompatibility, Linux live-owner preservation, participant-capacity consumption, same-name upgrade, and rollback tests in `tests/SharedMemoryStore.ContractTests/LockFreePackageContractTests.cs` and `tests/SharedMemoryStore.IntegrationTests/LockFreePackageIntegrationTests.cs` + +### Implementation and documentation + +- [X] T066 [US5] Implement engine-neutral/additive diagnostics and bounded v2 scans/counters without correctness dependencies in `src/SharedMemoryStore/Diagnostics/DiagnosticsSnapshot.cs`, `src/SharedMemoryStore/Diagnostics/StoreDiagnostics.cs`, and `src/SharedMemoryStore/LockFree/LockFreeDiagnostics.cs` until T062-T063 pass +- [X] T067 [P] [US5] Publish and verify the revised exact layout-2.0 offsets/state/memory-order fixtures, generation-tagged directory encodings, reserved bits, and lock-free slot ceiling in `protocol/layout-v2.0.md`, `protocol/fixtures/v2.0/manifest.json`, and `tests/SharedMemoryStore.ContractTests/LockFreeLayoutContractTests.cs` +- [X] T068 [P] [US5] Document resource protocol 2 and cold-only ordinary-lock participation in `protocol/resource-naming-v2.md` +- [X] T069 [US5] Update `protocol/compatibility.json` for C# layout 1.2/2.0 and C++/Python 1.2-only support, and implement fail-closed SMS2 header rejection in `src/cpp/src/protocol.cpp`, `src/cpp/src/store.cpp`, `src/cpp/src/platform_windows.cpp`, `src/cpp/src/platform_linux.cpp`, and the C-ABI-backed `src/python/shared_memory_store/store.py` only where T064 proves a gap +- [X] T070 [US5] Update package version/release notes and XML docs for every additive/changed public symbol—profile, protocol info, participant capacity/status, diagnostics, wait/remove semantics, reservation single-writer lifetime, and protocol identity—in `src/SharedMemoryStore/SharedMemoryStore.csproj`, `src/SharedMemoryStore/SharedMemoryStoreOptions.cs`, `src/SharedMemoryStore/StoreStatus.cs`, `src/SharedMemoryStore/StoreProtocolInfo.cs`, `src/SharedMemoryStore/MemoryStore.cs`, `src/SharedMemoryStore/ValueLease.cs`, `src/SharedMemoryStore/Ingest/ValueReservation.cs`, and `src/SharedMemoryStore/Diagnostics/DiagnosticsSnapshot.cs`; add all-new-public-symbol XML-doc assertions to `tests/SharedMemoryStore.ContractTests/LockFreeProfileApiContractTests.cs` until T065 passes +- [X] T071 [P] [US5] Update `README.md` and `protocol/README.md` with KV-only scope, lock-free versus wait-free meaning, profile selection, ownership, trust boundary, performance interpretation, migration, and rollback +- [X] T072 [P] [US5] Implement the compilable broker-key sample described by `quickstart.md` in `samples/LockFreeBrokerKeys/LockFreeBrokerKeys.csproj` and `samples/LockFreeBrokerKeys/Program.cs`; add it to `SharedMemoryStore.slnx` +- [X] T073 [US5] Add a sample validation test covering one producer, configurable 6-12 workers, observer, pending remove, missing key, and explicit recovery in `tests/SharedMemoryStore.IntegrationTests/LockFreeSampleValidationTests.cs` +- [X] T074 [US5] Run v1.2-only, v2-only, incompatible mixed-version, package, sample, upgrade, and rollback tests and record the matrix in `specs/009-lock-free-publish-read/compatibility-results.md` + +**Checkpoint**: User Story 5 provides observable operations and a safe explicit +deployment contract without making the store a stream or broker. + +--- + +## Phase 8: Cross-Story Correctness, Performance, and Release Qualification + +**Purpose**: Prove the combined protocol, success criteria, package, and +non-convergence gates. + +- [X] T075 Extend the already self-tested per-story reference model/checker with the combined reservation/lease/removal/recovery state space and deterministic failing-history minimization in `tests/SharedMemoryStore.LinearizabilityTests/ReferenceStoreModel.cs`, `tests/SharedMemoryStore.LinearizabilityTests/LinearizabilityChecker.cs`, and `tests/SharedMemoryStore.LinearizabilityTests/CheckerSelfTests.cs` +- [X] T076 Add seeded/minimized histories for publish/publish, commit/acquire, acquire/remove, release/reclaim, recovery/live action, disposal/operation, participant/value/lease capacity, cancellation, and stale tokens in `tests/SharedMemoryStore.LinearizabilityTests/LockFreeHistoryTests.cs`; pass deterministic and randomized configured tiers including one-million configured repetitions for every SC-011 race family +- [X] T077 [P] Implement and pass counting/throwing synchronization plus held-legacy-lock tests for every v2 operation in `tests/SharedMemoryStore.IntegrationTests/LockFreeNoOperationLockIntegrationTests.cs`, and a complete `StoreWaitOptions` matrix—open, publish/simple/segmented, reserve/advance/commit/abort, acquire/projection/release, remove/reclaim, diagnostics, recovery, disposal, NoWait/finite/infinite/timeout/cancellation before/after ordering, limit+250 ms, zero owner-controlled leakage—in `tests/SharedMemoryStore.IntegrationTests/LockFreeWaitPolicyMatrixIntegrationTests.cs` +- [X] T078 [P] Implement and self-test `scripts/validate-lock-free-os.ps1` subcommands for Windows/Linux architecture gating, primitive litmus, raw visibility, held-lock/no-lock trace, checkpoint/crash, Release tests, Docker/native/Python, samples, and pack; implement Linux marked-interval `strace` plus optional Docker pause coverage in `tests/SharedMemoryStore.IntegrationTests/LockFreeOsTraceIntegrationTests.cs`, with unavailable prerequisites reported as not-qualified +- [X] T079 Implement the complete process/affinity/JSON workload matrix and broker-key pipe protocol in `benchmarks/SharedMemoryStore.SyncProbe/Program.cs`, `benchmarks/SharedMemoryStore.SyncProbe/BenchmarkProtocol.cs`, and `benchmarks/SharedMemoryStore.SyncProbe/BenchmarkResults.cs` +- [X] T080 [P] Add BenchmarkDotNet profile/allocation/collision/recovery benchmarks in `benchmarks/SharedMemoryStore.Benchmarks/LockFreeProfileBenchmarks.cs` and register them in `benchmarks/SharedMemoryStore.Benchmarks/Program.cs` +- [X] T081 Implement and pass a production-no-op, raw Release full-protocol visibility/reuse smoke—sequence/complement/key/generation/full payload, independent publisher/readers/remover, no shared logging fences, aggressive reuse—in `tests/SharedMemoryStore.IntegrationTests/LockFreeRawVisibilityIntegrationTests.cs` and `tests/SharedMemoryStore.LockFreeAgent/Program.cs` +- [X] T082 Historical R5 freeze, superseded by T118-T121: freeze the then-current one-shot short correctness/raw-visibility/performance convergence contract in `specs/009-lock-free-publish-read/benchmark-results/short-report.md`; its closure would have been derived only from `artifacts/lock-free-qualification/009-final-r5-pr/{summary.json,sync-probe.json}` on the common clean commit, with no tracked post-run edit +- [X] T083 Create reproducible `pr`, `nightly`, and `release` tier orchestration with exact commands/timeouts/seeds in `scripts/run-lock-free-qualification.ps1` and `specs/009-lock-free-publish-read/qualification-config.json`, including bounded-operation limit-plus-250-ms and zero owner-controlled slot/lease/participant leakage assertions +- [X] T084 Freeze the configured nightly/release stress, recovery, churn, and bounded-wait execution contract in `specs/009-lock-free-publish-read/release-qualification.md`; final closure is machine-derived from the fixed ignored artifacts and exact config fields, with no tracked post-run edit +- [X] T085 Historical R5 freeze, superseded by T118-T121: freeze the then-current 60-second/three-trial, 100,000,000-operation, and 100,000 direct-1.3-MB-frame execution contract; authoritative raw JSON would remain in immutable ignored `artifacts/lock-free-qualification/009-final-r5-release/{summary.json,sync-probe.json}` rather than being copied into tracked `specs/`, with closure machine-derived and no tracked post-run edit +- [X] T086 Historical R5 freeze, superseded by T118-T121: freeze distinct Windows-x64 and Linux-x64 atomic/raw/no-lock/crash/tests/interop/sample/pack gates; its pass/fail/not-qualified result would have been derived from `artifacts/lock-free-qualification/009-final-r5-release/os-validation.json` and `artifacts/lock-free-os-validation/009-final-r5-linux-x64.json` with identical clean provenance and no tracked post-run edit +- [X] T087 Re-run `quickstart.md` end to end against the packed package and correct any API/sample drift in `specs/009-lock-free-publish-read/quickstart.md` and `samples/LockFreeBrokerKeys/` +- [X] T088 Freeze the independent concurrency-review acceptance contract covering public API, semver, atomic/control encodings, participant first-claim recovery, ABA, helping, cancellation cleanup, allocations, diagnostics, compatibility, and tests; conditional closure requires `code-review.md` to have no unresolved High or Medium finding for the exact clean provenance in final JSON, with no tracked post-run edit +- [X] T089 Freeze `specs/009-lock-free-publish-read/checklists/implementation.md` and the SC-001..SC-018 machine-evidence mapping; every checked/pass statement is conditional on linked JSON for the common clean commit, and no tracked post-run edit is allowed +- [X] T090 [P] Specify and freeze the required-feature-bit versioned-empty `SpillSummary` codec, exact Present/Empty transitions, layout field, protocol manifest, and old/new pre-release v2 mutual rejection in `spec.md`, `research.md`, `data-model.md`, `contracts/`, `protocol/`, and `LockFreeLayoutContractTests.cs` +- [X] T091 Implement summary-before-cell publication, stable same-canonical cleanup before mutation release, real overflow-scan telemetry, budgeted exact key equality, and conservative fail-closed uncertainty handling in `SpillSummary.cs`, `LockFreeKeyDirectory.cs`, `LockFreeDiagnostics.cs`, and engine/header wiring +- [X] T092 Add deterministic checkpoints and controlled tests for stale setter/clearer ABA, post-CAS validation loss, abort/fallback cleanup, exact clear ordering, normal churn convergence, late missing-scan avoidance, malformed codec words, and crash-agent routing in unit/integration/agent projects +- [X] T093 Rebuild Release, pass focused/full regression and independent code review, then run the strict three-trial 4,096-slot/10,000-cycle CR-M06 workload with nonzero cleanup diagnostics, zero late-window scans, zero correctness failures, and late/early p99 at most 2x; preserve before/after artifacts and conclusions +- [X] T094 Correct delayed directory helpers so a generation `G` insert/unlink can never clear or publish generation `G+1` location/operation state; add deterministic reuse schedules plus the 10-by-10-second publish/remove generation stress artifact +- [X] T095 Make reservation cancellation dominate insert helping at Prepared, TargetSelected, BindingChanged, overflow Empty, and completed-insert return windows; classify legal Aborting/Reclaiming/future-generation observations as `InvalidReservation` while retaining fail-closed `CorruptStore` for impossible same/lower-generation states +- [X] T096 Append and route checkpoints 52-55, expand SC-017 to all new mutation phases, and add deterministic primary/overflow insert-cancellation schedules proving no false corruption, no discoverable canceled key, directory drain, and capacity reuse +- [X] T097 Correct the reference model for documented retrying remove and add a production-backed sequential history proving repeated protected `RemovePending`, release/reclaim, then `NotFound` +- [X] T098 Freeze the post-T094-T097 focused/full, PR/nightly/release, dual-OS, and review rerun contract; final closure and evidence hashes are derived from the fixed ignored artifacts' manifests on one clean commit, with no tracked post-run edit +- [X] T099 Assign immutable per-slot `PublicationIntent` and required-feature bits 0/1, and make explicit-reservation versus atomic-publication ordering, duplicate classification, recovery, compatibility rejection, and layout fixtures agree across production code, contracts, protocol documents, and tests +- [X] T100 Prove the intent-aware create-conflict resolver and strict atomic-candidate cancellation protocol under NoWait/finite/infinite budgets, including a fresh-lookup retry charge after successful help and fail-closed handling of impossible same/lower-generation states +- [X] T101 Revalidate every invalid directory binding by exact source-word/slot-generation observation before returning corruption, and preserve the ten-by-ten-second plus deterministic primary/overflow/spill-summary stale-reference correction evidence +- [X] T102 Complete the physical `StoreFull` resource proof: per-handle nonblocking double-collect scratch, exact control-word equality ordering witness, `StoreBusy` on movement/competing local proof, malformed-control fail-closed behavior, reference-checker consumption, and deterministic/linearizability tests +- [X] T103 Harden `qualification-config.json`, `scripts/run-lock-free-qualification.ps1`, and `scripts/validate-lock-free-os.ps1` so configured counts/leak claims, full-suite TRX, exact performance rows/trials, restore/tool prerequisites, dual-platform OS results, source/toolchain hashes, immutable evidence paths, and nonzero not-qualified exits are machine-checkable; validate both scripts through non-executing dry runs +- [X] T104 Historical R5 freeze, superseded by T118-T121: freeze then-current one-shot PR/nightly/release execution at `009-final-r5-pr`, `009-final-r5-nightly`, and `009-final-r5-release` without overwriting preserved diagnostics or rejected earlier final candidates; its conditional closure required matching Windows-x64/Linux-x64 schema-v3 evidence and identical clean commit/source-manifest provenance, entirely machine-derived with no tracked post-run edit +- [X] T105 Freeze `benchmark-results/short-report.md`, `release-qualification.md`, and `checklists/implementation.md` before execution; every SC-001..SC-018 and review conclusion is conditionally closed only by linked final JSON and identical clean provenance, and failing/missing evidence invalidates the freeze rather than permitting a tracked post-run edit +- [X] T106 Fence Linux recovery by exact store/participant PID-namespace identities and monotonic Enabled/Mixed mode under required-feature bit 2 (mask 7); publish Mixed before a differing/unproven opener's first Registering CAS, preserve ordinary KV access, conservatively retain partial Registering owners, validate stable Active identities, and cover layout/checkpoint/crash behavior on Windows/Linux + +--- + +## Phase 9: Convergence + +- [X] T107 Harden the bounded C# cold create/open/close transaction so only physical creation authorizes initialization, Windows coordinates before mapping, and Linux orders lifecycle reconciliation, stale deletion, mapping lock, mapping/owner-anchor publication, reverse gate release, and post-gate failed-open cleanup within the caller's original wait budget; preserve conservative anchor/sidecar/release-marker liveness and hot key-value lock freedom. Correct directory joint-tuple validation and legal cancellation/location handoffs so delayed generation-G helpers cannot expose, clear, or overwrite generation-G+1 state; cover future-generation fail-closed behavior, first-valid-location arbitration, alternate cleanup, and post-CAS withdrawal with checkpoints 66-67 and the complete 67-entry Windows/Linux crash matrix. Correct the Linux raw tiny-operation workload to use two deterministic rotating keys per worker in distinct canonical buckets with a recorded catalog digest, unbiased fixed early/late reservoirs whose independent running maximum cannot evict a stall, exact host-tuple binding, mask-valid sparse affinity, paired-success cycle coherence, and zero checksum/corruption evidence. Bind exact one/eight-process raw trials and enforce one-process intrinsic-p99 non-regression, eight-process throughput non-regression, <=3x lock-free self-amplification, <=10 us absolute p99, and every-lock-free-trial <=10 ms sampled maximum for both scenarios; require the release importer to reproduce the decision and revalidate the exact OS evidence tree and command-log bindings. Replace process-associated Linux coordination with fail-closed OFD locks in current C# and native adapters, retain stable `.lock`/`.lifecycle` inodes, dispose synchronization before region/owner cleanup on every teardown path, and prove same-PID load-context/native exclusion plus concurrent final-close/reopen and foreign exclusion. Run focused regressions, anchor/release-marker tests, full Windows/Linux Release suites, a clean Linux `-Command all` diagnostic, and independent production/qualification-harness review with no unresolved High or Medium finding per FR-044, FR-054..FR-057, LC-009, SC-001..SC-018, and US1's independent test +- [X] T108 Historical R5 terminal freeze, superseded by T118-T121: freeze the then-current one-shot PR, nightly, release, Windows-x64, and Linux-x64 execution and machine-acceptance contract for one clean commit. This task's checkbox records only that the R5 commands, immutable paths, exact manifest/file-set/log binding, completion revalidation, identical clean commit/source-manifest provenance, and no-unresolved-High-or-Medium-review requirements were frozen; it does not assert that the ignored artifacts exist or passed. R5 approval would have been evidence-only at `artifacts/lock-free-qualification/009-final-r5-pr/{summary.json,sync-probe.json}`, `artifacts/lock-free-qualification/009-final-r5-nightly/{summary.json,sync-probe.json}`, `artifacts/lock-free-qualification/009-final-r5-release/{summary.json,sync-probe.json,os-validation.json}`, `artifacts/lock-free-os-validation/009-final-r5-linux-x64.json`, and `artifacts/lock-free-os-validation/009-final-r5-linux-x64.evidence/linux-tiny-performance.json`; T118 records its rejection and T121 is the only current terminal freeze +- [X] T109 Preserve the rejected first immutable candidate, correct the PR SC-017 directory-generation count from 46 to the source-owned 50-transition minimum, make the test assert its machine-readable transition-count contract, validate every PR/nightly/release tier against that count even in validation-only mode, reject a one-below-source negative case, and require exact start, 50 unique transition-pass markers, summed repetitions, completion, and zero corruption/false-miss/wrong-generation/leak evidence. Pass focused Release execution, PR/release validation-only checks, a complete non-final PR diagnostic, and independent harness review without changing production or hot-path code. +- [X] T110 Preserve the rejected non-final PR diagnostic and harden the raw mapped-atomic litmus controller without changing its 10,000 barriers or the agents' 30-second stall bound: replace the observed 45-second throughput cliff with a 120-second parent ceiling, launch incrementally under cleanup ownership, drain output from process start, capture role/PID/exit/output and mapped state before and after a timeout stop, report process/mapping cleanup failures, and pass repeated focused Windows execution plus mapped-atomic, Integration, full-solution, Windows/Linux atomic, and complete non-final PR diagnostics. +- [X] T111 Record the unbound full-solution diagnostic observation (no named immutable artifact or hash was emitted) that missed an instrumented reservation checkpoint before exercising its helper-ordering assertions; prove the production path and current mapped-atomic diff were uninvolved, replace the shared test-only 50-millisecond pre-checkpoint budget with a 2-second finite budget and the 150-millisecond post-checkpoint expiry delay with 2.25 seconds for all affected schedules, and pass focused repetition, the complete cancellation-race class, full UnitTests, full solution, and independent harness review without changing production code. +- [X] T112 Preserve the rejected `009-r2-pre-final2-pr` diagnostic whose full suite and both churn tests passed but whose focused churn importer correctly rejected two rows against a stale one-row contract. Prove production is uninvolved; bind both churn owner/leak mappings to the exact case-sensitive SC-016 collision-workload FQN and reject their joint drift to the existing fixed-key sibling; bind the source namespace, top-level class, and one direct `[Fact]` method as a single contract; keep the sibling regression in the full suite; label the configured environment value truthfully as total cycles; and make validation-only positive/negative cases exercise real XML parsing, wrapper failure-state mutation/cleanup, and rejection of role swaps, nested types/methods, distinct, missing, alternate-existing, extra-sibling, wrong, duplicate, or non-passed evidence. Pass PR/release validation-only checks, the exact focused churn test, the full solution, a new complete non-final PR diagnostic, and independent harness review without changing production or workload code. +- [X] T113 Preserve the rejected `009-r2-pre-final3-pr` diagnostic whose full suite, performance, capacity, and 52 other steady-state checkpoints (104/108 checkpoint-workload rows) passed before checkpoints 62-63 exposed two deterministic late-suspension contract gaps. Require `InvalidReservation` only for the checkpoint-62 pause protocol that deliberately begins reservation abort, preserve the primary reserve outcome across cleanup, and reject every other resumed outcome for that case. At checkpoint 63, keep the public finite remove probe and move the ownership budget check after the instrumentable post-lease-scan window so an expired participant remains universally helpable in `RemoveRequested` instead of claiming `Reclaiming` with a stale deadline. Add a deterministic two-second finite-deadline/2.25-second pause regression proving `RemovePending`, zero `Reclaiming`, cooperative same-key republish, and full capacity; pass focused checkpoint-62/63 suspension across distributed-key and mixed-churn workloads at both one-second and release 30-second pauses, the full solution, a new complete non-final PR diagnostic, and independent source/harness review. +- [X] T114 Freeze the second immutable candidate at non-overwriting `009-final-r2-*` paths after T109-T113 pass. Preserve the original `009-final-linux-x64` pass and every rejected final/non-final diagnostic as evidence only. This checkbox froze the then-current Linux x64, Windows PR, nightly, and release commands and all SC-001..SC-018, exact-tree, log-binding, completion-revalidation, dual-platform, and no-unresolved-High-or-Medium-review predicates; it did not assert that ignored final artifacts existed. The R2 Linux invocation was later rejected and superseded by T115 without overwriting it. +- [X] T115 Preserve the rejected `009-final-r2-linux-x64` attempt whose report correctly identified a Windows host: required native/Python rows were not qualified and Linux-only tiny/`strace`/SIGSTOP rows were optional and inapplicable after the Linux command was mistakenly launched by Windows PowerShell, so the report could not satisfy the intended Linux contract. Prove the store and completed rows were uninvolved; validate an explicit Ubuntu login-shell invocation as Linux x64 with all structural self-tests passed; freeze new non-overwriting `009-final-r3-*` paths and require that invocation before the unchanged full Linux, PR, nightly, release, SC-001..SC-018, common-provenance, and no-post-run-edit contract can qualify. +- [X] T116 Preserve the rejected `009-final-r3-linux-x64` attempt that reached Linux x64 but stopped before workloads because Ubuntu's package-managed SDK 10.0.109 consumed stale user-local workload-set 10.0.102 metadata and `dotnet --info` failed. Repair the empty-workload manifest set with `dotnet workload update` to 10.0.109.1, prove `dotnet --info`, workload listing, Docker/tool resolution, clean restore/build, and 45/45 architecture tests in an executable Linux diagnostic, add a non-artifact-consuming Linux prerequisite command before the final script, and freeze new non-overwriting `009-final-r4-*` paths under the unchanged full qualification contract. +- [X] T117 Preserve the R4 Linux pass and rejected R4 PR result without overwriting either artifact. Classify the sole PR failure as a disposal-test contract violation: the concurrent theory selected current-process lease recovery despite the override's process-wide quiescence precondition, while its reservation sibling carried the same latent defect. Use the normal concurrent-safe `false` mode for both recovery operations, stop dereferencing borrowed projection bytes because racing disposal can invalidate their lifetime, and retain exact content assertions on the unaffected live handle and dedicated data-path suites. Record the unbound local observation of 50 repeated 15-row disposal theories separately from durable evidence; pass 29/29 disposal-class tests, Integration 302/302, the full solution 1,014/1,014, independent H0/M0/L0 diff review, and the clean provenance-bound `009-r5-pre-final-pr` diagnostic with all 24 gates passed. Freeze new non-overwriting `009-final-r5-*` paths under the unchanged SC-001..SC-018, common-provenance, immutable-evidence, and no-post-run-edit contract; the R4 artifacts remain diagnostic only. +- [X] T118 Preserve the successful R5 Linux, PR, and nightly artifacts and the incomplete non-convergent R5 release directory without overwriting any of them. Diagnose the first Legacy mixed-churn trial through live process/CPU and read-only mapped-sequence evidence as progressing through the known named-semaphore baseline rather than deadlocked, and prove that three Legacy 100,000,000-operation trials plus Legacy 100,000-frame ingest and the remaining matrix cannot fit the six-hour whole-probe deadline. Reject R5 as a final sequence without changing product code or weakening SC-001/SC-009. +- [X] T119 Make benchmark completion policy explicit and convergent: config schema 5 selects only `LockFree` as count-bound; Legacy mixed-churn and large-ingest remain three 10-second-warm-up/60-second comparison trials; every LockFree mixed trial retains 100,000,000 operations and every LockFree large-ingest trial retains 100,000 frames. Emit schema 8 `CountBoundProfiles`, `OperationTarget`, and `FrameTarget`, 30-second progress heartbeats, and a controller-enforced warm-up-plus-duration-plus-exact-60-second-grace deadline armed before store setup. Use an atomic monotonic-deadline latch so delayed timer dispatch cannot accept overdue cleanup; on timeout, give tracked child-tree termination a bounded 100 ms budget and then unconditionally fail-fast the isolated probe process rather than unwinding an in-flight infinite store operation. Preserve standalone `--count-bound-profiles` default behavior as `both`. +- [X] T120 Fail closed on completion evidence in both importers: reject missing, dual, inherited, swapped, one-below, or unmet targets; independently require positive operations, usable early/late samples, and the configured measured duration for every ordinary duration row; update Linux raw fixtures to schema 8. Extract pure target resolution and pass 9/9 direct policy cases plus 5/5 direct watchdog cases, including deliberately delayed timer dispatch, a completing thread crossing the deadline, and a real timer forcing `Completing -> TimedOut` while that thread is blocked, and the release-runner and OS validation-only positive/negative suites, including short-duration and tampered-policy cases. +- [X] T121 Freeze the terminal R6 one-shot paths `009-final-r6-linux-x64`, `009-final-r6-pr`, `009-final-r6-nightly`, and `009-final-r6-release` on one clean commit. Conditional approval remains entirely machine-derived from the exact schema-8 target/timing evidence, schema-4 summaries, schema-3 OS reports, immutable manifests/log bindings, identical source provenance, all SC-001..SC-018 gates, and independent H0/M0 review; no tracked post-run edit or checkbox change may promote a result. + +**Final Checkpoint**: All correctness/package gates pass, required benchmark +evidence is recorded for qualified environments, review has no unresolved High +or Medium finding, and no repeated non-convergence condition remains. + +--- + +## Dependencies and Execution Order + +### Phase dependencies + +- Phase 1 has no dependencies. +- Phase 2 depends on Phase 1 and blocks every user story. +- User Story 2 depends on Phase 2 and provides committed v2 values. +- User Story 1 depends on the User Story 2 publication checkpoint for its public + independent test. +- User Story 3 depends on User Stories 1 and 2 because safe removal requires + published values and leases. +- User Story 4 depends on the minimal publication/lease/removal lifecycle from + User Stories 1-3. +- User Story 5 depends on all engine behavior whose diagnostics/contracts it + exposes. +- Phase 8 depends on all five story checkpoints. +- Phase 9 depends on Phase 8 and closes only after the convergence regressions, + clean diagnostic qualification, and independent review pass. + +### Within each phase + +1. Add and run the listed tests first; verify they fail for the intended missing + behavior, not because the test cannot compile for an unrelated reason. +2. Implement the smallest protocol slice that satisfies the tests. +3. Run the story's existing regression set before advancing. +4. Never leave a claimed descriptor/slot/lease merely to return cancellation or + `StoreBusy`. + +### Parallel opportunities + +- `[P]` contract/unit test files in a phase may be authored together. +- Layout codec work T015 can proceed beside facade/lifecycle work after their + failing tests exist. +- Story documentation/fixtures with `[P]` may proceed after the relevant public + contract is stable. +- Raw OS tracing T078 and BenchmarkDotNet T080 may proceed beside the main + multi-process harness after all stories pass. +- Tasks that touch `MemoryStore.cs`, `LockFreeStoreEngine.cs`, or shared protocol + state are deliberately sequential. + +## Requirement Coverage + +| Requirement group | Primary tasks | +|---|---| +| KV-only scope, exact keys, duplicate publication (FR-001..FR-007) | T020-T029, T071-T073 | +| Direct reservation/commit/lifetime (FR-008..FR-013) | T022-T030 | +| Shared zero-copy reads/leases (FR-014..FR-019) | T031-T040 | +| Logical remove/reclaim/reuse (FR-020..FR-024) | T042-T051 | +| Lock-free/no global owner/bounded retry (FR-025..FR-031) | T010, T020, T024-T029, T077-T084 | +| Failure scope/recovery/incarnations/no worker (FR-032..FR-036, FR-046) | T052-T061, T071-T073, T117 | +| Generation-tagged stale-helper fencing, spill-summary ABA prevention, cancellation, publication intent, exact-reference revalidation, PID-namespace recovery fencing, and v2 slot ceiling (FR-047..FR-053, LC-015..LC-016, SC-017..SC-018) | T007, T015, T020-T021, T026-T029, T043, T046, T048, T067, T082-T107 | +| Linux owner-anchor liveness, bounded cold-path convergence, and conservative orphan repair (FR-054) | T107-T108 | +| Persistent mapped-corruption latch and fail-closed propagation (FR-055) | T107-T108 | +| Physical-creator-only cold-open transaction and existing-unpublished fail-closed behavior (FR-056, LC-009) | T107-T108 | +| Linux OFD same-PID exclusion, stable lock inode, and synchronization-before-region teardown (FR-057) | T107-T108 | +| Allocation/diagnostics/disposal (FR-037..FR-041) | T030, T040, T051, T056, T062-T066, T117 | +| Public compatibility/docs/sample (FR-042..FR-045, LC-001..LC-016) | T005-T019, T062-T074, T086-T087, T090, T106 | +| Performance and success criteria SC-001..SC-018 | T039-T041, T045-T046, T055, T073, T075-T121 | + +## Implementation Strategy + +The minimal proving slice is Phase 2 plus one-key User Stories 2, 1, and 3: +reserve, commit, acquire, remove, release, reclaim, reuse. Do not broaden to full +API/diagnostics/benchmarks until this slice passes every controlled schedule and +the mapped atomic gate. Then add collision overflow, multi-process scaling, +recovery, compatibility, and release evidence incrementally. + +If a convergence gate repeats after two evidence-driven corrections, stop and +raise the exact invariant, minimal failing schedule, affected requirement, and +design choices needed from the user. Do not mark remaining tasks complete or +weaken the spec. diff --git a/src/SharedMemoryStore/Diagnostics/DiagnosticsSnapshot.cs b/src/SharedMemoryStore/Diagnostics/DiagnosticsSnapshot.cs index e3df05a..8955b39 100644 --- a/src/SharedMemoryStore/Diagnostics/DiagnosticsSnapshot.cs +++ b/src/SharedMemoryStore/Diagnostics/DiagnosticsSnapshot.cs @@ -1,3 +1,5 @@ +using SharedMemoryStore.Engines; + namespace SharedMemoryStore; /// @@ -32,7 +34,10 @@ internal DiagnosticsSnapshot( int maxObservedProbeLength, long indexCompactionCount, StoreStatus lastFailureStatus, - ReadOnlySpan failureCounts) + ReadOnlySpan failureCounts, + StoreProfile profile, + StoreProtocolInfo protocolInfo, + EngineMetrics engineMetrics) { TotalBytes = totalBytes; SlotCount = slotCount; @@ -60,6 +65,42 @@ internal DiagnosticsSnapshot( MaxObservedProbeLength = maxObservedProbeLength; IndexCompactionCount = indexCompactionCount; LastFailureStatus = lastFailureStatus; + Profile = profile; + ProtocolInfo = protocolInfo; + InitializingSlotCount = engineMetrics.InitializingSlotCount; + ReservedSlotCount = engineMetrics.ReservedSlotCount; + ReclaimingSlotCount = engineMetrics.ReclaimingSlotCount; + RetiredSlotCount = engineMetrics.RetiredSlotCount; + ClaimingLeaseCount = engineMetrics.ClaimingLeaseCount; + RecoveringLeaseCount = engineMetrics.RecoveringLeaseCount; + FreeLeaseCount = engineMetrics.FreeLeaseCount; + RetiredLeaseCount = engineMetrics.RetiredLeaseCount; + ParticipantRecordCount = engineMetrics.ParticipantRecordCount; + FreeParticipantCount = engineMetrics.FreeParticipantCount; + RegisteringParticipantCount = engineMetrics.RegisteringParticipantCount; + ActiveParticipantCount = engineMetrics.ActiveParticipantCount; + ClosingParticipantCount = engineMetrics.ClosingParticipantCount; + RecoveringParticipantCount = engineMetrics.RecoveringParticipantCount; + ReclaimingParticipantCount = engineMetrics.ReclaimingParticipantCount; + RetiredParticipantCount = engineMetrics.RetiredParticipantCount; + PrimaryDirectoryOccupancy = engineMetrics.PrimaryDirectoryOccupancy; + SpilledBucketCount = engineMetrics.SpilledBucketCount; + OverflowDirectoryOccupancy = engineMetrics.OverflowDirectoryOccupancy; + OverflowScanCount = engineMetrics.OverflowScanCount; + MaxObservedOverflowScanLength = engineMetrics.MaxObservedOverflowScanLength; + CasRetryCount = engineMetrics.CasRetryCount; + HelpedTransitionCount = engineMetrics.HelpedTransitionCount; + ContentionBudgetExhaustionCount = engineMetrics.ContentionBudgetExhaustionCount; + InvalidTokenCount = engineMetrics.InvalidTokenCount; + StaleTokenCount = engineMetrics.StaleTokenCount; + RecoveryAttemptCount = engineMetrics.RecoveryAttemptCount; + RecoveredTransitionCount = engineMetrics.RecoveredTransitionCount; + CurrentOwnerClassificationCount = engineMetrics.CurrentOwnerClassificationCount; + LiveOwnerClassificationCount = engineMetrics.LiveOwnerClassificationCount; + StaleOwnerClassificationCount = engineMetrics.StaleOwnerClassificationCount; + UnsupportedOwnerClassificationCount = engineMetrics.UnsupportedOwnerClassificationCount; + InconsistentOwnerClassificationCount = engineMetrics.InconsistentOwnerClassificationCount; + ChangingOwnerClassificationCount = engineMetrics.ChangingOwnerClassificationCount; _duplicateKeyFailures = failureCounts[(int)StoreStatus.DuplicateKey]; _notFoundFailures = failureCounts[(int)StoreStatus.NotFound]; _keyTooLargeFailures = failureCounts[(int)StoreStatus.KeyTooLarge]; @@ -107,6 +148,12 @@ internal DiagnosticsSnapshot( private readonly long _storeBusyFailures; private readonly long _operationCanceledFailures; + /// Gets the concurrency profile that produced this snapshot. + public StoreProfile Profile { get; } + + /// Gets the persisted layout and resource-protocol identity of the observed store. + public StoreProtocolInfo ProtocolInfo { get; } + /// Gets the configured mapped-region length. public long TotalBytes { get; } @@ -128,6 +175,18 @@ internal DiagnosticsSnapshot( /// Gets the number of slots currently reserved but not committed. public int ActiveReservationCount { get; } + /// Gets the number of v2 slots whose owner is initializing reservation metadata. + public int InitializingSlotCount { get; } + + /// Gets the number of v2 slots reserved for an unpublished value. + public int ReservedSlotCount { get; } + + /// Gets the number of v2 slots in abort or physical-reclamation transitions. + public int ReclaimingSlotCount { get; } + + /// Gets the number of v2 slots retired before generation reuse could wrap. + public int RetiredSlotCount { get; } + /// Gets the number of reservations aborted through this store handle. public long AbortedReservationCount { get; } @@ -155,6 +214,49 @@ internal DiagnosticsSnapshot( /// Gets the number of reservations recovery could not reclaim because shared state was inconsistent. public long FailedReservationRecoveryCount { get; } + /// Gets the number of v2 lease records whose first ownership claim is in progress. + public int ClaimingLeaseCount { get; } + + /// Gets the number of v2 lease records being released or explicitly recovered. + public int RecoveringLeaseCount { get; } + + /// Gets the number of v2 lease records currently available for a new claim. + public int FreeLeaseCount { get; } + + /// Gets the number of v2 lease records retired before incarnation reuse could wrap. + public int RetiredLeaseCount { get; } + + /// Gets the configured v2 participant-record capacity, or zero for legacy stores. + public int ParticipantRecordCount { get; } + + /// Gets the number of v2 participant records currently available to open handles. + public int FreeParticipantCount { get; } + + /// Gets the number of v2 participant records publishing a new process identity. + public int RegisteringParticipantCount { get; } + + /// Gets the number of active v2 participant records. + public int ActiveParticipantCount { get; } + + /// Gets the number of v2 participant records closing their local handle. + public int ClosingParticipantCount { get; } + + /// Gets the number of v2 participant records undergoing stale-owner recovery. + public int RecoveringParticipantCount { get; } + + /// Gets the number of unowned v2 participant records awaiting generation advance. + public int ReclaimingParticipantCount { get; } + + /// Gets the number of v2 participant records retired before token reuse could wrap. + public int RetiredParticipantCount { get; } + + /// + /// Gets whether a v2 snapshot observed no immediately free participant record. + /// The value is advisory because registrations may change concurrently. + /// + public bool IsParticipantTableExhausted => + ParticipantRecordCount > 0 && FreeParticipantCount == 0; + /// Gets the number of capacity-pressure failures observed by this store handle. public long CapacityPressureCount { get; } @@ -185,6 +287,60 @@ internal DiagnosticsSnapshot( /// Gets the number of synchronous key-index compactions completed by this handle. public long IndexCompactionCount { get; } + /// Gets the number of non-empty v2 primary-directory lanes observed. + public int PrimaryDirectoryOccupancy { get; } + + /// Gets the number of v2 primary buckets whose versioned spill summary is logically present. + public int SpilledBucketCount { get; } + + /// Gets the number of non-empty v2 overflow-directory cells observed. + public int OverflowDirectoryOccupancy { get; } + + /// Gets the number of overflow candidate scans recorded by this v2 handle. + public long OverflowScanCount { get; } + + /// Gets the largest overflow candidate scan length recorded by this v2 handle. + public int MaxObservedOverflowScanLength { get; } + + /// Gets the number of failed compare/exchange attempts recorded by this v2 handle. + public long CasRetryCount { get; } + + /// Gets the number of cooperative protocol transitions completed by this v2 handle. + public long HelpedTransitionCount { get; } + + /// Gets the number of local contention budgets exhausted by this v2 handle. + public long ContentionBudgetExhaustionCount { get; } + + /// Gets the number of structurally invalid reservation or lease tokens observed by this v2 handle. + public long InvalidTokenCount { get; } + + /// Gets the number of well-formed but no-longer-current tokens observed by this v2 handle. + public long StaleTokenCount { get; } + + /// Gets the number of explicit recovery records attempted by this v2 handle. + public long RecoveryAttemptCount { get; } + + /// Gets the number of exact recovery transitions completed by this v2 handle. + public long RecoveredTransitionCount { get; } + + /// Gets current-process owner classifications made by this v2 handle. + public long CurrentOwnerClassificationCount { get; } + + /// Gets other-live-process owner classifications made by this v2 handle. + public long LiveOwnerClassificationCount { get; } + + /// Gets safely stale owner classifications made by this v2 handle. + public long StaleOwnerClassificationCount { get; } + + /// Gets owner classifications that this v2 handle could not evaluate on the current platform. + public long UnsupportedOwnerClassificationCount { get; } + + /// Gets structurally inconsistent owner classifications made by this v2 handle. + public long InconsistentOwnerClassificationCount { get; } + + /// Gets owner classifications abandoned because the observed record changed concurrently. + public long ChangingOwnerClassificationCount { get; } + /// Gets the last non-success status observed by this store handle. public StoreStatus LastFailureStatus { get; } diff --git a/src/SharedMemoryStore/Diagnostics/StoreDiagnostics.cs b/src/SharedMemoryStore/Diagnostics/StoreDiagnostics.cs index 3de4b39..31bb607 100644 --- a/src/SharedMemoryStore/Diagnostics/StoreDiagnostics.cs +++ b/src/SharedMemoryStore/Diagnostics/StoreDiagnostics.cs @@ -1,4 +1,5 @@ using System.Threading; +using SharedMemoryStore.Engines; using SharedMemoryStore.Layout; namespace SharedMemoryStore.Diagnostics; @@ -107,7 +108,58 @@ public DiagnosticsSnapshot CreateSnapshot( indexState.MaxObservedProbeLength, indexCompactionCount, (StoreStatus)Volatile.Read(ref _lastFailureStatus), - counts); + counts, + StoreProfile.Legacy, + new StoreProtocolInfo(StoreProfile.Legacy, 1, 2, 1, 0, 0), + default); + } + + /// + /// Combines profile-neutral local counters with a bounded engine snapshot. + /// No correctness decision may depend on the resulting cross-instant view. + /// + internal DiagnosticsSnapshot CreateSnapshot( + StoreProfile profile, + StoreProtocolInfo protocolInfo, + in EngineMetrics metrics) + { + Span counts = stackalloc long[_failureCounts.Length]; + for (var i = 0; i < counts.Length; i++) + { + counts[i] = Volatile.Read(ref _failureCounts[i]); + } + + return new DiagnosticsSnapshot( + metrics.TotalBytes, + metrics.SlotCount, + metrics.FreeSlotCount, + metrics.PublishedSlotCount, + metrics.PendingRemovalCount, + metrics.ActiveLeaseCount, + checked(metrics.InitializingSlotCount + metrics.ReservedSlotCount), + Volatile.Read(ref _abortedReservationCount), + Volatile.Read(ref _recoveredLeaseCount), + Volatile.Read(ref _activeLeaseRecoveryCount), + Volatile.Read(ref _unsupportedLeaseRecoveryCount), + Volatile.Read(ref _failedLeaseRecoveryCount), + Volatile.Read(ref _recoveredReservationCount), + Volatile.Read(ref _activeReservationRecoveryCount), + Volatile.Read(ref _unsupportedReservationRecoveryCount), + Volatile.Read(ref _failedReservationRecoveryCount), + Volatile.Read(ref _capacityPressureCount), + metrics.IndexEntryCount, + metrics.OccupiedIndexEntryCount, + metrics.TombstoneIndexEntryCount, + metrics.EmptyIndexEntryCount, + metrics.UsableIndexCapacity, + metrics.LastObservedProbeLength, + metrics.MaxObservedProbeLength, + metrics.IndexCompactionCount, + (StoreStatus)Volatile.Read(ref _lastFailureStatus), + counts, + profile, + protocolInfo, + metrics); } private static void AddPositive(ref long field, int value) diff --git a/src/SharedMemoryStore/Engines/EngineMetrics.cs b/src/SharedMemoryStore/Engines/EngineMetrics.cs new file mode 100644 index 0000000..7774d31 --- /dev/null +++ b/src/SharedMemoryStore/Engines/EngineMetrics.cs @@ -0,0 +1,108 @@ +namespace SharedMemoryStore.Engines; + +/// +/// Profile-neutral instantaneous engine state consumed by facade diagnostics. +/// +/// +/// Metrics are observational only. No correctness decision may depend on this +/// potentially cross-instant snapshot. Profile-specific fields that do not +/// apply to the legacy engine remain zero. +/// +internal readonly record struct EngineMetrics +{ + internal long TotalBytes { get; init; } + + internal int SlotCount { get; init; } + + internal int FreeSlotCount { get; init; } + + internal int InitializingSlotCount { get; init; } + + internal int ReservedSlotCount { get; init; } + + internal int PublishedSlotCount { get; init; } + + internal int PendingRemovalCount { get; init; } + + internal int ReclaimingSlotCount { get; init; } + + internal int RetiredSlotCount { get; init; } + + internal int ActiveLeaseCount { get; init; } + + internal int ClaimingLeaseCount { get; init; } + + internal int RecoveringLeaseCount { get; init; } + + internal int FreeLeaseCount { get; init; } + + internal int RetiredLeaseCount { get; init; } + + internal int ParticipantRecordCount { get; init; } + + internal int FreeParticipantCount { get; init; } + + internal int RegisteringParticipantCount { get; init; } + + internal int ActiveParticipantCount { get; init; } + + internal int ClosingParticipantCount { get; init; } + + internal int RecoveringParticipantCount { get; init; } + + internal int ReclaimingParticipantCount { get; init; } + + internal int RetiredParticipantCount { get; init; } + + internal int IndexEntryCount { get; init; } + + internal int OccupiedIndexEntryCount { get; init; } + + internal int TombstoneIndexEntryCount { get; init; } + + internal int EmptyIndexEntryCount { get; init; } + + internal int UsableIndexCapacity { get; init; } + + internal int LastObservedProbeLength { get; init; } + + internal int MaxObservedProbeLength { get; init; } + + internal long IndexCompactionCount { get; init; } + + internal int PrimaryDirectoryOccupancy { get; init; } + + internal int SpilledBucketCount { get; init; } + + internal int OverflowDirectoryOccupancy { get; init; } + + internal long OverflowScanCount { get; init; } + + internal int MaxObservedOverflowScanLength { get; init; } + + internal long CasRetryCount { get; init; } + + internal long HelpedTransitionCount { get; init; } + + internal long ContentionBudgetExhaustionCount { get; init; } + + internal long InvalidTokenCount { get; init; } + + internal long StaleTokenCount { get; init; } + + internal long RecoveryAttemptCount { get; init; } + + internal long RecoveredTransitionCount { get; init; } + + internal long CurrentOwnerClassificationCount { get; init; } + + internal long LiveOwnerClassificationCount { get; init; } + + internal long StaleOwnerClassificationCount { get; init; } + + internal long UnsupportedOwnerClassificationCount { get; init; } + + internal long InconsistentOwnerClassificationCount { get; init; } + + internal long ChangingOwnerClassificationCount { get; init; } +} diff --git a/src/SharedMemoryStore/Engines/IStoreEngine.cs b/src/SharedMemoryStore/Engines/IStoreEngine.cs new file mode 100644 index 0000000..4847850 --- /dev/null +++ b/src/SharedMemoryStore/Engines/IStoreEngine.cs @@ -0,0 +1,109 @@ +using System.Buffers; + +namespace SharedMemoryStore.Engines; + +/// +/// Synchronous profile-neutral implementation boundary behind +/// . +/// +/// +/// Span-returning members validate their opaque handle synchronously and never +/// retain a caller span. Returned mapped-memory views remain governed by the +/// corresponding reservation or lease lifetime. Implementations must not use +/// asynchronous continuations or hidden background workers. +/// +internal interface IStoreEngine : IDisposable +{ + StoreProfile Profile { get; } + + StoreProtocolInfo ProtocolInfo { get; } + + /// + /// Records a non-success status produced by the facade before engine entry. + /// Implementations must update managed-local diagnostics only and must not + /// touch mapped memory, synchronization objects, or other disposable state. + /// + StoreStatus RecordFacadeStatus(StoreStatus status); + + /// + /// Creates the best available diagnostic snapshot after facade disposal has + /// closed operation entry. Implementations must use only immutable layout + /// metadata and managed-local counters; this member must never project or + /// scan disposed mapped memory or acquire disposable synchronization. + /// + DiagnosticsSnapshot CreateDisposedDiagnosticsSnapshot(); + + StoreStatus TryPublish( + ReadOnlySpan key, + ReadOnlySpan value, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions); + + StoreStatus TryReserve( + ReadOnlySpan key, + int payloadLength, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions, + out ReservationHandle reservation); + + StoreStatus TryPublishSegments( + ReadOnlySpan key, + ReadOnlySequence payload, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions, + out long copiedBytes); + + StoreStatus TryAcquire( + ReadOnlySpan key, + StoreWaitOptions waitOptions, + out LeaseHandle lease); + + StoreStatus TryRemove(ReadOnlySpan key, StoreWaitOptions waitOptions); + + StoreStatus TryRecoverLeases( + LeaseRecoveryOptions options, + StoreWaitOptions waitOptions, + out LeaseRecoveryReport report); + + StoreStatus TryRecoverReservations( + ReservationRecoveryOptions options, + StoreWaitOptions waitOptions, + out ReservationRecoveryReport report); + + StoreStatus TryGetMetrics(StoreWaitOptions waitOptions, out EngineMetrics metrics); + + StoreStatus TryGetDiagnostics(StoreWaitOptions waitOptions, out DiagnosticsSnapshot snapshot); + + bool IsReservationPending(ReservationHandle reservation); + + int GetReservationBytesWritten(ReservationHandle reservation); + + Span GetReservationSpan(ReservationHandle reservation, int sizeHint); + + Memory DangerousGetReservationMemory(ReservationHandle reservation, int sizeHint); + + StoreStatus AdvanceReservation( + ReservationHandle reservation, + int byteCount, + StoreWaitOptions waitOptions); + + StoreStatus CommitReservation( + ReservationHandle reservation, + StoreWaitOptions waitOptions); + + StoreStatus AbortReservation( + ReservationHandle reservation, + StoreWaitOptions waitOptions); + + bool IsLeaseActive(LeaseHandle lease); + + int GetValueLength(LeaseHandle lease); + + int GetDescriptorLength(LeaseHandle lease); + + ReadOnlySpan GetValueSpan(LeaseHandle lease); + + ReadOnlySpan GetDescriptorSpan(LeaseHandle lease); + + StoreStatus ReleaseLease(LeaseHandle lease, StoreWaitOptions waitOptions); +} diff --git a/src/SharedMemoryStore/Engines/LegacyV12/LegacyV12StoreEngine.cs b/src/SharedMemoryStore/Engines/LegacyV12/LegacyV12StoreEngine.cs new file mode 100644 index 0000000..74d6722 --- /dev/null +++ b/src/SharedMemoryStore/Engines/LegacyV12/LegacyV12StoreEngine.cs @@ -0,0 +1,103 @@ +using System.Buffers; + +namespace SharedMemoryStore.Engines.LegacyV12; + +/// +/// Layout-v1.2 engine adapter. The contained core preserves the frozen v1.2 +/// implementation while the public stays profile-neutral. +/// +internal sealed class LegacyV12StoreEngine : IStoreEngine +{ + internal LegacyV12StoreEngine(MemoryStore core) + { + Core = core; + } + + internal MemoryStore Core { get; } + + public StoreProfile Profile => StoreProfile.Legacy; + + public StoreProtocolInfo ProtocolInfo => new(StoreProfile.Legacy, 1, 2, 1, 0, 0); + + public StoreStatus RecordFacadeStatus(StoreStatus status) => + Core.RecordFacadeStatus(status); + + public DiagnosticsSnapshot CreateDisposedDiagnosticsSnapshot() => + Core.CreateDisposedSnapshot(); + + public StoreStatus TryPublish(ReadOnlySpan key, ReadOnlySpan value, ReadOnlySpan descriptor, StoreWaitOptions waitOptions) => + Core.TryPublish(key, value, descriptor, waitOptions); + + public StoreStatus TryReserve(ReadOnlySpan key, int payloadLength, ReadOnlySpan descriptor, StoreWaitOptions waitOptions, out ReservationHandle reservation) + { + var status = Core.TryReserve(key, payloadLength, descriptor, waitOptions, out var publicReservation); + reservation = status == StoreStatus.Success ? publicReservation.HandleForEngine : default; + return status; + } + + public StoreStatus TryPublishSegments(ReadOnlySpan key, ReadOnlySequence payload, ReadOnlySpan descriptor, StoreWaitOptions waitOptions, out long copiedBytes) => + Core.TryPublishSegments(key, payload, descriptor, waitOptions, out copiedBytes); + + public StoreStatus TryAcquire(ReadOnlySpan key, StoreWaitOptions waitOptions, out LeaseHandle lease) + { + var status = Core.TryAcquire(key, waitOptions, out var publicLease); + lease = status == StoreStatus.Success ? publicLease.HandleForEngine : default; + return status; + } + + public StoreStatus TryRemove(ReadOnlySpan key, StoreWaitOptions waitOptions) => Core.TryRemove(key, waitOptions); + + public StoreStatus TryRecoverLeases(LeaseRecoveryOptions options, StoreWaitOptions waitOptions, out LeaseRecoveryReport report) => + Core.TryRecoverLeases(options, waitOptions, out report); + + public StoreStatus TryRecoverReservations(ReservationRecoveryOptions options, StoreWaitOptions waitOptions, out ReservationRecoveryReport report) => + Core.TryRecoverReservations(options, waitOptions, out report); + + public StoreStatus TryGetMetrics(StoreWaitOptions waitOptions, out EngineMetrics metrics) + { + var status = Core.TryGetDiagnostics(waitOptions, out var snapshot); + metrics = status == StoreStatus.Success + ? new EngineMetrics + { + TotalBytes = snapshot.TotalBytes, + SlotCount = snapshot.SlotCount, + FreeSlotCount = snapshot.FreeSlotCount, + ReservedSlotCount = snapshot.ActiveReservationCount, + PublishedSlotCount = snapshot.PublishedSlotCount, + PendingRemovalCount = snapshot.PendingRemovalCount, + ActiveLeaseCount = snapshot.ActiveLeaseCount, + IndexEntryCount = snapshot.IndexEntryCount, + OccupiedIndexEntryCount = snapshot.OccupiedIndexEntryCount, + TombstoneIndexEntryCount = snapshot.TombstoneIndexEntryCount, + EmptyIndexEntryCount = snapshot.EmptyIndexEntryCount, + UsableIndexCapacity = snapshot.UsableIndexCapacity, + LastObservedProbeLength = snapshot.LastObservedProbeLength, + MaxObservedProbeLength = snapshot.MaxObservedProbeLength, + IndexCompactionCount = snapshot.IndexCompactionCount + } + : default; + return status; + } + + public StoreStatus TryGetDiagnostics(StoreWaitOptions waitOptions, out DiagnosticsSnapshot snapshot) => + Core.TryGetDiagnostics(waitOptions, out snapshot); + + public bool IsReservationPending(ReservationHandle reservation) => Core.IsReservationPending(reservation); + public int GetReservationBytesWritten(ReservationHandle reservation) => Core.GetReservationBytesWritten(reservation); + public Span GetReservationSpan(ReservationHandle reservation, int sizeHint) => Core.GetReservationSpan(reservation, sizeHint); + public Memory DangerousGetReservationMemory(ReservationHandle reservation, int sizeHint) => Core.GetReservationMemory(reservation, sizeHint); + public StoreStatus AdvanceReservation(ReservationHandle reservation, int byteCount, StoreWaitOptions waitOptions) => Core.AdvanceReservation(reservation, byteCount, waitOptions); + public StoreStatus CommitReservation(ReservationHandle reservation, StoreWaitOptions waitOptions) => Core.CommitReservation(reservation, waitOptions); + public StoreStatus AbortReservation(ReservationHandle reservation, StoreWaitOptions waitOptions) => Core.AbortReservation(reservation, countAbort: true, waitOptions); + public bool IsLeaseActive(LeaseHandle lease) => Core.IsLeaseActive(lease); + public int GetValueLength(LeaseHandle lease) => + Core.IsLeaseActive(lease) ? Core.GetValueLength(lease) : 0; + public int GetDescriptorLength(LeaseHandle lease) => + Core.IsLeaseActive(lease) ? Core.GetDescriptorLength(lease) : 0; + public ReadOnlySpan GetValueSpan(LeaseHandle lease) => + Core.IsLeaseActive(lease) ? Core.GetValueSpan(lease) : ReadOnlySpan.Empty; + public ReadOnlySpan GetDescriptorSpan(LeaseHandle lease) => + Core.IsLeaseActive(lease) ? Core.GetDescriptorSpan(lease) : ReadOnlySpan.Empty; + public StoreStatus ReleaseLease(LeaseHandle lease, StoreWaitOptions waitOptions) => Core.ReleaseLease(lease, waitOptions); + public void Dispose() => Core.Dispose(); +} diff --git a/src/SharedMemoryStore/Engines/StoreEngineFactory.cs b/src/SharedMemoryStore/Engines/StoreEngineFactory.cs new file mode 100644 index 0000000..28e9e35 --- /dev/null +++ b/src/SharedMemoryStore/Engines/StoreEngineFactory.cs @@ -0,0 +1,49 @@ +using SharedMemoryStore.Engines.LegacyV12; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.Engines; + +internal static class StoreEngineFactory +{ + internal static MemoryStore WrapLegacy(MemoryStore legacyCore) => + WrapOwnedEngine(new LegacyV12StoreEngine(legacyCore)); + + /// + /// Transfers one fully constructed engine into the public facade. If facade + /// initialization throws (including an engine property getter), ownership + /// remains here and the engine is disposed exactly once before rethrowing. + /// + internal static MemoryStore WrapOwnedEngine(IStoreEngine engine) + { + ArgumentNullException.ThrowIfNull(engine); + try + { + return new MemoryStore(engine); + } + catch + { + engine.Dispose(); + throw; + } + } + + internal static StoreOpenStatus TryCreateLockFreeUnderColdGate( + SharedMemoryStoreOptions options, + StoreWaitOptions waitOptions, + long waitStartTimestamp, + MemoryMappedStoreRegion region, + ISharedStoreSynchronization coldSynchronization, + RegionOpenDisposition disposition, + out IStoreEngine? engine) + { + return LockFreeStoreEngine.TryCreateOrOpenUnderColdGate( + options, + waitOptions, + waitStartTimestamp, + region, + coldSynchronization, + disposition, + out engine); + } +} diff --git a/src/SharedMemoryStore/Engines/StoreTokenHandles.cs b/src/SharedMemoryStore/Engines/StoreTokenHandles.cs new file mode 100644 index 0000000..f7d2dd6 --- /dev/null +++ b/src/SharedMemoryStore/Engines/StoreTokenHandles.cs @@ -0,0 +1,73 @@ +namespace SharedMemoryStore.Engines; + +/// +/// Engine-neutral identity for one pending reservation lifecycle. +/// +/// +/// The three unsigned words deliberately remain opaque to the public token. +/// Together they fence the mapping incarnation, participant incarnation, and +/// value-slot incarnation without exposing a layout-specific record type. +/// +internal readonly record struct ReservationHandle +{ + internal ReservationHandle( + ulong storeId, + ulong participantToken, + ulong slotBinding, + int payloadLength) + { + StoreId = storeId; + ParticipantToken = participantToken; + SlotBinding = slotBinding; + PayloadLength = payloadLength; + } + + internal ulong StoreId { get; } + + internal ulong ParticipantToken { get; } + + internal ulong SlotBinding { get; } + + internal int PayloadLength { get; } + + internal bool IsDefault => StoreId == 0 + && ParticipantToken == 0 + && SlotBinding == 0 + && PayloadLength == 0; +} + +/// +/// Engine-neutral identity for one active read-lease lifecycle. +/// +/// +/// The handle fences the mapping, participant, value slot, and lease record. +/// Layout-specific index/generation splits are interpreted only by the engine +/// that created it. +/// +internal readonly record struct LeaseHandle +{ + internal LeaseHandle( + ulong storeId, + ulong participantToken, + ulong slotBinding, + ulong leaseToken) + { + StoreId = storeId; + ParticipantToken = participantToken; + SlotBinding = slotBinding; + LeaseToken = leaseToken; + } + + internal ulong StoreId { get; } + + internal ulong ParticipantToken { get; } + + internal ulong SlotBinding { get; } + + internal ulong LeaseToken { get; } + + internal bool IsDefault => StoreId == 0 + && ParticipantToken == 0 + && SlotBinding == 0 + && LeaseToken == 0; +} diff --git a/src/SharedMemoryStore/Ingest/ReservationRecovery.cs b/src/SharedMemoryStore/Ingest/ReservationRecovery.cs index b4b8689..94d06b2 100644 --- a/src/SharedMemoryStore/Ingest/ReservationRecovery.cs +++ b/src/SharedMemoryStore/Ingest/ReservationRecovery.cs @@ -8,7 +8,12 @@ namespace SharedMemoryStore /// /// Options for explicit stale reservation recovery. /// - /// When true, current-process pending reservations may be recovered for tests and controlled shutdown. + /// + /// When true, current-process Reserved reservations may be recovered for tests + /// and controlled shutdown after the application has quiesced every writer. + /// Initializing reservations remain protected until their participant enters + /// the explicit Closing or Recovering handoff. + /// public readonly record struct ReservationRecoveryOptions(bool RecoverCurrentProcessReservations); /// diff --git a/src/SharedMemoryStore/Ingest/ValueReservation.cs b/src/SharedMemoryStore/Ingest/ValueReservation.cs index 3ac90b2..079fe49 100644 --- a/src/SharedMemoryStore/Ingest/ValueReservation.cs +++ b/src/SharedMemoryStore/Ingest/ValueReservation.cs @@ -1,115 +1,81 @@ +using SharedMemoryStore.Engines; using SharedMemoryStore.Layout; +using System.Runtime.CompilerServices; namespace SharedMemoryStore; /// -/// Lifecycle token for one pending store-owned payload reservation. +/// Exclusive single-producer lifecycle token for one pending store-owned payload reservation. +/// The struct may be copied for ordinary value passing, but concurrent lifecycle or writable-view +/// calls through copied tokens are unsupported. Safe writable views end at the next lifecycle +/// action and are invalid after commit, abort, recovery, token disposal, or store disposal. /// public struct ValueReservation : IDisposable { private readonly MemoryStore? _store; - private readonly int _slotIndex; - private readonly SlotLifecycleId _lifecycleId; - private readonly int _payloadLength; + private readonly ReservationHandle _handle; - internal ValueReservation(MemoryStore store, int slotIndex, SlotLifecycleId lifecycleId, int payloadLength) + internal ValueReservation(MemoryStore store, in ReservationHandle handle) { _store = store; - _slotIndex = slotIndex; - _lifecycleId = lifecycleId; - _payloadLength = payloadLength; + _handle = handle; + } + + internal ValueReservation(MemoryStore store, int slotIndex, SlotLifecycleId lifecycleId, int payloadLength) + : this(store, store.CreateLegacyReservationHandle(slotIndex, lifecycleId, payloadLength)) + { } /// Gets a value indicating whether this token still references a pending reservation. - public readonly bool IsValid => _store?.IsReservationPending(_slotIndex, _lifecycleId) == true; + public readonly bool IsValid => _store?.IsReservationPending(_handle) == true; /// Gets the announced payload length, in bytes. - public readonly int PayloadLength => IsValid ? _payloadLength : 0; + public readonly int PayloadLength => IsValid ? _handle.PayloadLength : 0; /// Gets the number of payload bytes advanced by the producer. - public readonly int BytesWritten => _store?.GetReservationBytesWritten(_slotIndex, _lifecycleId) ?? 0; + public readonly int BytesWritten => _store?.GetReservationBytesWritten(_handle) ?? 0; /// Gets the number of payload bytes that remain before the reservation can commit. public readonly int RemainingBytes => Math.Max(0, PayloadLength - BytesWritten); /// - /// Gets an immediate writable span over remaining store-owned payload bytes while the reservation is pending. + /// Gets an immediate writable span over remaining store-owned payload bytes while pending. + /// The span is borrowed until the next reservation lifecycle action. /// - /// Minimum useful remaining size requested by the caller, or zero for any remaining bytes. - public readonly Span GetSpan(int sizeHint = 0) - { - return _store is null - ? Span.Empty - : _store.GetReservationSpan(_slotIndex, _lifecycleId, sizeHint); - } + public readonly Span GetSpan(int sizeHint = 0) => + _store is null ? Span.Empty : _store.GetReservationSpan(_handle, sizeHint); /// - /// Gets advanced writable memory over remaining store-owned payload bytes while the reservation is pending. + /// Gets retained-capable writable memory whose logical lifetime is still bounded by this + /// reservation; accessing it after that lifetime is explicitly unsafe. /// - /// - /// This method is intended for trusted direct I/O adapters that require , - /// such as socket or stream reads into the store. The returned memory is retained-capable and must - /// not be used after commit, abort, recovery, disposal, store disposal, or slot reuse. - /// - /// Minimum useful remaining size requested by the caller, or zero for any remaining bytes. - public readonly Memory DangerousGetMemory(int sizeHint = 0) - { - return _store is null - ? Memory.Empty - : _store.GetReservationMemory(_slotIndex, _lifecycleId, sizeHint); - } + public readonly Memory DangerousGetMemory(int sizeHint = 0) => + _store is null ? Memory.Empty : _store.GetReservationMemory(_handle, sizeHint); - /// - /// Advances the exact number of payload bytes written into the current writable view. - /// - public readonly StoreStatus Advance(int byteCount) - { - return _store?.AdvanceReservation(_slotIndex, _lifecycleId, byteCount) ?? StoreStatus.InvalidReservation; - } + /// Advances the exact number of payload bytes written into the current writable view. + public readonly StoreStatus Advance(int byteCount) => Advance(byteCount, StoreWaitOptions.Default); - /// - /// Advances the exact number of payload bytes written using the supplied wait policy. - /// - public readonly StoreStatus Advance(int byteCount, StoreWaitOptions waitOptions) - { - return _store?.AdvanceReservation(_slotIndex, _lifecycleId, byteCount, waitOptions) ?? StoreStatus.InvalidReservation; - } + /// Advances written bytes using the supplied profile-specific bounded wait policy. + public readonly StoreStatus Advance(int byteCount, StoreWaitOptions waitOptions) => + _store?.AdvanceReservation(_handle, byteCount, waitOptions) ?? StoreStatus.InvalidReservation; - /// - /// Commits the reservation atomically after exactly the announced payload length has been advanced. - /// - public readonly StoreStatus Commit() - { - return _store?.CommitReservation(_slotIndex, _lifecycleId) ?? StoreStatus.InvalidReservation; - } + /// Commits the reservation after exactly the announced payload length has been advanced. + public readonly StoreStatus Commit() => Commit(StoreWaitOptions.Default); - /// - /// Commits the reservation using the supplied wait policy. - /// - public readonly StoreStatus Commit(StoreWaitOptions waitOptions) - { - return _store?.CommitReservation(_slotIndex, _lifecycleId, waitOptions) ?? StoreStatus.InvalidReservation; - } + /// Commits the reservation using the supplied profile-specific bounded wait policy. + public readonly StoreStatus Commit(StoreWaitOptions waitOptions) => + _store?.CommitReservation(_handle, waitOptions) ?? StoreStatus.InvalidReservation; - /// - /// Aborts the pending reservation, removes its pending key, and returns the slot to reusable storage. - /// - public readonly StoreStatus Abort() - { - return _store?.AbortReservation(_slotIndex, _lifecycleId, countAbort: true) ?? StoreStatus.InvalidReservation; - } + /// Aborts the pending reservation and makes its storage reusable. + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public readonly StoreStatus Abort() => Abort(StoreWaitOptions.Default); - /// - /// Aborts the reservation using the supplied wait policy. - /// - public readonly StoreStatus Abort(StoreWaitOptions waitOptions) - { - return _store?.AbortReservation(_slotIndex, _lifecycleId, countAbort: true, waitOptions) ?? StoreStatus.InvalidReservation; - } + /// Aborts the reservation using the supplied profile-specific bounded wait policy. + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public readonly StoreStatus Abort(StoreWaitOptions waitOptions) => + _store?.AbortReservation(_handle, countAbort: true, waitOptions) ?? StoreStatus.InvalidReservation; - /// - /// Aborts the reservation on a best-effort basis when it is still pending; completed reservations are left unchanged. - /// + /// Best-effort abort of a still-current reservation. public readonly void Dispose() { if (IsValid) @@ -118,7 +84,9 @@ public readonly void Dispose() } } - internal readonly int SlotIndexForTesting => _slotIndex; + internal readonly int SlotIndexForTesting => MemoryStore.DecodeLegacySlotIndex(_handle.SlotBinding); + + internal readonly SlotLifecycleId LifecycleIdForTesting => MemoryStore.DecodeLegacyLifecycle(_handle); - internal readonly SlotLifecycleId LifecycleIdForTesting => _lifecycleId; + internal readonly ReservationHandle HandleForEngine => _handle; } diff --git a/src/SharedMemoryStore/Interop/LinuxFileLock.cs b/src/SharedMemoryStore/Interop/LinuxFileLock.cs index e3920b2..b9f7ca1 100644 --- a/src/SharedMemoryStore/Interop/LinuxFileLock.cs +++ b/src/SharedMemoryStore/Interop/LinuxFileLock.cs @@ -1,26 +1,36 @@ -using System.Collections.Concurrent; using System.Diagnostics; +using System.Runtime.InteropServices; using System.Runtime.Versioning; +using Microsoft.Win32.SafeHandles; namespace SharedMemoryStore.Interop; [SupportedOSPlatform("linux")] internal sealed class LinuxFileLock : IDisposable { - private static readonly ConcurrentDictionary LocalLocks = new(StringComparer.Ordinal); + private const int OpenFileDescriptionSetLock = 37; + private const short WriteLock = 1; + private const short Unlock = 2; + private const short SeekSet = 0; + + private const int Interrupted = 4; + private const int TryAgain = 11; + private const int PermissionDenied = 13; + private const int InvalidArgument = 22; + private const int FunctionNotImplemented = 38; + private const int OperationNotSupported = 95; private readonly FileStream _stream; - private readonly string _localLockPath; - private readonly LocalLockEntry _localLockEntry; + private readonly SemaphoreSlim _localGate = new(1, 1); private bool _locked; private bool _localLockHeld; + private bool _streamClosed; + private bool _unusable; private bool _disposed; - private LinuxFileLock(string path, FileStream stream) + private LinuxFileLock(string path) { - _stream = stream; - _localLockPath = Path.GetFullPath(path); - _localLockEntry = AcquireLocalLockEntry(_localLockPath); + _stream = OpenLockFile(Path.GetFullPath(path)); } public static StoreStatus TryAcquire( @@ -34,10 +44,10 @@ public static StoreStatus TryAcquire( return StoreStatus.OperationCanceled; } - FileStream stream; + LinuxFileLock candidate; try { - stream = OpenLockFile(path); + candidate = new LinuxFileLock(path); } catch (UnauthorizedAccessException) { @@ -52,8 +62,7 @@ public static StoreStatus TryAcquire( return StoreStatus.UnknownFailure; } - var candidate = new LinuxFileLock(path, stream); - var status = candidate.TryAcquire(waitOptions); + StoreStatus status = candidate.TryAcquire(waitOptions); if (status != StoreStatus.Success) { candidate.Dispose(); @@ -66,12 +75,12 @@ public static StoreStatus TryAcquire( public StoreStatus TryAcquire(StoreWaitOptions waitOptions) { - if (_disposed) + if (_disposed || _unusable) { return StoreStatus.StoreDisposed; } - var startTimestamp = Stopwatch.GetTimestamp(); + long startTimestamp = Stopwatch.GetTimestamp(); if (!TryAcquireLocal(waitOptions, startTimestamp)) { return waitOptions.CancellationToken.IsCancellationRequested @@ -87,50 +96,52 @@ public StoreStatus TryAcquire(StoreWaitOptions waitOptions) return StoreStatus.OperationCanceled; } - try + var request = LinuxFlock.Create(WriteLock); + if (Fcntl(_stream.SafeFileHandle, OpenFileDescriptionSetLock, ref request) == 0) { - _stream.Lock(0, 1); _locked = true; return StoreStatus.Success; } - catch (IOException) + + int error = Marshal.GetLastPInvokeError(); + if (error == Interrupted) { - if (!waitOptions.IsInfinite && Stopwatch.GetElapsedTime(startTimestamp) >= waitOptions.Timeout) + if (!waitOptions.IsInfinite + && Stopwatch.GetElapsedTime(startTimestamp) >= waitOptions.Timeout) { Release(); return StoreStatus.StoreBusy; } - if (WaitBeforeRetry(waitOptions, startTimestamp)) - { - continue; - } - - var status = waitOptions.CancellationToken.IsCancellationRequested - ? StoreStatus.OperationCanceled - : StoreStatus.StoreBusy; - Release(); - return status; + continue; } - catch (UnauthorizedAccessException) + + if (error is InvalidArgument or FunctionNotImplemented or OperationNotSupported) { Release(); - return StoreStatus.AccessDenied; + return StoreStatus.UnsupportedPlatform; } - catch (PlatformNotSupportedException) + + if (error is not (PermissionDenied or TryAgain)) { Release(); - return StoreStatus.UnsupportedPlatform; + return StoreStatus.UnknownFailure; } - catch (ObjectDisposedException) + + if (!waitOptions.IsInfinite + && Stopwatch.GetElapsedTime(startTimestamp) >= waitOptions.Timeout) { Release(); - return StoreStatus.StoreDisposed; + return StoreStatus.StoreBusy; } - catch + + if (!WaitBeforeRetry(waitOptions, startTimestamp)) { + StoreStatus status = waitOptions.CancellationToken.IsCancellationRequested + ? StoreStatus.OperationCanceled + : StoreStatus.StoreBusy; Release(); - return StoreStatus.UnknownFailure; + return status; } } } @@ -140,9 +151,7 @@ public static StoreStatus TryOpen(string path, out LinuxFileLock? fileLock) fileLock = null; try { - var stream = OpenLockFile(path); - - fileLock = new LinuxFileLock(path, stream); + fileLock = new LinuxFileLock(path); return StoreStatus.Success; } catch (UnauthorizedAccessException) @@ -163,22 +172,43 @@ public void Release() { if (_locked) { - try - { - _stream.Unlock(0, 1); - } - catch + var request = LinuxFlock.Create(Unlock); + int result; + do { - // The stream is being torn down; callers receive operation status before this point. + result = Fcntl( + _stream.SafeFileHandle, + OpenFileDescriptionSetLock, + ref request); } + while (result != 0 && Marshal.GetLastPInvokeError() == Interrupted); + + bool unlocked = result == 0; _locked = false; + if (!unlocked) + { + // Releasing the process-local gate while an OFD lock might + // still be present would let local callers run while foreign + // callers remain excluded. Retire this descriptor first; + // close is the kernel-guaranteed OFD-lock release boundary. + _unusable = true; + try + { + CloseStream(); + } + catch + { + // This wrapper remains unusable even if descriptor close + // could not be confirmed; no local work can pass it. + } + } } if (_localLockHeld) { - Monitor.Exit(_localLockEntry.SyncRoot); _localLockHeld = false; + _localGate.Release(); } } @@ -191,21 +221,25 @@ public void Dispose() _disposed = true; Release(); - _stream.Dispose(); - ReleaseLocalLockEntry(_localLockPath, _localLockEntry); + CloseStream(); } private bool TryAcquireLocal(StoreWaitOptions waitOptions, long startTimestamp) { while (true) { - if (Monitor.TryEnter(_localLockEntry.SyncRoot)) + // OFD locks contend across separately opened descriptors, loaded + // assemblies, and native modules in one PID. One lock wrapper can + // still be shared by several local callers, so keep it explicitly + // non-reentrant before entering the kernel. + if (_localGate.Wait(0)) { _localLockHeld = true; return true; } - if (!waitOptions.IsInfinite && Stopwatch.GetElapsedTime(startTimestamp) >= waitOptions.Timeout) + if (!waitOptions.IsInfinite + && Stopwatch.GetElapsedTime(startTimestamp) >= waitOptions.Timeout) { return false; } @@ -222,7 +256,7 @@ private static bool WaitBeforeRetry(StoreWaitOptions waitOptions, long startTime var sleep = TimeSpan.FromMilliseconds(10); if (!waitOptions.IsInfinite) { - var remaining = waitOptions.Timeout - Stopwatch.GetElapsedTime(startTimestamp); + TimeSpan remaining = waitOptions.Timeout - Stopwatch.GetElapsedTime(startTimestamp); if (remaining <= TimeSpan.Zero) { return false; @@ -244,56 +278,62 @@ private static FileStream OpenLockFile(string path) Share = FileShare.ReadWrite | FileShare.Delete, UnixCreateMode = LinuxSharedMemoryDirectory.PrivateFileMode }); - File.SetUnixFileMode(path, LinuxSharedMemoryDirectory.PrivateFileMode); - return stream; - } - - private static LocalLockEntry AcquireLocalLockEntry(string path) - { - while (true) + try { - var entry = LocalLocks.GetOrAdd(path, static _ => new LocalLockEntry()); - lock (entry.ReferenceGate) - { - if (entry.Retired) - { - continue; - } - - entry.ReferenceCount++; - return entry; - } + File.SetUnixFileMode(path, LinuxSharedMemoryDirectory.PrivateFileMode); + return stream; + } + catch + { + stream.Dispose(); + throw; } } - private static void ReleaseLocalLockEntry(string path, LocalLockEntry entry) + private void CloseStream() { - var remove = false; - lock (entry.ReferenceGate) + if (_streamClosed) { - entry.ReferenceCount--; - if (entry.ReferenceCount == 0) - { - entry.Retired = true; - remove = true; - } + return; } - if (remove) - { - _ = ((ICollection>)LocalLocks).Remove( - new KeyValuePair(path, entry)); - } + _streamClosed = true; + _stream.Dispose(); } - private sealed class LocalLockEntry + [DllImport("libc", EntryPoint = "fcntl", SetLastError = true)] + private static extern int Fcntl( + SafeFileHandle fileDescriptor, + int command, + ref LinuxFlock request); + + // SharedMemoryStore requires a 64-bit process. Linux x64 and arm64 both + // use this LP64 struct flock ABI; OFD commands require l_pid to be zero. + [StructLayout(LayoutKind.Explicit, Size = 32)] + private struct LinuxFlock { - public object SyncRoot { get; } = new(); + [FieldOffset(0)] + internal short Type; - public object ReferenceGate { get; } = new(); + [FieldOffset(2)] + internal short Whence; - public int ReferenceCount { get; set; } + [FieldOffset(8)] + internal long Start; - public bool Retired { get; set; } + [FieldOffset(16)] + internal long Length; + + [FieldOffset(24)] + internal int ProcessId; + + internal static LinuxFlock Create(short type) => new() + { + Type = type, + Whence = SeekSet, + Start = 0, + Length = 1, + ProcessId = 0 + }; } } diff --git a/src/SharedMemoryStore/Interop/LinuxOwnerAnchor.cs b/src/SharedMemoryStore/Interop/LinuxOwnerAnchor.cs new file mode 100644 index 0000000..e0a700f --- /dev/null +++ b/src/SharedMemoryStore/Interop/LinuxOwnerAnchor.cs @@ -0,0 +1,398 @@ +using System.Collections.Concurrent; +using System.ComponentModel; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using Microsoft.Win32.SafeHandles; + +namespace SharedMemoryStore.Interop; + +internal enum LinuxOwnerAnchorState +{ + Missing, + Locked, + Unlocked, + Ambiguous +} + +internal readonly record struct LinuxOwnerAnchorArtifact(Guid OwnerToken, string Path); + +/// +/// Holds one private Linux owner-liveness anchor. The anchor deliberately uses +/// flock, not the resource protocol's POSIX record lock: it is private to +/// current managed owners, is safe to release from another thread, and a probe +/// through a separately opened file descriptor cannot acquire it reentrantly. +/// +[SupportedOSPlatform("linux")] +internal sealed class LinuxOwnerAnchor : IDisposable +{ + private const string AnchorSegment = ".anchor."; + private const int LockExclusive = 2; + private const int LockNonBlocking = 4; + private const int LockUnlock = 8; + private const int OpenReadWrite = 2; + private const int OpenCreate = 0x40; + private const int OpenExclusive = 0x80; + private const int OpenNonBlocking = 0x800; + private const int OpenNoFollow = 0x20000; + private const int OpenCloseOnExec = 0x80000; + private const int DuplicateDescriptorCloseOnExec = 1030; + private const int AtEmptyPath = 0x1000; + private const uint StatxType = 0x0001; + private const ushort FileTypeMask = 0xF000; + private const ushort RegularFileType = 0x8000; + private const uint OwnerReadWriteMode = 0x180; // 0600 + private const int NoEntryError = 2; + private const int WouldBlockError = 11; + + private static readonly ConcurrentDictionary LocalAnchors = + new(StringComparer.Ordinal); + + private readonly SafeFileHandle _handle; + private readonly string _path; + private int _disposed; + + private LinuxOwnerAnchor(string path, SafeFileHandle handle) + { + _path = path; + _handle = handle; + } + + internal string AnchorPath => _path; + + internal static LinuxOwnerAnchor Create(string ownersPath, Guid ownerToken) + { + string path = GetPath(ownersPath, ownerToken); + LinuxSharedMemoryDirectory.EnsureExists(Path.GetDirectoryName(path) ?? "."); + SafeFileHandle? handle = null; + var created = false; + try + { + int descriptor = Open( + path, + OpenReadWrite | OpenCreate | OpenExclusive | OpenNoFollow | OpenCloseOnExec, + OwnerReadWriteMode); + if (descriptor < 0) + { + throw new IOException( + "Unable to create the Linux owner anchor.", + new Win32Exception(Marshal.GetLastPInvokeError())); + } + + created = true; + handle = OwnDescriptor(descriptor); + File.SetUnixFileMode(path, LinuxSharedMemoryDirectory.PrivateFileMode); + if (Flock(handle, LockExclusive | LockNonBlocking) != 0) + { + throw new IOException( + "Unable to acquire the Linux owner anchor.", + new Win32Exception(Marshal.GetLastPInvokeError())); + } + + var anchor = new LinuxOwnerAnchor(path, handle); + handle = null; + if (!LocalAnchors.TryAdd(path, anchor)) + { + anchor.Dispose(); + throw new IOException("A local Linux owner anchor already uses the generated token."); + } + + return anchor; + } + catch + { + handle?.Dispose(); + if (created) + { + TryDeleteArtifact(path); + } + + throw; + } + } + + internal static LinuxOwnerAnchorState Probe(string ownersPath, Guid ownerToken) => + Probe(ownersPath, ownerToken, honorLocalRegistry: true); + + internal static LinuxOwnerAnchorState Probe( + string ownersPath, + Guid ownerToken, + bool honorLocalRegistry) + { + string path = GetPath(ownersPath, ownerToken); + if (honorLocalRegistry + && LocalAnchors.TryGetValue(path, out LinuxOwnerAnchor? local) + && Volatile.Read(ref local._disposed) == 0) + { + return LinuxOwnerAnchorState.Locked; + } + + return ProbePath(path, deleteWhenUnlocked: false); + } + + private static LinuxOwnerAnchorState ProbePath(string path, bool deleteWhenUnlocked) + { + try + { + int descriptor = Open( + path, + OpenReadWrite | OpenNonBlocking | OpenNoFollow | OpenCloseOnExec, + OwnerReadWriteMode); + if (descriptor < 0) + { + return Marshal.GetLastPInvokeError() == NoEntryError + ? LinuxOwnerAnchorState.Missing + : LinuxOwnerAnchorState.Ambiguous; + } + + using SafeFileHandle handle = OwnDescriptor(descriptor); + if (!IsRegularFile(handle)) + { + return LinuxOwnerAnchorState.Ambiguous; + } + + if (Flock(handle, LockExclusive | LockNonBlocking) == 0) + { + try + { + if (deleteWhenUnlocked) + { + // Keep the separately opened description locked until the + // pathname is removed. Cooperative creators cannot replace a + // same-store anchor while the caller holds .lifecycle. + TryDeleteArtifact(path); + } + } + finally + { + _ = Flock(handle, LockUnlock); + } + + return LinuxOwnerAnchorState.Unlocked; + } + + return Marshal.GetLastPInvokeError() == WouldBlockError + ? LinuxOwnerAnchorState.Locked + : LinuxOwnerAnchorState.Ambiguous; + } + catch (FileNotFoundException) + { + return LinuxOwnerAnchorState.Missing; + } + catch (DirectoryNotFoundException) + { + return LinuxOwnerAnchorState.Missing; + } + catch + { + return LinuxOwnerAnchorState.Ambiguous; + } + } + + private static bool IsRegularFile(SafeFileHandle handle) + { + try + { + int descriptor = handle.DangerousGetHandle().ToInt32(); + int result = Statx( + descriptor, + string.Empty, + AtEmptyPath, + StatxType, + out LinuxStatx metadata); + GC.KeepAlive(handle); + return result == 0 + && (metadata.Mask & StatxType) != 0 + && (metadata.Mode & FileTypeMask) == RegularFileType; + } + catch + { + // A missing libc entry point, unsupported kernel operation, invalid + // descriptor, or marshaling uncertainty must retain the artifact. + return false; + } + } + + internal static string GetPath(string ownersPath, Guid ownerToken) => + ownersPath + AnchorSegment + ownerToken.ToString("N"); + + /// + /// Removes only well-formed, unreferenced anchor artifacts whose lock can + /// be acquired through a separate open description. Malformed names and + /// locked or ambiguous artifacts are deliberately retained. + /// + internal static void SweepUnreferencedArtifacts( + string ownersPath, + IReadOnlySet referencedOwnerTokens) + { + foreach (LinuxOwnerAnchorArtifact artifact in EnumerateWellFormedArtifacts(ownersPath)) + { + if (referencedOwnerTokens.Contains(artifact.OwnerToken)) + { + continue; + } + + // This deliberately bypasses LocalAnchors. The independently + // opened descriptor is the authoritative cross-process probe. + _ = ProbePath(artifact.Path, deleteWhenUnlocked: true); + } + } + + private static SafeFileHandle OwnDescriptor(int descriptor) + { + // SafeFileHandle treats zero as invalid. Native open may legitimately + // return fd 0 when standard input is closed, so duplicate that one onto + // a close-on-exec descriptor which SafeFileHandle can own safely. + if (descriptor == 0) + { + int duplicate = Fcntl( + descriptor, + DuplicateDescriptorCloseOnExec, + minimumDescriptor: 1); + int error = Marshal.GetLastPInvokeError(); + _ = Close(descriptor); + if (duplicate < 0) + { + throw new IOException( + "Unable to own the Linux anchor descriptor.", + new Win32Exception(error)); + } + + descriptor = duplicate; + } + + return new SafeFileHandle((IntPtr)descriptor, ownsHandle: true); + } + + internal static LinuxOwnerAnchorArtifact[] EnumerateWellFormedArtifacts(string ownersPath) + { + string directory = Path.GetDirectoryName(ownersPath) ?? "."; + if (!Directory.Exists(directory)) + { + return []; + } + + string prefix = Path.GetFileName(ownersPath) + AnchorSegment; + try + { + var artifacts = new List(); + foreach (string path in Directory.GetFileSystemEntries( + directory, + prefix + "*", + SearchOption.TopDirectoryOnly)) + { + string name = Path.GetFileName(path); + if (!name.StartsWith(prefix, StringComparison.Ordinal) + || name.Length != prefix.Length + 32) + { + continue; + } + + string tokenText = name[prefix.Length..]; + if (Guid.TryParseExact(tokenText, "N", out Guid token) + && string.Equals(tokenText, token.ToString("N"), StringComparison.Ordinal)) + { + artifacts.Add(new LinuxOwnerAnchorArtifact(token, path)); + } + } + + return artifacts.ToArray(); + } + catch + { + // Enumeration is advisory cleanup. Failure must retain artifacts, + // never turn uncertainty into deletion. + return []; + } + } + + internal static void ReleaseLocalAfterOwnerAbsent(string ownersPath, Guid ownerToken) + { + string path = GetPath(ownersPath, ownerToken); + if (LocalAnchors.TryGetValue(path, out LinuxOwnerAnchor? anchor)) + { + anchor.Dispose(); + return; + } + + _ = ProbePath(path, deleteWhenUnlocked: true); + } + + internal static void TryDeleteArtifact(string path) + { + try + { + File.Delete(path); + } + catch (FileNotFoundException) + { + } + catch (DirectoryNotFoundException) + { + } + catch + { + // Anchor artifacts are advisory only after their exact owner line is + // committed absent. A later lifecycle cleanup can retry deletion. + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _ = ((ICollection>)LocalAnchors).Remove( + new KeyValuePair(_path, this)); + try + { + _ = Flock(_handle, LockUnlock); + } + catch + { + } + + _handle.Dispose(); + TryDeleteArtifact(_path); + } + + [DllImport("libc", EntryPoint = "open", SetLastError = true)] + private static extern int Open( + [MarshalAs(UnmanagedType.LPUTF8Str)] string path, + int flags, + uint mode); + + [DllImport("libc", EntryPoint = "fcntl", SetLastError = true)] + private static extern int Fcntl( + int fileDescriptor, + int command, + int minimumDescriptor); + + [DllImport("libc", EntryPoint = "close", SetLastError = true)] + private static extern int Close(int fileDescriptor); + + [DllImport("libc", EntryPoint = "statx", SetLastError = true)] + private static extern int Statx( + int directoryFileDescriptor, + [MarshalAs(UnmanagedType.LPUTF8Str)] string path, + int flags, + uint mask, + out LinuxStatx metadata); + + [DllImport("libc", EntryPoint = "flock", SetLastError = true)] + private static extern int Flock(SafeFileHandle fileDescriptor, int operation); + + // Linux statx is an architecture-neutral, versioned 256-byte ABI. Only the + // returned-mask and file-type fields are projected; explicit offsets avoid + // depending on the architecture-specific layout of struct stat. + [StructLayout(LayoutKind.Explicit, Size = 256)] + private struct LinuxStatx + { + [FieldOffset(0)] + internal uint Mask; + + [FieldOffset(28)] + internal ushort Mode; + } +} diff --git a/src/SharedMemoryStore/Interop/LinuxSharedMemoryRegion.cs b/src/SharedMemoryStore/Interop/LinuxSharedMemoryRegion.cs index 9964b55..ff4441b 100644 --- a/src/SharedMemoryStore/Interop/LinuxSharedMemoryRegion.cs +++ b/src/SharedMemoryStore/Interop/LinuxSharedMemoryRegion.cs @@ -1,12 +1,18 @@ using System.Diagnostics; using System.IO.MemoryMappedFiles; using System.Runtime.Versioning; +using System.Text; namespace SharedMemoryStore.Interop; [SupportedOSPlatform("linux")] internal static class LinuxSharedMemoryRegion { + private const string ReleaseMarkerSegment = ".released."; + private const string FinalizedReleaseMarkerSuffix = ".ready"; + private const int MaximumReleaseMarkerBytes = 1024; + private static readonly StoreWaitOptions OwnerReleaseWaitOptions = new(TimeSpan.FromMilliseconds(250)); + public static StoreOpenStatus TryOpen( PlatformResourceName resourceName, SharedMemoryStoreOptions options, @@ -28,24 +34,17 @@ public static StoreOpenStatus TryOpen( { try { - LinuxSharedMemoryDirectory.EnsureExists(Path.GetDirectoryName(resourceName.LinuxRegionPath) ?? "."); - var liveOwners = ReadLiveOwnerRecords(resourceName.LinuxOwnersPath); - var hasLiveResource = File.Exists(resourceName.LinuxRegionPath) && liveOwners.Count > 0; - if (!hasLiveResource) - { - DeleteStaleResources(resourceName); - } - - return options.OpenMode switch - { - OpenMode.CreateNew when hasLiveResource => StoreOpenStatus.AlreadyExists, - OpenMode.OpenExisting when !hasLiveResource => StoreOpenStatus.NotFound, - OpenMode.CreateNew => CreateRegion(resourceName, options, out region), - OpenMode.OpenExisting => OpenExistingRegion(resourceName, options, out region), - _ => hasLiveResource - ? OpenExistingRegion(resourceName, options, out region) - : CreateRegion(resourceName, options, out region) - }; + PrepareOpen( + resourceName, + out List committedOwners, + out bool hasLiveResource); + return OpenPreparedRegion( + resourceName, + options, + committedOwners, + hasLiveResource, + out region, + out _); } catch (UnauthorizedAccessException) { @@ -65,6 +64,7 @@ public static StoreOpenStatus TryOpen( private static StoreOpenStatus CreateRegion( PlatformResourceName resourceName, SharedMemoryStoreOptions options, + IReadOnlyList committedOwners, out MemoryMappedStoreRegion? region) { region = null; @@ -78,7 +78,6 @@ private static StoreOpenStatus CreateRegion( Share = FileShare.ReadWrite | FileShare.Delete, UnixCreateMode = LinuxSharedMemoryDirectory.PrivateFileMode }); - File.SetUnixFileMode(resourceName.LinuxRegionPath, LinuxSharedMemoryDirectory.PrivateFileMode); } catch (IOException) when (File.Exists(resourceName.LinuxRegionPath)) { @@ -87,8 +86,16 @@ private static StoreOpenStatus CreateRegion( try { + File.SetUnixFileMode( + resourceName.LinuxRegionPath, + LinuxSharedMemoryDirectory.PrivateFileMode); stream.SetLength(options.TotalBytes); - return CreateMappedRegion(resourceName, options.TotalBytes, stream, out region); + return CreateMappedRegion( + resourceName, + options.TotalBytes, + stream, + committedOwners, + out region); } catch { @@ -101,6 +108,7 @@ private static StoreOpenStatus CreateRegion( private static StoreOpenStatus OpenExistingRegion( PlatformResourceName resourceName, SharedMemoryStoreOptions options, + IReadOnlyList committedOwners, out MemoryMappedStoreRegion? region) { region = null; @@ -114,28 +122,42 @@ private static StoreOpenStatus OpenExistingRegion( FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite | FileShare.Delete); - File.SetUnixFileMode(resourceName.LinuxRegionPath, LinuxSharedMemoryDirectory.PrivateFileMode); - - if (stream.Length < options.TotalBytes) + try + { + File.SetUnixFileMode( + resourceName.LinuxRegionPath, + LinuxSharedMemoryDirectory.PrivateFileMode); + } + catch { stream.Dispose(); - return StoreOpenStatus.IncompatibleLayout; + throw; } - return CreateMappedRegion(resourceName, stream.Length, stream, out region); + // Always map the existing file at its actual capacity. Header validation decides + // whether the requested dimensions/profile are compatible; the requested size must + // never prevent probing a readable existing mapping. + return CreateMappedRegion( + resourceName, + stream.Length, + stream, + committedOwners, + out region); } private static StoreOpenStatus CreateMappedRegion( PlatformResourceName resourceName, long mappingCapacity, FileStream stream, + IReadOnlyList committedOwners, out MemoryMappedStoreRegion? region) { region = null; - var ownerRecord = CreateOwnerRecord(); + var ownerRecord = CreateOwnerRecord(out Guid ownerToken); MemoryMappedFile? mapping = null; MemoryMappedViewAccessor? accessor = null; MemoryMappedStoreRegion? candidate = null; + LinuxOwnerAnchor? ownerAnchor = null; var ownerRegistered = false; try { @@ -148,28 +170,49 @@ private static StoreOpenStatus CreateMappedRegion( leaveOpen: false); accessor = mapping.CreateViewAccessor(0, mappingCapacity, MemoryMappedFileAccess.ReadWrite); + ownerAnchor = LinuxOwnerAnchor.Create(resourceName.LinuxOwnersPath, ownerToken); + LinuxOwnerAnchor registeredAnchor = ownerAnchor; candidate = MemoryMappedStoreRegion.Create( mapping, accessor, - mappingCapacity, () => { - if (ownerRegistered) + OwnerReleaseOutcome releaseOutcome = ownerRegistered + ? ReleaseOwner(resourceName, ownerRecord) + : OwnerReleaseOutcome.OwnerAbsent; + if (releaseOutcome != OwnerReleaseOutcome.Failed) { - ReleaseOwner(resourceName, ownerRecord); + // The exact owner line is either absent or covered by a + // durable finalized release marker. Both are authoritative + // after the mapped view has already been unmapped. + registeredAnchor.Dispose(); } }); mapping = null; accessor = null; - RegisterOwner(resourceName.LinuxOwnersPath, ownerRecord); + CommitOwnerRegistration(resourceName, committedOwners, ownerRecord); + // Publish the callback state at the sidecar replacement commit point. + // The pre-registration sweep already covered every unreferenced + // artifact visible at lifecycle entry. A crash after anchor creation + // but before this commit is repaired by the next cold lifecycle. ownerRegistered = true; region = candidate; candidate = null; + ownerAnchor = null; return StoreOpenStatus.Success; } catch { - candidate?.Dispose(); + if (candidate is not null) + { + candidate.Dispose(); + candidate = null; + // The callback owns the anchor outcome, including deliberate + // retention after a bounded release-marker fallback. + ownerAnchor = null; + } + + ownerAnchor?.Dispose(); accessor?.Dispose(); mapping?.Dispose(); stream.Dispose(); @@ -177,55 +220,355 @@ private static StoreOpenStatus CreateMappedRegion( } } - private static string CreateOwnerRecord() + /// + /// Begins a Linux cold-open transaction. The retained lifecycle lock is + /// acquired before the ordinary operation lock, so stale-resource cleanup + /// cannot race the stable operation-lock rendezvous while the scope holds it. + /// Both locks remain held across mapped initialization/validation. + /// + internal static StoreOpenStatus TryBeginColdOpen( + PlatformResourceName resourceName, + SharedMemoryStoreOptions options, + StoreWaitOptions waitOptions, + long waitStartTimestamp, + out SharedStoreOpenScope? scope) { + scope = null; + LinuxFileLock? lifecycleLock = null; + LinuxSharedStoreSynchronization? synchronization = null; + MemoryMappedStoreRegion? region = null; + bool synchronizationEntered = false; + + try + { + StoreStatus remainingStatus = SharedStorePlatform.TryGetRemainingWaitOptions( + waitOptions, + waitStartTimestamp, + out StoreWaitOptions remainingWait); + if (remainingStatus != StoreStatus.Success) + { + return ToOpenStatus(remainingStatus); + } + + StoreStatus lifecycleLockStatus = LinuxFileLock.TryAcquire( + resourceName.LinuxLifecycleLockPath, + remainingWait, + out lifecycleLock); + if (lifecycleLockStatus != StoreStatus.Success || lifecycleLock is null) + { + return ToOpenStatus(lifecycleLockStatus); + } + + PrepareOpen( + resourceName, + out List committedOwners, + out bool hasLiveResource); + + remainingStatus = SharedStorePlatform.TryGetRemainingWaitOptions( + waitOptions, + waitStartTimestamp, + out remainingWait); + if (remainingStatus != StoreStatus.Success) + { + return ToOpenStatus(remainingStatus); + } + + if (options.OpenMode == OpenMode.CreateNew && hasLiveResource) + { + return StoreOpenStatus.AlreadyExists; + } + + if (options.OpenMode == OpenMode.OpenExisting && !hasLiveResource) + { + return StoreOpenStatus.NotFound; + } + + // Open the ordinary lock only after stale data-resource deletion + // has completed under .lifecycle. Current cleanup retains this + // stable rendezvous inode; ordering also protects older clients. + synchronization = new LinuxSharedStoreSynchronization(resourceName); + StoreStatus enterStatus = synchronization.TryEnter(remainingWait); + if (enterStatus != StoreStatus.Success) + { + return ToOpenStatus(enterStatus); + } + + synchronizationEntered = true; + + // The ordinary lock wait is part of the same end-to-end open + // budget. Do not map the region or publish an owner marker after + // that wait consumed the deadline (or cancellation was observed). + remainingStatus = SharedStorePlatform.TryGetRemainingWaitOptions( + waitOptions, + waitStartTimestamp, + out _); + if (remainingStatus != StoreStatus.Success) + { + return ToOpenStatus(remainingStatus); + } + + StoreOpenStatus openStatus = OpenPreparedRegion( + resourceName, + options, + committedOwners, + hasLiveResource, + out region, + out RegionOpenDisposition disposition); + if (openStatus != StoreOpenStatus.Success || region is null) + { + return openStatus; + } + + scope = new SharedStoreOpenScope( + region, + synchronization, + lifecycleLock, + disposition); + region = null; + synchronization = null; + lifecycleLock = null; + synchronizationEntered = false; + return StoreOpenStatus.Success; + } + catch (UnauthorizedAccessException) + { + return StoreOpenStatus.AccessDenied; + } + catch (PlatformNotSupportedException) + { + return StoreOpenStatus.UnsupportedPlatform; + } + catch + { + return StoreOpenStatus.MappingFailed; + } + finally + { + try + { + if (synchronizationEntered) + { + synchronization?.Exit(); + } + } + finally + { + try + { + lifecycleLock?.Dispose(); + } + finally + { + try + { + // Close the ordinary descriptor before region cleanup + // can reacquire .lifecycle and publish final-owner state. + synchronization?.Dispose(); + } + finally + { + // The region callback may acquire .lifecycle, so mapping + // cleanup follows lifecycle release on every failure. + region?.Dispose(); + } + } + } + } + } + + private static void PrepareOpen( + PlatformResourceName resourceName, + out List committedOwners, + out bool hasLiveResource) + { + LinuxSharedMemoryDirectory.EnsureExists(Path.GetDirectoryName(resourceName.LinuxRegionPath) ?? "."); + ReconcileReleaseMarkers(resourceName); + OwnerSnapshot ownerSnapshot = ReadOwnerSnapshot(resourceName); + committedOwners = ownerSnapshot.CommittedOwners; + // A live witness makes the existing sidecar an already-committed + // conservative owner set. It need not be rewritten or fully + // reclassified merely to attach another handle. When no owner is + // live, commit the filtered empty set before stale-anchor cleanup. + if (!ownerSnapshot.HasLiveOwner) + { + WriteOwners(resourceName.LinuxOwnersPath, committedOwners); + } + + SweepUnreferencedOwnerAnchors(resourceName.LinuxOwnersPath, committedOwners); + hasLiveResource = File.Exists(resourceName.LinuxRegionPath) + && ownerSnapshot.HasLiveOwner; + if (!hasLiveResource) + { + DeleteStaleResources(resourceName); + committedOwners = []; + } + } + + private static StoreOpenStatus OpenPreparedRegion( + PlatformResourceName resourceName, + SharedMemoryStoreOptions options, + IReadOnlyList committedOwners, + bool hasLiveResource, + out MemoryMappedStoreRegion? region, + out RegionOpenDisposition disposition) + { + region = null; + disposition = default; + if (options.OpenMode == OpenMode.CreateNew && hasLiveResource) + { + return StoreOpenStatus.AlreadyExists; + } + + if (options.OpenMode == OpenMode.OpenExisting && !hasLiveResource) + { + return StoreOpenStatus.NotFound; + } + + bool createNew = options.OpenMode == OpenMode.CreateNew + || (options.OpenMode == OpenMode.CreateOrOpen && !hasLiveResource); + StoreOpenStatus status = createNew + ? CreateRegion(resourceName, options, committedOwners, out region) + : OpenExistingRegion(resourceName, options, committedOwners, out region); + if (status == StoreOpenStatus.Success) + { + disposition = createNew + ? RegionOpenDisposition.CreatedNew + : RegionOpenDisposition.OpenedExisting; + } + + return status; + } + + private static string CreateOwnerRecord(out Guid ownerToken) + { + ownerToken = Guid.NewGuid(); return string.Join( ':', Environment.ProcessId.ToString(System.Globalization.CultureInfo.InvariantCulture), GetProcessStartToken(Environment.ProcessId), - Guid.NewGuid().ToString("N")); + ownerToken.ToString("N")); } - private static void RegisterOwner(string ownersPath, string ownerRecord) + private static void CommitOwnerRegistration( + PlatformResourceName resourceName, + IReadOnlyList committedOwners, + string ownerRecord) { - var owners = ReadLiveOwnerRecords(ownersPath); - owners.Add(ownerRecord); - WriteOwners(ownersPath, owners); + var nextOwners = new List(committedOwners.Count + 1); + nextOwners.AddRange(committedOwners); + nextOwners.Add(ownerRecord); + WriteOwners(resourceName.LinuxOwnersPath, nextOwners); } - private static void ReleaseOwner(PlatformResourceName resourceName, string ownerRecord) + private static OwnerReleaseOutcome ReleaseOwner( + PlatformResourceName resourceName, + string ownerRecord) { + var ownerCommittedAbsent = false; try { var lifecycleLockStatus = LinuxFileLock.TryAcquire( resourceName.LinuxLifecycleLockPath, - StoreWaitOptions.Infinite, + OwnerReleaseWaitOptions, out var lifecycleLock); if (lifecycleLockStatus != StoreStatus.Success || lifecycleLock is null) { - return; + return TryPublishReleaseMarker(resourceName, ownerRecord) + ? OwnerReleaseOutcome.FinalizedMarkerPublished + : OwnerReleaseOutcome.Failed; } using (lifecycleLock) { - var owners = ReadLiveOwnerRecords(resourceName.LinuxOwnersPath); + ReconcileReleaseMarkers(resourceName); + OwnerScan ownerScan = ReadLiveOwnerRecords(resourceName); + var owners = ownerScan.LiveOwners; owners.RemoveAll(owner => string.Equals(owner, ownerRecord, StringComparison.Ordinal)); + // The sidecar replacement is the commit point. Marker deletion and stale + // resource cleanup must happen only after this exact owner is absent there. + WriteOwners(resourceName.LinuxOwnersPath, owners); + ownerCommittedAbsent = true; + SweepUnreferencedOwnerAnchors(resourceName.LinuxOwnersPath, owners); if (owners.Count == 0) { DeleteStaleResources(resourceName); - return; } - - WriteOwners(resourceName.LinuxOwnersPath, owners); } } catch { - // Cleanup is best effort; later opens re-check owner liveness before reusing stale files. + // The finalized marker below makes this exact release replayable by a later C# opener. + } + + if (ownerCommittedAbsent) + { + return OwnerReleaseOutcome.OwnerAbsent; + } + + return TryPublishReleaseMarker(resourceName, ownerRecord) + ? OwnerReleaseOutcome.FinalizedMarkerPublished + : OwnerReleaseOutcome.Failed; + } + + private static OwnerScan ReadLiveOwnerRecords(PlatformResourceName resourceName) + { + var owners = new List(); + foreach (var trimmed in ReadOwnerRecords(resourceName.LinuxOwnersPath)) + { + if (IsOwnerRecordLive(resourceName.LinuxOwnersPath, trimmed)) + { + owners.Add(trimmed); + } + } + + return new OwnerScan(owners); + } + + private static OwnerSnapshot ReadOwnerSnapshot(PlatformResourceName resourceName) + { + List committedOwners = ReadOwnerRecords(resourceName.LinuxOwnersPath); + foreach (string owner in committedOwners) + { + if (IsOwnerRecordLive(resourceName.LinuxOwnersPath, owner)) + { + // One authoritative live witness is sufficient to prove that the + // mapping remains owned. Preserve the complete committed sidecar + // so attach cost does not grow with every prior process. Full + // stale-record pruning remains a release/no-live responsibility. + return new OwnerSnapshot(committedOwners, HasLiveOwner: true); + } + } + + return new OwnerSnapshot([], HasLiveOwner: false); + } + + private static bool IsOwnerRecordLive(string ownersPath, string ownerRecord) + { + if (!TryReadOwnerIdentity(ownerRecord, out int processId, out string? startToken)) + { + return false; } + + if (TryReadOwnerToken(ownerRecord, out Guid ownerToken)) + { + LinuxOwnerAnchorState anchorState = LinuxOwnerAnchor.Probe(ownersPath, ownerToken); + if (anchorState is LinuxOwnerAnchorState.Locked or LinuxOwnerAnchorState.Ambiguous) + { + return true; + } + + if (anchorState == LinuxOwnerAnchorState.Unlocked) + { + return false; + } + } + + // Missing anchors are expected for C++/Python and older managed + // owners. Preserve the resource-v1 PID/start-token classification. + return IsProcessLive(processId, startToken); } - private static List ReadLiveOwnerRecords(string ownersPath) + private static List ReadOwnerRecords(string ownersPath) { if (!File.Exists(ownersPath)) { @@ -236,19 +579,187 @@ private static List ReadLiveOwnerRecords(string ownersPath) foreach (var line in File.ReadAllLines(ownersPath)) { var trimmed = line.Trim(); - if (trimmed.Length == 0) + if (trimmed.Length != 0) + { + owners.Add(trimmed); + } + } + + return owners; + } + + private static void ReconcileReleaseMarkers(PlatformResourceName resourceName) + { + var finalizedMarkers = EnumerateReleaseMarkerArtifacts(resourceName, finalizedOnly: true); + if (finalizedMarkers.Length == 0) + { + return; + } + + var owners = ReadOwnerRecords(resourceName.LinuxOwnersPath); + var reconciledMarkers = new List(finalizedMarkers.Length); + var releasedTokens = new List(finalizedMarkers.Length); + foreach (var markerPath in finalizedMarkers) + { + if (!TryReadReleaseMarker(resourceName, markerPath, out var releasedOwner)) { - continue; + // A finalized marker is a protocol record, not disposable debris. Retain + // malformed state and fail the cold operation closed rather than risking + // deletion of a still-owned mapping. + throw new InvalidDataException($"Invalid owner-release marker '{Path.GetFileName(markerPath)}'."); } - if (TryReadOwnerIdentity(trimmed, out var processId, out var startToken) - && IsProcessLive(processId, startToken)) + owners.RemoveAll(owner => string.Equals(owner, releasedOwner, StringComparison.Ordinal)); + reconciledMarkers.Add(markerPath); + if (TryReadOwnerToken(releasedOwner, out Guid releasedToken)) { - owners.Add(trimmed); + releasedTokens.Add(releasedToken); } } - return owners; + if (reconciledMarkers.Count == 0) + { + return; + } + + // Rewrite even when every exact line was already absent. This makes a crash after + // the prior rewrite but before marker deletion idempotently replayable. + WriteOwners(resourceName.LinuxOwnersPath, owners); + foreach (Guid releasedToken in releasedTokens) + { + // A same-process bounded close may have retained its flock until a + // later lifecycle action committed this exact line absent. + LinuxOwnerAnchor.ReleaseLocalAfterOwnerAbsent( + resourceName.LinuxOwnersPath, + releasedToken); + } + + foreach (var markerPath in reconciledMarkers) + { + DeleteIfExists(markerPath); + } + } + + private static bool TryReadReleaseMarker( + PlatformResourceName resourceName, + string markerPath, + out string ownerRecord) + { + ownerRecord = string.Empty; + var markerInfo = new FileInfo(markerPath); + if (!markerInfo.Exists + || markerInfo.Length <= 0 + || markerInfo.Length > MaximumReleaseMarkerBytes + || markerInfo.LinkTarget is not null + || (markerInfo.Attributes & FileAttributes.ReparsePoint) != 0) + { + return false; + } + + var markerName = Path.GetFileName(markerPath); + var prefix = Path.GetFileName(resourceName.LinuxOwnersPath) + ReleaseMarkerSegment; + if (!markerName.StartsWith(prefix, StringComparison.Ordinal) + || !markerName.EndsWith(FinalizedReleaseMarkerSuffix, StringComparison.Ordinal)) + { + return false; + } + + var uniqueToken = markerName[ + prefix.Length..^FinalizedReleaseMarkerSuffix.Length]; + if (!Guid.TryParseExact(uniqueToken, "N", out _)) + { + return false; + } + + var candidate = File.ReadAllText(markerPath, Encoding.UTF8).Trim(); + if (candidate.Length == 0 || candidate.IndexOfAny('\r', '\n') >= 0) + { + return false; + } + + var parts = candidate.Split(':', 3); + if (parts.Length != 3 + || !Guid.TryParseExact(parts[2], "N", out _) + || !string.Equals(parts[2], uniqueToken, StringComparison.OrdinalIgnoreCase) + || !TryReadOwnerIdentity(candidate, out var processId, out _) + || processId <= 0) + { + return false; + } + + ownerRecord = candidate; + return true; + } + + private static bool TryPublishReleaseMarker( + PlatformResourceName resourceName, + string ownerRecord) + { + string? temporaryPath = null; + var published = false; + try + { + var ownerParts = ownerRecord.Split(':', 3); + if (ownerParts.Length != 3 || !Guid.TryParseExact(ownerParts[2], "N", out var uniqueToken)) + { + return false; + } + + var directory = Path.GetDirectoryName(resourceName.LinuxOwnersPath) ?? "."; + LinuxSharedMemoryDirectory.EnsureExists(directory); + var finalPath = resourceName.LinuxOwnersPath + + ReleaseMarkerSegment + + uniqueToken.ToString("N") + + FinalizedReleaseMarkerSuffix; + temporaryPath = finalPath + ".tmp." + Guid.NewGuid().ToString("N"); + using (var stream = new FileStream(temporaryPath, new FileStreamOptions + { + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + UnixCreateMode = LinuxSharedMemoryDirectory.PrivateFileMode + })) + { + using (var writer = new StreamWriter( + stream, + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + bufferSize: 256, + leaveOpen: true)) + { + writer.WriteLine(ownerRecord); + writer.Flush(); + } + + stream.Flush(flushToDisk: true); + } + + File.SetUnixFileMode(temporaryPath, LinuxSharedMemoryDirectory.PrivateFileMode); + File.Move(temporaryPath, finalPath, overwrite: true); + temporaryPath = null; + published = true; + File.SetUnixFileMode(finalPath, LinuxSharedMemoryDirectory.PrivateFileMode); + } + catch + { + // Unmapping/Dispose must complete even if the private resource directory is damaged. + // The live owner record remains conservative when marker publication is impossible. + } + finally + { + if (temporaryPath is not null) + { + try + { + DeleteIfExists(temporaryPath); + } + catch + { + // Stale-resource cleanup removes release-marker temporary artifacts. + } + } + } + + return published; } private static void WriteOwners(string ownersPath, List owners) @@ -310,6 +821,14 @@ private static bool TryReadOwnerIdentity(string ownerRecord, out int processId, return true; } + private static bool TryReadOwnerToken(string ownerRecord, out Guid ownerToken) + { + ownerToken = default; + var parts = ownerRecord.Split(':', 3); + return parts.Length == 3 + && Guid.TryParseExact(parts[2], "N", out ownerToken); + } + private static bool IsProcessLive(int processId, string? startToken) { if (processId <= 0) @@ -382,10 +901,81 @@ private static string GetProcessStartToken(int processId) private static void DeleteStaleResources(PlatformResourceName resourceName) { + // Callers commit an owner-sidecar rewrite before entering this method. A marker + // arriving after their reconciliation is safe to remove only when its owner was + // also absent from the just-classified live set. DeleteIfExists(resourceName.LinuxRegionPath); - DeleteIfExists(resourceName.LinuxSynchronizationPath); + // Keep the ordinary lock inode as a permanent rendezvous, just like + // .lifecycle. It carries no store generation state, costs one empty + // mode-0600 file, and cannot split active and reopening participants + // across unlinked/replacement inodes. DeleteIfExists(resourceName.LinuxOwnersPath); DeleteIfExists(resourceName.LinuxOwnersPath + ".tmp"); + foreach (var markerPath in EnumerateReleaseMarkerArtifacts(resourceName, finalizedOnly: false)) + { + DeleteIfExists(markerPath); + } + } + + private static void SweepUnreferencedOwnerAnchors( + string ownersPath, + IEnumerable committedOwners) + { + // Production callers hold .lifecycle and invoke this only with the + // unchanged committed sidecar or after a replacement sidecar commit. + // It repairs a crash between anchor creation/locking and publication of + // that anchor's owner line without adding work to a key-value path. + var referencedOwnerTokens = new HashSet(); + foreach (string owner in committedOwners) + { + if (TryReadExactOwnerToken(owner, out Guid ownerToken)) + { + referencedOwnerTokens.Add(ownerToken); + } + } + + LinuxOwnerAnchor.SweepUnreferencedArtifacts(ownersPath, referencedOwnerTokens); + } + + private static bool TryReadExactOwnerToken(string ownerRecord, out Guid ownerToken) + { + ownerToken = default; + string[] parts = ownerRecord.Split(':', 3); + if (parts.Length != 3 + || !int.TryParse( + parts[0], + System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, + out int processId) + || processId <= 0 + || !string.Equals( + parts[0], + processId.ToString(System.Globalization.CultureInfo.InvariantCulture), + StringComparison.Ordinal) + || !Guid.TryParseExact(parts[2], "N", out ownerToken) + || !string.Equals(parts[2], ownerToken.ToString("N"), StringComparison.Ordinal)) + { + ownerToken = default; + return false; + } + + return true; + } + + private static string[] EnumerateReleaseMarkerArtifacts( + PlatformResourceName resourceName, + bool finalizedOnly) + { + var directory = Path.GetDirectoryName(resourceName.LinuxOwnersPath) ?? "."; + if (!Directory.Exists(directory)) + { + return []; + } + + var pattern = Path.GetFileName(resourceName.LinuxOwnersPath) + + ReleaseMarkerSegment + + (finalizedOnly ? "*" + FinalizedReleaseMarkerSuffix : "*"); + return Directory.GetFiles(directory, pattern, SearchOption.TopDirectoryOnly); } private static void DeleteIfExists(string path) @@ -408,4 +998,15 @@ private static StoreOpenStatus ToOpenStatus(StoreStatus status) _ => StoreOpenStatus.MappingFailed }; } + + private sealed record OwnerScan(List LiveOwners); + + private sealed record OwnerSnapshot(List CommittedOwners, bool HasLiveOwner); + + private enum OwnerReleaseOutcome + { + OwnerAbsent, + FinalizedMarkerPublished, + Failed + } } diff --git a/src/SharedMemoryStore/Interop/MemoryMappedStoreRegion.cs b/src/SharedMemoryStore/Interop/MemoryMappedStoreRegion.cs index 4348985..612a6de 100644 --- a/src/SharedMemoryStore/Interop/MemoryMappedStoreRegion.cs +++ b/src/SharedMemoryStore/Interop/MemoryMappedStoreRegion.cs @@ -14,13 +14,12 @@ internal sealed unsafe class MemoryMappedStoreRegion : ISharedStoreRegion private MemoryMappedStoreRegion( MemoryMappedFile mapping, MemoryMappedViewAccessor accessor, - long capacity, Action? disposeCallback) { _mapping = mapping; _accessor = accessor; _disposeCallback = disposeCallback; - Capacity = capacity; + Capacity = accessor.Capacity; _accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref _pointer); } @@ -38,10 +37,9 @@ public byte* Pointer public static MemoryMappedStoreRegion Create( MemoryMappedFile mapping, MemoryMappedViewAccessor accessor, - long capacity, Action? disposeCallback = null) { - return new MemoryMappedStoreRegion(mapping, accessor, capacity, disposeCallback); + return new MemoryMappedStoreRegion(mapping, accessor, disposeCallback); } public static StoreOpenStatus TryOpen(SharedMemoryStoreOptions options, out MemoryMappedStoreRegion? region) @@ -57,14 +55,33 @@ public void Dispose() } _disposed = true; - if (_pointer is not null) + try { - _accessor.SafeMemoryMappedViewHandle.ReleasePointer(); - _pointer = null; + if (_pointer is not null) + { + _accessor.SafeMemoryMappedViewHandle.ReleasePointer(); + _pointer = null; + } + } + finally + { + try + { + _accessor.Dispose(); + } + finally + { + try + { + _mapping.Dispose(); + } + finally + { + // Linux owner cleanup must run only after the view is unmapped, and it + // must still run when an earlier local-handle teardown reports an error. + _disposeCallback?.Invoke(); + } + } } - - _accessor.Dispose(); - _mapping.Dispose(); - _disposeCallback?.Invoke(); } } diff --git a/src/SharedMemoryStore/Interop/RegionOpenDisposition.cs b/src/SharedMemoryStore/Interop/RegionOpenDisposition.cs new file mode 100644 index 0000000..593704b --- /dev/null +++ b/src/SharedMemoryStore/Interop/RegionOpenDisposition.cs @@ -0,0 +1,12 @@ +namespace SharedMemoryStore.Interop; + +/// +/// Identifies whether the current cold-open attempt created the physical +/// mapping or attached to a mapping that was already owned by another handle. +/// Only the physical creator is allowed to initialize an unpublished header. +/// +internal enum RegionOpenDisposition +{ + CreatedNew, + OpenedExisting +} diff --git a/src/SharedMemoryStore/Interop/SharedStoreOpenScope.cs b/src/SharedMemoryStore/Interop/SharedStoreOpenScope.cs new file mode 100644 index 0000000..dbaa7cf --- /dev/null +++ b/src/SharedMemoryStore/Interop/SharedStoreOpenScope.cs @@ -0,0 +1,82 @@ +namespace SharedMemoryStore.Interop; + +/// +/// Owns one cold create/open transaction. The ordinary synchronization gate +/// and, on Linux, the outer lifecycle gate remain held until header validation +/// and participant registration finish. Mapped resource ownership can then be +/// transferred to the constructed engine without relying on lock reentrancy. +/// +internal sealed class SharedStoreOpenScope : IDisposable +{ + private readonly IDisposable? _outerLifecycleGate; + private int _disposed; + private bool _resourcesTransferred; + + internal SharedStoreOpenScope( + MemoryMappedStoreRegion region, + ISharedStoreSynchronization synchronization, + IDisposable? outerLifecycleGate, + RegionOpenDisposition disposition) + { + Region = region; + Synchronization = synchronization; + _outerLifecycleGate = outerLifecycleGate; + Disposition = disposition; + } + + internal MemoryMappedStoreRegion Region { get; } + + internal ISharedStoreSynchronization Synchronization { get; } + + internal RegionOpenDisposition Disposition { get; } + + /// + /// Transfers the mapped region and ordinary synchronization object to a + /// fully constructed core/engine. The scope continues to own only the held + /// gate state and releases it when disposed. + /// + internal void TransferResourceOwnership() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + _resourcesTransferred = true; + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + try + { + Synchronization.Exit(); + } + finally + { + try + { + // Linux acquires .lifecycle before .lock, so release the + // ordinary gate before the outer lifecycle rendezvous. + _outerLifecycleGate?.Dispose(); + } + finally + { + if (!_resourcesTransferred) + { + try + { + // Retire the ordinary descriptor before mapped-owner + // cleanup can re-enter Linux .lifecycle. Both held + // gates have already been released at this point. + Synchronization.Dispose(); + } + finally + { + Region.Dispose(); + } + } + } + } + } +} diff --git a/src/SharedMemoryStore/Interop/SharedStorePlatform.cs b/src/SharedMemoryStore/Interop/SharedStorePlatform.cs index 109a310..22733c2 100644 --- a/src/SharedMemoryStore/Interop/SharedStorePlatform.cs +++ b/src/SharedMemoryStore/Interop/SharedStorePlatform.cs @@ -2,44 +2,118 @@ namespace SharedMemoryStore.Interop; internal static class SharedStorePlatform { - public static StoreOpenStatus TryOpen( + public static StoreOpenStatus TryBeginOpen( SharedMemoryStoreOptions options, StoreWaitOptions waitOptions, - out MemoryMappedStoreRegion? region, - out ISharedStoreSynchronization? synchronization) + long waitStartTimestamp, + out SharedStoreOpenScope? scope) { - region = null; - synchronization = null; + scope = null; + if (!Environment.Is64BitProcess || !BitConverter.IsLittleEndian) + { + return StoreOpenStatus.UnsupportedPlatform; + } - var status = TryOpenRegion(options, waitOptions, out region); - if (status != StoreOpenStatus.Success || region is null) + PlatformResourceName resourceName = PlatformResourceName.Create(options.Name); + if (OperatingSystem.IsLinux()) { - return status; + return LinuxSharedMemoryRegion.TryBeginColdOpen( + resourceName, + options, + waitOptions, + waitStartTimestamp, + out scope); + } + + if (!OperatingSystem.IsWindows()) + { + return StoreOpenStatus.UnsupportedPlatform; } + WindowsSharedStoreSynchronization? synchronization = null; + MemoryMappedStoreRegion? region = null; + bool synchronizationEntered = false; try { - synchronization = CreateSynchronization(PlatformResourceName.Create(options.Name)); + synchronization = new WindowsSharedStoreSynchronization(resourceName); + StoreStatus remainingStatus = TryGetRemainingWaitOptions( + waitOptions, + waitStartTimestamp, + out StoreWaitOptions remainingWait); + if (remainingStatus != StoreStatus.Success) + { + return ToOpenStatus(remainingStatus); + } + + StoreStatus enterStatus = synchronization.TryEnter(remainingWait); + if (enterStatus != StoreStatus.Success) + { + return ToOpenStatus(enterStatus); + } + + synchronizationEntered = true; + remainingStatus = TryGetRemainingWaitOptions( + waitOptions, + waitStartTimestamp, + out _); + if (remainingStatus != StoreStatus.Success) + { + return ToOpenStatus(remainingStatus); + } + + StoreOpenStatus openStatus = WindowsSharedMemoryRegion.TryOpen( + resourceName, + options, + out region, + out RegionOpenDisposition disposition); + if (openStatus != StoreOpenStatus.Success || region is null) + { + return openStatus; + } + + scope = new SharedStoreOpenScope( + region, + synchronization, + outerLifecycleGate: null, + disposition); + region = null; + synchronization = null; + synchronizationEntered = false; return StoreOpenStatus.Success; } catch (UnauthorizedAccessException) { - region.Dispose(); - region = null; return StoreOpenStatus.AccessDenied; } catch (PlatformNotSupportedException) { - region.Dispose(); - region = null; return StoreOpenStatus.UnsupportedPlatform; } catch { - region.Dispose(); - region = null; return StoreOpenStatus.MappingFailed; } + finally + { + try + { + if (synchronizationEntered) + { + synchronization?.Exit(); + } + } + finally + { + try + { + synchronization?.Dispose(); + } + finally + { + region?.Dispose(); + } + } + } } public static StoreOpenStatus TryOpenRegion( @@ -81,4 +155,46 @@ public static ISharedStoreSynchronization CreateSynchronization(PlatformResource throw new PlatformNotSupportedException("SharedMemoryStore supports Linux and Windows shared synchronization."); } + + internal static StoreStatus TryGetRemainingWaitOptions( + StoreWaitOptions waitOptions, + long waitStartTimestamp, + out StoreWaitOptions remainingWait) + { + remainingWait = default; + if (waitOptions.CancellationToken.IsCancellationRequested) + { + return StoreStatus.OperationCanceled; + } + + if (waitOptions.IsInfinite || waitOptions.Timeout == TimeSpan.Zero) + { + remainingWait = waitOptions; + return StoreStatus.Success; + } + + TimeSpan elapsed = System.Diagnostics.Stopwatch.GetElapsedTime(waitStartTimestamp); + if (elapsed >= waitOptions.Timeout) + { + return StoreStatus.StoreBusy; + } + + remainingWait = new StoreWaitOptions( + waitOptions.Timeout - elapsed, + waitOptions.CancellationToken); + return StoreStatus.Success; + } + + private static StoreOpenStatus ToOpenStatus(StoreStatus status) + { + return status switch + { + StoreStatus.Success => StoreOpenStatus.Success, + StoreStatus.StoreBusy => StoreOpenStatus.StoreBusy, + StoreStatus.OperationCanceled => StoreOpenStatus.OperationCanceled, + StoreStatus.AccessDenied => StoreOpenStatus.AccessDenied, + StoreStatus.UnsupportedPlatform => StoreOpenStatus.UnsupportedPlatform, + _ => StoreOpenStatus.MappingFailed + }; + } } diff --git a/src/SharedMemoryStore/Interop/WindowsSharedMemoryRegion.cs b/src/SharedMemoryStore/Interop/WindowsSharedMemoryRegion.cs index a27fc9f..25de019 100644 --- a/src/SharedMemoryStore/Interop/WindowsSharedMemoryRegion.cs +++ b/src/SharedMemoryStore/Interop/WindowsSharedMemoryRegion.cs @@ -11,30 +11,56 @@ public static StoreOpenStatus TryOpen( PlatformResourceName resourceName, SharedMemoryStoreOptions options, out MemoryMappedStoreRegion? region) + { + return TryOpen(resourceName, options, out region, out _); + } + + internal static StoreOpenStatus TryOpen( + PlatformResourceName resourceName, + SharedMemoryStoreOptions options, + out MemoryMappedStoreRegion? region, + out RegionOpenDisposition disposition) { region = null; + disposition = default; MemoryMappedFile? mapping = null; MemoryMappedViewAccessor? accessor = null; try { - mapping = options.OpenMode switch + switch (options.OpenMode) { - OpenMode.CreateNew => MemoryMappedFile.CreateNew( - resourceName.WindowsRegionName, - options.TotalBytes, - MemoryMappedFileAccess.ReadWrite), - OpenMode.OpenExisting => MemoryMappedFile.OpenExisting( - resourceName.WindowsRegionName, - MemoryMappedFileRights.ReadWrite), - _ => MemoryMappedFile.CreateOrOpen( - resourceName.WindowsRegionName, - options.TotalBytes, - MemoryMappedFileAccess.ReadWrite) - }; + case OpenMode.CreateNew: + mapping = MemoryMappedFile.CreateNew( + resourceName.WindowsRegionName, + options.TotalBytes, + MemoryMappedFileAccess.ReadWrite); + disposition = RegionOpenDisposition.CreatedNew; + break; - accessor = mapping.CreateViewAccessor(0, options.TotalBytes, MemoryMappedFileAccess.ReadWrite); - region = MemoryMappedStoreRegion.Create(mapping, accessor, options.TotalBytes); + case OpenMode.OpenExisting: + mapping = MemoryMappedFile.OpenExisting( + resourceName.WindowsRegionName, + MemoryMappedFileRights.ReadWrite); + disposition = RegionOpenDisposition.OpenedExisting; + break; + + default: + mapping = OpenExistingOrCreate( + resourceName.WindowsRegionName, + options.TotalBytes, + out bool createdNew); + disposition = createdNew + ? RegionOpenDisposition.CreatedNew + : RegionOpenDisposition.OpenedExisting; + break; + } + + // A zero-length view projects the actual mapping capacity. In particular, an + // opener with larger requested dimensions can still inspect an existing small + // mapping's header and report IncompatibleLayout instead of failing view creation. + accessor = mapping.CreateViewAccessor(0, 0, MemoryMappedFileAccess.ReadWrite); + region = MemoryMappedStoreRegion.Create(mapping, accessor); mapping = null; accessor = null; return StoreOpenStatus.Success; @@ -69,4 +95,41 @@ public static StoreOpenStatus TryOpen( mapping?.Dispose(); } } + + private static MemoryMappedFile OpenExistingOrCreate( + string mappingName, + long requestedCapacity, + out bool createdNew) + { + try + { + MemoryMappedFile existing = MemoryMappedFile.OpenExisting( + mappingName, + MemoryMappedFileRights.ReadWrite); + createdNew = false; + return existing; + } + catch (FileNotFoundException) + { + try + { + MemoryMappedFile created = MemoryMappedFile.CreateNew( + mappingName, + requestedCapacity, + MemoryMappedFileAccess.ReadWrite); + createdNew = true; + return created; + } + catch (IOException) + { + // Another creator won after the initial probe. Opening its mapping also + // avoids projecting our requested capacity onto that existing resource. + MemoryMappedFile existing = MemoryMappedFile.OpenExisting( + mappingName, + MemoryMappedFileRights.ReadWrite); + createdNew = false; + return existing; + } + } + } } diff --git a/src/SharedMemoryStore/Layout/StoreKey.cs b/src/SharedMemoryStore/Layout/StoreKey.cs index dd784b9..b192197 100644 --- a/src/SharedMemoryStore/Layout/StoreKey.cs +++ b/src/SharedMemoryStore/Layout/StoreKey.cs @@ -19,7 +19,16 @@ public static StoreStatus Validate(ReadOnlySpan key, int maxKeyBytes) public static ulong Hash(ReadOnlySpan key) { - var hash = OffsetBasis; + return ContinueHash(OffsetBasis, key); + } + + /// + /// Continues the canonical FNV-1a key hash across one caller-selected + /// chunk. Lock-free callers use this to observe an operation budget + /// without changing persisted hash identity. + /// + internal static ulong ContinueHash(ulong hash, ReadOnlySpan key) + { foreach (var value in key) { hash ^= value; @@ -29,6 +38,8 @@ public static ulong Hash(ReadOnlySpan key) return hash; } + internal static ulong HashSeed => OffsetBasis; + public static bool Equals(byte* storedKey, int storedLength, ReadOnlySpan key) { return storedLength == key.Length diff --git a/src/SharedMemoryStore/LayoutV2/LayoutV2Constants.cs b/src/SharedMemoryStore/LayoutV2/LayoutV2Constants.cs new file mode 100644 index 0000000..bf1dc83 --- /dev/null +++ b/src/SharedMemoryStore/LayoutV2/LayoutV2Constants.cs @@ -0,0 +1,60 @@ +using System.Runtime.InteropServices; + +namespace SharedMemoryStore.LayoutV2; + +internal static class LayoutV2Constants +{ + public const uint Magic = 0x3253_4D53; + public const int LayoutMajorVersion = 2; + public const int LayoutMinorVersion = 0; + public const int ResourceProtocolVersion = 2; + public const int HeaderLength = 512; + public const int AtomicAlignment = 8; + public const int CacheLineSize = 64; + public const int ParticipantRecordStride = 64; + public const int PrimaryDirectoryBucketStride = 128; + public const int PrimaryLanesPerBucket = 8; + public const int OverflowBindingStride = 8; + public const int LeaseRecordStride = 64; + public const int ValueSlotMetadataStride = 128; + public const int MaximumSlotCount = 1_048_575; + public const int MaximumParticipantRecordCount = 1_048_575; + public const int SlotGenerationBits = 33; + public const int ParticipantTokenBits = 28; + public const ulong SpillSummaryVersionedEmptyRequiredFeature = 1UL << 0; + public const ulong PublicationIntentRequiredFeature = 1UL << 1; + public const ulong PidNamespaceIdentityRequiredFeature = 1UL << 2; + public const ulong RequiredFeatures = + SpillSummaryVersionedEmptyRequiredFeature + | PublicationIntentRequiredFeature + | PidNamespaceIdentityRequiredFeature; + public const ulong OptionalFeatures = 0; + + public const long StoreInitializing = 1; + public const long StoreReady = 2; + public const long StoreCorrupt = 3; + public const long StoreUnsupported = 4; + + public const long PidNamespaceRecoveryEnabled = 1; + public const long PidNamespaceRecoveryMixed = 2; + + public const int ParticipantFree = 0; + public const int ParticipantRegistering = 1; + public const int ParticipantActive = 2; + public const int ParticipantClosing = 3; + public const int ParticipantRecovering = 4; + public const int ParticipantReclaiming = 5; + public const int ParticipantRetired = 6; + + public const int IdentityUnknown = 0; + public const int IdentityWindowsProcessCreationFileTime = 1; + public const int IdentityLinuxProcStartTicks = 2; + + public const int SlotFree = 0; + public const int LeaseFree = 0; + + public static bool MatchesRequiredFeatures(ulong mappedRequiredFeatures) => + mappedRequiredFeatures == RequiredFeatures; + + public static bool IsSupportedArchitecture(Architecture architecture) => architecture == Architecture.X64; +} diff --git a/src/SharedMemoryStore/LayoutV2/SharedRecordsV2.cs b/src/SharedMemoryStore/LayoutV2/SharedRecordsV2.cs new file mode 100644 index 0000000..1d51fed --- /dev/null +++ b/src/SharedMemoryStore/LayoutV2/SharedRecordsV2.cs @@ -0,0 +1,108 @@ +using System.Runtime.InteropServices; + +namespace SharedMemoryStore.LayoutV2; + +[StructLayout(LayoutKind.Explicit, Size = LayoutV2Constants.ParticipantRecordStride)] +internal struct ParticipantRecordV2 +{ + [FieldOffset(0)] public long Control; + [FieldOffset(8)] public int IdentityKind; + [FieldOffset(12)] public int Reserved; + [FieldOffset(16)] public long ProcessStartValue; + [FieldOffset(24)] public long OpenSequence; + [FieldOffset(32)] public ulong PidNamespaceId; +} + +[StructLayout(LayoutKind.Explicit, Size = LayoutV2Constants.PrimaryDirectoryBucketStride)] +internal unsafe struct PrimaryDirectoryBucketV2 +{ + [FieldOffset(0)] public long SpillSummary; + [FieldOffset(8)] public long Mutation; + [FieldOffset(16)] public fixed long Lanes[LayoutV2Constants.PrimaryLanesPerBucket]; +} + +[StructLayout(LayoutKind.Explicit, Size = LayoutV2Constants.LeaseRecordStride)] +internal struct LeaseRecordV2 +{ + [FieldOffset(0)] public long Control; + [FieldOffset(8)] public ulong SlotBinding; + [FieldOffset(16)] public long AcquireSequence; +} + +[StructLayout(LayoutKind.Explicit, Size = LayoutV2Constants.ValueSlotMetadataStride)] +internal struct ValueSlotMetadataV2 +{ + [FieldOffset(0)] public long Control; + [FieldOffset(8)] public ulong DirectoryBinding; + [FieldOffset(16)] public long DirectoryLocation; + [FieldOffset(24)] public long DirectoryOperation; + [FieldOffset(32)] public ulong KeyHash; + [FieldOffset(40)] public int KeyLength; + [FieldOffset(44)] public int DescriptorLength; + [FieldOffset(48)] public int ValueLength; + [FieldOffset(52)] public int PublicationIntent; + [FieldOffset(56)] public long BytesAdvanced; + [FieldOffset(64)] public long CommitSequence; + [FieldOffset(72)] public long KeyOffset; + [FieldOffset(80)] public long DescriptorOffset; + [FieldOffset(88)] public long PayloadOffset; +} + +internal enum SlotPublicationIntent +{ + None = 0, + ExplicitReservation = 1, + AtomicPublication = 2, +} + +[StructLayout(LayoutKind.Explicit, Size = LayoutV2Constants.HeaderLength)] +internal struct StoreHeaderV2 +{ + [FieldOffset(0)] public uint Magic; + [FieldOffset(4)] public ushort LayoutMajorVersion; + [FieldOffset(6)] public ushort LayoutMinorVersion; + [FieldOffset(8)] public int HeaderLength; + [FieldOffset(12)] public int ResourceProtocolVersion; + [FieldOffset(16)] public ulong RequiredFeatures; + [FieldOffset(24)] public ulong OptionalFeatures; + [FieldOffset(32)] public long TotalBytes; + [FieldOffset(40)] public ulong StoreId; + [FieldOffset(48)] public long Control; + [FieldOffset(56)] public long Sequence; + [FieldOffset(64)] public int SlotCount; + [FieldOffset(68)] public int LeaseRecordCount; + [FieldOffset(72)] public int ParticipantRecordCount; + [FieldOffset(76)] public int MaxKeyBytes; + [FieldOffset(80)] public int MaxDescriptorBytes; + [FieldOffset(84)] public int MaxValueBytes; + [FieldOffset(88)] public int ParticipantIndexBits; + [FieldOffset(92)] public int ParticipantGenerationBits; + [FieldOffset(96)] public long ParticipantOffset; + [FieldOffset(104)] public long ParticipantLength; + [FieldOffset(112)] public int ParticipantStride; + [FieldOffset(116)] public int PrimaryLaneCount; + [FieldOffset(120)] public int PrimaryBucketCount; + [FieldOffset(124)] public int PrimaryBucketStride; + [FieldOffset(128)] public long PrimaryDirectoryOffset; + [FieldOffset(136)] public long PrimaryDirectoryLength; + [FieldOffset(144)] public long OverflowDirectoryOffset; + [FieldOffset(152)] public long OverflowDirectoryLength; + [FieldOffset(160)] public int OverflowStride; + [FieldOffset(164)] public int LeaseStride; + [FieldOffset(168)] public long LeaseRegistryOffset; + [FieldOffset(176)] public long LeaseRegistryLength; + [FieldOffset(184)] public int SlotMetadataStride; + [FieldOffset(188)] public int KeyStride; + [FieldOffset(192)] public long SlotMetadataOffset; + [FieldOffset(200)] public long SlotMetadataLength; + [FieldOffset(208)] public long KeyStorageOffset; + [FieldOffset(216)] public long KeyStorageLength; + [FieldOffset(224)] public int DescriptorStride; + [FieldOffset(228)] public int PayloadStride; + [FieldOffset(232)] public long DescriptorStorageOffset; + [FieldOffset(240)] public long DescriptorStorageLength; + [FieldOffset(248)] public long PayloadStorageOffset; + [FieldOffset(256)] public long PayloadStorageLength; + [FieldOffset(264)] public ulong PidNamespaceId; + [FieldOffset(272)] public long PidNamespaceMode; +} diff --git a/src/SharedMemoryStore/LayoutV2/StoreLayoutV2.cs b/src/SharedMemoryStore/LayoutV2/StoreLayoutV2.cs new file mode 100644 index 0000000..39ea36d --- /dev/null +++ b/src/SharedMemoryStore/LayoutV2/StoreLayoutV2.cs @@ -0,0 +1,233 @@ +namespace SharedMemoryStore.LayoutV2; + +internal readonly struct StoreLayoutV2 +{ + public StoreLayoutV2( + long totalBytes, + int slotCount, + int leaseRecordCount, + int participantRecordCount, + int maxKeyBytes, + int maxDescriptorBytes, + int maxValueBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(slotCount, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(slotCount, LayoutV2Constants.MaximumSlotCount); + ArgumentOutOfRangeException.ThrowIfLessThan(leaseRecordCount, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(participantRecordCount, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(participantRecordCount, LayoutV2Constants.MaximumParticipantRecordCount); + ArgumentOutOfRangeException.ThrowIfLessThan(maxKeyBytes, 1); + ArgumentOutOfRangeException.ThrowIfNegative(maxDescriptorBytes); + ArgumentOutOfRangeException.ThrowIfLessThan(maxValueBytes, 1); + + checked + { + TotalBytes = totalBytes; + SlotCount = slotCount; + LeaseRecordCount = leaseRecordCount; + ParticipantRecordCount = participantRecordCount; + MaxKeyBytes = maxKeyBytes; + MaxDescriptorBytes = maxDescriptorBytes; + MaxValueBytes = maxValueBytes; + HeaderLength = LayoutV2Constants.HeaderLength; + + ParticipantIndexBits = RequiredBits(participantRecordCount + 1); + ParticipantGenerationBits = LayoutV2Constants.ParticipantTokenBits - ParticipantIndexBits; + if (ParticipantGenerationBits < 8) + { + throw new ArgumentOutOfRangeException(nameof(participantRecordCount)); + } + + ParticipantIndexMask = (1 << ParticipantIndexBits) - 1; + ParticipantGenerationMask = (1 << ParticipantGenerationBits) - 1; + + ParticipantStride = LayoutV2Constants.ParticipantRecordStride; + ParticipantOffset = HeaderLength; + ParticipantLength = (long)participantRecordCount * ParticipantStride; + + PrimaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + PrimaryBucketCount = PrimaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket; + PrimaryBucketStride = LayoutV2Constants.PrimaryDirectoryBucketStride; + PrimaryDirectoryOffset = Align64(ParticipantOffset + ParticipantLength); + PrimaryDirectoryLength = (long)PrimaryBucketCount * PrimaryBucketStride; + + OverflowStride = LayoutV2Constants.OverflowBindingStride; + OverflowDirectoryOffset = Align8(PrimaryDirectoryOffset + PrimaryDirectoryLength); + OverflowDirectoryLength = (long)slotCount * OverflowStride; + + LeaseStride = LayoutV2Constants.LeaseRecordStride; + LeaseRegistryOffset = Align64(OverflowDirectoryOffset + OverflowDirectoryLength); + LeaseRegistryLength = (long)leaseRecordCount * LeaseStride; + + SlotMetadataStride = LayoutV2Constants.ValueSlotMetadataStride; + SlotMetadataOffset = Align64(LeaseRegistryOffset + LeaseRegistryLength); + SlotMetadataLength = (long)slotCount * SlotMetadataStride; + + KeyStride = checked((int)Align8(Math.Max(1, maxKeyBytes))); + KeyStorageOffset = Align8(SlotMetadataOffset + SlotMetadataLength); + KeyStorageLength = (long)slotCount * KeyStride; + + DescriptorStride = checked((int)Align8(Math.Max(1, maxDescriptorBytes))); + DescriptorStorageOffset = Align8(KeyStorageOffset + KeyStorageLength); + DescriptorStorageLength = (long)slotCount * DescriptorStride; + + PayloadStride = checked((int)Align8(Math.Max(1, maxValueBytes))); + PayloadStorageOffset = Align8(DescriptorStorageOffset + DescriptorStorageLength); + PayloadStorageLength = (long)slotCount * PayloadStride; + RequiredBytes = Align8(PayloadStorageOffset + PayloadStorageLength); + } + } + + public long TotalBytes { get; } + public int SlotCount { get; } + public int LeaseRecordCount { get; } + public int ParticipantRecordCount { get; } + public int MaxKeyBytes { get; } + public int MaxDescriptorBytes { get; } + public int MaxValueBytes { get; } + public int HeaderLength { get; } + public int ParticipantIndexBits { get; } + public int ParticipantGenerationBits { get; } + public int ParticipantIndexMask { get; } + public int ParticipantGenerationMask { get; } + public int ParticipantStride { get; } + public long ParticipantOffset { get; } + public long ParticipantLength { get; } + public int PrimaryLaneCount { get; } + public int PrimaryBucketCount { get; } + public int BucketCount => PrimaryBucketCount; + public int PrimaryBucketStride { get; } + public long PrimaryDirectoryOffset { get; } + public long PrimaryDirectoryLength { get; } + public int OverflowStride { get; } + public long OverflowDirectoryOffset { get; } + public long OverflowDirectoryLength { get; } + public int LeaseStride { get; } + public long LeaseRegistryOffset { get; } + public long LeaseRegistryLength { get; } + public int SlotMetadataStride { get; } + public long SlotMetadataOffset { get; } + public long SlotMetadataLength { get; } + public int KeyStride { get; } + public long KeyStorageOffset { get; } + public long KeyStorageLength { get; } + public int DescriptorStride { get; } + public long DescriptorStorageOffset { get; } + public long DescriptorStorageLength { get; } + public int PayloadStride { get; } + public long PayloadStorageOffset { get; } + public long PayloadStorageLength { get; } + public long RequiredBytes { get; } + + public bool FitsWithinTotalBytes() => TotalBytes >= RequiredBytes && TotalBytes > 0; + + public bool MatchesHeader(in StoreHeaderV2 header) + { + return header.Magic == LayoutV2Constants.Magic + && header.LayoutMajorVersion == LayoutV2Constants.LayoutMajorVersion + && header.LayoutMinorVersion == LayoutV2Constants.LayoutMinorVersion + && header.HeaderLength == HeaderLength + && header.ResourceProtocolVersion == LayoutV2Constants.ResourceProtocolVersion + && LayoutV2Constants.MatchesRequiredFeatures(header.RequiredFeatures) + && header.TotalBytes == TotalBytes + && header.StoreId != 0 + && header.SlotCount == SlotCount + && header.LeaseRecordCount == LeaseRecordCount + && header.ParticipantRecordCount == ParticipantRecordCount + && header.MaxKeyBytes == MaxKeyBytes + && header.MaxDescriptorBytes == MaxDescriptorBytes + && header.MaxValueBytes == MaxValueBytes + && header.ParticipantIndexBits == ParticipantIndexBits + && header.ParticipantGenerationBits == ParticipantGenerationBits + && header.ParticipantOffset == ParticipantOffset + && header.ParticipantLength == ParticipantLength + && header.ParticipantStride == ParticipantStride + && header.PrimaryLaneCount == PrimaryLaneCount + && header.PrimaryBucketCount == PrimaryBucketCount + && header.PrimaryBucketStride == PrimaryBucketStride + && header.PrimaryDirectoryOffset == PrimaryDirectoryOffset + && header.PrimaryDirectoryLength == PrimaryDirectoryLength + && header.OverflowDirectoryOffset == OverflowDirectoryOffset + && header.OverflowDirectoryLength == OverflowDirectoryLength + && header.OverflowStride == OverflowStride + && header.LeaseStride == LeaseStride + && header.LeaseRegistryOffset == LeaseRegistryOffset + && header.LeaseRegistryLength == LeaseRegistryLength + && header.SlotMetadataStride == SlotMetadataStride + && header.KeyStride == KeyStride + && header.SlotMetadataOffset == SlotMetadataOffset + && header.SlotMetadataLength == SlotMetadataLength + && header.KeyStorageOffset == KeyStorageOffset + && header.KeyStorageLength == KeyStorageLength + && header.DescriptorStride == DescriptorStride + && header.PayloadStride == PayloadStride + && header.DescriptorStorageOffset == DescriptorStorageOffset + && header.DescriptorStorageLength == DescriptorStorageLength + && header.PayloadStorageOffset == PayloadStorageOffset + && header.PayloadStorageLength == PayloadStorageLength; + } + + public static long CalculateRequiredBytes( + int slotCount, + int maxValueBytes, + int maxDescriptorBytes, + int maxKeyBytes, + int leaseRecordCount, + int participantRecordCount = 64) + { + return new StoreLayoutV2( + 0, + slotCount, + leaseRecordCount, + participantRecordCount, + maxKeyBytes, + maxDescriptorBytes, + maxValueBytes).RequiredBytes; + } + + public static StoreLayoutV2 FromOptions(SharedMemoryStoreOptions options) + { + return new StoreLayoutV2( + options.TotalBytes, + options.SlotCount, + options.LeaseRecordCount, + options.ParticipantRecordCount, + options.MaxKeyBytes, + options.MaxDescriptorBytes, + options.MaxValueBytes); + } + + private static int RequiredBits(int distinctValues) + { + var bits = 0; + var value = distinctValues - 1; + do + { + bits++; + value >>= 1; + } + while (value != 0); + + return bits; + } + + private static int NextPowerOfTwo(int value) + { + if (value <= 0 || value > 1 << 30) + { + throw new OverflowException("The requested primary directory cannot be represented."); + } + + var result = 1; + while (result < value) + { + result = checked(result << 1); + } + + return result; + } + + private static long Align8(long value) => checked(value + 7) & ~7L; + + private static long Align64(long value) => checked(value + 63) & ~63L; +} diff --git a/src/SharedMemoryStore/Leasing/LeaseOwnerClassifier.cs b/src/SharedMemoryStore/Leasing/LeaseOwnerClassifier.cs index da2674d..90f85ce 100644 --- a/src/SharedMemoryStore/Leasing/LeaseOwnerClassifier.cs +++ b/src/SharedMemoryStore/Leasing/LeaseOwnerClassifier.cs @@ -1,4 +1,6 @@ using System.Diagnostics; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; namespace SharedMemoryStore.Leasing; @@ -63,4 +65,349 @@ public static LeaseOwnerClassification Classify(int ownerProcessId) return new LeaseOwnerClassification(LeaseOwnerKind.Unsupported, ownerProcessId); } } + + /// + /// Classifies an exact participant identity. Unlike the legacy PID-only + /// overload, this overload never treats PID existence alone as proof that + /// the stored incarnation is live. + /// + public static LeaseOwnerClassification Classify(ParticipantIncarnation participant) + { + if (participant.ProcessId <= 0 + || participant.ProcessStartValue < 0 + || participant.IdentityKind is < LayoutV2Constants.IdentityUnknown + or > LayoutV2Constants.IdentityLinuxProcStartTicks) + { + return new LeaseOwnerClassification(LeaseOwnerKind.UnsafeRecord, participant.ProcessId); + } + + // A Linux PID has meaning only inside its PID namespace. Prove that + // the caller observes the exact namespace captured by the owner before + // consulting /proc/ or Process, otherwise the same numeric PID + // could name an unrelated process (or appear absent) in this caller's + // namespace. Windows records must retain the protocol's zero value. + if (OperatingSystem.IsLinux()) + { + if (participant.PidNamespaceId == 0 + || !TryObserveLinuxPidNamespaceId(Environment.ProcessId, out ulong currentNamespaceId) + || currentNamespaceId != participant.PidNamespaceId) + { + return new LeaseOwnerClassification(LeaseOwnerKind.Unsupported, participant.ProcessId); + } + } + else if (participant.PidNamespaceId != 0) + { + return new LeaseOwnerClassification(LeaseOwnerKind.UnsafeRecord, participant.ProcessId); + } + + if (participant.IdentityKind == LayoutV2Constants.IdentityUnknown) + { + return new LeaseOwnerClassification(LeaseOwnerKind.Unsupported, participant.ProcessId); + } + + if (participant.ProcessStartValue == 0) + { + return new LeaseOwnerClassification(LeaseOwnerKind.UnsafeRecord, participant.ProcessId); + } + + IdentityObservation observation = ObserveIdentity( + participant.ProcessId, + participant.IdentityKind, + out long observedStartValue); + if (observation == IdentityObservation.Missing) + { + return new LeaseOwnerClassification(LeaseOwnerKind.StaleProcess, participant.ProcessId); + } + + if (observation != IdentityObservation.Available) + { + return new LeaseOwnerClassification(LeaseOwnerKind.Unsupported, participant.ProcessId); + } + + if (observedStartValue != participant.ProcessStartValue) + { + return new LeaseOwnerClassification(LeaseOwnerKind.StaleProcess, participant.ProcessId); + } + + return new LeaseOwnerClassification( + participant.ProcessId == Environment.ProcessId + ? LeaseOwnerKind.CurrentProcess + : LeaseOwnerKind.OtherLiveProcess, + participant.ProcessId); + } + + /// + /// Captures the current process identity used when publishing an Active + /// participant record. Failure deliberately degrades registration to the + /// Unknown identity kind, which remains usable but unrecoverable. + /// + internal static bool TryCaptureCurrentProcessIdentity( + out int identityKind, + out long processStartValue, + out ulong pidNamespaceId) + { + pidNamespaceId = 0; + identityKind = OperatingSystem.IsWindows() + ? LayoutV2Constants.IdentityWindowsProcessCreationFileTime + : OperatingSystem.IsLinux() + ? LayoutV2Constants.IdentityLinuxProcStartTicks + : LayoutV2Constants.IdentityUnknown; + processStartValue = 0; + if (identityKind == LayoutV2Constants.IdentityUnknown) + { + return false; + } + + if (OperatingSystem.IsLinux() + && !TryObserveLinuxPidNamespaceId(Environment.ProcessId, out pidNamespaceId)) + { + identityKind = LayoutV2Constants.IdentityUnknown; + processStartValue = 0; + pidNamespaceId = 0; + return false; + } + + if (ObserveIdentity(Environment.ProcessId, identityKind, out processStartValue) + == IdentityObservation.Available) + { + return true; + } + + identityKind = LayoutV2Constants.IdentityUnknown; + processStartValue = 0; + return false; + } + + internal static bool TryCaptureCurrentProcessIdentity( + out int identityKind, + out long processStartValue) => + TryCaptureCurrentProcessIdentity( + out identityKind, + out processStartValue, + out _); + + /// + /// Conservative fallback for a process stopped in Registering before it + /// wrote a process-start identity. Only definite PID absence is stale; + /// PID presence cannot distinguish reuse and is therefore unsupported. + /// + internal static LeaseOwnerClassification ClassifyPresenceOnly( + int processId, + ulong pidNamespaceId) + { + if (processId <= 0) + { + return new LeaseOwnerClassification(LeaseOwnerKind.UnsafeRecord, processId); + } + + if (!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + { + return new LeaseOwnerClassification(LeaseOwnerKind.Unsupported, processId); + } + + if (OperatingSystem.IsLinux()) + { + if (pidNamespaceId == 0 + || !TryObserveLinuxPidNamespaceId(Environment.ProcessId, out ulong currentNamespaceId) + || currentNamespaceId != pidNamespaceId) + { + return new LeaseOwnerClassification(LeaseOwnerKind.Unsupported, processId); + } + } + else if (pidNamespaceId != 0) + { + return new LeaseOwnerClassification(LeaseOwnerKind.Unsupported, processId); + } + + try + { + using Process process = Process.GetProcessById(processId); + if (process.HasExited) + { + return new LeaseOwnerClassification(LeaseOwnerKind.StaleProcess, processId); + } + + return new LeaseOwnerClassification( + processId == Environment.ProcessId + ? LeaseOwnerKind.CurrentProcess + : LeaseOwnerKind.Unsupported, + processId); + } + catch (ArgumentException) + { + return new LeaseOwnerClassification(LeaseOwnerKind.StaleProcess, processId); + } + catch (InvalidOperationException) + { + return new LeaseOwnerClassification(LeaseOwnerKind.StaleProcess, processId); + } + catch + { + return new LeaseOwnerClassification(LeaseOwnerKind.Unsupported, processId); + } + } + + /// + /// Reads Linux's stable numeric PID-namespace inode token from the procfs + /// namespace symlink (for example, pid:[4026531836]). No PID/start + /// liveness decision is made here. + /// + internal static bool TryObserveLinuxPidNamespaceId( + int processId, + out ulong pidNamespaceId) + { + pidNamespaceId = 0; + if (!OperatingSystem.IsLinux() || processId <= 0) + { + return false; + } + + try + { + string? target = new FileInfo($"/proc/{processId}/ns/pid").LinkTarget; + const string prefix = "pid:["; + if (target is null + || !target.StartsWith(prefix, StringComparison.Ordinal) + || target.Length <= prefix.Length + 1 + || target[^1] != ']' + || !ulong.TryParse( + target.AsSpan(prefix.Length, target.Length - prefix.Length - 1), + System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, + out pidNamespaceId) + || pidNamespaceId == 0) + { + pidNamespaceId = 0; + return false; + } + + return true; + } + catch + { + pidNamespaceId = 0; + return false; + } + } + + private static IdentityObservation ObserveIdentity( + int processId, + int identityKind, + out long processStartValue) + { + processStartValue = 0; + if (identityKind == LayoutV2Constants.IdentityWindowsProcessCreationFileTime) + { + return OperatingSystem.IsWindows() + ? ObserveWindowsIdentity(processId, out processStartValue) + : IdentityObservation.Unsupported; + } + + if (identityKind == LayoutV2Constants.IdentityLinuxProcStartTicks) + { + return OperatingSystem.IsLinux() + ? ObserveLinuxIdentity(processId, out processStartValue) + : IdentityObservation.Unsupported; + } + + return IdentityObservation.Unsupported; + } + + private static IdentityObservation ObserveWindowsIdentity( + int processId, + out long processStartValue) + { + processStartValue = 0; + try + { + using Process process = Process.GetProcessById(processId); + if (process.HasExited) + { + return IdentityObservation.Missing; + } + + processStartValue = process.StartTime.ToUniversalTime().ToFileTimeUtc(); + return processStartValue > 0 + ? IdentityObservation.Available + : IdentityObservation.Unsupported; + } + catch (ArgumentException) + { + return IdentityObservation.Missing; + } + catch (InvalidOperationException) + { + return IdentityObservation.Missing; + } + catch (PlatformNotSupportedException) + { + return IdentityObservation.Unsupported; + } + catch + { + return IdentityObservation.Unsupported; + } + } + + private static IdentityObservation ObserveLinuxIdentity( + int processId, + out long processStartValue) + { + processStartValue = 0; + string processDirectory = $"/proc/{processId}"; + try + { + string stat = File.ReadAllText($"{processDirectory}/stat"); + int commandEnd = stat.LastIndexOf(')'); + if (commandEnd < 0 || commandEnd + 2 >= stat.Length) + { + return IdentityObservation.Unsupported; + } + + string[] fields = stat[(commandEnd + 2)..] + .Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (fields.Length <= 19 + || !long.TryParse( + fields[19], + System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, + out processStartValue) + || processStartValue <= 0) + { + processStartValue = 0; + return IdentityObservation.Unsupported; + } + + return IdentityObservation.Available; + } + catch (FileNotFoundException) + { + return IdentityObservation.Missing; + } + catch (DirectoryNotFoundException) + { + return IdentityObservation.Missing; + } + catch (UnauthorizedAccessException) + { + return IdentityObservation.Unsupported; + } + catch (IOException) + { + return Directory.Exists(processDirectory) + ? IdentityObservation.Unsupported + : IdentityObservation.Missing; + } + catch + { + return IdentityObservation.Unsupported; + } + } + + private enum IdentityObservation + { + Available, + Missing, + Unsupported + } } diff --git a/src/SharedMemoryStore/Lifecycle/StoreLifecycleGate.cs b/src/SharedMemoryStore/Lifecycle/StoreLifecycleGate.cs index 8963808..e319c02 100644 --- a/src/SharedMemoryStore/Lifecycle/StoreLifecycleGate.cs +++ b/src/SharedMemoryStore/Lifecycle/StoreLifecycleGate.cs @@ -1,58 +1,132 @@ +using System.Diagnostics; + namespace SharedMemoryStore.Lifecycle; internal sealed class StoreLifecycleGate { - private const int Open = 0; - private const int Disposing = 1; - private const int Disposed = 2; + private const long EntryClosed = long.MinValue; + private const long ActiveCountMask = long.MaxValue; - private readonly object _sync = new(); - private int _state; - private int _activeOperations; + private readonly ManualResetEventSlim _disposeCompleted = new(initialState: false); + private long _lifetimeWord; + private int _isDisposed; - public bool IsDisposed => Volatile.Read(ref _state) == Disposed; + public bool IsDisposed => Volatile.Read(ref _isDisposed) != 0; - public bool IsDisposingOrDisposed => Volatile.Read(ref _state) != Open; + public bool IsDisposingOrDisposed => (Volatile.Read(ref _lifetimeWord) & EntryClosed) != 0; public bool TryEnter(out Operation operation) { - lock (_sync) + while (true) { - if (_state != Open) + var observed = Volatile.Read(ref _lifetimeWord); + if ((observed & EntryClosed) != 0) { operation = default; return false; } - _activeOperations++; - operation = new Operation(this); - return true; + if ((observed & ActiveCountMask) == ActiveCountMask) + { + throw new InvalidOperationException("The active operation count is exhausted."); + } + + if (Interlocked.CompareExchange(ref _lifetimeWord, observed + 1, observed) == observed) + { + operation = new Operation(this); + return true; + } } } - public bool TryBeginDispose() + /// + /// Attempts to enter a public operation without allowing process-local CAS + /// contention to sit outside the caller's operation-wide wait/work bound. + /// A zero timeout receives one CAS attempt; finite and infinite policies + /// retain the same cancellation semantics as the shared protocol budget. + /// + public StoreStatus TryEnter( + StoreWaitOptions waitOptions, + long started, + out Operation operation) { - lock (_sync) + var spinner = new SpinWait(); + while (true) { - if (_state == Disposed) + var observed = Volatile.Read(ref _lifetimeWord); + if ((observed & EntryClosed) != 0) { - return false; + operation = default; + return StoreStatus.StoreDisposed; } - if (_state == Disposing) + if (!waitOptions.IsValid) { - while (_state != Disposed) + operation = default; + return StoreStatus.UnknownFailure; + } + + if (waitOptions.CancellationToken.IsCancellationRequested) + { + operation = default; + return StoreStatus.OperationCanceled; + } + + if (!waitOptions.IsInfinite + && waitOptions.Timeout > TimeSpan.Zero + && Stopwatch.GetElapsedTime(started) >= waitOptions.Timeout) + { + operation = default; + return StoreStatus.StoreBusy; + } + + if ((observed & ActiveCountMask) == ActiveCountMask) + { + throw new InvalidOperationException("The active operation count is exhausted."); + } + + if (Interlocked.CompareExchange(ref _lifetimeWord, observed + 1, observed) == observed) + { + operation = new Operation(this); + return StoreStatus.Success; + } + + if (waitOptions.Timeout == TimeSpan.Zero) + { + operation = default; + return (Volatile.Read(ref _lifetimeWord) & EntryClosed) != 0 + ? StoreStatus.StoreDisposed + : StoreStatus.StoreBusy; + } + + spinner.SpinOnce(); + } + } + + public bool TryBeginDispose() + { + while (true) + { + var observed = Volatile.Read(ref _lifetimeWord); + if ((observed & EntryClosed) != 0) + { + if (Volatile.Read(ref _isDisposed) == 0) { - Monitor.Wait(_sync); + _disposeCompleted.Wait(); } return false; } - _state = Disposing; - while (_activeOperations != 0) + if (Interlocked.CompareExchange(ref _lifetimeWord, observed | EntryClosed, observed) != observed) { - Monitor.Wait(_sync); + continue; + } + + var spinner = new SpinWait(); + while ((Volatile.Read(ref _lifetimeWord) & ActiveCountMask) != 0) + { + spinner.SpinOnce(); } return true; @@ -61,22 +135,16 @@ public bool TryBeginDispose() public void CompleteDispose() { - lock (_sync) - { - _state = Disposed; - Monitor.PulseAll(_sync); - } + Volatile.Write(ref _isDisposed, 1); + _disposeCompleted.Set(); } private void Exit() { - lock (_sync) + var remaining = Interlocked.Decrement(ref _lifetimeWord); + if ((remaining & ActiveCountMask) == ActiveCountMask) { - _activeOperations--; - if (_activeOperations == 0) - { - Monitor.PulseAll(_sync); - } + throw new InvalidOperationException("An operation exited without a matching entry."); } } diff --git a/src/SharedMemoryStore/LockFree/AtomicControlWord.cs b/src/SharedMemoryStore/LockFree/AtomicControlWord.cs new file mode 100644 index 0000000..5d68b45 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/AtomicControlWord.cs @@ -0,0 +1,55 @@ +namespace SharedMemoryStore.LockFree; + +internal static class AtomicControlWord +{ + public static ulong EncodeParticipant(int state, int incarnation, int pid) + { + if (state is < 0 or > 7) + { + throw new ArgumentOutOfRangeException(nameof(state)); + } + + if (incarnation is < 0 or > 0x0fff_ffff) + { + throw new ArgumentOutOfRangeException(nameof(incarnation)); + } + + ArgumentOutOfRangeException.ThrowIfNegative(pid); + return (uint)state | ((ulong)(uint)incarnation << 3) | ((ulong)(uint)pid << 31); + } + + public static ulong EncodeSlot(int state, long generation, int participantToken) => + EncodeOwnedLifecycle(state, generation, participantToken); + + public static ulong EncodeLease(int state, long generation, int participantToken) => + EncodeOwnedLifecycle(state, generation, participantToken); + + public static long LoadAcquire(ref long location) => Volatile.Read(ref location); + + public static void StoreRelease(ref long location, long value) => Volatile.Write(ref location, value); + + public static long CompareExchange(ref long location, long value, long comparand) => + Interlocked.CompareExchange(ref location, value, comparand); + + public static long Exchange(ref long location, long value) => Interlocked.Exchange(ref location, value); + + private static ulong EncodeOwnedLifecycle(int state, long generation, int participantToken) + { + if (state is < 0 or > 7) + { + throw new ArgumentOutOfRangeException(nameof(state)); + } + + if (generation is < 1 or > 0x1_ffff_ffffL) + { + throw new ArgumentOutOfRangeException(nameof(generation)); + } + + if (participantToken is < 0 or > 0x0fff_ffff) + { + throw new ArgumentOutOfRangeException(nameof(participantToken)); + } + + return (uint)state | ((ulong)generation << 3) | ((ulong)(uint)participantToken << 36); + } +} diff --git a/src/SharedMemoryStore/LockFree/DirectoryLocation.cs b/src/SharedMemoryStore/LockFree/DirectoryLocation.cs new file mode 100644 index 0000000..cbfb7e5 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/DirectoryLocation.cs @@ -0,0 +1,64 @@ +namespace SharedMemoryStore.LockFree; + +internal readonly struct DirectoryLocation +{ + private const ulong IndexMask = (1UL << 22) - 1; + private const ulong GenerationMask = (1UL << 33) - 1; + private const ulong UsedBitsMask = (1UL << 57) - 1; + + private DirectoryLocation(ulong value, int kind, long index, long generation) + { + Value = value; + Kind = kind; + Index = index; + Generation = generation; + } + + public ulong Value { get; } + public int Kind { get; } + public long Index { get; } + public long Generation { get; } + + public static ulong Encode(int kind, long index, long generation) + { + if (kind is < 1 or > 2) + { + throw new ArgumentOutOfRangeException(nameof(kind)); + } + + ArgumentOutOfRangeException.ThrowIfNegative(index); + if ((ulong)index > IndexMask) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + ArgumentOutOfRangeException.ThrowIfLessThan(generation, 1); + if ((ulong)generation > GenerationMask) + { + throw new ArgumentOutOfRangeException(nameof(generation)); + } + + return (uint)kind | ((ulong)index << 2) | ((ulong)generation << 24); + } + + public static DirectoryLocation Decode(ulong raw) + { + if (raw == 0) + { + return default; + } + + var kind = (int)(raw & 0x3); + var generation = (raw >> 24) & GenerationMask; + if (kind is < 1 or > 2 || generation == 0 || (raw & ~UsedBitsMask) != 0) + { + throw new ArgumentOutOfRangeException(nameof(raw)); + } + + return new DirectoryLocation( + raw, + kind, + checked((long)((raw >> 2) & IndexMask)), + checked((long)generation)); + } +} diff --git a/src/SharedMemoryStore/LockFree/DirectoryOperation.cs b/src/SharedMemoryStore/LockFree/DirectoryOperation.cs new file mode 100644 index 0000000..1082112 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/DirectoryOperation.cs @@ -0,0 +1,88 @@ +namespace SharedMemoryStore.LockFree; + +internal readonly struct DirectoryOperation +{ + private const ulong IndexMask = (1UL << 22) - 1; + private const ulong GenerationMask = (1UL << 33) - 1; + private const ulong UsedBitsMask = (1UL << 62) - 1; + + private DirectoryOperation(ulong value, int intent, int phase, int kind, long index, long generation) + { + Value = value; + Intent = intent; + Phase = phase; + Kind = kind; + Index = index; + Generation = generation; + } + + public ulong Value { get; } + public int Intent { get; } + public int Phase { get; } + public int Kind { get; } + public long Index { get; } + public long Generation { get; } + + public static ulong Encode(int intent, int phase, int targetKind, long targetIndex, long generation) + { + if (intent is < 0 or > 2) + { + throw new ArgumentOutOfRangeException(nameof(intent)); + } + + if (phase is < 0 or > 5) + { + throw new ArgumentOutOfRangeException(nameof(phase)); + } + + if (targetKind is < 0 or > 2) + { + throw new ArgumentOutOfRangeException(nameof(targetKind)); + } + + ArgumentOutOfRangeException.ThrowIfNegative(targetIndex); + if ((ulong)targetIndex > IndexMask) + { + throw new ArgumentOutOfRangeException(nameof(targetIndex)); + } + + if (intent == 0 && phase == 0 && targetKind == 0 && targetIndex == 0 && generation == 0) + { + return 0; + } + + ArgumentOutOfRangeException.ThrowIfLessThan(generation, 1); + if ((ulong)generation > GenerationMask) + { + throw new ArgumentOutOfRangeException(nameof(generation)); + } + + return (uint)intent + | ((ulong)(uint)phase << 2) + | ((ulong)(uint)targetKind << 5) + | ((ulong)targetIndex << 7) + | ((ulong)generation << 29); + } + + public static DirectoryOperation Decode(ulong raw) + { + if (raw == 0) + { + return default; + } + + var generation = (raw >> 29) & GenerationMask; + if (generation == 0 || (raw & ~UsedBitsMask) != 0) + { + throw new ArgumentOutOfRangeException(nameof(raw)); + } + + return new DirectoryOperation( + raw, + (int)(raw & 0x3), + (int)((raw >> 2) & 0x7), + (int)((raw >> 5) & 0x3), + checked((long)((raw >> 7) & IndexMask)), + checked((long)generation)); + } +} diff --git a/src/SharedMemoryStore/LockFree/IndexBinding.cs b/src/SharedMemoryStore/LockFree/IndexBinding.cs new file mode 100644 index 0000000..5e8de63 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/IndexBinding.cs @@ -0,0 +1,39 @@ +namespace SharedMemoryStore.LockFree; + +internal readonly struct IndexBinding +{ + private const ulong IndexMask = 0x7fff_ffffUL; + private const ulong GenerationMask = 0x1_ffff_ffffUL; + + private IndexBinding(ulong value, int slotIndex, long generation) + { + Value = value; + SlotIndex = slotIndex; + Generation = generation; + } + + public ulong Value { get; } + public int SlotIndex { get; } + public long Generation { get; } + + public static ulong Encode(int slotIndex, long generation) + { + ArgumentOutOfRangeException.ThrowIfNegative(slotIndex); + ArgumentOutOfRangeException.ThrowIfGreaterThan(slotIndex, int.MaxValue - 1); + ArgumentOutOfRangeException.ThrowIfLessThan(generation, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(generation, checked((long)GenerationMask)); + return ((ulong)generation << 31) | checked((uint)(slotIndex + 1)); + } + + public static IndexBinding Decode(ulong raw) + { + var indexPlusOne = raw & IndexMask; + var generation = raw >> 31; + if (indexPlusOne == 0 || generation == 0) + { + throw new ArgumentOutOfRangeException(nameof(raw)); + } + + return new IndexBinding(raw, checked((int)indexPlusOne - 1), checked((long)generation)); + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeByteOperations.cs b/src/SharedMemoryStore/LockFree/LockFreeByteOperations.cs new file mode 100644 index 0000000..ad95f38 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeByteOperations.cs @@ -0,0 +1,73 @@ +using SharedMemoryStore.Layout; + +namespace SharedMemoryStore.LockFree; + +/// +/// Budget-aware byte-linear primitives for the layout-v2 engine. Chunking is +/// local-only work control; it does not change the canonical hash or mapped +/// byte representation. +/// +internal static class LockFreeByteOperations +{ + private const int ChunkBytes = 64; + + internal static StoreStatus TryHash( + ReadOnlySpan key, + in LockFreeOperationBudget budget, + out ulong hash) + { + hash = StoreKey.HashSeed; + int chunkIndex = 0; + for (var offset = 0; offset < key.Length; offset += ChunkBytes, chunkIndex++) + { + StoreStatus bound = budget.CheckPeriodic(chunkIndex); + if (bound != StoreStatus.Success) + { + hash = 0; + return bound; + } + + int length = Math.Min(ChunkBytes, key.Length - offset); + hash = StoreKey.ContinueHash(hash, key.Slice(offset, length)); + } + + return budget.CheckPeriodic(chunkIndex); + } + + internal static StoreStatus TryCopy( + ReadOnlySpan source, + Span destination, + in LockFreeOperationBudget budget) + { + return TryCopy(source, destination, budget, out _); + } + + internal static StoreStatus TryCopy( + ReadOnlySpan source, + Span destination, + in LockFreeOperationBudget budget, + out int copiedBytes) + { + copiedBytes = 0; + if (destination.Length < source.Length) + { + return StoreStatus.CorruptStore; + } + + int chunkIndex = 0; + for (var offset = 0; offset < source.Length; offset += ChunkBytes, chunkIndex++) + { + StoreStatus bound = budget.CheckPeriodic(chunkIndex); + if (bound != StoreStatus.Success) + { + return bound; + } + + int length = Math.Min(ChunkBytes, source.Length - offset); + source.Slice(offset, length).CopyTo(destination.Slice(offset, length)); + copiedBytes += length; + } + + return budget.CheckPeriodic(chunkIndex); + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeCheckpoint.cs b/src/SharedMemoryStore/LockFree/LockFreeCheckpoint.cs new file mode 100644 index 0000000..25afb3a --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeCheckpoint.cs @@ -0,0 +1,813 @@ +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; + +namespace SharedMemoryStore.LockFree; + +/// +/// Stable identifiers for deterministic pauses immediately around protocol +/// transitions. Numeric assignments are append-only test/protocol identities. +/// +internal enum LockFreeCheckpointId +{ + PublishBeforeSlotClaim = 1, + PublishAfterCommitPublication = 2, + ReserveBeforeSlotClaim = 3, + ReserveAfterReservationPublication = 4, + CommitBeforePublicationCas = 5, + CommitAfterPublicationCas = 6, + AbortBeforeAbortCas = 7, + AbortAfterUnlinkCompletion = 8, + AcquireBeforeLeaseClaimCas = 9, + AcquireAfterPublishedRevalidation = 10, + ProjectBeforeHandleValidation = 11, + ProjectAfterSpanProjection = 12, + ReleaseBeforeActiveReleaseCas = 13, + ReleaseAfterRecordRecycle = 14, + RemoveBeforeLogicalRemovalCas = 15, + RemoveAfterLeaseClassification = 16, + ReclaimBeforeOwnershipCas = 17, + ReclaimAfterGenerationAdvance = 18, + DirectoryBeforeDescriptorPublication = 19, + DirectoryAfterDescriptorClear = 20, + DiagnosticsBeforeBoundedScan = 21, + DiagnosticsAfterSnapshotAssembly = 22, + RecoveryBeforeOwnerClassification = 23, + RecoveryAfterExactRecoveryCas = 24, + DisposalBeforeLocalGateClose = 25, + DisposalAfterParticipantRelease = 26, + ParticipantBeforeRegisteringCas = 27, + ParticipantAfterActivePublication = 28, + DirectoryAfterOperationValidation = 29, + DirectoryAfterLocationValidation = 30, + ReclaimAfterMetadataValidation = 31, + ParticipantAfterIdentityKindWrite = 32, + ParticipantAfterReservedWrite = 33, + ParticipantAfterProcessStartWrite = 34, + ParticipantAfterOpenSequenceWrite = 35, + AbortAfterOwnershipReleaseCas = 36, + SlotClaimAfterParticipantRecheck = 37, + ReleaseAfterOwnershipReleaseCas = 38, + AcquireAfterLeaseActivationBeforeFinalLookup = 39, + ReserveAfterExistingLookup = 40, + DirectoryBeforeSpillSummaryPublicationCas = 41, + DirectoryAfterSpillSummaryPublication = 42, + DirectoryAfterEmptySpillSummaryScan = 43, + DirectoryAfterSpillSummaryClear = 44, + ParticipantAfterRecoveryFenceBeforeReferenceScan = 45, + AdvanceBeforeBytesAdvancedCas = 46, + AdvanceAfterBytesAdvancedCas = 47, + DisposalAfterParticipantClosingPublication = 48, + ParticipantAfterRegistrationBeforeEngineConstruction = 49, + DirectoryAfterUnlinkOperationValidationBeforeLocationRead = 50, + DirectoryAfterLocationPublisherBindingValidation = 51, + DirectoryAfterCurrentOperationRevalidationBeforeDispatch = 52, + DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication = 53, + DirectoryAfterInsertCompletionStateValidationBeforeLocationRead = 54, + ReserveAfterDirectoryInsertBeforePendingClassification = 55, + DirectoryBeforeInsertOuterLoopBudgetCheck = 56, + DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation = 57, + StoreFullAfterFirstCollectBeforeVerification = 58, + StoreFullAfterExactDoubleCollect = 59, + DirectoryAfterUnlinkDescriptorClearBeforeGenerationAdvance = 60, + ParticipantAfterPidNamespaceWrite = 61, + DirectoryAfterCancelLocationClearBeforeDescriptorRejection = 62, + ReclaimAfterLeaseScanBeforeOwnershipCas = 63, + ParticipantBeforeReclaimGenerationAdvanceCas = 64, + ProjectAfterMetadataReadBeforeControlRevalidation = 65, + DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas = 66, + DirectoryAfterLocationPublicationBeforeSourceRevalidation = 67 +} + +internal enum LockFreeCheckpointFamily +{ + Publish, + Reserve, + Commit, + Abort, + Acquire, + Project, + Release, + Remove, + Reclaim, + Directory, + Diagnostics, + Recovery, + Disposal, + Participant, + Advance +} + +internal enum LockFreeCheckpointPosition +{ + Before, + After +} + +internal enum LockFreePauseClassification +{ + Unspecified, + NoSharedOwnership, + BoundedOwnership, + PublishedState +} + +internal enum LockFreeCrashClassification +{ + Unspecified, + NoSharedEffect, + ExplicitRecovery, + Helpable, + DurableOutcome +} + +internal enum LockFreeRaceClassification +{ + Unspecified, + ValidationWindow, + OrderingPoint, + HelpWindow, + ProjectionLifetime, + SnapshotWindow +} + +/// One canonical deterministic checkpoint and its safety metadata. +internal readonly record struct LockFreeCheckpointEntry( + LockFreeCheckpointId Id, + LockFreeCheckpointFamily Family, + LockFreeCheckpointPosition Position, + LockFreePauseClassification Pause, + LockFreeCrashClassification Crash, + LockFreeRaceClassification Race, + bool IsPublicOrderingPoint, + string Description); + +/// +/// Physical value-slot lifecycle transitions exposed only to friend-test +/// instrumentation. These observations are not protocol state and never +/// change the public API or the shared-memory layout. +/// +internal enum LockFreeSlotResourceEventKind +{ + Claim, + Free, + Retire +} + +/// +/// One exact winning slot-control CAS. identifies the +/// occupied lifecycle: a release to Free(g + 1) is therefore reported as +/// Free(g), pairing it with the claim that occupied generation g. +/// +internal readonly record struct LockFreeSlotResourceEvent( + LockFreeSlotResourceEventKind Kind, + int SlotIndex, + long Generation); + +/// +/// Test-only semantic recorder for the StoreFull candidate instant and its +/// later verification. Production's statically specialized no-op strategy does +/// not create a token, callback, allocation, or process-wide counter. +/// +internal interface ILockFreeStoreFullProofObserver +{ + long BeginCandidate(int slotCount); + + void CompleteCandidate(long token, bool confirmed); +} + +/// +/// Test-only semantic recorder for the LeaseTableFull candidate instant and +/// its later verification. Production's statically specialized no-op strategy +/// does not create a token, callback, allocation, or process-wide counter. +/// +internal interface ILockFreeLeaseTableFullProofObserver +{ + long BeginCandidate(int leaseRecordCount); + + void CompleteCandidate(long token, bool confirmed); +} + +/// +/// Canonical checkpoint inventory consumed by schedulers and cross-process +/// agents. New transition identifiers must be appended and classified here. +/// +internal static class LockFreeCheckpointCatalog +{ + private static readonly ReadOnlyCollection Catalog = Array.AsReadOnly( + new[] + { + Before(LockFreeCheckpointId.PublishBeforeSlotClaim, LockFreeCheckpointFamily.Publish, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.ValidationWindow, "Before a simple publisher claims a value slot."), + After(LockFreeCheckpointId.PublishAfterCommitPublication, LockFreeCheckpointFamily.Publish, + LockFreePauseClassification.PublishedState, LockFreeCrashClassification.DurableOutcome, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "After the value generation becomes published."), + + Before(LockFreeCheckpointId.ReserveBeforeSlotClaim, LockFreeCheckpointFamily.Reserve, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.ValidationWindow, "Before a reservation claims a value slot."), + After(LockFreeCheckpointId.ReserveAfterReservationPublication, LockFreeCheckpointFamily.Reserve, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "After exact-key reservation ownership is published."), + + Before(LockFreeCheckpointId.CommitBeforePublicationCas, LockFreeCheckpointFamily.Commit, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.OrderingPoint, "Immediately before the reservation publication CAS."), + After(LockFreeCheckpointId.CommitAfterPublicationCas, LockFreeCheckpointFamily.Commit, + LockFreePauseClassification.PublishedState, LockFreeCrashClassification.DurableOutcome, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "Immediately after the reservation publication CAS."), + + Before(LockFreeCheckpointId.AbortBeforeAbortCas, LockFreeCheckpointFamily.Abort, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.OrderingPoint, "Before reservation ownership changes to aborting."), + After(LockFreeCheckpointId.AbortAfterUnlinkCompletion, LockFreeCheckpointFamily.Abort, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.HelpWindow, "After the aborted key binding is unlinked."), + + Before(LockFreeCheckpointId.AcquireBeforeLeaseClaimCas, LockFreeCheckpointFamily.Acquire, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.OrderingPoint, "Before claiming a lease record."), + After(LockFreeCheckpointId.AcquireAfterPublishedRevalidation, LockFreeCheckpointFamily.Acquire, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "After an active lease revalidates the published generation."), + + Before(LockFreeCheckpointId.ProjectBeforeHandleValidation, LockFreeCheckpointFamily.Project, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.ProjectionLifetime, "Before a token validates its exact incarnation."), + After(LockFreeCheckpointId.ProjectAfterSpanProjection, LockFreeCheckpointFamily.Project, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.DurableOutcome, + LockFreeRaceClassification.ProjectionLifetime, "After a validated mapped-memory span is projected."), + + Before(LockFreeCheckpointId.ReleaseBeforeActiveReleaseCas, LockFreeCheckpointFamily.Release, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.OrderingPoint, "Before ending active lease protection."), + After(LockFreeCheckpointId.ReleaseAfterRecordRecycle, LockFreeCheckpointFamily.Release, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.HelpWindow, orderingPoint: true, + "After the exact lease-record incarnation is recycled or retired."), + + Before(LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas, LockFreeCheckpointFamily.Remove, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.OrderingPoint, "Before published changes to logically removed."), + After(LockFreeCheckpointId.RemoveAfterLeaseClassification, LockFreeCheckpointFamily.Remove, + LockFreePauseClassification.PublishedState, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "After the stable exact-lease classification scan."), + + Before(LockFreeCheckpointId.ReclaimBeforeOwnershipCas, LockFreeCheckpointFamily.Reclaim, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.OrderingPoint, "Before one helper claims exact reclamation."), + After(LockFreeCheckpointId.ReclaimAfterGenerationAdvance, LockFreeCheckpointFamily.Reclaim, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.HelpWindow, orderingPoint: true, + "After exact helper cleanup and generation advance."), + + Before(LockFreeCheckpointId.DirectoryBeforeDescriptorPublication, LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.HelpWindow, "Before publishing a complete directory mutation descriptor."), + After(LockFreeCheckpointId.DirectoryAfterDescriptorClear, LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.HelpWindow, orderingPoint: true, + "After completing and clearing the exact directory descriptor."), + + Before(LockFreeCheckpointId.DiagnosticsBeforeBoundedScan, LockFreeCheckpointFamily.Diagnostics, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.SnapshotWindow, "Before a diagnostics caller begins bounded scans."), + After(LockFreeCheckpointId.DiagnosticsAfterSnapshotAssembly, LockFreeCheckpointFamily.Diagnostics, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.SnapshotWindow, "After the moment-in-time snapshot is assembled."), + + Before(LockFreeCheckpointId.RecoveryBeforeOwnerClassification, LockFreeCheckpointFamily.Recovery, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.ValidationWindow, "Before classifying an exact participant incarnation."), + After(LockFreeCheckpointId.RecoveryAfterExactRecoveryCas, LockFreeCheckpointFamily.Recovery, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.HelpWindow, orderingPoint: true, + "After the exact stale-owner recovery CAS."), + + Before(LockFreeCheckpointId.DisposalBeforeLocalGateClose, LockFreeCheckpointFamily.Disposal, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.OrderingPoint, "Before the local handle rejects new operations."), + After(LockFreeCheckpointId.DisposalAfterParticipantRelease, LockFreeCheckpointFamily.Disposal, + LockFreePauseClassification.PublishedState, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.HelpWindow, orderingPoint: true, + "After bounded local cleanup and the exact participant retirement attempt."), + + Before(LockFreeCheckpointId.ParticipantBeforeRegisteringCas, LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.OrderingPoint, "Before publishing PID and participant incarnation."), + After(LockFreeCheckpointId.ParticipantAfterActivePublication, LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.DurableOutcome, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "After the participant record becomes active and usable by data claims."), + + After(LockFreeCheckpointId.DirectoryAfterOperationValidation, LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After exact operation/binding/control generation validation and before a helper side effect."), + After(LockFreeCheckpointId.DirectoryAfterLocationValidation, LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After exact location generation validation and before unlink side effects."), + After(LockFreeCheckpointId.ReclaimAfterMetadataValidation, LockFreeCheckpointFamily.Reclaim, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After exact reclaim-generation validation and before the final generation-advance CAS."), + + After(LockFreeCheckpointId.ParticipantAfterIdentityKindWrite, LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After a Registering owner writes identity kind while older ordinary fields may remain."), + After(LockFreeCheckpointId.ParticipantAfterReservedWrite, LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After a Registering owner clears the reserved field while older ordinary fields may remain."), + After(LockFreeCheckpointId.ParticipantAfterProcessStartWrite, LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After a Registering owner writes process-start identity before Active publication."), + After(LockFreeCheckpointId.ParticipantAfterOpenSequenceWrite, LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After a Registering owner writes open sequence before Active publication."), + After(LockFreeCheckpointId.AbortAfterOwnershipReleaseCas, LockFreeCheckpointFamily.Abort, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.HelpWindow, orderingPoint: true, + "After reservation ownership changes to universally helpable Aborting."), + After(LockFreeCheckpointId.SlotClaimAfterParticipantRecheck, LockFreeCheckpointFamily.Reserve, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After a slot claim revalidates its participant and before ordinary metadata writes."), + After(LockFreeCheckpointId.ReleaseAfterOwnershipReleaseCas, LockFreeCheckpointFamily.Release, + LockFreePauseClassification.PublishedState, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.HelpWindow, + "After an active lease publishes unowned Releasing and before exact-incarnation recycle."), + After(LockFreeCheckpointId.AcquireAfterLeaseActivationBeforeFinalLookup, LockFreeCheckpointFamily.Acquire, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After lease activation and before the acquire operation's final directory revalidation."), + After(LockFreeCheckpointId.ReserveAfterExistingLookup, LockFreeCheckpointFamily.Reserve, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.ValidationWindow, + "After reserve/publish observes an existing key and before its final existing-generation lookup."), + Before(LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After loading the prior spill-summary version and revalidating the exact insert, before its publication CAS."), + After(LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "After publishing Present(candidate) and before any overflow-cell publication CAS."), + After(LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After a stable empty full overflow scan and before the exact versioned-empty clear CAS."), + After(LockFreeCheckpointId.DirectoryAfterSpillSummaryClear, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "After Present(X) becomes Empty(X) and before releasing the exact canonical mutation."), + After(LockFreeCheckpointId.ParticipantAfterRecoveryFenceBeforeReferenceScan, + LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After fencing one stale participant as Recovering and before scanning exact owned references."), + Before(LockFreeCheckpointId.AdvanceBeforeBytesAdvancedCas, + LockFreeCheckpointFamily.Advance, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.OrderingPoint, + "After exact reservation and range validation, immediately before the BytesAdvanced CAS."), + After(LockFreeCheckpointId.AdvanceAfterBytesAdvancedCas, + LockFreeCheckpointFamily.Advance, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "Immediately after the checked BytesAdvanced CAS advances the exact reservation."), + After(LockFreeCheckpointId.DisposalAfterParticipantClosingPublication, + LockFreeCheckpointFamily.Disposal, + LockFreePauseClassification.PublishedState, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.OrderingPoint, orderingPoint: true, + "After exact Active-to-Closing publication and before bounded local resource cleanup."), + After(LockFreeCheckpointId.ParticipantAfterRegistrationBeforeEngineConstruction, + LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After participant Active publication returns successfully and before the engine can escape construction."), + After(LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After an unlink helper validates its exact operation and before reading the generation-tagged location word."), + After(LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After a location publisher validates its exact slot binding and before reading or publishing the location word."), + After(LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After a helper revalidates the exact current directory operation and slot state, before dispatching the phase-specific side effect."), + After(LockFreeCheckpointId.DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After a BindingChanged insert helper validates a non-canceling slot state and before publishing Reserved."), + After(LockFreeCheckpointId.DirectoryAfterInsertCompletionStateValidationBeforeLocationRead, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After TryInsert validates a completed non-canceling insert and before reading its generation-tagged location."), + After(LockFreeCheckpointId.ReserveAfterDirectoryInsertBeforePendingClassification, + LockFreeCheckpointFamily.Reserve, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After directory insertion succeeds and before the reserve path classifies whether the reservation remains pending."), + Before(LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.HelpWindow, + "Immediately before TryInsert checks its operation-wide budget at the outer helper-loop boundary."), + After(LockFreeCheckpointId.DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After an invalid binding's exact source word is confirmed and before the binding and source are jointly revalidated."), + After(LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification, + LockFreeCheckpointFamily.Reserve, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.ValidationWindow, + "After the first all-occupied slot-control collect and before the exact verification collect."), + After(LockFreeCheckpointId.StoreFullAfterExactDoubleCollect, + LockFreeCheckpointFamily.Reserve, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.NoSharedEffect, + LockFreeRaceClassification.ValidationWindow, + "After the second collect confirms the StoreFull candidate observed between the two collects."), + After(LockFreeCheckpointId.DirectoryAfterUnlinkDescriptorClearBeforeGenerationAdvance, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After an exact unlink descriptor is clear and before the reclaiming slot generation advances."), + After(LockFreeCheckpointId.ParticipantAfterPidNamespaceWrite, + LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ValidationWindow, + "After a Registering owner writes its Linux PID-namespace identity before Active publication."), + After(LockFreeCheckpointId.DirectoryAfterCancelLocationClearBeforeDescriptorRejection, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After a canceled insert clears its exact cell/location and before publishing the Rejected descriptor."), + After(LockFreeCheckpointId.ReclaimAfterLeaseScanBeforeOwnershipCas, + LockFreeCheckpointFamily.Reclaim, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After proving no exact active lease remains and before RemoveRequested changes to Reclaiming."), + Before(LockFreeCheckpointId.ParticipantBeforeReclaimGenerationAdvanceCas, + LockFreeCheckpointFamily.Participant, + LockFreePauseClassification.NoSharedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "Immediately before an unowned Reclaiming participant record advances or retires its generation."), + After(LockFreeCheckpointId.ProjectAfterMetadataReadBeforeControlRevalidation, + LockFreeCheckpointFamily.Project, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.ExplicitRecovery, + LockFreeRaceClassification.ProjectionLifetime, + "After lease projection metadata is read and before its exact slot control is revalidated."), + After(LockFreeCheckpointId.DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.BoundedOwnership, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, + "After an empty location and its exact publication source are revalidated, immediately before the zero-to-location CAS."), + After(LockFreeCheckpointId.DirectoryAfterLocationPublicationBeforeSourceRevalidation, + LockFreeCheckpointFamily.Directory, + LockFreePauseClassification.PublishedState, LockFreeCrashClassification.Helpable, + LockFreeRaceClassification.ValidationWindow, orderingPoint: true, + "Immediately after a zero-to-location CAS succeeds and before the publisher revalidates its exact source.") + }); + + internal static IReadOnlyList Entries => Catalog; + + internal static LockFreeCheckpointEntry Get(LockFreeCheckpointId id) + { + int index = (int)id - 1; + if ((uint)index >= (uint)Catalog.Count || Catalog[index].Id != id) + { + throw new ArgumentOutOfRangeException(nameof(id)); + } + + return Catalog[index]; + } + + private static LockFreeCheckpointEntry Before( + LockFreeCheckpointId id, + LockFreeCheckpointFamily family, + LockFreePauseClassification pause, + LockFreeCrashClassification crash, + LockFreeRaceClassification race, + string description) => + new(id, family, LockFreeCheckpointPosition.Before, pause, crash, race, false, description); + + private static LockFreeCheckpointEntry After( + LockFreeCheckpointId id, + LockFreeCheckpointFamily family, + LockFreePauseClassification pause, + LockFreeCrashClassification crash, + LockFreeRaceClassification race, + string description) => + After(id, family, pause, crash, race, orderingPoint: false, description); + + private static LockFreeCheckpointEntry After( + LockFreeCheckpointId id, + LockFreeCheckpointFamily family, + LockFreePauseClassification pause, + LockFreeCrashClassification crash, + LockFreeRaceClassification race, + bool orderingPoint, + string description) => + new(id, family, LockFreeCheckpointPosition.After, pause, crash, race, orderingPoint, description); +} + +/// +/// Static strategy contract used by generic protocol code. The strategy value +/// is passed by reference so instrumented tests can carry scheduler state while +/// the empty no-op specialization is completely elidable. +/// +internal interface ILockFreeCheckpointStrategy + where TSelf : struct, ILockFreeCheckpointStrategy +{ + static abstract void Reach(ref TSelf strategy, LockFreeCheckpointId checkpoint); + + static abstract void ObserveSlotResource( + ref TSelf strategy, + LockFreeSlotResourceEventKind kind, + int slotIndex, + long generation); + + static abstract long BeginStoreFullProof(ref TSelf strategy, int slotCount); + + static abstract void CompleteStoreFullProof( + ref TSelf strategy, + long token, + bool confirmed); + + static abstract long BeginLeaseTableFullProof( + ref TSelf strategy, + int leaseRecordCount); + + static abstract void CompleteLeaseTableFullProof( + ref TSelf strategy, + long token, + bool confirmed); +} + +/// Ordinary production specialization; deliberately contains no state or side effect. +internal struct NoOpLockFreeCheckpoint : ILockFreeCheckpointStrategy +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Reach(ref NoOpLockFreeCheckpoint strategy, LockFreeCheckpointId checkpoint) + { + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ObserveSlotResource( + ref NoOpLockFreeCheckpoint strategy, + LockFreeSlotResourceEventKind kind, + int slotIndex, + long generation) + { + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long BeginStoreFullProof(ref NoOpLockFreeCheckpoint strategy, int slotCount) => 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void CompleteStoreFullProof( + ref NoOpLockFreeCheckpoint strategy, + long token, + bool confirmed) + { + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long BeginLeaseTableFullProof( + ref NoOpLockFreeCheckpoint strategy, + int leaseRecordCount) => 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void CompleteLeaseTableFullProof( + ref NoOpLockFreeCheckpoint strategy, + long token, + bool confirmed) + { + } +} + +/// Friend-test specialization forwarding checkpoints to a controlled observer. +internal struct InstrumentedLockFreeCheckpoint : ILockFreeCheckpointStrategy +{ + private readonly Action _observer; + private readonly Action? _slotResourceObserver; + private readonly ILockFreeStoreFullProofObserver? _storeFullProofObserver; + private readonly ILockFreeLeaseTableFullProofObserver? _leaseTableFullProofObserver; + + internal InstrumentedLockFreeCheckpoint(Action observer) + : this( + observer, + slotResourceObserver: null, + storeFullProofObserver: null, + leaseTableFullProofObserver: null) + { + } + + internal InstrumentedLockFreeCheckpoint( + Action observer, + Action? slotResourceObserver, + ILockFreeStoreFullProofObserver? storeFullProofObserver = null, + ILockFreeLeaseTableFullProofObserver? leaseTableFullProofObserver = null) + { + ArgumentNullException.ThrowIfNull(observer); + _observer = observer; + _slotResourceObserver = slotResourceObserver; + _storeFullProofObserver = storeFullProofObserver; + _leaseTableFullProofObserver = leaseTableFullProofObserver; + } + + public static void Reach(ref InstrumentedLockFreeCheckpoint strategy, LockFreeCheckpointId checkpoint) + { + strategy._observer(LockFreeCheckpointCatalog.Get(checkpoint)); + } + + public static void ObserveSlotResource( + ref InstrumentedLockFreeCheckpoint strategy, + LockFreeSlotResourceEventKind kind, + int slotIndex, + long generation) + { + strategy._slotResourceObserver?.Invoke(new LockFreeSlotResourceEvent( + kind, + slotIndex, + generation)); + } + + public static long BeginStoreFullProof( + ref InstrumentedLockFreeCheckpoint strategy, + int slotCount) => strategy._storeFullProofObserver?.BeginCandidate(slotCount) ?? 0; + + public static void CompleteStoreFullProof( + ref InstrumentedLockFreeCheckpoint strategy, + long token, + bool confirmed) + { + if (token != 0) + { + strategy._storeFullProofObserver?.CompleteCandidate(token, confirmed); + } + } + + public static long BeginLeaseTableFullProof( + ref InstrumentedLockFreeCheckpoint strategy, + int leaseRecordCount) => + strategy._leaseTableFullProofObserver?.BeginCandidate(leaseRecordCount) ?? 0; + + public static void CompleteLeaseTableFullProof( + ref InstrumentedLockFreeCheckpoint strategy, + long token, + bool confirmed) + { + if (token != 0) + { + strategy._leaseTableFullProofObserver?.CompleteCandidate(token, confirmed); + } + } +} + +/// Inlining gateway used by generic lock-free protocol components. +internal static class LockFreeCheckpoint +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void Reach(ref TCheckpoint strategy, LockFreeCheckpointId checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + TCheckpoint.Reach(ref strategy, checkpoint); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void ObserveSlotResource( + ref TCheckpoint strategy, + LockFreeSlotResourceEventKind kind, + int slotIndex, + long generation) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + TCheckpoint.ObserveSlotResource(ref strategy, kind, slotIndex, generation); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long BeginStoreFullProof( + ref TCheckpoint strategy, + int slotCount) + where TCheckpoint : struct, ILockFreeCheckpointStrategy => + TCheckpoint.BeginStoreFullProof(ref strategy, slotCount); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void CompleteStoreFullProof( + ref TCheckpoint strategy, + long token, + bool confirmed) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + TCheckpoint.CompleteStoreFullProof(ref strategy, token, confirmed); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long BeginLeaseTableFullProof( + ref TCheckpoint strategy, + int leaseRecordCount) + where TCheckpoint : struct, ILockFreeCheckpointStrategy => + TCheckpoint.BeginLeaseTableFullProof(ref strategy, leaseRecordCount); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void CompleteLeaseTableFullProof( + ref TCheckpoint strategy, + long token, + bool confirmed) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + TCheckpoint.CompleteLeaseTableFullProof(ref strategy, token, confirmed); + } +} + +/// +/// Factory intentionally exposed only through friend assemblies. Public store +/// construction never accepts checkpoint instrumentation. +/// +internal static class LockFreeCheckpointFactory +{ + internal static InstrumentedLockFreeCheckpoint CreateInstrumented( + Action observer) => new(observer); + + internal static InstrumentedLockFreeCheckpoint CreateInstrumented( + Action observer, + Action slotResourceObserver) + { + ArgumentNullException.ThrowIfNull(slotResourceObserver); + return new InstrumentedLockFreeCheckpoint(observer, slotResourceObserver); + } + + internal static InstrumentedLockFreeCheckpoint CreateInstrumented( + Action observer, + Action slotResourceObserver, + ILockFreeStoreFullProofObserver storeFullProofObserver) + { + ArgumentNullException.ThrowIfNull(slotResourceObserver); + ArgumentNullException.ThrowIfNull(storeFullProofObserver); + return new InstrumentedLockFreeCheckpoint( + observer, + slotResourceObserver, + storeFullProofObserver); + } + + internal static InstrumentedLockFreeCheckpoint CreateInstrumented( + Action observer, + ILockFreeLeaseTableFullProofObserver leaseTableFullProofObserver) + { + ArgumentNullException.ThrowIfNull(leaseTableFullProofObserver); + return new InstrumentedLockFreeCheckpoint( + observer, + slotResourceObserver: null, + storeFullProofObserver: null, + leaseTableFullProofObserver); + } + + internal static InstrumentedLockFreeCheckpoint CreateInstrumented( + Action observer, + Action slotResourceObserver, + ILockFreeStoreFullProofObserver storeFullProofObserver, + ILockFreeLeaseTableFullProofObserver leaseTableFullProofObserver) + { + ArgumentNullException.ThrowIfNull(slotResourceObserver); + ArgumentNullException.ThrowIfNull(storeFullProofObserver); + ArgumentNullException.ThrowIfNull(leaseTableFullProofObserver); + return new InstrumentedLockFreeCheckpoint( + observer, + slotResourceObserver, + storeFullProofObserver, + leaseTableFullProofObserver); + } +} + +/// +/// Friend-instrumentation bridge for facade ordering points that occur before +/// control enters the layout-v2 engine. Production engines still execute the +/// statically elidable no-op checkpoint path. +/// +internal interface ILockFreeCheckpointEmitter +{ + void ReachCheckpoint(LockFreeCheckpointId checkpoint); +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeCorruptionTrace.cs b/src/SharedMemoryStore/LockFree/LockFreeCorruptionTrace.cs new file mode 100644 index 0000000..0d5f4f6 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeCorruptionTrace.cs @@ -0,0 +1,35 @@ +using System.Runtime.CompilerServices; + +namespace SharedMemoryStore.LockFree; + +/// +/// Retains the first structural-failure origin on the current thread. The +/// success path never touches this state; stress harnesses can therefore +/// distinguish rare protocol failures without adding shared-memory traffic. +/// +internal static class LockFreeCorruptionTrace +{ + [ThreadStatic] + private static CorruptionOrigin? _first; + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static StoreStatus Corrupt( + string component, + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) + { + _first ??= new CorruptionOrigin(component, member, line); + return StoreStatus.CorruptStore; + } + + internal static string? Consume() + { + CorruptionOrigin? origin = _first; + _first = null; + return origin is { } value + ? $"{value.Component}.{value.Member}:{value.Line}" + : null; + } + + private readonly record struct CorruptionOrigin(string Component, string Member, int Line); +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeDiagnostics.cs b/src/SharedMemoryStore/LockFree/LockFreeDiagnostics.cs new file mode 100644 index 0000000..7035c4e --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeDiagnostics.cs @@ -0,0 +1,640 @@ +using SharedMemoryStore.Diagnostics; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Observational layout-v2 diagnostics. Every shared-state scan has a fixed +/// layout-derived bound, performs acquire reads only, and never helps or changes +/// protocol ownership in pursuit of a consistent snapshot. +/// +internal sealed unsafe class LockFreeDiagnostics +{ + private readonly byte* _mappingBase; + private readonly StoreLayoutV2 _layout; + private readonly StoreProtocolInfo _protocolInfo; + private readonly StoreDiagnostics _local = new(); + private readonly LockFreeTelemetry _telemetry; + private readonly LockFreeStoreControl? _storeControl; + private MetricsCache _lastMetrics; + + internal LockFreeDiagnostics( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + StoreProtocolInfo protocolInfo) + : this(region, layout, protocolInfo, new LockFreeTelemetry()) + { + } + + internal LockFreeDiagnostics( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + StoreProtocolInfo protocolInfo, + LockFreeTelemetry telemetry, + LockFreeStoreControl? storeControl = null) + { + ArgumentNullException.ThrowIfNull(region); + if (protocolInfo.Profile != StoreProfile.LockFree) + { + throw new ArgumentException("Layout-v2 diagnostics require the lock-free profile.", nameof(protocolInfo)); + } + + _mappingBase = region.Pointer; + _layout = layout; + _protocolInfo = protocolInfo; + _telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry)); + _storeControl = storeControl; + _lastMetrics = new MetricsCache(new EngineMetrics + { + TotalBytes = layout.TotalBytes, + SlotCount = layout.SlotCount, + ParticipantRecordCount = layout.ParticipantRecordCount, + IndexEntryCount = checked(layout.PrimaryLaneCount + layout.SlotCount) + }); + } + + /// Creates a bounded, cross-instant public snapshot. + internal DiagnosticsSnapshot CreateSnapshot() + { + ReachBeforeBoundedScan(); + _ = TryCreateSnapshotAfterBoundedPrecheck( + LockFreeOperationBudget.UnboundedScan, + out DiagnosticsSnapshot snapshot); + return snapshot; + } + + /// + /// Creates a disposal-safe snapshot without dereferencing the mapped region. + /// Dynamic shared occupancy is the last successfully scanned observation (or + /// zero when no scan occurred); immutable sizing and all managed-local + /// failure/telemetry counters remain observable. + /// + internal DiagnosticsSnapshot CreateDisposedSnapshot() + { + EngineMetrics metrics = Volatile.Read(ref _lastMetrics).Value with + { + LastObservedProbeLength = _telemetry.LastObservedOverflowScanLength, + MaxObservedProbeLength = _telemetry.MaxObservedOverflowScanLength, + OverflowScanCount = _telemetry.OverflowScanCount, + MaxObservedOverflowScanLength = _telemetry.MaxObservedOverflowScanLength, + CasRetryCount = _telemetry.CasRetryCount, + HelpedTransitionCount = _telemetry.HelpedTransitionCount, + ContentionBudgetExhaustionCount = _telemetry.ContentionBudgetExhaustionCount, + InvalidTokenCount = _telemetry.InvalidTokenCount, + StaleTokenCount = _telemetry.StaleTokenCount, + RecoveryAttemptCount = _telemetry.RecoveryAttemptCount, + RecoveredTransitionCount = _telemetry.RecoveredTransitionCount, + CurrentOwnerClassificationCount = _telemetry.CurrentOwnerClassificationCount, + LiveOwnerClassificationCount = _telemetry.LiveOwnerClassificationCount, + StaleOwnerClassificationCount = _telemetry.StaleOwnerClassificationCount, + UnsupportedOwnerClassificationCount = _telemetry.UnsupportedOwnerClassificationCount, + InconsistentOwnerClassificationCount = _telemetry.InconsistentOwnerClassificationCount, + ChangingOwnerClassificationCount = _telemetry.ChangingOwnerClassificationCount + }; + + return _local.CreateSnapshot( + StoreProfile.LockFree, + _protocolInfo, + metrics); + } + + /// Creates the bounded engine-neutral portion of a snapshot. + internal EngineMetrics ScanMetrics() + { + ReachBeforeBoundedScan(); + _ = TryScanMetricsAfterBoundedPrecheck( + LockFreeOperationBudget.UnboundedScan, + out EngineMetrics metrics); + return metrics; + } + + internal void ReachBeforeBoundedScan() + { + NoOpLockFreeCheckpoint checkpoint = default; + ReachBeforeBoundedScan(ref checkpoint); + } + + internal void ReachBeforeBoundedScan(ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy => + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.DiagnosticsBeforeBoundedScan); + + internal DiagnosticsSnapshot CreateSnapshotAfterBoundedPrecheck() + { + _ = TryCreateSnapshotAfterBoundedPrecheck( + LockFreeOperationBudget.UnboundedScan, + out DiagnosticsSnapshot snapshot); + return snapshot; + } + + internal StoreStatus TryCreateSnapshotAfterBoundedPrecheck( + in LockFreeOperationBudget budget, + out DiagnosticsSnapshot snapshot) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryCreateSnapshotAfterBoundedPrecheck(budget, ref checkpoint, out snapshot); + } + + internal StoreStatus TryCreateSnapshotAfterBoundedPrecheck( + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out DiagnosticsSnapshot snapshot) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + snapshot = default; + StoreStatus status = TryScanMetricsCore(budget, out EngineMetrics metrics); + if (status != StoreStatus.Success) + { + return status; + } + + snapshot = _local.CreateSnapshot( + StoreProfile.LockFree, + _protocolInfo, + metrics); + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.DiagnosticsAfterSnapshotAssembly); + return StoreStatus.Success; + } + + internal EngineMetrics ScanMetricsAfterBoundedPrecheck() + { + _ = TryScanMetricsAfterBoundedPrecheck( + LockFreeOperationBudget.UnboundedScan, + out EngineMetrics metrics); + return metrics; + } + + internal StoreStatus TryScanMetricsAfterBoundedPrecheck( + in LockFreeOperationBudget budget, + out EngineMetrics metrics) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryScanMetricsAfterBoundedPrecheck(budget, ref checkpoint, out metrics); + } + + internal StoreStatus TryScanMetricsAfterBoundedPrecheck( + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out EngineMetrics metrics) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + StoreStatus status = TryScanMetricsCore(budget, out metrics); + if (status != StoreStatus.Success) + { + return status; + } + + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.DiagnosticsAfterSnapshotAssembly); + return StoreStatus.Success; + } + + private StoreStatus TryScanMetricsCore( + in LockFreeOperationBudget budget, + out EngineMetrics metrics) + { + metrics = default; + var freeSlots = 0; + var initializingSlots = 0; + var reservedSlots = 0; + var publishedSlots = 0; + var pendingRemovalSlots = 0; + var reclaimingSlots = 0; + var retiredSlots = 0; + for (var index = 0; index < _layout.SlotCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref Slot(index).Control); + if (!LockFreeSlotTable.TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out _)) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeDiagnostics)); + } + + int state = State(control); + switch (state) + { + case LockFreeSlotTable.FreeState: + freeSlots++; + break; + case LockFreeSlotTable.InitializingState: + initializingSlots++; + break; + case LockFreeSlotTable.ReservedState: + reservedSlots++; + break; + case LockFreeSlotTable.PublishedState: + publishedSlots++; + break; + case LockFreeSlotTable.RemoveRequestedState: + pendingRemovalSlots++; + break; + case LockFreeSlotTable.AbortingState: + case LockFreeSlotTable.ReclaimingState: + reclaimingSlots++; + break; + case LockFreeSlotTable.RetiredState: + retiredSlots++; + break; + } + } + + var freeLeases = 0; + var claimingLeases = 0; + var activeLeases = 0; + var recoveringLeases = 0; + var retiredLeases = 0; + for (var index = 0; index < _layout.LeaseRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref Lease(index).Control); + if (!LockFreeLeaseRegistry.TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out _)) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeDiagnostics)); + } + + int state = State(control); + switch (state) + { + case LockFreeLeaseRegistry.FreeState: + freeLeases++; + break; + case LockFreeLeaseRegistry.ClaimingState: + claimingLeases++; + break; + case LockFreeLeaseRegistry.ActiveState: + activeLeases++; + break; + case LockFreeLeaseRegistry.ReleasingState: + case LockFreeLeaseRegistry.RecoveringState: + recoveringLeases++; + break; + case LockFreeLeaseRegistry.RetiredState: + retiredLeases++; + break; + } + } + + var freeParticipants = 0; + var registeringParticipants = 0; + var activeParticipants = 0; + var closingParticipants = 0; + var recoveringParticipants = 0; + var reclaimingParticipants = 0; + var retiredParticipants = 0; + for (var index = 0; index < _layout.ParticipantRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref Participant(index).Control); + if (!LockFreeParticipantRegistry.IsStructuralControlValid( + control, + _layout.ParticipantGenerationMask)) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeDiagnostics)); + } + + int state = State(control); + switch (state) + { + case LayoutV2Constants.ParticipantFree: + freeParticipants++; + break; + case LayoutV2Constants.ParticipantRegistering: + registeringParticipants++; + break; + case LayoutV2Constants.ParticipantActive: + activeParticipants++; + break; + case LayoutV2Constants.ParticipantClosing: + closingParticipants++; + break; + case LayoutV2Constants.ParticipantRecovering: + recoveringParticipants++; + break; + case LayoutV2Constants.ParticipantReclaiming: + reclaimingParticipants++; + break; + case LayoutV2Constants.ParticipantRetired: + retiredParticipants++; + break; + } + } + + var primaryOccupancy = 0; + var spilledBuckets = 0; + for (var bucket = 0; bucket < _layout.PrimaryBucketCount; bucket++) + { + StoreStatus bound = budget.CheckPeriodic(bucket); + if (bound != StoreStatus.Success) + { + return bound; + } + + ulong spillSummaryRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref BucketSpillSummary(bucket))); + SpillSummary spillSummary; + try + { + spillSummary = SpillSummary.Decode(spillSummaryRaw); + } + catch (ArgumentOutOfRangeException) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeDiagnostics)); + } + catch (OverflowException) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeDiagnostics)); + } + + if (!spillSummary.IsInitial + && (uint)spillSummary.SlotIndex >= (uint)_layout.SlotCount) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeDiagnostics)); + } + + spilledBuckets += spillSummary.IsPresent ? 1 : 0; + StoreStatus mutationStatus = TryReadStructurallyValidBindingReference( + ref BucketMutation(bucket), + out _); + if (mutationStatus != StoreStatus.Success) + { + return mutationStatus; + } + + for (var lane = 0; lane < LayoutV2Constants.PrimaryLanesPerBucket; lane++) + { + StoreStatus laneStatus = TryReadStructurallyValidBindingReference( + ref PrimaryLane(bucket, lane), + out ulong laneBinding); + if (laneStatus != StoreStatus.Success) + { + return laneStatus; + } + + primaryOccupancy += laneBinding == 0 ? 0 : 1; + } + } + + var overflowOccupancy = 0; + for (var index = 0; index < _layout.SlotCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + StoreStatus cellStatus = TryReadStructurallyValidBindingReference( + ref OverflowCell(index), + out ulong cellBinding); + if (cellStatus != StoreStatus.Success) + { + return cellStatus; + } + + overflowOccupancy += cellBinding == 0 ? 0 : 1; + } + + int indexEntryCount = checked(_layout.PrimaryLaneCount + _layout.SlotCount); + int occupiedIndexEntryCount = checked(primaryOccupancy + overflowOccupancy); + int lastOverflowScanLength = _telemetry.LastObservedOverflowScanLength; + int maxOverflowScanLength = _telemetry.MaxObservedOverflowScanLength; + metrics = new EngineMetrics + { + TotalBytes = _layout.TotalBytes, + SlotCount = _layout.SlotCount, + FreeSlotCount = freeSlots, + InitializingSlotCount = initializingSlots, + ReservedSlotCount = reservedSlots, + PublishedSlotCount = publishedSlots, + PendingRemovalCount = pendingRemovalSlots, + ReclaimingSlotCount = reclaimingSlots, + RetiredSlotCount = retiredSlots, + ActiveLeaseCount = activeLeases, + ClaimingLeaseCount = claimingLeases, + RecoveringLeaseCount = recoveringLeases, + FreeLeaseCount = freeLeases, + RetiredLeaseCount = retiredLeases, + ParticipantRecordCount = _layout.ParticipantRecordCount, + FreeParticipantCount = freeParticipants, + RegisteringParticipantCount = registeringParticipants, + ActiveParticipantCount = activeParticipants, + ClosingParticipantCount = closingParticipants, + RecoveringParticipantCount = recoveringParticipants, + ReclaimingParticipantCount = reclaimingParticipants, + RetiredParticipantCount = retiredParticipants, + IndexEntryCount = indexEntryCount, + OccupiedIndexEntryCount = occupiedIndexEntryCount, + TombstoneIndexEntryCount = 0, + EmptyIndexEntryCount = Math.Max(0, indexEntryCount - occupiedIndexEntryCount), + UsableIndexCapacity = freeSlots, + LastObservedProbeLength = lastOverflowScanLength, + MaxObservedProbeLength = maxOverflowScanLength, + IndexCompactionCount = 0, + PrimaryDirectoryOccupancy = primaryOccupancy, + SpilledBucketCount = spilledBuckets, + OverflowDirectoryOccupancy = overflowOccupancy, + OverflowScanCount = _telemetry.OverflowScanCount, + MaxObservedOverflowScanLength = maxOverflowScanLength, + CasRetryCount = _telemetry.CasRetryCount, + HelpedTransitionCount = _telemetry.HelpedTransitionCount, + ContentionBudgetExhaustionCount = _telemetry.ContentionBudgetExhaustionCount, + InvalidTokenCount = _telemetry.InvalidTokenCount, + StaleTokenCount = _telemetry.StaleTokenCount, + RecoveryAttemptCount = _telemetry.RecoveryAttemptCount, + RecoveredTransitionCount = _telemetry.RecoveredTransitionCount, + CurrentOwnerClassificationCount = _telemetry.CurrentOwnerClassificationCount, + LiveOwnerClassificationCount = _telemetry.LiveOwnerClassificationCount, + StaleOwnerClassificationCount = _telemetry.StaleOwnerClassificationCount, + UnsupportedOwnerClassificationCount = _telemetry.UnsupportedOwnerClassificationCount, + InconsistentOwnerClassificationCount = _telemetry.InconsistentOwnerClassificationCount, + ChangingOwnerClassificationCount = _telemetry.ChangingOwnerClassificationCount + }; + + Volatile.Write(ref _lastMetrics, new MetricsCache(metrics)); + + return StoreStatus.Success; + } + + private sealed class MetricsCache(EngineMetrics value) + { + internal EngineMetrics Value { get; } = value; + } + + /// Records one public status and returns it unchanged for call-site composition. + internal StoreStatus RecordStatus(StoreStatus status) + { + _local.Record(status); + if (status == StoreStatus.StoreBusy) + { + _telemetry.RecordContentionBudgetExhaustion(); + } + + return status; + } + + internal void RecordReservationAbort() => _local.RecordReservationAbort(); + + internal void RecordLeaseRecoveryResults(in LeaseRecoveryReport report) + { + _local.RecordLeaseRecoveryResults( + report.RecoveredLeaseCount, + report.ActiveLeaseCount, + report.UnsupportedLeaseCount, + report.FailedRecoveryCount); + RecordRecoveryAttempt(report.ScannedRecordCount); + RecordRecoveredTransition(report.RecoveredLeaseCount); + } + + internal void RecordReservationRecoveryResults(in ReservationRecoveryReport report) + { + _local.RecordReservationRecoveryResults( + report.RecoveredReservationCount, + report.ActiveReservationCount, + report.UnsupportedReservationCount, + report.FailedRecoveryCount); + RecordRecoveryAttempt(report.ScannedReservationCount); + RecordRecoveredTransition(report.RecoveredReservationCount); + } + + internal void RecordOverflowScan(int scannedCellCount) => + _telemetry.RecordOverflowScan(scannedCellCount); + + internal void RecordCasRetry(int count = 1) => _telemetry.RecordCasLoss(count); + + internal void RecordHelpedTransition(int count = 1) => + _telemetry.RecordHelpedTransition(count); + + internal void RecordInvalidToken(bool stale) + { + if (stale) + { + _telemetry.RecordInvalidToken(stale: true); + } + else + { + _telemetry.RecordInvalidToken(stale: false); + } + } + + internal void RecordRecoveryAttempt(int count = 1) => + _telemetry.RecordRecoveryAttempt(count); + + internal void RecordRecoveredTransition(int count = 1) => + _telemetry.RecordRecoveredTransition(count); + + internal void RecordOwnerClassification(ParticipantClassificationKind kind) + { + _telemetry.RecordOwnerClassification(kind); + } + + private ref ParticipantRecordV2 Participant(int index) => + ref *(ParticipantRecordV2*)( + _mappingBase + _layout.ParticipantOffset + ((long)index * _layout.ParticipantStride)); + + private ref LeaseRecordV2 Lease(int index) => + ref *(LeaseRecordV2*)( + _mappingBase + _layout.LeaseRegistryOffset + ((long)index * _layout.LeaseStride)); + + private ref ValueSlotMetadataV2 Slot(int index) => + ref *(ValueSlotMetadataV2*)( + _mappingBase + _layout.SlotMetadataOffset + ((long)index * _layout.SlotMetadataStride)); + + private ref long BucketSpillSummary(int bucket) => + ref *(long*)( + _mappingBase + _layout.PrimaryDirectoryOffset + ((long)bucket * _layout.PrimaryBucketStride)); + + private ref long BucketMutation(int bucket) => + ref *(long*)( + _mappingBase + + _layout.PrimaryDirectoryOffset + + ((long)bucket * _layout.PrimaryBucketStride) + + sizeof(long)); + + private ref long PrimaryLane(int bucket, int lane) => + ref *(long*)( + _mappingBase + + _layout.PrimaryDirectoryOffset + + ((long)bucket * _layout.PrimaryBucketStride) + + 16 + + (lane * sizeof(long))); + + private ref long OverflowCell(int index) => + ref *(long*)( + _mappingBase + _layout.OverflowDirectoryOffset + ((long)index * _layout.OverflowStride)); + + private StoreStatus TryReadStructurallyValidBindingReference( + ref long reference, + out ulong observed) + { + for (var attempt = 0; attempt < 8; attempt++) + { + observed = unchecked((ulong)AtomicControlWord.LoadAcquire(ref reference)); + if (observed == 0 || IsStructurallyValidBinding(observed)) + { + return StoreStatus.Success; + } + + if (unchecked((ulong)AtomicControlWord.LoadAcquire(ref reference)) == observed) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeDiagnostics)); + } + } + + observed = 0; + return StoreStatus.StoreBusy; + } + + private bool IsStructurallyValidBinding(ulong raw) + { + try + { + IndexBinding binding = IndexBinding.Decode(raw); + return (uint)binding.SlotIndex < (uint)_layout.SlotCount; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + catch (OverflowException) + { + return false; + } + } + + private static int State(long control) => (int)((ulong)control & 0x7UL); +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeInstrumentedStoreFactory.cs b/src/SharedMemoryStore/LockFree/LockFreeInstrumentedStoreFactory.cs new file mode 100644 index 0000000..d3f9344 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeInstrumentedStoreFactory.cs @@ -0,0 +1,94 @@ +using SharedMemoryStore.Interop; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Options; +using System.Diagnostics; + +namespace SharedMemoryStore.LockFree; + +/// +/// Friend-assembly-only construction seam for deterministic protocol pauses. +/// Ordinary public construction always uses the statically empty checkpoint +/// specialization. +/// +internal static class LockFreeInstrumentedStoreFactory +{ + internal static StoreOpenStatus TryCreateOrOpen( + SharedMemoryStoreOptions options, + InstrumentedLockFreeCheckpoint checkpoint, + out MemoryStore? store) + { + store = null; + if (options.Profile != StoreProfile.LockFree) + { + return StoreOpenStatus.InvalidOptions; + } + + StoreOpenStatus validation = SharedMemoryStoreOptionsValidator.Validate(options, out _); + if (validation != StoreOpenStatus.Success) + { + return validation; + } + + long waitStartTimestamp = Stopwatch.GetTimestamp(); + StoreOpenStatus mapped = SharedStorePlatform.TryBeginOpen( + options, + StoreWaitOptions.Default, + waitStartTimestamp, + out SharedStoreOpenScope? openScope); + if (mapped != StoreOpenStatus.Success || openScope is null) + { + return mapped; + } + + LockFreeStoreEngine? engine = null; + StoreOpenStatus status; + try + { + using (openScope) + { + status = LockFreeStoreEngine.TryCreateOrOpenUnderColdGate( + options, + StoreWaitOptions.Default, + waitStartTimestamp, + openScope.Region, + openScope.Synchronization, + openScope.Disposition, + checkpoint, + out engine); + if (status == StoreOpenStatus.Success && engine is not null) + { + openScope.TransferResourceOwnership(); + } + } + } + catch (UnauthorizedAccessException) + { + engine?.Dispose(); + return StoreOpenStatus.AccessDenied; + } + catch + { + engine?.Dispose(); + return StoreOpenStatus.MappingFailed; + } + + if (status != StoreOpenStatus.Success || engine is null) + { + return status; + } + + try + { + store = StoreEngineFactory.WrapOwnedEngine(engine); + return StoreOpenStatus.Success; + } + catch (UnauthorizedAccessException) + { + return StoreOpenStatus.AccessDenied; + } + catch + { + return StoreOpenStatus.MappingFailed; + } + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeKeyDirectory.cs b/src/SharedMemoryStore/LockFree/LockFreeKeyDirectory.cs new file mode 100644 index 0000000..f404605 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeKeyDirectory.cs @@ -0,0 +1,5273 @@ +using System.Runtime.CompilerServices; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Owns the layout-v2 binding directory. Slot lifecycle ownership remains in +/// the slot table; this component validates exact slot/key +/// incarnations and makes directory mutations helpable through the canonical +/// bucket descriptor. +/// +internal sealed unsafe class LockFreeKeyDirectory +{ + private const int IntentInsert = 1; + private const int IntentUnlink = 2; + private const int PhasePrepared = 1; + private const int PhaseTargetSelected = 2; + private const int PhaseBindingChanged = 3; + private const int PhaseRejected = 4; + private const int PhaseComplete = 5; + private const int TargetPrimary = 1; + private const int TargetOverflow = 2; + private const int SlotInitializing = 1; + private const int SlotReserved = 2; + private const int SlotPublished = 3; + private const int SlotRemoveRequested = 4; + private const int SlotAborting = 5; + private const int SlotReclaiming = 6; + private const int SlotRetired = 7; + private const int DefaultRetryBudget = 128; + private const ulong SlotGenerationMask = 0x1_ffff_ffffUL; + private const ulong SlotParticipantMask = 0x0fff_ffffUL; + + private readonly ISharedStoreRegion _region; + private readonly StoreLayoutV2 _layout; + private readonly int _bucketMask; + private readonly LockFreeTelemetry _telemetry; + private readonly LockFreeStoreControl? _storeControl; + + internal LockFreeKeyDirectory( + ISharedStoreRegion region, + StoreLayoutV2 layout) + : this(region, layout, new LockFreeTelemetry()) + { + } + + internal LockFreeKeyDirectory( + ISharedStoreRegion region, + StoreLayoutV2 layout, + LockFreeTelemetry telemetry, + LockFreeStoreControl? storeControl = null) + { + ArgumentNullException.ThrowIfNull(region); + if (!layout.FitsWithinTotalBytes() || region.Capacity < layout.RequiredBytes) + { + throw new ArgumentOutOfRangeException(nameof(layout)); + } + + _region = region; + _layout = layout; + _telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry)); + _storeControl = storeControl; + _bucketMask = layout.PrimaryBucketCount - 1; + if (layout.PrimaryBucketCount < 2 + || (layout.PrimaryBucketCount & _bucketMask) != 0 + || layout.OverflowDirectoryLength / layout.OverflowStride != layout.SlotCount) + { + throw new ArgumentException("Layout-v2 directory dimensions are inconsistent.", nameof(layout)); + } + } + + internal StoreStatus TryLookup( + ReadOnlySpan key, + ulong keyHash, + out ulong binding, + out DirectoryLocation location) => + TryLookup(key, keyHash, LockFreeOperationBudget.StructuralAttempt, out binding, out location); + + internal StoreStatus TryLookup( + ReadOnlySpan key, + ulong keyHash, + in LockFreeOperationBudget budget, + out ulong binding, + out DirectoryLocation location) + { + return FindExact(key, keyHash, excludedBinding: 0, budget, out binding, out location); + } + + /// + /// Reads the exact directory cell that supplied a successful lookup. This + /// is deliberately a source-word check only: slot lifecycle + /// classification remains owned by , while + /// the engine composes the two observations when a cached lookup would + /// otherwise be reported as structural corruption. + /// + internal StoreStatus TryConfirmExactLookupReference( + DirectoryLocation location, + ulong exactBinding, + out bool remainsExact) + { + remainsExact = false; + if (!TryDecodeBinding(exactBinding, out IndexBinding binding) + || binding.SlotIndex >= _layout.SlotCount + || location.Value == 0 + || location.Generation != binding.Generation + || !TryGetTargetCell(location.Kind, location.Index, out CellReference cell)) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + remainsExact = unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell.Value)) + == exactBinding; + return StoreStatus.Success; + } + + internal StoreStatus TryInsert( + ReadOnlySpan key, + ulong keyHash, + ulong candidateBinding, + out DirectoryLocation location) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryInsert( + key, + keyHash, + candidateBinding, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint, + out location); + } + + internal StoreStatus TryInsert( + ReadOnlySpan key, + ulong keyHash, + ulong candidateBinding, + in LockFreeOperationBudget budget, + out DirectoryLocation location) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryInsert(key, keyHash, candidateBinding, budget, ref checkpoint, out location); + } + + internal StoreStatus TryInsert( + ReadOnlySpan key, + ulong keyHash, + ulong candidateBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out DirectoryLocation location) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + location = default; + if (!TryDecodeBinding(candidateBinding, out var decoded) + || decoded.SlotIndex >= _layout.SlotCount) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + ref var slot = ref Slot(decoded.SlotIndex); + StoreStatus validationStatus = ValidateBinding( + candidateBinding, + keyHash, + key, + budget, + out BindingValidation validation); + if (validationStatus != StoreStatus.Success) + { + return validationStatus; + } + + if (validation == BindingValidation.Stale) + { + return StoreStatus.InvalidReservation; + } + + CurrentSlotStatus currentSlot = TryReadCurrentSlotStatus( + candidateBinding, + ref slot, + out int currentState, + out _); + if (validation != BindingValidation.Exact + || currentSlot == CurrentSlotStatus.Invalid) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (currentSlot != CurrentSlotStatus.Current) + { + return currentSlot == CurrentSlotStatus.Retry + ? StoreStatus.StoreBusy + : StoreStatus.InvalidReservation; + } + + if (currentState is not (SlotInitializing or SlotReserved)) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + var prepared = DirectoryOperation.Encode( + IntentInsert, + PhasePrepared, + targetKind: 0, + targetIndex: 0, + generation: decoded.Generation); + var prepareStatus = PrepareOperation( + ref slot, + candidateBinding, + prepared, + IntentInsert, + budget); + if (prepareStatus != StoreStatus.Success) + { + return prepareStatus; + } + + GetBuckets(keyHash, out var canonicalBucket, out _); + var claimStatus = ClaimMutation(canonicalBucket, candidateBinding, budget, ref checkpoint); + if (claimStatus != StoreStatus.Success) + { + return claimStatus; + } + + for (var attempt = 0; ; attempt++) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck); + StoreStatus observedOutcome = ObserveInsertOutcomeBeforeBudget( + candidateBinding, + decoded.Generation, + ref slot, + ref checkpoint, + out bool hasObservedOutcome, + out location); + if (hasObservedOutcome) + { + return observedOutcome; + } + + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + var helpStatus = HelpMutation(canonicalBucket, budget, ref checkpoint, maxSteps: 8); + if (helpStatus is not (StoreStatus.Success or StoreStatus.StoreBusy)) + { + return helpStatus; + } + + observedOutcome = ObserveInsertOutcomeBeforeBudget( + candidateBinding, + decoded.Generation, + ref slot, + ref checkpoint, + out hasObservedOutcome, + out location); + if (hasObservedOutcome) + { + return observedOutcome; + } + + var operationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)); + bool operationDecoded = TryDecodeOperation(operationRaw, out var operation); + if (!operationDecoded + || operation.Intent != IntentInsert + || operation.Generation != decoded.Generation) + { + if (IsCanceledInsertObservation( + candidateBinding, + ref slot, + operationRaw, + operationDecoded, + operation)) + { + return StoreStatus.InvalidReservation; + } + + CurrentSlotStatus bindingStatus = TryReadCurrentSlotStatus( + candidateBinding, + ref slot, + out _, + out _); + return bindingStatus switch + { + CurrentSlotStatus.Current or CurrentSlotStatus.Invalid => + CorruptFrom(nameof(LockFreeKeyDirectory)), + CurrentSlotStatus.Retry => StoreStatus.StoreBusy, + _ => StoreStatus.InvalidReservation, + }; + } + + if (IsCanceledInsertObservation( + candidateBinding, + ref slot, + operationRaw, + operationDecoded, + operation)) + { + return StoreStatus.InvalidReservation; + } + + if (operation.Phase == PhaseRejected) + { + return StoreStatus.DuplicateKey; + } + + if (operation.Phase == PhaseComplete) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterInsertCompletionStateValidationBeforeLocationRead); + var locationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + if (!TryDecodeLocation(locationRaw, out location) + || location.Generation != decoded.Generation) + { + StoreStatus revalidatedOutcome = ObserveInsertOutcomeBeforeBudget( + candidateBinding, + decoded.Generation, + ref slot, + ref checkpoint, + out bool hasRevalidatedOutcome, + out location); + if (hasRevalidatedOutcome) + { + return revalidatedOutcome; + } + + continue; + } + + return StoreStatus.Success; + } + + if (attempt + 1 >= DefaultRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + } + } + + internal StoreStatus TryUnlink(ulong exactBinding) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryUnlink( + exactBinding, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint); + } + + internal StoreStatus TryUnlink( + ulong exactBinding, + in LockFreeOperationBudget budget) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryUnlink(exactBinding, budget, ref checkpoint); + } + + internal StoreStatus TryUnlink( + ulong exactBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (!TryDecodeBinding(exactBinding, out var decoded) || decoded.SlotIndex >= _layout.SlotCount) + { + return CorruptHere(); + } + + ref var slot = ref Slot(decoded.SlotIndex); + CurrentSlotStatus currentSlot = TryReadCurrentSlotStatus( + exactBinding, + ref slot, + out int currentState, + out ulong keyHash); + if (currentSlot == CurrentSlotStatus.Invalid) + { + return CorruptHere(); + } + + if (currentSlot != CurrentSlotStatus.Current) + { + return currentSlot == CurrentSlotStatus.Retry + ? StoreStatus.StoreBusy + : StoreStatus.NotFound; + } + + if (currentState is not (SlotAborting or SlotReclaiming)) + { + return StoreStatus.StoreBusy; + } + + var prepared = DirectoryOperation.Encode( + IntentUnlink, + PhasePrepared, + targetKind: 0, + targetIndex: 0, + generation: decoded.Generation); + var prepareStatus = PrepareOperation( + ref slot, + exactBinding, + prepared, + IntentUnlink, + budget); + if (prepareStatus != StoreStatus.Success) + { + return prepareStatus; + } + + GetBuckets(keyHash, out var canonicalBucket, out _); + var claimStatus = ClaimMutation(canonicalBucket, exactBinding, budget, ref checkpoint); + if (claimStatus != StoreStatus.Success) + { + return claimStatus; + } + + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + ulong mutationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref BucketMutation(canonicalBucket))); + if (mutationRaw != exactBinding) + { + StoreStatus reclaimMutation = ClaimMutation( + canonicalBucket, + exactBinding, + budget, + ref checkpoint); + if (reclaimMutation == StoreStatus.StoreBusy) + { + Thread.SpinWait(4 << Math.Min(attempt, 10)); + continue; + } + + if (reclaimMutation != StoreStatus.Success) + { + return reclaimMutation; + } + } + + var helpStatus = HelpMutation(canonicalBucket, budget, ref checkpoint, maxSteps: 8); + if (helpStatus is not (StoreStatus.Success or StoreStatus.StoreBusy)) + { + return helpStatus; + } + + var operationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)); + if (operationRaw == 0) + { + return StoreStatus.Success; + } + + if (!TryDecodeOperation(operationRaw, out var operation) + || operation.Intent != IntentUnlink + || operation.Generation != decoded.Generation) + { + CurrentSlotStatus bindingStatus = TryReadCurrentSlotStatus( + exactBinding, + ref slot, + out _, + out _); + return bindingStatus is CurrentSlotStatus.Current or CurrentSlotStatus.Invalid + ? CorruptHere() + : bindingStatus == CurrentSlotStatus.Retry + ? StoreStatus.StoreBusy + : StoreStatus.Success; + } + + if (operation.Phase == PhaseComplete) + { + StoreStatus finishStatus = FinishUnlink( + canonicalBucket, + exactBinding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + if (finishStatus is not (StoreStatus.Success or StoreStatus.StoreBusy)) + { + return finishStatus; + } + } + + if (attempt + 1 >= DefaultRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + } + } + + internal StoreStatus HelpMutation(int canonicalBucketIndex, int maxSteps = DefaultRetryBudget) + { + NoOpLockFreeCheckpoint checkpoint = default; + return HelpMutation( + canonicalBucketIndex, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint, + maxSteps); + } + + internal StoreStatus HelpMutation( + int canonicalBucketIndex, + in LockFreeOperationBudget budget, + int maxSteps = DefaultRetryBudget) + { + NoOpLockFreeCheckpoint checkpoint = default; + return HelpMutation(canonicalBucketIndex, budget, ref checkpoint, maxSteps); + } + + internal StoreStatus HelpMutationForKeyHash( + ulong keyHash, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + int maxSteps = DefaultRetryBudget) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + GetBuckets(keyHash, out int canonicalBucketIndex, out _); + return HelpMutation( + canonicalBucketIndex, + budget, + ref checkpoint, + maxSteps); + } + + internal StoreStatus HelpMutation( + int canonicalBucketIndex, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + int maxSteps = DefaultRetryBudget) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if ((uint)canonicalBucketIndex >= (uint)_layout.PrimaryBucketCount || maxSteps <= 0) + { + return CorruptHere(); + } + + ref var mutation = ref BucketMutation(canonicalBucketIndex); + for (var step = 0; step < maxSteps; step++) + { + StoreStatus bound = budget.CheckPeriodic(step); + if (bound != StoreStatus.Success) + { + return bound; + } + + var mutationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref mutation)); + if (mutationRaw == 0) + { + return StoreStatus.Success; + } + + if (!TryDecodeBinding(mutationRaw, out var decoded) || decoded.SlotIndex >= _layout.SlotCount) + { + if (unchecked((ulong)AtomicControlWord.LoadAcquire(ref mutation)) == mutationRaw) + { + return CorruptHere(); + } + + continue; + } + + ref var slot = ref Slot(decoded.SlotIndex); + MutationSnapshotStatus snapshotStatus = TryReadMutationSnapshot( + mutationRaw, + ref slot, + out ulong keyHash, + out ulong operationRaw, + out DirectoryOperation operation, + out int slotState, + out _); + if (snapshotStatus == MutationSnapshotStatus.Retry) + { + continue; + } + + if (snapshotStatus == MutationSnapshotStatus.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref mutation, + mutationRaw, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + + continue; + } + + if (snapshotStatus == MutationSnapshotStatus.Invalid) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (operationRaw == 0) + { + if (slotState is SlotAborting or SlotReclaiming) + { + StoreStatus cleanup = TryClearBindingReference( + ref mutation, + mutationRaw, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + + continue; + } + + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (operation.Generation != decoded.Generation) + { + if (operation.Generation > decoded.Generation) + { + CurrentSlotStatus bindingStatus = TryReadCurrentSlotStatus( + mutationRaw, + ref slot, + out _, + out _); + if (bindingStatus == CurrentSlotStatus.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref mutation, + mutationRaw, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + + continue; + } + + if (bindingStatus == CurrentSlotStatus.Retry) + { + continue; + } + + // Never erase a future-generation descriptor on behalf of + // an older mutation. Seeing both as current is structural + // corruption rather than cleanup authority. + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + StoreStatus operationCleanup = TryClearOperationReference( + ref slot.DirectoryOperation, + operationRaw, + out _); + if (operationCleanup != StoreStatus.Success) + { + return operationCleanup; + } + + continue; + } + + GetBuckets(keyHash, out var actualCanonical, out _); + if (actualCanonical != canonicalBucketIndex) + { + CurrentSlotStatus bindingStatus = TryReadCurrentSlotStatus( + mutationRaw, + ref slot, + out _, + out _); + if (bindingStatus == CurrentSlotStatus.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref mutation, + mutationRaw, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + + continue; + } + + if (bindingStatus == CurrentSlotStatus.Retry) + { + continue; + } + + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterOperationValidation); + StoreStatus currentOperation = ClassifyCurrentOperation( + mutationRaw, + ref slot, + operationRaw, + operation, + out slotState); + if (currentOperation != StoreStatus.Success) + { + if (currentOperation == StoreStatus.CorruptStore) + { + return currentOperation; + } + + // The operation word may have advanced legitimately after the + // snapshot. Retain the discoverable mutation while this exact + // binding generation is current, and help its newer phase. + CurrentSlotStatus bindingStatus = TryReadCurrentSlotStatus( + mutationRaw, + ref slot, + out _, + out _); + if (bindingStatus == CurrentSlotStatus.Invalid) + { + // The operation comparison above is intentionally allowed to + // lose to a newer helper phase. Do not turn a split + // operation/slot snapshot into terminal corruption unless the + // exact canonical mutation, exact operation, location, control, + // and immutable binding tuple all remain stable together. + ulong observedLocation = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryLocation)); + ExactCanonicalSourceStatus sourceStatus = RevalidateExactCanonicalSource( + canonicalBucketIndex, + mutationRaw, + ref slot, + operationRaw, + operation, + requiredLocationRaw: observedLocation, + publicationLocationRaw: null, + sourceMetadataValid: true); + if (sourceStatus == ExactCanonicalSourceStatus.Invalid) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (sourceStatus == ExactCanonicalSourceStatus.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref mutation, + mutationRaw, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + } + + // Current means the first Invalid observation was transient; + // Advanced means the exact source was legitimately consumed; + // Retry means the tuple kept moving. In every case the outer + // loop must take a fresh canonical snapshot. + continue; + } + + if (bindingStatus == CurrentSlotStatus.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref mutation, + mutationRaw, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + } + + continue; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch); + + StoreStatus status = operation.Intent switch + { + IntentInsert => HelpInsert( + canonicalBucketIndex, + mutationRaw, + ref slot, + operationRaw, + operation, + slotState, + budget, + ref checkpoint), + IntentUnlink => HelpUnlink( + canonicalBucketIndex, + mutationRaw, + ref slot, + operationRaw, + operation, + keyHash, + budget, + ref checkpoint), + _ => CorruptFrom(nameof(LockFreeKeyDirectory)) + }; + if (status != StoreStatus.Success) + { + return status; + } + } + + return unchecked((ulong)AtomicControlWord.LoadAcquire(ref mutation)) == 0 + ? StoreStatus.Success + : StoreStatus.StoreBusy; + } + + internal int PrimaryOccupancy + { + get + { + var count = 0; + for (var index = 0; index < _layout.PrimaryLaneCount; index++) + { + count += AtomicControlWord.LoadAcquire(ref PrimaryCell(index)) == 0 ? 0 : 1; + } + + return count; + } + } + + internal int OverflowOccupancy + { + get + { + var count = 0; + for (var index = 0; index < _layout.SlotCount; index++) + { + count += AtomicControlWord.LoadAcquire(ref OverflowCell(index)) == 0 ? 0 : 1; + } + + return count; + } + } + + internal int SpilledBucketCount + { + get + { + var count = 0; + for (var index = 0; index < _layout.PrimaryBucketCount; index++) + { + ulong raw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref BucketSpillSummary(index))); + count += TryDecodeSpillSummary(raw, out SpillSummary summary) && summary.IsPresent ? 1 : 0; + } + + return count; + } + } + + internal ulong ReadSpillSummary(int canonicalBucketIndex) + { + ArgumentOutOfRangeException.ThrowIfNegative(canonicalBucketIndex); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( + canonicalBucketIndex, + _layout.PrimaryBucketCount); + return unchecked((ulong)AtomicControlWord.LoadAcquire( + ref BucketSpillSummary(canonicalBucketIndex))); + } + + internal ulong ReadCanonicalMutation(int canonicalBucketIndex) + { + ArgumentOutOfRangeException.ThrowIfNegative(canonicalBucketIndex); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( + canonicalBucketIndex, + _layout.PrimaryBucketCount); + return unchecked((ulong)AtomicControlWord.LoadAcquire( + ref BucketMutation(canonicalBucketIndex))); + } + + internal StoreStatus ContainsExactBindingReference( + ulong exactBinding, + in LockFreeOperationBudget budget, + out bool containsReference) + { + containsReference = false; + if (!TryDecodeBinding(exactBinding, out IndexBinding binding) + || binding.SlotIndex >= _layout.SlotCount) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + var probe = 0; + for (var bucketIndex = 0; bucketIndex < _layout.PrimaryBucketCount; bucketIndex++) + { + StoreStatus bound = budget.CheckPeriodic(probe++); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref long mutation = ref BucketMutation(bucketIndex); + StoreStatus referenceStatus = TryReadStructurallyValidBindingReference( + ref mutation, + out ulong observed); + if (referenceStatus != StoreStatus.Success) + { + return referenceStatus; + } + + if (observed == exactBinding) + { + containsReference = true; + return StoreStatus.Success; + } + } + + for (var cellIndex = 0; cellIndex < _layout.PrimaryLaneCount; cellIndex++) + { + StoreStatus bound = budget.CheckPeriodic(probe++); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref long cell = ref PrimaryCell(cellIndex); + StoreStatus referenceStatus = TryReadStructurallyValidBindingReference( + ref cell, + out ulong observed); + if (referenceStatus != StoreStatus.Success) + { + return referenceStatus; + } + + if (observed == exactBinding) + { + containsReference = true; + return StoreStatus.Success; + } + } + + for (var cellIndex = 0; cellIndex < _layout.SlotCount; cellIndex++) + { + StoreStatus bound = budget.CheckPeriodic(probe++); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref long cell = ref OverflowCell(cellIndex); + StoreStatus referenceStatus = TryReadStructurallyValidBindingReference( + ref cell, + out ulong observed); + if (referenceStatus != StoreStatus.Success) + { + return referenceStatus; + } + + if (observed == exactBinding) + { + containsReference = true; + return StoreStatus.Success; + } + } + + return StoreStatus.Success; + } + + private StoreStatus ClaimMutation( + int canonicalBucketIndex, + ulong binding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + ref var mutation = ref BucketMutation(canonicalBucketIndex); + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + var observed = unchecked((ulong)AtomicControlWord.LoadAcquire(ref mutation)); + if (observed == binding) + { + return StoreStatus.Success; + } + + if (observed == 0 + && AtomicControlWord.CompareExchange( + ref mutation, + unchecked((long)binding), + comparand: 0) == 0) + { + return StoreStatus.Success; + } + + var helpStatus = HelpMutation( + canonicalBucketIndex, + budget, + ref checkpoint, + maxSteps: 8); + if (helpStatus is not (StoreStatus.Success or StoreStatus.StoreBusy)) + { + return helpStatus; + } + + if (attempt + 1 >= DefaultRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + } + } + + private StoreStatus HelpInsert( + int canonicalBucketIndex, + ulong binding, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + DirectoryOperation operation, + int slotState, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (IsInsertCancellationState(slotState)) + { + return CancelInsert( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + } + + if (operation.Phase is PhaseRejected or PhaseComplete) + { + return CompleteMutationRelease( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + } + + if (operation.Phase == PhasePrepared) + { + StoreStatus currentAfterCellClaim = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out slotState); + if (currentAfterCellClaim != StoreStatus.Success) + { + return currentAfterCellClaim == StoreStatus.NotFound + ? StoreStatus.Success + : currentAfterCellClaim; + } + + if (IsInsertCancellationState(slotState)) + { + return CancelInsert( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + } + + if (slotState != SlotInitializing || !TryGetKey(ref slot, out var key)) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + ulong keyHash = slot.KeyHash; + var lookupStatus = FindExact( + key, + keyHash, + excludedBinding: 0, + budget, + ref checkpoint, + out var existing, + out var existingLocation); + ulong next; + if (lookupStatus == StoreStatus.Success) + { + next = existing == binding + ? DirectoryOperation.Encode( + IntentInsert, + PhaseTargetSelected, + existingLocation.Kind, + existingLocation.Index, + operation.Generation) + : DirectoryOperation.Encode( + IntentInsert, + PhaseRejected, + targetKind: 0, + targetIndex: 0, + generation: operation.Generation); + } + else if (lookupStatus == StoreStatus.NotFound) + { + var targetStatus = SelectInsertTarget( + keyHash, + operation.Generation, + budget, + ref checkpoint, + out var target); + if (targetStatus != StoreStatus.Success) + { + return targetStatus; + } + + next = DirectoryOperation.Encode( + IntentInsert, + PhaseTargetSelected, + target.Kind, + target.Index, + operation.Generation); + } + else + { + return lookupStatus; + } + + ulong observedNext = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)next), + unchecked((long)operationRaw))); + return ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + next, + observedNext); + } + + if (operation.Phase == PhaseTargetSelected) + { + if (!TryGetTargetCell(operation.Kind, operation.Index, out var cell)) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + StoreStatus currentAfterCellClaim = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out slotState); + if (currentAfterCellClaim != StoreStatus.Success) + { + return currentAfterCellClaim == StoreStatus.NotFound + ? StoreStatus.Success + : currentAfterCellClaim; + } + + if (IsInsertCancellationState(slotState)) + { + return CancelInsert( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + } + + if (operation.Kind == TargetOverflow) + { + StoreStatus summaryStatus = PrepareOverflowPublication( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint, + out bool mayPublish); + if (summaryStatus != StoreStatus.Success || !mayPublish) + { + return summaryStatus; + } + } + + var observed = unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell.Value)); + if (observed != binding) + { + if (observed == 0) + { + observed = unchecked((ulong)AtomicControlWord.CompareExchange( + ref cell.Value, + unchecked((long)binding), + comparand: 0)); + } + + if (observed != 0 && observed != binding) + { + StoreStatus validationStatus = ValidateBinding( + observed, + expectedHash: null, + expectedKey: default, + budget, + out BindingValidation observedValidation); + if (validationStatus != StoreStatus.Success) + { + return validationStatus; + } + + if (observedValidation == BindingValidation.Stale) + { + return TryClearBindingReference( + ref cell.Value, + observed, + out _); + } + + var prepared = DirectoryOperation.Encode( + IntentInsert, + PhasePrepared, + targetKind: 0, + targetIndex: 0, + generation: operation.Generation); + ulong observedPrepared = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)prepared), + unchecked((long)operationRaw))); + return ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + prepared, + observedPrepared); + } + } + + StoreStatus postClaimOperationStatus = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out slotState); + if (postClaimOperationStatus != StoreStatus.Success) + { + // Another helper may have consumed this exact cell and advanced + // the same insert. In that case the cell is now the published + // directory entry, not residue owned by this delayed helper. + // Only roll it back when the descriptor did not reach the + // side-effect-committed phase. + ulong currentOperationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + if (!IsSameOrLaterInsertPhase(currentOperationRaw, operation)) + { + StoreStatus cleanup = TryClearBindingReference( + ref cell.Value, + binding, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + } + + return postClaimOperationStatus == StoreStatus.NotFound + ? StoreStatus.Success + : postClaimOperationStatus; + } + + if (IsInsertCancellationState(slotState)) + { + return CancelInsert( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + } + + ulong locationRaw = DirectoryLocation.Encode( + operation.Kind, + operation.Index, + operation.Generation); + StoreStatus locationPublication = TryPublishExactLocation( + canonicalBucketIndex, + ref slot, + binding, + operationRaw, + operation, + locationRaw, + ref checkpoint, + out bool locationPublished); + if (locationPublication != StoreStatus.Success) + { + return locationPublication; + } + + if (!locationPublished) + { + // Losing the exact operation source is not proof that the + // claimed cell is abandoned. Another helper may already have + // published this same binding and advanced the descriptor; in + // that case clearing the cell makes a committed key disappear + // and permits a second same-key insert to report success. + // Cancellation/recovery owns abandoned-cell cleanup while its + // exact generation is still canonical. + return StoreStatus.Success; + } + + var changed = DirectoryOperation.Encode( + IntentInsert, + PhaseBindingChanged, + operation.Kind, + operation.Index, + operation.Generation); + ulong observedOperation = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)changed), + unchecked((long)operationRaw))); + StoreStatus operationCas = ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + changed, + observedOperation); + if (operationCas != StoreStatus.Success) + { + return operationCas; + } + + if (observedOperation != operationRaw + && !IsSameOrLaterInsertPhase(observedOperation, operation)) + { + StoreStatus cellCleanup = TryClearBindingReference( + ref cell.Value, + binding, + out _); + if (cellCleanup != StoreStatus.Success) + { + return cellCleanup; + } + + StoreStatus locationCleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + locationRaw, + out _); + if (locationCleanup != StoreStatus.Success) + { + return locationCleanup; + } + } + + return StoreStatus.Success; + } + + if (operation.Phase == PhaseBindingChanged) + { + StoreStatus currentOperation = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out slotState); + if (currentOperation != StoreStatus.Success) + { + return currentOperation == StoreStatus.NotFound + ? StoreStatus.Success + : currentOperation; + } + + if (IsInsertCancellationState(slotState)) + { + return CancelInsert( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication); + + var reserveStatus = PublishReserved(ref slot, binding); + if (reserveStatus != StoreStatus.Success) + { + currentOperation = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out int currentState); + if (currentOperation != StoreStatus.Success) + { + return currentOperation == StoreStatus.NotFound + ? StoreStatus.Success + : currentOperation; + } + + if (IsInsertCancellationState(currentState)) + { + return CancelInsert( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + } + + return reserveStatus == StoreStatus.CorruptStore + ? CorruptFrom(nameof(LockFreeKeyDirectory)) + : reserveStatus; + } + + var complete = DirectoryOperation.Encode( + IntentInsert, + PhaseComplete, + operation.Kind, + operation.Index, + operation.Generation); + ulong observedComplete = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)complete), + unchecked((long)operationRaw))); + return ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + complete, + observedComplete); + } + + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + private StoreStatus PrepareOverflowPublication( + int canonicalBucketIndex, + ulong binding, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + DirectoryOperation operation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out bool mayPublish) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + mayPublish = false; + ulong desired; + try + { + desired = SpillSummary.EncodePresent(binding); + } + catch (ArgumentOutOfRangeException) + { + return CorruptHere(); + } + catch (OverflowException) + { + return CorruptHere(); + } + + ref long summaryWord = ref BucketSpillSummary(canonicalBucketIndex); + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + StoreStatus canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + out int initialState); + if (canonicalOperation != StoreStatus.Success + || IsInsertCancellationState(initialState)) + { + return canonicalOperation == StoreStatus.NotFound + ? StoreStatus.Success + : canonicalOperation; + } + + ulong observedRaw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref summaryWord)); + if (!TryDecodeSpillSummary(observedRaw, out SpillSummary observed)) + { + return CorruptHere(); + } + + if (observed.Binding == binding) + { + if (!observed.IsPresent) + { + // Empty(binding) is a terminal version for this exact + // insertion lifecycle. Cancellation may have published it + // while this TargetSelected helper was between state + // validations; that legal transition suppresses further + // publication. Otherwise re-publishing it would recreate + // an ABA value and therefore fails closed. + StoreStatus currentOperation = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out int currentState); + if (currentOperation != StoreStatus.Success + || IsInsertCancellationState(currentState)) + { + return currentOperation == StoreStatus.NotFound + ? StoreStatus.Success + : currentOperation; + } + + return CorruptHere(); + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication); + canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + out int publishedState); + if (canonicalOperation == StoreStatus.Success + && !IsInsertCancellationState(publishedState) + && unchecked((ulong)AtomicControlWord.LoadAcquire(ref summaryWord)) == desired) + { + mayPublish = true; + } + + if (canonicalOperation is not (StoreStatus.Success or StoreStatus.NotFound)) + { + return canonicalOperation; + } + + return StoreStatus.Success; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas); + canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + out int preCasState); + if (canonicalOperation != StoreStatus.Success + || IsInsertCancellationState(preCasState)) + { + return canonicalOperation == StoreStatus.NotFound + ? StoreStatus.Success + : canonicalOperation; + } + + ulong exchanged = unchecked((ulong)AtomicControlWord.CompareExchange( + ref summaryWord, + unchecked((long)desired), + unchecked((long)observedRaw))); + StoreStatus summaryCas = ValidateSpillSummaryCasObservation( + ref summaryWord, + observedRaw, + desired, + exchanged); + if (summaryCas != StoreStatus.Success) + { + return summaryCas; + } + + if (exchanged != observedRaw) + { + _telemetry.RecordCasLoss(); + continue; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication); + canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + out int postCasState); + if (canonicalOperation == StoreStatus.Success + && !IsInsertCancellationState(postCasState) + && unchecked((ulong)AtomicControlWord.LoadAcquire(ref summaryWord)) == desired) + { + mayPublish = true; + } + + if (canonicalOperation is not (StoreStatus.Success or StoreStatus.NotFound)) + { + return canonicalOperation; + } + + // A failed post-CAS validation deliberately retains Present(binding). + // It is a conservative positive and must never be rolled back by a + // setter that no longer owns the exact canonical mutation. + return StoreStatus.Success; + } + } + + private StoreStatus HelpUnlink( + int canonicalBucketIndex, + ulong binding, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + DirectoryOperation operation, + ulong keyHash, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (operation.Phase == PhaseComplete) + { + return FinishUnlink( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + } + + if (operation.Phase == PhasePrepared) + { + StoreStatus currentOperation = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out int slotState); + if (currentOperation != StoreStatus.Success + || slotState is not (SlotAborting or SlotReclaiming)) + { + return currentOperation == StoreStatus.NotFound + ? StoreStatus.Success + : currentOperation; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead); + var locationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + DirectoryLocation location; + if (locationRaw == 0) + { + StoreStatus findLocation = TryFindExactBindingLocation( + binding, + keyHash, + operation.Generation, + budget, + out location); + if (findLocation == StoreStatus.NotFound) + { + var completeWithoutTarget = DirectoryOperation.Encode( + IntentUnlink, + PhaseComplete, + targetKind: 0, + targetIndex: 0, + generation: operation.Generation); + ulong observedOperation = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)completeWithoutTarget), + unchecked((long)operationRaw))); + return ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + completeWithoutTarget, + observedOperation); + } + + if (findLocation != StoreStatus.Success) + { + return findLocation; + } + + ulong recoveredLocation = location.Value; + StoreStatus locationPublication = TryPublishExactLocation( + canonicalBucketIndex, + ref slot, + binding, + operationRaw, + operation, + recoveredLocation, + ref checkpoint, + out bool locationPublished); + if (locationPublication != StoreStatus.Success) + { + return locationPublication; + } + + if (!locationPublished) + { + return StoreStatus.Success; + } + + locationRaw = recoveredLocation; + } + else if (!TryDecodeLocation(locationRaw, out location)) + { + ExactCanonicalSourceStatus malformedSource = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: locationRaw, + publicationLocationRaw: null, + sourceMetadataValid: false); + return CompleteLocationSourceLoss( + malformedSource, + ref slot, + binding, + operation, + exactLocation: locationRaw, + cleanupExactLocation: false); + } + + if (location.Generation != operation.Generation) + { + bool olderLocationValid = location.Generation < operation.Generation + && TryGetTargetCell(location.Kind, location.Index, out _); + ExactCanonicalSourceStatus generationSource = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: locationRaw, + publicationLocationRaw: null, + sourceMetadataValid: olderLocationValid); + if (generationSource != ExactCanonicalSourceStatus.Current) + { + return CompleteLocationSourceLoss( + generationSource, + ref slot, + binding, + operation, + exactLocation: locationRaw, + cleanupExactLocation: false); + } + + // Current is possible only for a structurally valid older + // residue. A future location with an otherwise exact old-G + // tuple is impossible and was classified Invalid above. + StoreStatus locationCleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + locationRaw, + out _); + if (locationCleanup != StoreStatus.Success) + { + return locationCleanup; + } + + return StoreStatus.Success; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterLocationValidation); + currentOperation = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out slotState); + if (currentOperation != StoreStatus.Success) + { + return currentOperation == StoreStatus.NotFound + ? StoreStatus.Success + : currentOperation; + } + + var target = DirectoryOperation.Encode( + IntentUnlink, + PhaseTargetSelected, + location.Kind, + location.Index, + operation.Generation); + ulong observedTarget = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)target), + unchecked((long)operationRaw))); + return ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + target, + observedTarget); + } + + if (operation.Phase == PhaseTargetSelected) + { + if (!TryGetTargetCell(operation.Kind, operation.Index, out var cell)) + { + return CorruptHere(); + } + + StoreStatus currentOperation = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out int slotState); + if (currentOperation != StoreStatus.Success + || slotState is not (SlotAborting or SlotReclaiming)) + { + return currentOperation == StoreStatus.NotFound + ? StoreStatus.Success + : currentOperation; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead); + ulong expectedLocation = DirectoryLocation.Encode( + operation.Kind, + operation.Index, + operation.Generation); + ulong observedLocation = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + if (observedLocation != 0 && observedLocation != expectedLocation) + { + if (!TryDecodeLocation(observedLocation, out var otherLocation)) + { + ExactCanonicalSourceStatus malformedSource = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: observedLocation, + publicationLocationRaw: null, + sourceMetadataValid: false); + return CompleteLocationSourceLoss( + malformedSource, + ref slot, + binding, + operation, + expectedLocation, + cleanupExactLocation: false); + } + + if (otherLocation.Generation != operation.Generation) + { + bool olderLocationValid = otherLocation.Generation < operation.Generation + && TryGetTargetCell(otherLocation.Kind, otherLocation.Index, out _); + ExactCanonicalSourceStatus generationSource = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: observedLocation, + publicationLocationRaw: null, + sourceMetadataValid: olderLocationValid); + if (generationSource != ExactCanonicalSourceStatus.Current) + { + return CompleteLocationSourceLoss( + generationSource, + ref slot, + binding, + operation, + expectedLocation, + cleanupExactLocation: false); + } + + StoreStatus staleLocationCleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + observedLocation, + out _); + if (staleLocationCleanup != StoreStatus.Success) + { + return staleLocationCleanup; + } + + return StoreStatus.Success; + } + + bool otherTargetValid = TryGetTargetCell( + otherLocation.Kind, + otherLocation.Index, + out CellReference otherCell); + ExactCanonicalSourceStatus conflictSource = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: observedLocation, + publicationLocationRaw: expectedLocation, + sourceMetadataValid: otherTargetValid, + publicationTargetMustMatchBinding: false, + secondaryStructuralLocationRaw: observedLocation); + if (conflictSource != ExactCanonicalSourceStatus.Current) + { + return CompleteLocationSourceLoss( + conflictSource, + ref slot, + binding, + operation, + expectedLocation, + cleanupExactLocation: false); + } + + // A delayed Unlink/Prepared publisher may install a second + // same-generation location after this descriptor selected its + // first target. Both witnesses belong to the same removal. Use + // exact comparands for both cells and the observed location, so + // a replacement or reused generation is never overwritten. + StoreStatus selectedCleanup = TryClearBindingReference( + ref cell.Value, + binding, + out _); + if (selectedCleanup != StoreStatus.Success) + { + return selectedCleanup; + } + + StoreStatus otherCleanup = TryClearBindingReference( + ref otherCell.Value, + binding, + out _); + if (otherCleanup != StoreStatus.Success) + { + return otherCleanup; + } + + StoreStatus conflictLocationCleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + observedLocation, + out _); + if (conflictLocationCleanup != StoreStatus.Success) + { + return conflictLocationCleanup; + } + + var conflictChanged = DirectoryOperation.Encode( + IntentUnlink, + PhaseBindingChanged, + operation.Kind, + operation.Index, + operation.Generation); + ulong observedConflictChanged = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)conflictChanged), + unchecked((long)operationRaw))); + return ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + conflictChanged, + observedConflictChanged); + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterLocationValidation); + currentOperation = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out slotState); + if (currentOperation != StoreStatus.Success) + { + return currentOperation == StoreStatus.NotFound + ? StoreStatus.Success + : currentOperation; + } + + StoreStatus cellCleanup = TryClearBindingReference( + ref cell.Value, + binding, + out _); + if (cellCleanup != StoreStatus.Success) + { + return cellCleanup; + } + + StoreStatus locationCleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + expectedLocation, + out _); + if (locationCleanup != StoreStatus.Success) + { + return locationCleanup; + } + + var changed = DirectoryOperation.Encode( + IntentUnlink, + PhaseBindingChanged, + operation.Kind, + operation.Index, + operation.Generation); + ulong observedChanged = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)changed), + unchecked((long)operationRaw))); + return ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + changed, + observedChanged); + } + + if (operation.Phase == PhaseBindingChanged) + { + StoreStatus currentOperation = ClassifyCurrentOperation( + binding, + ref slot, + operationRaw, + operation, + out int slotState); + if (currentOperation != StoreStatus.Success + || slotState is not (SlotAborting or SlotReclaiming)) + { + return currentOperation == StoreStatus.NotFound + ? StoreStatus.Success + : currentOperation; + } + + var complete = DirectoryOperation.Encode( + IntentUnlink, + PhaseComplete, + operation.Kind, + operation.Index, + operation.Generation); + ulong observedComplete = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)complete), + unchecked((long)operationRaw))); + return ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + complete, + observedComplete); + } + + return CorruptHere(); + } + + private StoreStatus FinishUnlink( + int canonicalBucketIndex, + ulong binding, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + DirectoryOperation operation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (operation.Kind is TargetPrimary or TargetOverflow) + { + ulong expectedLocation = DirectoryLocation.Encode( + operation.Kind, + operation.Index, + operation.Generation); + StoreStatus locationCleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + expectedLocation, + out _); + if (locationCleanup != StoreStatus.Success) + { + return locationCleanup; + } + } + + StoreStatus releaseStatus = CompleteMutationRelease( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + if (releaseStatus != StoreStatus.Success) + { + return releaseStatus; + } + + StoreStatus operationCleanup = TryClearOperationReference( + ref slot.DirectoryOperation, + operationRaw, + out bool clearedOperation); + if (operationCleanup != StoreStatus.Success) + { + return operationCleanup; + } + + if (clearedOperation) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterUnlinkDescriptorClearBeforeGenerationAdvance); + } + + return StoreStatus.Success; + } + + private StoreStatus CancelInsert( + int canonicalBucketIndex, + ulong binding, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + DirectoryOperation operation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (operation.Kind is TargetPrimary or TargetOverflow + && TryGetTargetCell(operation.Kind, operation.Index, out var cell)) + { + StoreStatus cellCleanup = TryClearBindingReference( + ref cell.Value, + binding, + out _); + if (cellCleanup != StoreStatus.Success) + { + return cellCleanup; + } + + ulong expectedLocation = DirectoryLocation.Encode( + operation.Kind, + operation.Index, + operation.Generation); + StoreStatus locationCleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + expectedLocation, + out _); + if (locationCleanup != StoreStatus.Success) + { + return locationCleanup; + } + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterCancelLocationClearBeforeDescriptorRejection); + + StoreStatus summaryStatus = PrepareSpillSummaryForMutationRelease( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + if (summaryStatus != StoreStatus.Success) + { + return summaryStatus; + } + + StoreStatus canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + out _); + if (canonicalOperation != StoreStatus.Success) + { + return canonicalOperation == StoreStatus.NotFound + ? StoreStatus.Success + : canonicalOperation; + } + + ulong rejected = DirectoryOperation.Encode( + IntentInsert, + PhaseRejected, + targetKind: 0, + targetIndex: 0, + generation: operation.Generation); + ulong exchanged = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)rejected), + unchecked((long)operationRaw))); + StoreStatus operationCas = ValidateOperationCasObservation( + ref slot.DirectoryOperation, + operationRaw, + rejected, + exchanged); + if (operationCas != StoreStatus.Success) + { + return operationCas; + } + + if (exchanged == operationRaw) + { + return TryClearMutationWord(canonicalBucketIndex, binding); + } + + return StoreStatus.Success; + } + + private StoreStatus CompleteMutationRelease( + int canonicalBucketIndex, + ulong binding, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + DirectoryOperation operation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + StoreStatus summaryStatus = PrepareSpillSummaryForMutationRelease( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + budget, + ref checkpoint); + if (summaryStatus != StoreStatus.Success) + { + return summaryStatus; + } + + StoreStatus canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + out _); + if (canonicalOperation == StoreStatus.Success) + { + return TryClearMutationWord(canonicalBucketIndex, binding); + } + + if (canonicalOperation is not (StoreStatus.Success or StoreStatus.NotFound)) + { + return canonicalOperation; + } + + return StoreStatus.Success; + } + + private StoreStatus PrepareSpillSummaryForMutationRelease( + int canonicalBucketIndex, + ulong binding, + ref ValueSlotMetadataV2 operationSlot, + ulong operationRaw, + DirectoryOperation operation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + StoreStatus canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + binding, + ref operationSlot, + operationRaw, + operation, + out _); + if (canonicalOperation != StoreStatus.Success) + { + return canonicalOperation == StoreStatus.NotFound + ? StoreStatus.Success + : canonicalOperation; + } + + ref long summaryWord = ref BucketSpillSummary(canonicalBucketIndex); + ulong capturedRaw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref summaryWord)); + if (!TryDecodeSpillSummary(capturedRaw, out SpillSummary captured)) + { + return CorruptHere(); + } + + if (!captured.IsPresent) + { + return StoreStatus.Success; + } + + StoreStatus witnessStatus = TryRetainPresentForExactOverflowWitness( + canonicalBucketIndex, + captured, + binding, + ref operationSlot, + operationRaw, + operation, + budget, + ref checkpoint, + out bool retainedForExactWitness); + if (witnessStatus != StoreStatus.Success || retainedForExactWitness) + { + return witnessStatus; + } + + var scannedCellCount = 0; + for (var index = 0; index < _layout.SlotCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return bound; + } + + ref long cell = ref OverflowCell(index); + ulong cellBinding = unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell)); + scannedCellCount++; + if (cellBinding == 0) + { + continue; + } + + StoreStatus validationStatus = ValidateBinding( + cellBinding, + expectedHash: null, + expectedKey: default, + budget, + out BindingValidation validation); + if (validationStatus != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return validationStatus; + } + + if (validation == BindingValidation.Invalid) + { + StoreStatus revalidationStatus = RevalidateInvalidBindingReference( + ref cell, + cellBinding, + cellBinding, + expectedHash: null, + expectedKey: default, + budget, + ref checkpoint, + out validation, + out bool referenceRemainsExact); + if (revalidationStatus != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return revalidationStatus; + } + + if (!referenceRemainsExact) + { + continue; + } + } + + if (validation == BindingValidation.Invalid) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (validation == BindingValidation.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref cell, + cellBinding, + out _); + if (cleanup != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return cleanup; + } + + continue; + } + + if (!TryDecodeBinding(cellBinding, out IndexBinding decoded) + || decoded.SlotIndex >= _layout.SlotCount) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return CorruptHere(); + } + + ref ValueSlotMetadataV2 candidateSlot = ref Slot(decoded.SlotIndex); + CurrentSlotStatus currentSlot = TryReadCurrentSlotStatus( + cellBinding, + ref candidateSlot, + out _, + out ulong keyHash); + if (currentSlot != CurrentSlotStatus.Current) + { + StoreStatus revalidationStatus = ValidateBinding( + cellBinding, + expectedHash: null, + expectedKey: default, + budget, + out BindingValidation revalidation); + if (revalidationStatus != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return revalidationStatus; + } + + if (revalidation == BindingValidation.Invalid) + { + revalidationStatus = RevalidateInvalidBindingReference( + ref cell, + cellBinding, + cellBinding, + expectedHash: null, + expectedKey: default, + budget, + ref checkpoint, + out revalidation, + out bool referenceRemainsExact); + if (revalidationStatus != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return revalidationStatus; + } + + if (!referenceRemainsExact) + { + continue; + } + } + + if (revalidation == BindingValidation.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref cell, + cellBinding, + out _); + if (cleanup != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return cleanup; + } + + continue; + } + + _telemetry.RecordOverflowScan(scannedCellCount); + return revalidation == BindingValidation.Invalid + ? CorruptHere() + : StoreStatus.StoreBusy; + } + + GetBuckets(keyHash, out int candidateCanonicalBucket, out _); + if (candidateCanonicalBucket == canonicalBucketIndex) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return TryRepointPresentSpillSummary( + canonicalBucketIndex, + capturedRaw, + captured, + cellBinding, + binding, + ref operationSlot, + operationRaw, + operation); + } + } + + _telemetry.RecordOverflowScan(scannedCellCount); + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan); + canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + binding, + ref operationSlot, + operationRaw, + operation, + out _); + if (canonicalOperation != StoreStatus.Success) + { + return canonicalOperation == StoreStatus.NotFound + ? StoreStatus.Success + : canonicalOperation; + } + + ulong empty = captured.EmptyValue; + ulong exchanged = unchecked((ulong)AtomicControlWord.CompareExchange( + ref summaryWord, + unchecked((long)empty), + unchecked((long)capturedRaw))); + StoreStatus summaryCas = ValidateSpillSummaryCasObservation( + ref summaryWord, + capturedRaw, + empty, + exchanged); + if (summaryCas != StoreStatus.Success) + { + return summaryCas; + } + + if (exchanged != capturedRaw && exchanged != empty) + { + return StoreStatus.StoreBusy; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterSpillSummaryClear); + return StoreStatus.Success; + } + + private StoreStatus TryRetainPresentForExactOverflowWitness( + int canonicalBucketIndex, + SpillSummary captured, + ulong operationBinding, + ref ValueSlotMetadataV2 operationSlot, + ulong operationRaw, + DirectoryOperation operation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out bool retained) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + retained = false; + ulong witnessBinding = captured.Binding; + StoreStatus validationStatus = ValidateBinding( + witnessBinding, + expectedHash: null, + expectedKey: default, + budget, + out BindingValidation validation); + if (validationStatus != StoreStatus.Success) + { + return validationStatus; + } + + if (validation == BindingValidation.Invalid) + { + ref long summaryWord = ref BucketSpillSummary(canonicalBucketIndex); + StoreStatus revalidationStatus = RevalidateInvalidBindingReference( + ref summaryWord, + captured.Value, + witnessBinding, + expectedHash: null, + expectedKey: default, + budget, + ref checkpoint, + out validation, + out bool referenceRemainsExact); + if (revalidationStatus != StoreStatus.Success) + { + return revalidationStatus; + } + + if (!referenceRemainsExact) + { + return StoreStatus.Success; + } + } + + if (validation == BindingValidation.Invalid) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (validation == BindingValidation.Stale + || !TryDecodeBinding(witnessBinding, out IndexBinding witness) + || witness.SlotIndex >= _layout.SlotCount) + { + return StoreStatus.Success; + } + + ref ValueSlotMetadataV2 witnessSlot = ref Slot(witness.SlotIndex); + CurrentSlotStatus witnessStatus = TryReadCurrentSlotStatus( + witnessBinding, + ref witnessSlot, + out _, + out ulong witnessHash); + if (witnessStatus != CurrentSlotStatus.Current) + { + return witnessStatus == CurrentSlotStatus.Invalid + ? CorruptFrom(nameof(LockFreeKeyDirectory)) + : witnessStatus == CurrentSlotStatus.Retry + ? StoreStatus.StoreBusy + : StoreStatus.Success; + } + + GetBuckets(witnessHash, out int witnessCanonicalBucket, out _); + ulong locationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref witnessSlot.DirectoryLocation)); + if (witnessCanonicalBucket != canonicalBucketIndex + || !TryDecodeLocation(locationRaw, out DirectoryLocation location) + || location.Kind != TargetOverflow + || location.Generation != witness.Generation + || !TryGetTargetCell(location.Kind, location.Index, out CellReference cell) + || unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell.Value)) != witnessBinding) + { + return StoreStatus.Success; + } + + // If another helper already released this operation, conservative + // Present is still safe and this stale helper must not do further work. + StoreStatus canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + operationBinding, + ref operationSlot, + operationRaw, + operation, + out _); + if (canonicalOperation != StoreStatus.Success) + { + retained = true; + return canonicalOperation == StoreStatus.NotFound + ? StoreStatus.Success + : canonicalOperation; + } + + // While the exact canonical mutation remains ours, no unlink for the + // witness can pass its own canonical-operation validation. Re-read the + // complete witness tuple before retaining Present without a table scan. + witnessStatus = TryReadCurrentSlotStatus( + witnessBinding, + ref witnessSlot, + out _, + out ulong revalidatedHash); + if (witnessStatus == CurrentSlotStatus.Invalid) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + operationBinding, + ref operationSlot, + operationRaw, + operation, + out _); + if (canonicalOperation is not (StoreStatus.Success or StoreStatus.NotFound)) + { + return canonicalOperation; + } + + if (witnessStatus == CurrentSlotStatus.Current + && revalidatedHash == witnessHash + && unchecked((ulong)AtomicControlWord.LoadAcquire( + ref witnessSlot.DirectoryLocation)) == locationRaw + && unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell.Value)) == witnessBinding + && canonicalOperation == StoreStatus.Success) + { + retained = true; + } + + return StoreStatus.Success; + } + + private StoreStatus TryRepointPresentSpillSummary( + int canonicalBucketIndex, + ulong capturedRaw, + SpillSummary captured, + ulong witnessBinding, + ulong operationBinding, + ref ValueSlotMetadataV2 operationSlot, + ulong operationRaw, + DirectoryOperation operation) + { + if (witnessBinding == captured.Binding) + { + return StoreStatus.Success; + } + + StoreStatus canonicalOperation = ClassifyCurrentCanonicalOperation( + canonicalBucketIndex, + operationBinding, + ref operationSlot, + operationRaw, + operation, + out _); + if (canonicalOperation != StoreStatus.Success) + { + return canonicalOperation == StoreStatus.NotFound + ? StoreStatus.Success + : canonicalOperation; + } + + ulong desired; + try + { + desired = SpillSummary.EncodePresent(witnessBinding); + } + catch (ArgumentOutOfRangeException) + { + return CorruptHere(); + } + catch (OverflowException) + { + return CorruptHere(); + } + + ref long summaryWord = ref BucketSpillSummary(canonicalBucketIndex); + ulong exchanged = unchecked((ulong)AtomicControlWord.CompareExchange( + ref summaryWord, + unchecked((long)desired), + unchecked((long)capturedRaw))); + StoreStatus summaryCas = ValidateSpillSummaryCasObservation( + ref summaryWord, + capturedRaw, + desired, + exchanged); + if (summaryCas != StoreStatus.Success) + { + return summaryCas; + } + + if (exchanged != capturedRaw && exchanged != desired) + { + _telemetry.RecordCasLoss(); + return StoreStatus.Success; + } + + return StoreStatus.Success; + } + + private StoreStatus ValidateSpillSummaryCasObservation( + ref long summaryWord, + ulong expected, + ulong desired, + ulong observed) + { + if (observed == expected + || observed == desired + || TryDecodeSpillSummary(observed, out _)) + { + return StoreStatus.Success; + } + + // A losing CAS returns a moment-in-time value. Confirm the exact raw + // word before turning a malformed sample into durable corruption. + if (unchecked((ulong)AtomicControlWord.LoadAcquire(ref summaryWord)) != observed) + { + return StoreStatus.Success; + } + + return CorruptHere(); + } + + private StoreStatus TryClearMutationWord(int canonicalBucketIndex, ulong binding) + { + if ((uint)canonicalBucketIndex >= (uint)_layout.PrimaryBucketCount) + { + return CorruptHere(); + } + + ref long mutation = ref BucketMutation(canonicalBucketIndex); + return TryClearBindingReference(ref mutation, binding, out _); + } + + /// + /// Clears one exact directory binding without treating a valid replacement + /// as cleanup authority or corruption. A malformed CAS winner is corrupt + /// only when the exact same word remains installed on confirmation. + /// + private StoreStatus TryClearBindingReference( + ref long reference, + ulong expected, + out bool clearedExact) => + TryClearExactReference( + ref reference, + expected, + CleanupReferenceKind.Binding, + out clearedExact); + + private StoreStatus TryClearLocationReference( + ref long reference, + ulong expected, + out bool clearedExact) => + TryClearExactReference( + ref reference, + expected, + CleanupReferenceKind.Location, + out clearedExact); + + private StoreStatus TryClearOperationReference( + ref long reference, + ulong expected, + out bool clearedExact) => + TryClearExactReference( + ref reference, + expected, + CleanupReferenceKind.Operation, + out clearedExact); + + private StoreStatus TryClearExactReference( + ref long reference, + ulong expected, + CleanupReferenceKind kind, + out bool clearedExact) + { + clearedExact = false; + ulong observed = unchecked((ulong)AtomicControlWord.CompareExchange( + ref reference, + value: 0, + comparand: unchecked((long)expected))); + if (observed == expected) + { + clearedExact = true; + return StoreStatus.Success; + } + + if (observed == 0 || IsStructurallyValidCleanupWinner(observed, kind)) + { + _telemetry.RecordCasLoss(); + return StoreStatus.Success; + } + + // The CAS observation may already have been replaced. Only a stable + // exact malformed winner is durable corruption; transient samples are + // an ordinary legal race and are never rewritten by this helper. + if (unchecked((ulong)AtomicControlWord.LoadAcquire(ref reference)) != observed) + { + _telemetry.RecordCasLoss(); + return StoreStatus.Success; + } + + return CorruptHere(); + } + + private StoreStatus ValidateOperationCasObservation( + ref long reference, + ulong expected, + ulong desired, + ulong observed) + { + if (observed == expected || observed == desired || observed == 0 + || IsStructurallyValidCleanupWinner(observed, CleanupReferenceKind.Operation)) + { + if (observed != expected) + { + _telemetry.RecordCasLoss(); + } + + return StoreStatus.Success; + } + + if (unchecked((ulong)AtomicControlWord.LoadAcquire(ref reference)) != observed) + { + _telemetry.RecordCasLoss(); + return StoreStatus.Success; + } + + return CorruptHere(); + } + + private bool IsStructurallyValidCleanupWinner( + ulong raw, + CleanupReferenceKind kind) + { + switch (kind) + { + case CleanupReferenceKind.Binding: + return TryDecodeBinding(raw, out IndexBinding binding) + && (uint)binding.SlotIndex < (uint)_layout.SlotCount; + case CleanupReferenceKind.Location: + return TryDecodeLocation(raw, out DirectoryLocation location) + && TryGetTargetCell(location.Kind, location.Index, out _); + case CleanupReferenceKind.Operation: + return TryDecodeOperation(raw, out DirectoryOperation operation) + && IsOperationTargetInBounds(operation); + default: + return false; + } + } + + private StoreStatus TryPublishExactLocation( + int canonicalBucketIndex, + ref ValueSlotMetadataV2 slot, + ulong binding, + ulong operationRaw, + DirectoryOperation operation, + ulong exactLocation, + ref TCheckpoint checkpoint, + out bool published) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + published = false; + IndexBinding decodedBinding = default; + DirectoryOperation decodedOperation = default; + DirectoryLocation decodedLocation = default; + bool metadataValid = TryDecodeBinding(binding, out decodedBinding) + && TryDecodeOperation(operationRaw, out decodedOperation) + && decodedOperation.Value == operation.Value + && decodedOperation.Value == operationRaw + && TryDecodeLocation(exactLocation, out decodedLocation) + && decodedBinding.Generation == decodedLocation.Generation + && operation.Generation == decodedBinding.Generation + && ((operation.Intent == IntentInsert + && operation.Phase == PhaseTargetSelected + && operation.Kind == decodedLocation.Kind + && operation.Index == decodedLocation.Index) + || (operation.Intent == IntentUnlink + && operation.Phase == PhasePrepared)); + ExactCanonicalSourceStatus sourceStatus = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: null, + publicationLocationRaw: exactLocation, + sourceMetadataValid: metadataValid); + if (sourceStatus != ExactCanonicalSourceStatus.Current) + { + return CompleteLocationSourceLoss( + sourceStatus, + ref slot, + binding, + operation, + exactLocation, + cleanupExactLocation: false); + } + + for (var attempt = 0; attempt < DefaultRetryBudget; attempt++) + { + sourceStatus = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: null, + publicationLocationRaw: exactLocation, + sourceMetadataValid: true); + if (sourceStatus != ExactCanonicalSourceStatus.Current) + { + return CompleteLocationSourceLoss( + sourceStatus, + ref slot, + binding, + operation, + exactLocation, + cleanupExactLocation: false); + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation); + ulong observed = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + if (observed == exactLocation) + { + sourceStatus = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: exactLocation, + publicationLocationRaw: exactLocation, + sourceMetadataValid: true); + if (sourceStatus == ExactCanonicalSourceStatus.Current) + { + published = true; + return StoreStatus.Success; + } + + return CompleteLocationSourceLoss( + sourceStatus, + ref slot, + binding, + operation, + exactLocation, + cleanupExactLocation: sourceStatus == ExactCanonicalSourceStatus.Stale + || (sourceStatus == ExactCanonicalSourceStatus.Advanced + && operation.Intent == IntentUnlink + && operation.Phase == PhasePrepared)); + } + + if (observed == 0) + { + sourceStatus = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: 0, + publicationLocationRaw: exactLocation, + sourceMetadataValid: true); + if (sourceStatus != ExactCanonicalSourceStatus.Current) + { + return CompleteLocationSourceLoss( + sourceStatus, + ref slot, + binding, + operation, + exactLocation, + cleanupExactLocation: false); + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas); + + ulong exchanged = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryLocation, + unchecked((long)exactLocation), + comparand: 0)); + if (exchanged == 0) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterLocationPublicationBeforeSourceRevalidation); + sourceStatus = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: exactLocation, + publicationLocationRaw: exactLocation, + sourceMetadataValid: true); + if (sourceStatus == ExactCanonicalSourceStatus.Current) + { + published = true; + return StoreStatus.Success; + } + + return CompleteLocationSourceLoss( + sourceStatus, + ref slot, + binding, + operation, + exactLocation, + cleanupExactLocation: sourceStatus == ExactCanonicalSourceStatus.Stale + || (sourceStatus == ExactCanonicalSourceStatus.Advanced + && operation.Intent == IntentUnlink + && operation.Phase == PhasePrepared)); + } + + continue; + } + + bool observedLocationValid = TryDecodeLocation(observed, out var existing); + bool sameGenerationUnlinkWinner = observedLocationValid + && existing.Generation == decodedBinding.Generation + && operation.Intent == IntentUnlink + && operation.Phase == PhasePrepared + && TryGetTargetCell(existing.Kind, existing.Index, out _); + bool observedMetadataValid = observedLocationValid + && (existing.Generation < decodedBinding.Generation + || sameGenerationUnlinkWinner); + sourceStatus = RevalidateExactCanonicalSource( + canonicalBucketIndex, + binding, + ref slot, + operationRaw, + operation, + requiredLocationRaw: observed, + publicationLocationRaw: exactLocation, + sourceMetadataValid: observedMetadataValid, + secondaryStructuralLocationRaw: sameGenerationUnlinkWinner + ? observed + : null); + if (sourceStatus != ExactCanonicalSourceStatus.Current) + { + return CompleteLocationSourceLoss( + sourceStatus, + ref slot, + binding, + operation, + exactLocation, + cleanupExactLocation: false); + } + + // Unlink/Prepared does not yet encode a selected target. Independent + // helpers may therefore recover different exact cells while the + // location word is still zero. The first valid location publisher + // is the arbitration winner even if its target has since become + // empty or contains another structurally valid binding: a helper + // may already be paused immediately before selecting that target. + // Withdraw only this helper's distinct recovered binding and let + // the winner drive the descriptor. Insert/TargetSelected already + // names one exact target, so its different same-generation location + // remains structural corruption. + if (sameGenerationUnlinkWinner) + { + if (!TryGetTargetCell( + decodedLocation.Kind, + decodedLocation.Index, + out CellReference proposedTarget)) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + return TryClearBindingReference( + ref proposedTarget.Value, + binding, + out _); + } + + // The stable exact source proof above deliberately includes the + // observed location. Reaching this branch therefore means damaged + // same-generation metadata was not a CAS loss or later lifecycle. + if (!observedLocationValid + || existing.Generation >= decodedBinding.Generation) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + StoreStatus staleCleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + observed, + out _); + if (staleCleanup != StoreStatus.Success) + { + return staleCleanup; + } + } + + return StoreStatus.StoreBusy; + } + + /// + /// Re-proves the complete source of a delayed directory helper without + /// publishing the terminal corruption latch. A source word that moved is + /// ordinary lock-free progress; a later slot generation is stale work. An + /// invalid result is returned only after no-op CAS confirmation of every + /// atomic source word and a second immutable-binding read. + /// + private ExactCanonicalSourceStatus RevalidateExactCanonicalSource( + int canonicalBucketIndex, + ulong bindingRaw, + ref ValueSlotMetadataV2 slot, + ulong expectedOperationRaw, + DirectoryOperation expectedOperation, + ulong? requiredLocationRaw, + ulong? publicationLocationRaw, + bool sourceMetadataValid, + bool publicationTargetMustMatchBinding = true, + ulong? secondaryStructuralLocationRaw = null) + { + if ((uint)canonicalBucketIndex >= (uint)_layout.PrimaryBucketCount) + { + return ExactCanonicalSourceStatus.Invalid; + } + + bool bindingValid = TryDecodeBinding(bindingRaw, out IndexBinding binding) + && binding.SlotIndex < _layout.SlotCount; + bool targetValid = true; + CellReference target = default; + if (publicationLocationRaw is ulong publicationRaw) + { + targetValid = bindingValid + && TryDecodeLocation(publicationRaw, out DirectoryLocation publicationLocation) + && publicationLocation.Generation == binding.Generation + && TryGetTargetCell(publicationLocation.Kind, publicationLocation.Index, out target); + } + + bool secondaryTargetValid = true; + CellReference secondaryTarget = default; + if (secondaryStructuralLocationRaw is ulong secondaryRaw) + { + secondaryTargetValid = bindingValid + && TryDecodeLocation(secondaryRaw, out DirectoryLocation secondaryLocation) + && secondaryLocation.Generation == binding.Generation + && TryGetTargetCell( + secondaryLocation.Kind, + secondaryLocation.Index, + out secondaryTarget); + } + + ref long mutation = ref BucketMutation(canonicalBucketIndex); + for (var attempt = 0; attempt < 8; attempt++) + { + ulong mutation1 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref mutation)); + ulong operation1 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)); + ulong location1 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + ulong control1 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ulong directoryBinding1 = Volatile.Read(ref slot.DirectoryBinding); + ulong target1 = publicationLocationRaw.HasValue && targetValid + ? unchecked((ulong)AtomicControlWord.LoadAcquire(ref target.Value)) + : bindingRaw; + ulong secondaryTarget1 = secondaryStructuralLocationRaw.HasValue + && secondaryTargetValid + ? unchecked((ulong)AtomicControlWord.LoadAcquire(ref secondaryTarget.Value)) + : 0; + + ulong secondaryTarget2 = secondaryStructuralLocationRaw.HasValue + && secondaryTargetValid + ? unchecked((ulong)AtomicControlWord.LoadAcquire(ref secondaryTarget.Value)) + : 0; + ulong target2 = publicationLocationRaw.HasValue && targetValid + ? unchecked((ulong)AtomicControlWord.LoadAcquire(ref target.Value)) + : bindingRaw; + ulong directoryBinding2 = Volatile.Read(ref slot.DirectoryBinding); + ulong control2 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ulong location2 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + ulong operation2 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)); + ulong mutation2 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref mutation)); + if (mutation1 != mutation2 + || operation1 != operation2 + || location1 != location2 + || control1 != control2 + || directoryBinding1 != directoryBinding2 + || target1 != target2 + || secondaryTarget1 != secondaryTarget2) + { + continue; + } + + if (mutation1 != bindingRaw + || operation1 != expectedOperationRaw + || (requiredLocationRaw.HasValue && location1 != requiredLocationRaw.Value)) + { + return ExactCanonicalSourceStatus.Advanced; + } + + ExactCanonicalSourceStatus result; + bool publicationTargetStructurallyValid = !publicationLocationRaw.HasValue + || target1 == 0 + || IsStructurallyValidCleanupWinner( + target1, + CleanupReferenceKind.Binding); + bool secondaryTargetStructurallyValid = !secondaryStructuralLocationRaw.HasValue + || secondaryTarget1 == 0 + || IsStructurallyValidCleanupWinner( + secondaryTarget1, + CleanupReferenceKind.Binding); + if (!sourceMetadataValid + || !bindingValid + || !targetValid + || !secondaryTargetValid + || !publicationTargetStructurallyValid + || !secondaryTargetStructurallyValid) + { + result = ExactCanonicalSourceStatus.Invalid; + } + else + { + ControlBindingStatus controlStatus = ClassifyControlBinding( + control1, + binding.Generation); + result = controlStatus switch + { + ControlBindingStatus.Stale => ExactCanonicalSourceStatus.Stale, + ControlBindingStatus.Invalid => ExactCanonicalSourceStatus.Invalid, + _ when directoryBinding1 != bindingRaw => ExactCanonicalSourceStatus.Invalid, + // PrepareOperation may replace an exact Insert descriptor + // with Unlink/Prepared while a previously validated insert + // cancellation helper is delayed. That old helper can + // then exact-clear the target before this unlink location + // publisher resumes. The same witness rule applies to an + // Insert publisher racing its cancellation: empty, or a + // structurally valid new binding that claimed the cleared + // cell, is ordinary progress in a canceling lifecycle. A + // stable malformed or out-of-range target word still fails + // closed below. + _ when publicationTargetMustMatchBinding + && publicationLocationRaw.HasValue + && target1 != bindingRaw + && IsInsertCancellationState((int)(control1 & 0x7UL)) + && (expectedOperation.Intent == IntentInsert + || (expectedOperation.Intent == IntentUnlink + && expectedOperation.Phase == PhasePrepared)) + && (target1 == 0 + || IsStructurallyValidCleanupWinner( + target1, + CleanupReferenceKind.Binding)) => + ExactCanonicalSourceStatus.Advanced, + _ when publicationTargetMustMatchBinding + && publicationLocationRaw.HasValue + && target1 != bindingRaw => + ExactCanonicalSourceStatus.Invalid, + _ => ExactCanonicalSourceStatus.Current, + }; + } + + if (result != ExactCanonicalSourceStatus.Invalid) + { + return result; + } + + // Invalid is terminal only when the exact tuple survives atomic + // confirmation. A CAS loss is progress and must be retried without + // touching the winner's generation. + if (unchecked((ulong)AtomicControlWord.CompareExchange( + ref mutation, + unchecked((long)mutation1), + unchecked((long)mutation1))) != mutation1 + || unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)operation1), + unchecked((long)operation1))) != operation1 + || unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryLocation, + unchecked((long)location1), + unchecked((long)location1))) != location1 + || unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.Control, + unchecked((long)control1), + unchecked((long)control1))) != control1 + || (publicationLocationRaw.HasValue + && targetValid + && unchecked((ulong)AtomicControlWord.CompareExchange( + ref target.Value, + unchecked((long)target1), + unchecked((long)target1))) != target1) + || (secondaryStructuralLocationRaw.HasValue + && secondaryTargetValid + && unchecked((ulong)AtomicControlWord.CompareExchange( + ref secondaryTarget.Value, + unchecked((long)secondaryTarget1), + unchecked((long)secondaryTarget1))) != secondaryTarget1) + || Volatile.Read(ref slot.DirectoryBinding) != directoryBinding1 + || unchecked((ulong)AtomicControlWord.LoadAcquire(ref mutation)) != mutation1 + || unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)) != operation1 + || unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)) != location1 + || unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)) != control1 + || (publicationLocationRaw.HasValue + && targetValid + && unchecked((ulong)AtomicControlWord.LoadAcquire(ref target.Value)) != target1) + || (secondaryStructuralLocationRaw.HasValue + && secondaryTargetValid + && unchecked((ulong)AtomicControlWord.LoadAcquire(ref secondaryTarget.Value)) + != secondaryTarget1)) + { + continue; + } + + return ExactCanonicalSourceStatus.Invalid; + } + + return ExactCanonicalSourceStatus.Retry; + } + + private StoreStatus CompleteLocationSourceLoss( + ExactCanonicalSourceStatus sourceStatus, + ref ValueSlotMetadataV2 slot, + ulong binding, + DirectoryOperation operation, + ulong exactLocation, + bool cleanupExactLocation) + { + if (sourceStatus == ExactCanonicalSourceStatus.Invalid) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (sourceStatus == ExactCanonicalSourceStatus.Retry) + { + return StoreStatus.StoreBusy; + } + + // Every successor of an exact Unlink/Prepared source removes this + // binding. A publisher that loses that source therefore withdraws its + // recovered target even when it never won DirectoryLocation. This also + // closes the late 0->location CAS window after another helper has + // already completed unlink. Exact CAS preserves replacements and slot + // reuse; Insert successors deliberately do not use this rule because + // BindingChanged/Complete may represent a committed publication. + if (operation.Intent == IntentUnlink + && operation.Phase == PhasePrepared + && sourceStatus is ExactCanonicalSourceStatus.Advanced + or ExactCanonicalSourceStatus.Stale) + { + if (!TryDecodeLocation(exactLocation, out DirectoryLocation location) + || !TryGetTargetCell(location.Kind, location.Index, out CellReference target)) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + StoreStatus targetCleanup = TryClearBindingReference( + ref target.Value, + binding, + out _); + if (targetCleanup != StoreStatus.Success) + { + return targetCleanup; + } + } + + if (cleanupExactLocation) + { + StoreStatus cleanup = TryClearLocationReference( + ref slot.DirectoryLocation, + exactLocation, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + } + + return StoreStatus.Success; + } + + private StoreStatus TryFindExactBindingLocation( + ulong binding, + ulong keyHash, + long generation, + in LockFreeOperationBudget budget, + out DirectoryLocation location) + { + if (!TryDecodeBinding(binding, out IndexBinding decodedBinding) + || decodedBinding.SlotIndex >= _layout.SlotCount) + { + location = default; + return CorruptHere(); + } + + GetBuckets(keyHash, out int first, out int second); + StoreStatus primaryStatus = TryFindExactPrimaryBinding( + first, + binding, + generation, + out location); + if (primaryStatus == StoreStatus.Success) + { + return StoreStatus.Success; + } + + if (primaryStatus != StoreStatus.NotFound) + { + return primaryStatus; + } + + primaryStatus = TryFindExactPrimaryBinding( + second, + binding, + generation, + out location); + if (primaryStatus == StoreStatus.Success) + { + return StoreStatus.Success; + } + + if (primaryStatus != StoreStatus.NotFound) + { + return primaryStatus; + } + + int start = OverflowStart(keyHash); + var scannedCellCount = 0; + for (var offset = 0; offset < _layout.SlotCount; offset++) + { + StoreStatus bound = budget.CheckPeriodic(offset); + if (bound != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + location = default; + return bound; + } + + int index = (start + offset) % _layout.SlotCount; + scannedCellCount++; + ref long cell = ref OverflowCell(index); + StoreStatus referenceStatus = TryReadStructurallyValidBindingReference( + ref cell, + out ulong observed); + if (referenceStatus != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + location = default; + return referenceStatus; + } + + if (observed == binding) + { + _telemetry.RecordOverflowScan(scannedCellCount); + location = DirectoryLocation.Decode( + DirectoryLocation.Encode(TargetOverflow, index, generation)); + return StoreStatus.Success; + } + } + + _telemetry.RecordOverflowScan(scannedCellCount); + location = default; + return StoreStatus.NotFound; + } + + private StoreStatus TryFindExactPrimaryBinding( + int bucketIndex, + ulong binding, + long generation, + out DirectoryLocation location) + { + int firstLane = bucketIndex * LayoutV2Constants.PrimaryLanesPerBucket; + for (var lane = 0; lane < LayoutV2Constants.PrimaryLanesPerBucket; lane++) + { + int index = firstLane + lane; + ref long cell = ref PrimaryCell(index); + StoreStatus referenceStatus = TryReadStructurallyValidBindingReference( + ref cell, + out ulong observed); + if (referenceStatus != StoreStatus.Success) + { + location = default; + return referenceStatus; + } + + if (observed == binding) + { + location = DirectoryLocation.Decode( + DirectoryLocation.Encode(TargetPrimary, index, generation)); + return StoreStatus.Success; + } + } + + location = default; + return StoreStatus.NotFound; + } + + private bool IsSameOrLaterInsertPhase( + ulong observedRaw, + DirectoryOperation expected) + { + return TryDecodeOperation(observedRaw, out var observed) + && observed.Intent == IntentInsert + && observed.Generation == expected.Generation + && observed.Kind == expected.Kind + && observed.Index == expected.Index + && observed.Phase is PhaseBindingChanged or PhaseComplete; + } + + private StoreStatus FindExact( + ReadOnlySpan key, + ulong keyHash, + ulong excludedBinding, + in LockFreeOperationBudget budget, + out ulong binding, + out DirectoryLocation location) + { + NoOpLockFreeCheckpoint checkpoint = default; + return FindExact( + key, + keyHash, + excludedBinding, + budget, + ref checkpoint, + out binding, + out location); + } + + private StoreStatus FindExact( + ReadOnlySpan key, + ulong keyHash, + ulong excludedBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out ulong binding, + out DirectoryLocation location) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + binding = 0; + location = default; + GetBuckets(keyHash, out var first, out var second); + var status = ScanPrimaryBucket( + first, + key, + keyHash, + excludedBinding, + budget, + ref checkpoint, + out binding, + out location); + if (status != StoreStatus.NotFound) + { + return status; + } + + status = ScanPrimaryBucket( + second, + key, + keyHash, + excludedBinding, + budget, + ref checkpoint, + out binding, + out location); + if (status != StoreStatus.NotFound) + { + return status; + } + + ulong summaryRaw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref BucketSpillSummary(first))); + if (!TryDecodeSpillSummary(summaryRaw, out SpillSummary summary)) + { + return CorruptHere(); + } + + if (!summary.IsPresent) + { + return StoreStatus.NotFound; + } + + var start = OverflowStart(keyHash); + var scannedCellCount = 0; + for (var offset = 0; offset < _layout.SlotCount; offset++) + { + StoreStatus bound = budget.CheckPeriodic(offset); + if (bound != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return bound; + } + + var index = (start + offset) % _layout.SlotCount; + scannedCellCount++; + status = InspectCell( + ref OverflowCell(index), + TargetOverflow, + index, + key, + keyHash, + excludedBinding, + budget, + ref checkpoint, + out binding, + out location); + if (status != StoreStatus.NotFound) + { + _telemetry.RecordOverflowScan(scannedCellCount); + return status; + } + } + + _telemetry.RecordOverflowScan(scannedCellCount); + return StoreStatus.NotFound; + } + + private StoreStatus ScanPrimaryBucket( + int bucketIndex, + ReadOnlySpan key, + ulong keyHash, + ulong excludedBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out ulong binding, + out DirectoryLocation location) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + binding = 0; + location = default; + var firstLane = bucketIndex * LayoutV2Constants.PrimaryLanesPerBucket; + for (var lane = 0; lane < LayoutV2Constants.PrimaryLanesPerBucket; lane++) + { + var cellIndex = firstLane + lane; + var status = InspectCell( + ref PrimaryCell(cellIndex), + TargetPrimary, + cellIndex, + key, + keyHash, + excludedBinding, + budget, + ref checkpoint, + out binding, + out location); + if (status != StoreStatus.NotFound) + { + return status; + } + } + + return StoreStatus.NotFound; + } + + private StoreStatus InspectCell( + ref long cell, + int kind, + long index, + ReadOnlySpan key, + ulong keyHash, + ulong excludedBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out ulong binding, + out DirectoryLocation location) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + binding = unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell)); + location = default; + if (binding == 0 || binding == excludedBinding) + { + binding = 0; + return StoreStatus.NotFound; + } + + StoreStatus validationStatus = ValidateBinding( + binding, + keyHash, + key, + budget, + out BindingValidation validation); + if (validationStatus != StoreStatus.Success) + { + return validationStatus; + } + + if (validation == BindingValidation.Invalid) + { + StoreStatus revalidationStatus = RevalidateInvalidBindingReference( + ref cell, + binding, + binding, + keyHash, + key, + budget, + ref checkpoint, + out validation, + out bool referenceRemainsExact); + if (revalidationStatus != StoreStatus.Success) + { + return revalidationStatus; + } + + if (!referenceRemainsExact) + { + binding = 0; + return StoreStatus.NotFound; + } + } + + if (validation == BindingValidation.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref cell, + binding, + out _); + if (cleanup != StoreStatus.Success) + { + return cleanup; + } + + binding = 0; + return StoreStatus.NotFound; + } + + if (validation == BindingValidation.Invalid) + { + return CorruptHere(); + } + + if (validation != BindingValidation.Exact + || unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell)) != binding) + { + binding = 0; + return StoreStatus.NotFound; + } + + if (!TryDecodeBinding(binding, out var decoded)) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + location = DirectoryLocation.Decode(DirectoryLocation.Encode(kind, index, decoded.Generation)); + return StoreStatus.Success; + } + + private StoreStatus SelectInsertTarget( + ulong keyHash, + long generation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out DirectoryLocation target) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + GetBuckets(keyHash, out var first, out var second); + StoreStatus primaryStatus = TrySelectPrimary( + first, + generation, + budget, + ref checkpoint, + out target); + if (primaryStatus == StoreStatus.Success) + { + return StoreStatus.Success; + } + + if (primaryStatus != StoreStatus.NotFound) + { + return primaryStatus; + } + + primaryStatus = TrySelectPrimary( + second, + generation, + budget, + ref checkpoint, + out target); + if (primaryStatus == StoreStatus.Success) + { + return StoreStatus.Success; + } + + if (primaryStatus != StoreStatus.NotFound) + { + return primaryStatus; + } + + var start = OverflowStart(keyHash); + var scannedCellCount = 0; + for (var offset = 0; offset < _layout.SlotCount; offset++) + { + StoreStatus bound = budget.CheckPeriodic(offset); + if (bound != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + target = default; + return bound; + } + + var index = (start + offset) % _layout.SlotCount; + scannedCellCount++; + ref var cell = ref OverflowCell(index); + var raw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell)); + if (raw == 0) + { + _telemetry.RecordOverflowScan(scannedCellCount); + target = DirectoryLocation.Decode( + DirectoryLocation.Encode(TargetOverflow, index, generation)); + return StoreStatus.Success; + } + + StoreStatus validationStatus = ValidateBinding( + raw, + expectedHash: null, + expectedKey: default, + budget, + out BindingValidation validation); + if (validationStatus != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + target = default; + return validationStatus; + } + + if (validation == BindingValidation.Invalid) + { + StoreStatus revalidationStatus = RevalidateInvalidBindingReference( + ref cell, + raw, + raw, + expectedHash: null, + expectedKey: default, + budget, + ref checkpoint, + out validation, + out bool referenceRemainsExact); + if (revalidationStatus != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + target = default; + return revalidationStatus; + } + + if (!referenceRemainsExact) + { + continue; + } + } + + if (validation == BindingValidation.Invalid) + { + _telemetry.RecordOverflowScan(scannedCellCount); + target = default; + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (validation == BindingValidation.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref cell, + raw, + out _); + if (cleanup != StoreStatus.Success) + { + _telemetry.RecordOverflowScan(scannedCellCount); + target = default; + return cleanup; + } + + if (AtomicControlWord.LoadAcquire(ref cell) == 0) + { + _telemetry.RecordOverflowScan(scannedCellCount); + target = DirectoryLocation.Decode( + DirectoryLocation.Encode(TargetOverflow, index, generation)); + return StoreStatus.Success; + } + } + } + + _telemetry.RecordOverflowScan(scannedCellCount); + target = default; + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + private StoreStatus TrySelectPrimary( + int bucketIndex, + long generation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out DirectoryLocation target) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + var firstLane = bucketIndex * LayoutV2Constants.PrimaryLanesPerBucket; + for (var lane = 0; lane < LayoutV2Constants.PrimaryLanesPerBucket; lane++) + { + var index = firstLane + lane; + ref var cell = ref PrimaryCell(index); + var raw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref cell)); + if (raw == 0) + { + target = DirectoryLocation.Decode( + DirectoryLocation.Encode(TargetPrimary, index, generation)); + return StoreStatus.Success; + } + + StoreStatus validationStatus = ValidateBinding( + raw, + expectedHash: null, + expectedKey: default, + budget, + out BindingValidation validation); + if (validationStatus != StoreStatus.Success) + { + target = default; + return validationStatus; + } + + if (validation == BindingValidation.Invalid) + { + StoreStatus revalidationStatus = RevalidateInvalidBindingReference( + ref cell, + raw, + raw, + expectedHash: null, + expectedKey: default, + budget, + ref checkpoint, + out validation, + out bool referenceRemainsExact); + if (revalidationStatus != StoreStatus.Success) + { + target = default; + return revalidationStatus; + } + + if (!referenceRemainsExact) + { + continue; + } + } + + if (validation == BindingValidation.Invalid) + { + target = default; + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (validation == BindingValidation.Stale) + { + StoreStatus cleanup = TryClearBindingReference( + ref cell, + raw, + out _); + if (cleanup != StoreStatus.Success) + { + target = default; + return cleanup; + } + + if (AtomicControlWord.LoadAcquire(ref cell) == 0) + { + target = DirectoryLocation.Decode( + DirectoryLocation.Encode(TargetPrimary, index, generation)); + return StoreStatus.Success; + } + } + } + + target = default; + return StoreStatus.NotFound; + } + + private StoreStatus ValidateBinding( + ulong raw, + ulong? expectedHash, + ReadOnlySpan expectedKey, + in LockFreeOperationBudget budget, + out BindingValidation validation) + { + validation = BindingValidation.Invalid; + if (!TryDecodeBinding(raw, out var binding) || binding.SlotIndex >= _layout.SlotCount) + { + return StoreStatus.Success; + } + + ref var slot = ref Slot(binding.SlotIndex); + for (var attempt = 0; attempt < 8; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + var control1 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ControlBindingStatus controlStatus1 = ClassifyControlBinding( + control1, + binding.Generation); + if (controlStatus1 != ControlBindingStatus.Current) + { + validation = controlStatus1 == ControlBindingStatus.Stale + ? BindingValidation.Stale + : BindingValidation.Invalid; + return StoreStatus.Success; + } + + if (slot.DirectoryBinding != raw) + { + ulong controlAfterBindingChange = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.Control)); + if (control1 != controlAfterBindingChange) + { + continue; + } + + ControlBindingStatus controlStatus2 = ClassifyControlBinding( + controlAfterBindingChange, + binding.Generation); + validation = controlStatus2 == ControlBindingStatus.Stale + ? BindingValidation.Stale + : BindingValidation.Invalid; + return StoreStatus.Success; + } + + ulong observedHash = slot.KeyHash; + ReadOnlySpan storedKey = default; + bool validKey = expectedHash is null || TryGetKey(ref slot, out storedKey); + bool equal = false; + if (expectedHash is not null + && observedHash == expectedHash.Value + && validKey) + { + StoreStatus equalityStatus = KeysEqual( + storedKey, + expectedKey, + budget, + out equal); + if (equalityStatus != StoreStatus.Success) + { + return equalityStatus; + } + } + + var control2 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + if (control1 == control2 && slot.DirectoryBinding == raw) + { + if (expectedHash is null) + { + validation = BindingValidation.CurrentOther; + return StoreStatus.Success; + } + + if (observedHash == expectedHash.Value && !validKey) + { + validation = BindingValidation.Invalid; + return StoreStatus.Success; + } + + validation = equal ? BindingValidation.Exact : BindingValidation.CurrentOther; + return StoreStatus.Success; + } + + // State changes such as Initializing -> Reserved -> Published do + // not make an exact-generation directory cell stale. Retry the + // snapshot; only a generation/binding change permits cell cleanup. + ControlBindingStatus revalidation = ClassifyControlBinding( + control2, + binding.Generation); + if (revalidation != ControlBindingStatus.Current + || slot.DirectoryBinding != raw) + { + validation = revalidation == ControlBindingStatus.Stale + ? BindingValidation.Stale + : BindingValidation.Invalid; + return StoreStatus.Success; + } + } + + // Persistent same-generation movement is current contention, never + // evidence that a helper may erase the cell. + validation = BindingValidation.CurrentOther; + return StoreStatus.Success; + } + + /// + /// Confirms that a would-be corrupt binding is still named by the exact + /// directory/summary word that supplied it. A sampled word may be removed + /// or replaced while its old slot is concurrently reclaimed; that obsolete + /// sample is contention, not evidence that the current store is corrupt. + /// The second exact-word read makes fail-closed classification depend on a + /// freshly validated reference rather than the original scan observation. + /// + private StoreStatus RevalidateInvalidBindingReference( + ref long reference, + ulong expectedReference, + ulong binding, + ulong? expectedHash, + ReadOnlySpan expectedKey, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out BindingValidation validation, + out bool referenceRemainsExact) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + validation = BindingValidation.Invalid; + referenceRemainsExact = false; + if (unchecked((ulong)AtomicControlWord.LoadAcquire(ref reference)) != expectedReference) + { + return StoreStatus.Success; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation); + + StoreStatus status = ValidateBinding( + binding, + expectedHash, + expectedKey, + budget, + out validation); + if (status != StoreStatus.Success) + { + return status; + } + + referenceRemainsExact = unchecked((ulong)AtomicControlWord.LoadAcquire(ref reference)) + == expectedReference; + return StoreStatus.Success; + } + + private static StoreStatus KeysEqual( + ReadOnlySpan storedKey, + ReadOnlySpan expectedKey, + in LockFreeOperationBudget budget, + out bool equal) + { + equal = false; + if (storedKey.Length != expectedKey.Length) + { + return StoreStatus.Success; + } + + const int ComparisonChunkBytes = 64; + var chunkCount = 0; + for (var offset = 0; offset < storedKey.Length; offset += ComparisonChunkBytes) + { + StoreStatus bound = budget.CheckPeriodic(chunkCount); + if (bound != StoreStatus.Success) + { + return bound; + } + + int length = Math.Min(ComparisonChunkBytes, storedKey.Length - offset); + if (!storedKey.Slice(offset, length).SequenceEqual(expectedKey.Slice(offset, length))) + { + return StoreStatus.Success; + } + + chunkCount++; + } + + StoreStatus completionStatus = budget.CheckPeriodic(chunkCount); + if (completionStatus != StoreStatus.Success) + { + return completionStatus; + } + + equal = true; + return StoreStatus.Success; + } + + private bool ControlMatchesBinding(ulong control, long generation) => + ClassifyControlBinding(control, generation) == ControlBindingStatus.Current; + + private ControlBindingStatus ClassifyControlBinding( + ulong control, + long expectedGeneration) + { + long observedGeneration = (long)((control >> 3) & SlotGenerationMask); + ulong participant = (control >> 36) & SlotParticipantMask; + int state = (int)(control & 0x7UL); + bool structurallyValid = state switch + { + LayoutV2Constants.SlotFree => participant == 0, + SlotInitializing or SlotReserved => ParticipantToken.IsStructurallyValid( + participant, + _layout.ParticipantRecordCount), + SlotPublished or SlotRemoveRequested or SlotAborting or SlotReclaiming => + participant == 0, + SlotRetired => participant == 0 + && observedGeneration == LockFreeSlotTable.TerminalGeneration, + _ => false, + }; + if (!structurallyValid || observedGeneration is < 1 or > LockFreeSlotTable.TerminalGeneration) + { + return ControlBindingStatus.Invalid; + } + + if (observedGeneration > expectedGeneration) + { + return ControlBindingStatus.Stale; + } + + if (observedGeneration < expectedGeneration) + { + return ControlBindingStatus.Invalid; + } + + return state switch + { + LayoutV2Constants.SlotFree => ControlBindingStatus.Invalid, + SlotRetired => ControlBindingStatus.Stale, + _ => ControlBindingStatus.Current, + }; + } + + private static bool IsInsertCancellationState(int state) => + state is SlotAborting or SlotReclaiming; + + private StoreStatus ObserveInsertOutcomeBeforeBudget( + ulong bindingRaw, + long expectedGeneration, + ref ValueSlotMetadataV2 slot, + ref TCheckpoint checkpoint, + out bool hasOutcome, + out DirectoryLocation location) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + hasOutcome = false; + location = default; + + // This observation deliberately precedes the operation-wide budget + // check. A helper may have crossed the insert's ordering point while + // this caller was descheduled, and an expired caller must not rewrite + // that already-ordered outcome as StoreBusy/OperationCanceled. + for (var attempt = 0; attempt < 8; attempt++) + { + ulong control1 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ControlBindingStatus controlStatus = ClassifyControlBinding( + control1, + expectedGeneration); + if (controlStatus != ControlBindingStatus.Current) + { + hasOutcome = true; + return controlStatus == ControlBindingStatus.Stale + ? StoreStatus.InvalidReservation + : CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + ulong observedBinding = slot.DirectoryBinding; + ulong operationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + ulong locationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryLocation)); + ulong control2 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ulong operationRaw2 = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + ulong locationRaw2 = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryLocation)); + + if (control1 != control2 + || operationRaw != operationRaw2 + || locationRaw != locationRaw2) + { + continue; + } + + if (slot.DirectoryBinding != observedBinding) + { + continue; + } + + if (observedBinding != bindingRaw) + { + hasOutcome = true; + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + int state = (int)(control2 & 0x7UL); + bool operationDecoded = TryDecodeOperation(operationRaw, out DirectoryOperation operation); + if (IsInsertCancellationState(state)) + { + // Complete plus its exact location proves that Reserved was + // published before cancellation. Every other valid canceling + // insert/unlink observation is still pre-order and therefore + // an invalid reservation, not corruption. + if (operationDecoded + && operation.Intent == IntentInsert + && operation.Generation == expectedGeneration + && (operation.Phase is PhaseBindingChanged or PhaseComplete)) + { + hasOutcome = true; + if (locationRaw == 0) + { + // CancelInsert clears the exact location before it can + // replace BindingChanged/Complete with Rejected. This + // stable zero is a legal pre-order cleanup window. + return StoreStatus.InvalidReservation; + } + + bool exactLocation = TryValidateInsertLocation( + operation, + locationRaw, + out location); + if (!exactLocation) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + return operation.Phase == PhaseComplete + ? StoreStatus.Success + : StoreStatus.InvalidReservation; + } + + if (operationRaw == 0 + || (operationDecoded + && operation.Generation == expectedGeneration + && operation.Intent is IntentInsert or IntentUnlink + && IsOperationTargetInBounds(operation))) + { + hasOutcome = true; + return StoreStatus.InvalidReservation; + } + + hasOutcome = true; + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (!operationDecoded + || operation.Intent != IntentInsert + || operation.Generation != expectedGeneration) + { + hasOutcome = true; + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (state == SlotInitializing) + { + if (operation.Phase == PhaseRejected) + { + hasOutcome = true; + return StoreStatus.DuplicateKey; + } + + if (operation.Phase is PhasePrepared or PhaseTargetSelected or PhaseBindingChanged) + { + return StoreStatus.Success; + } + + hasOutcome = true; + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (state == SlotReserved + && operation.Phase is PhaseBindingChanged or PhaseComplete) + { + if (operation.Phase == PhaseComplete) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.DirectoryAfterInsertCompletionStateValidationBeforeLocationRead); + } + + hasOutcome = true; + return TryValidateInsertLocation( + operation, + locationRaw, + out location) + ? StoreStatus.Success + : CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + hasOutcome = true; + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + // A rapidly changing, still pre-order snapshot remains ordinary + // contention. The caller's normal budget check decides that outcome. + return StoreStatus.Success; + } + + private bool TryValidateInsertLocation( + DirectoryOperation operation, + ulong locationRaw, + out DirectoryLocation location) + { + location = default; + if (!TryDecodeLocation(locationRaw, out DirectoryLocation decodedLocation) + || decodedLocation.Kind != operation.Kind + || decodedLocation.Index != operation.Index + || decodedLocation.Generation != operation.Generation + || !TryGetTargetCell(operation.Kind, operation.Index, out _)) + { + return false; + } + + location = decodedLocation; + return true; + } + + private bool IsCanceledInsertObservation( + ulong bindingRaw, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + bool operationDecoded, + DirectoryOperation operation) + { + CurrentSlotStatus slotStatus = TryReadCurrentSlotStatus( + bindingRaw, + ref slot, + out int state, + out _); + if (!TryDecodeBinding(bindingRaw, out IndexBinding binding) + || slotStatus != CurrentSlotStatus.Current + || !IsInsertCancellationState(state)) + { + return false; + } + + if (operationRaw == 0) + { + return true; + } + + if (!operationDecoded + || operation.Generation != binding.Generation + || operation.Intent is not (IntentInsert or IntentUnlink) + || !IsOperationTargetInBounds(operation)) + { + return false; + } + + // Exact Insert/Complete is an ordered outcome, not a generic canceled + // descriptor. Its location must be validated by the stable tuple path; + // excluding it here prevents malformed/zero location metadata from + // being normalized to InvalidReservation. + return operation.Intent != IntentInsert || operation.Phase != PhaseComplete; + } + + private bool IsOperationTargetInBounds(DirectoryOperation operation) => + operation.Phase is PhasePrepared or PhaseRejected + || (operation.Intent == IntentUnlink + && operation.Phase == PhaseComplete + && operation.Kind == 0 + && operation.Index == 0) + || TryGetTargetCell(operation.Kind, operation.Index, out _); + + private CurrentSlotStatus TryReadCurrentSlotStatus( + ulong raw, + ref ValueSlotMetadataV2 slot, + out int state, + out ulong keyHash) + { + state = SlotRetired; + keyHash = 0; + if (!TryDecodeBinding(raw, out var binding)) + { + return CurrentSlotStatus.Invalid; + } + + for (var attempt = 0; attempt < 8; attempt++) + { + ulong control1 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ControlBindingStatus controlStatus1 = ClassifyControlBinding( + control1, + binding.Generation); + if (controlStatus1 != ControlBindingStatus.Current) + { + return controlStatus1 == ControlBindingStatus.Invalid + ? CurrentSlotStatus.Invalid + : CurrentSlotStatus.Stale; + } + + ulong directoryBinding = slot.DirectoryBinding; + ulong observedKeyHash = slot.KeyHash; + ulong control2 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + if (control1 == control2 && directoryBinding == raw) + { + state = (int)(control2 & 0x7UL); + keyHash = observedKeyHash; + return CurrentSlotStatus.Current; + } + + ControlBindingStatus controlStatus2 = ClassifyControlBinding( + control2, + binding.Generation); + if (controlStatus2 != ControlBindingStatus.Current) + { + return controlStatus2 == ControlBindingStatus.Invalid + ? CurrentSlotStatus.Invalid + : CurrentSlotStatus.Stale; + } + + if (slot.DirectoryBinding != raw) + { + return CurrentSlotStatus.Invalid; + } + } + + ulong control = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ControlBindingStatus controlStatus = ClassifyControlBinding(control, binding.Generation); + if (controlStatus != ControlBindingStatus.Current) + { + return controlStatus == ControlBindingStatus.Invalid + ? CurrentSlotStatus.Invalid + : CurrentSlotStatus.Stale; + } + + ulong finalBinding = slot.DirectoryBinding; + ulong finalHash = slot.KeyHash; + ulong confirmedControl = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ControlBindingStatus confirmedStatus = ClassifyControlBinding( + confirmedControl, + binding.Generation); + if (confirmedStatus != ControlBindingStatus.Current) + { + return confirmedStatus == ControlBindingStatus.Invalid + ? CurrentSlotStatus.Invalid + : CurrentSlotStatus.Stale; + } + + if (confirmedControl != control) + { + return CurrentSlotStatus.Retry; + } + + if (finalBinding != raw) + { + return CurrentSlotStatus.Invalid; + } + + state = (int)(control & 0x7UL); + keyHash = finalHash; + return CurrentSlotStatus.Current; + } + + private MutationSnapshotStatus TryReadMutationSnapshot( + ulong bindingRaw, + ref ValueSlotMetadataV2 slot, + out ulong keyHash, + out ulong operationRaw, + out DirectoryOperation operation, + out int state, + out SlotPublicationIntent publicationIntent) + { + keyHash = 0; + operationRaw = 0; + operation = default; + state = SlotRetired; + publicationIntent = SlotPublicationIntent.None; + if (!TryDecodeBinding(bindingRaw, out var binding)) + { + return MutationSnapshotStatus.Invalid; + } + + for (var attempt = 0; attempt < 8; attempt++) + { + ulong control1 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + ControlBindingStatus controlStatus1 = ClassifyControlBinding( + control1, + binding.Generation); + if (controlStatus1 != ControlBindingStatus.Current) + { + return controlStatus1 == ControlBindingStatus.Invalid + ? MutationSnapshotStatus.Invalid + : MutationSnapshotStatus.Stale; + } + + ulong directoryBinding = slot.DirectoryBinding; + ulong observedKeyHash = slot.KeyHash; + int rawPublicationIntent = Volatile.Read(ref slot.PublicationIntent); + ulong observedOperation = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + ulong control2 = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.Control)); + if (control1 != control2 || directoryBinding != bindingRaw) + { + ControlBindingStatus controlStatus2 = ClassifyControlBinding( + control2, + binding.Generation); + if (controlStatus2 != ControlBindingStatus.Current + || slot.DirectoryBinding != bindingRaw) + { + return controlStatus2 == ControlBindingStatus.Invalid + ? MutationSnapshotStatus.Invalid + : MutationSnapshotStatus.Stale; + } + + continue; + } + + state = (int)(control2 & 0x7UL); + keyHash = observedKeyHash; + operationRaw = observedOperation; + publicationIntent = (SlotPublicationIntent)rawPublicationIntent; + if (publicationIntent is not ( + SlotPublicationIntent.ExplicitReservation + or SlotPublicationIntent.AtomicPublication)) + { + publicationIntent = SlotPublicationIntent.None; + return MutationSnapshotStatus.Invalid; + } + + if (operationRaw == 0) + { + return MutationSnapshotStatus.Current; + } + + return TryDecodeOperation(operationRaw, out operation) + ? MutationSnapshotStatus.Current + : MutationSnapshotStatus.Invalid; + } + + return MutationSnapshotStatus.Retry; + } + + private StoreStatus ClassifyCurrentOperation( + ulong bindingRaw, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + DirectoryOperation operation, + out int state) + { + state = SlotRetired; + if (!TryDecodeBinding(bindingRaw, out var binding) + || operation.Generation != binding.Generation) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)) != operationRaw) + { + return StoreStatus.NotFound; + } + + CurrentSlotStatus slotStatus = TryReadCurrentSlotStatus( + bindingRaw, + ref slot, + out state, + out _); + if (slotStatus != CurrentSlotStatus.Current) + { + if (slotStatus == CurrentSlotStatus.Invalid + && unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)) == operationRaw) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + return slotStatus == CurrentSlotStatus.Retry + ? StoreStatus.StoreBusy + : StoreStatus.NotFound; + } + + return unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)) == operationRaw + ? StoreStatus.Success + : StoreStatus.NotFound; + } + + private StoreStatus ClassifyCurrentCanonicalOperation( + int canonicalBucketIndex, + ulong bindingRaw, + ref ValueSlotMetadataV2 slot, + ulong operationRaw, + DirectoryOperation operation, + out int state) + { + state = SlotRetired; + if (unchecked((ulong)AtomicControlWord.LoadAcquire( + ref BucketMutation(canonicalBucketIndex))) != bindingRaw) + { + return StoreStatus.NotFound; + } + + StoreStatus operationStatus = ClassifyCurrentOperation( + bindingRaw, + ref slot, + operationRaw, + operation, + out state); + if (operationStatus != StoreStatus.Success) + { + return operationStatus; + } + + return unchecked((ulong)AtomicControlWord.LoadAcquire( + ref BucketMutation(canonicalBucketIndex))) == bindingRaw + ? StoreStatus.Success + : StoreStatus.NotFound; + } + + private StoreStatus PublishReserved(ref ValueSlotMetadataV2 slot, ulong bindingRaw) + { + if (!TryDecodeBinding(bindingRaw, out var binding)) + { + return StoreStatus.CorruptStore; + } + + for (var attempt = 0; attempt < DefaultRetryBudget; attempt++) + { + long observedControl = AtomicControlWord.LoadAcquire(ref slot.Control); + if (!LockFreeSlotTable.TryClassifyStructuralControl( + observedControl, + _layout.ParticipantRecordCount, + out _)) + { + return StoreStatus.CorruptStore; + } + + var observed = unchecked((ulong)observedControl); + if (((observed >> 3) & SlotGenerationMask) != (ulong)binding.Generation) + { + return StoreStatus.CorruptStore; + } + + var state = (int)(observed & 0x7); + if (state == SlotReserved) + { + return StoreStatus.Success; + } + + if (state != SlotInitializing) + { + return StoreStatus.CorruptStore; + } + + var desired = (observed & ~0x7UL) | SlotReserved; + long publicationObservation = AtomicControlWord.CompareExchange( + ref slot.Control, + unchecked((long)desired), + unchecked((long)observed)); + if (!LockFreeSlotTable.TryClassifyStructuralControl( + publicationObservation, + _layout.ParticipantRecordCount, + out _)) + { + return StoreStatus.CorruptStore; + } + + if (unchecked((ulong)publicationObservation) == observed) + { + return StoreStatus.Success; + } + } + + return StoreStatus.StoreBusy; + } + + private StoreStatus PrepareOperation( + ref ValueSlotMetadataV2 slot, + ulong binding, + ulong prepared, + int intent, + in LockFreeOperationBudget budget) + { + if (!TryDecodeBinding(binding, out var decodedBinding) + || !TryDecodeOperation(prepared, out var preparedOperation) + || preparedOperation.Intent != intent + || preparedOperation.Phase != PhasePrepared + || preparedOperation.Generation != decodedBinding.Generation) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + CurrentSlotStatus slotStatus = TryReadCurrentSlotStatus( + binding, + ref slot, + out int slotState, + out _); + if (slotStatus != CurrentSlotStatus.Current) + { + return CurrentSlotFailure(slotStatus, intent); + } + + var observed = unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)); + if (observed == prepared) + { + slotStatus = TryReadCurrentSlotStatus(binding, ref slot, out _, out _); + return slotStatus == CurrentSlotStatus.Current + ? StoreStatus.Success + : CurrentSlotFailure(slotStatus, intent); + } + + if (observed == 0) + { + slotStatus = TryReadCurrentSlotStatus(binding, ref slot, out _, out _); + if (slotStatus != CurrentSlotStatus.Current) + { + return CurrentSlotFailure(slotStatus, intent); + } + + ulong publicationObservation = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)prepared), + comparand: 0)); + StoreStatus publicationCas = ValidateOperationCasObservation( + ref slot.DirectoryOperation, + expected: 0, + desired: prepared, + publicationObservation); + if (publicationCas != StoreStatus.Success) + { + return publicationCas; + } + + if (publicationObservation == 0) + { + slotStatus = TryReadCurrentSlotStatus(binding, ref slot, out _, out _); + if (slotStatus == CurrentSlotStatus.Current) + { + return StoreStatus.Success; + } + + if (slotStatus == CurrentSlotStatus.Invalid) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + // Ownership changed after publication. Withdraw only this + // exact generation-tagged descriptor; a newer operation + // cannot compare equal. + StoreStatus operationCleanup = TryClearOperationReference( + ref slot.DirectoryOperation, + prepared, + out _); + if (operationCleanup != StoreStatus.Success) + { + return operationCleanup; + } + + return CurrentSlotFailure(slotStatus, intent); + } + + continue; + } + + if (!TryDecodeOperation(observed, out var operation)) + { + slotStatus = TryReadCurrentSlotStatus(binding, ref slot, out _, out _); + return slotStatus is CurrentSlotStatus.Current or CurrentSlotStatus.Invalid + ? CorruptFrom(nameof(LockFreeKeyDirectory)) + : CurrentSlotFailure(slotStatus, intent); + } + + if (operation.Generation != decodedBinding.Generation) + { + slotStatus = TryReadCurrentSlotStatus(binding, ref slot, out _, out _); + bool stillCurrent = slotStatus == CurrentSlotStatus.Current; + if (slotStatus == CurrentSlotStatus.Invalid) + { + return CorruptFrom(nameof(LockFreeKeyDirectory)); + } + + if (operation.Generation > decodedBinding.Generation) + { + // A future descriptor belongs to a reused slot. If the old + // binding somehow still appears current this is structural + // corruption; otherwise this helper simply lost ownership. + return stillCurrent + ? CorruptFrom(nameof(LockFreeKeyDirectory)) + : intent == IntentInsert + ? StoreStatus.InvalidReservation + : StoreStatus.NotFound; + } + + if (!stillCurrent) + { + return intent == IntentInsert + ? StoreStatus.InvalidReservation + : StoreStatus.NotFound; + } + + StoreStatus operationCleanup = TryClearOperationReference( + ref slot.DirectoryOperation, + observed, + out _); + if (operationCleanup != StoreStatus.Success) + { + return operationCleanup; + } + + continue; + } + + if (intent == IntentInsert) + { + if (operation.Intent == IntentInsert) + { + return StoreStatus.Success; + } + + return StoreStatus.InvalidReservation; + } + + if (operation.Intent == IntentUnlink) + { + return StoreStatus.Success; + } + + if (operation.Intent != IntentInsert + || slotState is not (SlotAborting or SlotReclaiming)) + { + return StoreStatus.StoreBusy; + } + + ulong replacementObservation = unchecked((ulong)AtomicControlWord.CompareExchange( + ref slot.DirectoryOperation, + unchecked((long)prepared), + unchecked((long)observed))); + StoreStatus replacementCas = ValidateOperationCasObservation( + ref slot.DirectoryOperation, + observed, + prepared, + replacementObservation); + if (replacementCas != StoreStatus.Success) + { + return replacementCas; + } + + if (replacementObservation == observed) + { + return StoreStatus.Success; + } + + if (attempt + 1 >= DefaultRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + } + } + + private StoreStatus CurrentSlotFailure(CurrentSlotStatus status, int intent) => + status switch + { + CurrentSlotStatus.Invalid => CorruptFrom(nameof(LockFreeKeyDirectory)), + CurrentSlotStatus.Retry => StoreStatus.StoreBusy, + _ => intent == IntentInsert + ? StoreStatus.InvalidReservation + : StoreStatus.NotFound, + }; + + private bool TryGetKey(ref ValueSlotMetadataV2 slot, out ReadOnlySpan key) + { + key = default; + if (slot.KeyLength <= 0 || slot.KeyLength > _layout.MaxKeyBytes) + { + return false; + } + + if (!TryDecodeBinding(slot.DirectoryBinding, out var binding) || binding.SlotIndex >= _layout.SlotCount) + { + return false; + } + + var expectedOffset = _layout.KeyStorageOffset + ((long)binding.SlotIndex * _layout.KeyStride); + if (slot.KeyOffset != expectedOffset + || expectedOffset < _layout.KeyStorageOffset + || expectedOffset + slot.KeyLength > _layout.KeyStorageOffset + _layout.KeyStorageLength) + { + return false; + } + + key = new ReadOnlySpan(_region.Pointer + expectedOffset, slot.KeyLength); + return true; + } + + private bool TryGetTargetCell(int kind, long index, out CellReference cell) + { + if (kind == TargetPrimary && index >= 0 && index < _layout.PrimaryLaneCount) + { + cell = new CellReference(_region.Pointer + PrimaryCellOffset((int)index)); + return true; + } + + if (kind == TargetOverflow && index >= 0 && index < _layout.SlotCount) + { + cell = new CellReference(_region.Pointer + OverflowCellOffset((int)index)); + return true; + } + + cell = default; + return false; + } + + private void GetBuckets(ulong hash, out int first, out int second) + { + first = (int)(Mix(hash) & (uint)_bucketMask); + second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & (uint)_bucketMask); + if (second == first) + { + second = (first + 1) & _bucketMask; + } + } + + private int OverflowStart(ulong hash) => (int)(Mix(hash ^ 0xd6e8_feb8_6659_fd93UL) % (uint)_layout.SlotCount); + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private ref ValueSlotMetadataV2 Slot(int slotIndex) => + ref *(ValueSlotMetadataV2*)(_region.Pointer + _layout.SlotMetadataOffset + ((long)slotIndex * _layout.SlotMetadataStride)); + + private ref long BucketSpillSummary(int bucketIndex) => + ref *(long*)(_region.Pointer + _layout.PrimaryDirectoryOffset + ((long)bucketIndex * _layout.PrimaryBucketStride)); + + private ref long BucketMutation(int bucketIndex) => + ref *(long*)(_region.Pointer + _layout.PrimaryDirectoryOffset + ((long)bucketIndex * _layout.PrimaryBucketStride) + 8); + + private ref long PrimaryCell(int absoluteCellIndex) => + ref *(long*)(_region.Pointer + PrimaryCellOffset(absoluteCellIndex)); + + private long PrimaryCellOffset(int absoluteCellIndex) + { + var bucket = absoluteCellIndex / LayoutV2Constants.PrimaryLanesPerBucket; + var lane = absoluteCellIndex % LayoutV2Constants.PrimaryLanesPerBucket; + return _layout.PrimaryDirectoryOffset + + ((long)bucket * _layout.PrimaryBucketStride) + + 16 + + (lane * sizeof(long)); + } + + private ref long OverflowCell(int index) => + ref *(long*)(_region.Pointer + OverflowCellOffset(index)); + + private long OverflowCellOffset(int index) => + _layout.OverflowDirectoryOffset + ((long)index * _layout.OverflowStride); + + /// + /// Reads a directory binding word and rejects only a malformed value that + /// remains in the exact same location on confirmation. A concurrently + /// replaced malformed sample is contention; a valid binding for an older + /// slot generation remains ordinary stale directory state. + /// + private StoreStatus TryReadStructurallyValidBindingReference( + ref long reference, + out ulong observed) + { + for (var attempt = 0; attempt < 8; attempt++) + { + observed = unchecked((ulong)AtomicControlWord.LoadAcquire(ref reference)); + if (observed == 0 + || (TryDecodeBinding(observed, out IndexBinding binding) + && binding.SlotIndex < _layout.SlotCount)) + { + return StoreStatus.Success; + } + + if (unchecked((ulong)AtomicControlWord.LoadAcquire(ref reference)) == observed) + { + return CorruptHere(); + } + } + + observed = 0; + return StoreStatus.StoreBusy; + } + + private static bool TryDecodeBinding(ulong raw, out IndexBinding binding) + { + try + { + binding = IndexBinding.Decode(raw); + return true; + } + catch (ArgumentOutOfRangeException) + { + binding = default; + return false; + } + catch (OverflowException) + { + binding = default; + return false; + } + } + + private bool TryDecodeSpillSummary(ulong raw, out SpillSummary summary) + { + try + { + summary = SpillSummary.Decode(raw); + return summary.IsInitial || (uint)summary.SlotIndex < (uint)_layout.SlotCount; + } + catch (ArgumentOutOfRangeException) + { + summary = default; + return false; + } + catch (OverflowException) + { + summary = default; + return false; + } + } + + private static bool TryDecodeOperation(ulong raw, out DirectoryOperation operation) + { + try + { + operation = DirectoryOperation.Decode(raw); + if (raw == 0 + || operation.Intent is not (IntentInsert or IntentUnlink) + || operation.Phase is < PhasePrepared or > PhaseComplete) + { + return false; + } + + if (operation.Phase == PhasePrepared) + { + return operation.Kind == 0 && operation.Index == 0; + } + + if (operation.Phase == PhaseRejected) + { + return operation.Intent == IntentInsert + && operation.Kind == 0 + && operation.Index == 0; + } + + if (operation.Phase == PhaseComplete + && operation.Intent == IntentUnlink + && operation.Kind == 0) + { + return operation.Index == 0; + } + + return operation.Kind is TargetPrimary or TargetOverflow; + } + catch (ArgumentOutOfRangeException) + { + operation = default; + return false; + } + catch (OverflowException) + { + operation = default; + return false; + } + } + + private static bool TryDecodeLocation(ulong raw, out DirectoryLocation location) + { + try + { + location = DirectoryLocation.Decode(raw); + return raw != 0 && location.Kind is TargetPrimary or TargetOverflow; + } + catch (ArgumentOutOfRangeException) + { + location = default; + return false; + } + catch (OverflowException) + { + location = default; + return false; + } + } + + private StoreStatus CorruptHere( + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) => + LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeKeyDirectory), + member, + line); + + private StoreStatus CorruptFrom( + string component, + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) => + LockFreeStoreControl.ReportCorruption( + _storeControl, + component, + member, + line); + + private enum BindingValidation + { + Stale, + CurrentOther, + Exact, + Invalid + } + + private enum ControlBindingStatus + { + Current, + Stale, + Invalid, + } + + private enum CurrentSlotStatus + { + Current, + Stale, + Retry, + Invalid, + } + + private enum ExactCanonicalSourceStatus + { + Current, + Advanced, + Stale, + Retry, + Invalid, + } + + private enum MutationSnapshotStatus + { + Stale, + Current, + Retry, + Invalid + } + + private enum CleanupReferenceKind + { + Binding, + Location, + Operation, + } + + private readonly unsafe struct CellReference + { + private readonly long* _pointer; + + internal CellReference(byte* pointer) + { + _pointer = (long*)pointer; + } + + internal ref long Value => ref *_pointer; + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeLeaseRegistry.cs b/src/SharedMemoryStore/LockFree/LockFreeLeaseRegistry.cs new file mode 100644 index 0000000..6ea0837 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeLeaseRegistry.cs @@ -0,0 +1,1586 @@ +using System.Runtime.CompilerServices; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Incarnation-fenced layout-v2 lease records. Claim and activation are +/// deliberately separate so the engine can relinquish an internal lease when +/// its final directory/slot revalidation fails. +/// +internal sealed unsafe class LockFreeLeaseRegistry +{ + internal const int FreeState = 0; + internal const int ClaimingState = 1; + internal const int ActiveState = 2; + internal const int ReleasingState = 3; + internal const int RecoveringState = 4; + internal const int RetiredState = 5; + internal const long TerminalIncarnation = 0x1_ffff_ffffL; + + private const ulong IncarnationMask = 0x1_ffff_ffffUL; + private const ulong ParticipantMask = 0x0fff_ffffUL; + private const ulong RecordIndexMask = 0x7fff_ffffUL; + private const int MaxRecoveryAttemptsPerRecord = 8; + + private readonly byte* _mappingBase; + private readonly StoreLayoutV2 _layout; + private readonly LockFreeParticipantRegistry.Registration _participant; + private readonly LockFreeParticipantRegistry _participants; + private readonly LockFreeTelemetry _telemetry; + private readonly LockFreeStoreControl? _storeControl; + private readonly ulong _storeId; + private readonly long[] _leaseTableFullSnapshot; + private int _leaseTableFullProofGate; + + internal LockFreeLeaseRegistry( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + in LockFreeParticipantRegistry.Registration participant, + LockFreeParticipantRegistry participants) + : this(region, layout, participant, participants, new LockFreeTelemetry()) + { + } + + internal LockFreeLeaseRegistry( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + in LockFreeParticipantRegistry.Registration participant, + LockFreeParticipantRegistry participants, + LockFreeTelemetry telemetry, + LockFreeStoreControl? storeControl = null) + { + ArgumentNullException.ThrowIfNull(region); + ArgumentNullException.ThrowIfNull(participants); + if (!participant.IsValid || participant.Token > ParticipantMask) + { + throw new ArgumentOutOfRangeException(nameof(participant)); + } + + _mappingBase = region.Pointer; + _layout = layout; + _participant = participant; + _participants = participants; + _telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry)); + _storeControl = storeControl; + // An exhausted claim scan is not a simultaneous-capacity witness: a + // released record can rotate behind the scanner. Keep one eager, + // process-local snapshot per open handle for the rare exact proof path. + // The buffer adds eight bytes per configured lease record, is reused by + // every operation, and never appears in mapped state. + _leaseTableFullSnapshot = + GC.AllocateUninitializedArray(layout.LeaseRecordCount); + _storeId = ((StoreHeaderV2*)_mappingBase)->StoreId; + if (_storeId == 0) + { + throw new ArgumentException("The layout-v2 mapping has no store incarnation.", nameof(region)); + } + } + + /// + /// Claims a free record with the complete participant token in the first + /// shared CAS, revalidates that participant, and fills the exact binding. + /// + internal StoreStatus TryClaim( + ulong slotBinding, + long acquireSequence, + out LeaseHandle lease) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryClaim( + slotBinding, + acquireSequence, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint, + out lease); + } + + internal StoreStatus TryClaim( + ulong slotBinding, + long acquireSequence, + in LockFreeOperationBudget budget, + out LeaseHandle lease) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryClaim( + slotBinding, + acquireSequence, + budget, + ref checkpoint, + out lease); + } + + internal StoreStatus TryClaim( + ulong slotBinding, + long acquireSequence, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out LeaseHandle lease) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + lease = default; + if (!IsParticipantActive()) + { + return ParticipantUnavailableStatus(); + } + + if (!IsValidSlotBinding(slotBinding)) + { + return CorruptHere(); + } + + var capacityRetryAttempt = 0; + RetryClaim: + if (capacityRetryAttempt != 0 && !IsParticipantActive()) + { + return ParticipantUnavailableStatus(); + } + + // Prefer this participant's stable home record. Sequential readers then + // reuse a cache line that no other live participant normally writes, + // while multiple outstanding leases still fall through to the complete + // bounded table scan. This is placement only, never exclusive ownership. + int start = _participant.RecordIndex % _layout.LeaseRecordCount; + for (var visited = 0; visited < _layout.LeaseRecordCount; visited++) + { + StoreStatus bound = budget.CheckPeriodic(visited); + if (bound != StoreStatus.Success) + { + return bound; + } + + var index = start + visited; + if (index >= _layout.LeaseRecordCount) + { + index -= _layout.LeaseRecordCount; + } + + ref var record = ref Record(index); + var observed = AtomicControlWord.LoadAcquire(ref record.Control); + StoreStatus structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + int observedState = State(observed); + long observedIncarnation = Incarnation(observed); + if (observedState is ReleasingState or RecoveringState + && Participant(observed) == 0 + && observedIncarnation is >= 1 and <= TerminalIncarnation) + { + // Releasing/Recovering has no owner-only writes left. Any + // claimant may finish the exact incarnation before deciding + // that the table is full; a delayed recycler is fenced by its + // old expected control once this record is claimed again. + structure = TryRecycle( + index, + observedIncarnation, + observed, + out bool recycled); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (recycled) + { + _telemetry.RecordHelpedTransition(); + } + + observed = AtomicControlWord.LoadAcquire(ref record.Control); + structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + } + + if (State(observed) != FreeState) + { + continue; + } + + var incarnation = Incarnation(observed); + var claiming = OwnedControl(ClaimingState, incarnation, _participant.Token); + long claimObservation = AtomicControlWord.CompareExchange( + ref record.Control, + claiming, + observed); + if (claimObservation != observed) + { + _telemetry.RecordCasLoss(); + structure = ValidateControl(claimObservation); + if (structure != StoreStatus.Success) + { + return structure; + } + + continue; + } + + var leaseToken = IndexBinding.Encode(index, incarnation); + lease = new LeaseHandle(_storeId, _participant.Token, slotBinding, leaseToken); + + // A crash at the CAS above is recoverable because its complete owner + // token is already in shared state. A participant that begins closing + // here cannot leave a legitimate owner-controlled claim behind. + if (!IsParticipantActive()) + { + StoreStatus unavailable = ParticipantUnavailableStatus(); + StoreStatus cancel = unavailable == StoreStatus.CorruptStore + ? unavailable + : TryCancelClaim(lease); + lease = default; + return unavailable == StoreStatus.CorruptStore + || cancel == StoreStatus.CorruptStore + ? CorruptHere() + : StoreStatus.StoreDisposed; + } + + record.SlotBinding = slotBinding; + record.AcquireSequence = acquireSequence; + return StoreStatus.Success; + } + + // A sequential exhausted scan is only a candidate: a free record can + // rotate behind it while another reader releases and reclaims records. + // Only two identical, structurally valid all-occupied collects expose + // LeaseTableFull. Every other result is ordinary contention governed by + // the caller's operation-wide wait budget. + StoreStatus proof = TryProveLeaseTableFull( + budget, + ref checkpoint, + out bool provenFull); + if (proof != StoreStatus.Success) + { + return proof; + } + + if (provenFull) + { + return StoreStatus.LeaseTableFull; + } + + if (!budget.TryContinueAfterContention( + capacityRetryAttempt++, + out StoreStatus capacityTerminal)) + { + return capacityTerminal; + } + + goto RetryClaim; + } + + /// + /// Converts scan exhaustion into an exact physical lease-capacity result. + /// Equal all-occupied controls in the same order prove a common point + /// between the collects at which no lease record was reusable. A free + /// record, movement, or another local proof holding the buffer is transient. + /// + private StoreStatus TryProveLeaseTableFull( + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out bool provenFull) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + provenFull = false; + if (Interlocked.CompareExchange(ref _leaseTableFullProofGate, 1, 0) != 0) + { + return StoreStatus.Success; + } + + long proofToken = 0; + var candidateObserved = false; + var proofConfirmed = false; + try + { + for (var index = 0; index < _layout.LeaseRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref Record(index).Control); + StoreStatus classification = ClassifyCapacityControl( + control, + out bool occupied); + if (classification != StoreStatus.Success) + { + return classification; + } + + if (!occupied) + { + return StoreStatus.Success; + } + + _leaseTableFullSnapshot[index] = control; + } + + proofToken = LockFreeCheckpoint.BeginLeaseTableFullProof( + ref checkpoint, + _layout.LeaseRecordCount); + candidateObserved = true; + + for (var index = 0; index < _layout.LeaseRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref Record(index).Control); + StoreStatus classification = ClassifyCapacityControl( + control, + out bool occupied); + if (classification != StoreStatus.Success) + { + return classification; + } + + if (!occupied || control != _leaseTableFullSnapshot[index]) + { + return StoreStatus.Success; + } + } + + proofConfirmed = true; + LockFreeCheckpoint.CompleteLeaseTableFullProof( + ref checkpoint, + proofToken, + confirmed: true); + provenFull = true; + return StoreStatus.Success; + } + finally + { + if (candidateObserved && !proofConfirmed) + { + LockFreeCheckpoint.CompleteLeaseTableFullProof( + ref checkpoint, + proofToken, + confirmed: false); + } + + Volatile.Write(ref _leaseTableFullProofGate, 0); + } + } + + /// Publishes a fully initialized claimed record as active. + internal StoreStatus TryActivate(in LeaseHandle lease) + { + if (!TryDecodeHandle(lease, out var index, out var incarnation)) + { + return StoreStatus.InvalidLease; + } + + ref var record = ref Record(index); + var claiming = OwnedControl(ClaimingState, incarnation, lease.ParticipantToken); + long observed = AtomicControlWord.LoadAcquire(ref record.Control); + StoreStatus structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed != claiming || record.SlotBinding != lease.SlotBinding) + { + StoreStatus cancel = TryCancelClaim(lease); + return cancel == StoreStatus.CorruptStore + ? CorruptHere() + : StoreStatus.InvalidLease; + } + + if (!IsParticipantActive()) + { + StoreStatus unavailable = ParticipantUnavailableStatus(); + StoreStatus cancel = unavailable == StoreStatus.CorruptStore + ? unavailable + : TryCancelClaim(lease); + return cancel == StoreStatus.CorruptStore + ? CorruptHere() + : StoreStatus.StoreDisposed; + } + + var active = OwnedControl(ActiveState, incarnation, lease.ParticipantToken); + observed = AtomicControlWord.CompareExchange(ref record.Control, active, claiming); + if (observed != claiming) + { + return LeaseStatus(observed, incarnation); + } + + if (IsParticipantActive()) + { + return StoreStatus.Success; + } + + StoreStatus participantStatus = ParticipantUnavailableStatus(); + if (participantStatus == StoreStatus.CorruptStore) + { + return CorruptHere(); + } + + var recovering = UnownedControl(RecoveringState, incarnation); + observed = AtomicControlWord.CompareExchange(ref record.Control, recovering, active); + if (observed == active) + { + _ = TryRecycle(index, incarnation, recovering, out _); + } + else if (ValidateControl(observed) != StoreStatus.Success) + { + return CorruptHere(); + } + + return participantStatus; + } + + internal StoreStatus TryClaimAndActivate( + ulong slotBinding, + long acquireSequence, + out LeaseHandle lease) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryClaimAndActivate( + slotBinding, + acquireSequence, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint, + out lease); + } + + internal StoreStatus TryClaimAndActivate( + ulong slotBinding, + long acquireSequence, + in LockFreeOperationBudget budget, + out LeaseHandle lease) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryClaimAndActivate( + slotBinding, + acquireSequence, + budget, + ref checkpoint, + out lease); + } + + internal StoreStatus TryClaimAndActivate( + ulong slotBinding, + long acquireSequence, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out LeaseHandle lease) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + var status = TryClaim( + slotBinding, + acquireSequence, + budget, + ref checkpoint, + out lease); + if (status != StoreStatus.Success) + { + return status; + } + + status = TryActivate(lease); + if (status != StoreStatus.Success) + { + lease = default; + } + + return status; + } + + /// Relinquishes a claim that never became a public acquire. + internal StoreStatus TryCancelClaim(in LeaseHandle lease) + { + if (!TryDecodeHandle(lease, out var index, out var incarnation)) + { + return StoreStatus.InvalidLease; + } + + ref var record = ref Record(index); + var claiming = OwnedControl(ClaimingState, incarnation, lease.ParticipantToken); + var recovering = UnownedControl(RecoveringState, incarnation); + var observed = AtomicControlWord.CompareExchange(ref record.Control, recovering, claiming); + StoreStatus structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed == claiming || observed == recovering) + { + _ = TryRecycle(index, incarnation, recovering, out _); + return StoreStatus.Success; + } + + return LeaseStatus(observed, incarnation); + } + + internal bool IsActive(in LeaseHandle lease) + { + return TryReadActiveSlotBinding(lease, out _); + } + + internal bool TryGetActiveSlotBinding(in LeaseHandle lease, out ulong slotBinding) + { + return TryReadActiveSlotBinding(lease, out slotBinding); + } + + private bool TryReadActiveSlotBinding(in LeaseHandle lease, out ulong slotBinding) + { + slotBinding = 0; + if (!TryDecodeHandle(lease, out int index, out long incarnation) + || !IsParticipantActive()) + { + return false; + } + + ref LeaseRecordV2 record = ref Record(index); + long active = OwnedControl(ActiveState, incarnation, lease.ParticipantToken); + long control1 = AtomicControlWord.LoadAcquire(ref record.Control); + if (ValidateControl(control1) != StoreStatus.Success || control1 != active) + { + return false; + } + + ulong observedBinding = Volatile.Read(ref record.SlotBinding); + long control2 = AtomicControlWord.LoadAcquire(ref record.Control); + if (ValidateControl(control2) != StoreStatus.Success || control2 != control1) + { + return false; + } + + if (!IsValidSlotBinding(observedBinding) || observedBinding != lease.SlotBinding) + { + _ = CorruptHere(); + return false; + } + + if (!IsParticipantActive() + || !TryConfirmStructuralControl(ref record.Control, control1)) + { + return false; + } + + slotBinding = observedBinding; + return true; + } + + /// Ends protection at the exact Active-to-Releasing CAS. + internal StoreStatus TryRelease(in LeaseHandle lease) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryRelease(lease, ref checkpoint); + } + + internal StoreStatus TryRelease( + in LeaseHandle lease, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (!TryDecodeHandle(lease, out var index, out var incarnation)) + { + return StoreStatus.InvalidLease; + } + + ref var record = ref Record(index); + var active = OwnedControl(ActiveState, incarnation, lease.ParticipantToken); + var releasing = UnownedControl(ReleasingState, incarnation); + const int confirmationAttempts = 8; + for (var attempt = 0; attempt < confirmationAttempts; attempt++) + { + var observed = AtomicControlWord.CompareExchange(ref record.Control, releasing, active); + if (observed == active) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.ReleaseAfterOwnershipReleaseCas); + + // Active-to-Releasing is the public release ordering point. + // Cleanup corruption is latched by TryRecycle but cannot undo + // the already-completed release observed by this caller. + _ = TryRecycle(index, incarnation, releasing, out _); + return StoreStatus.Success; + } + + _telemetry.RecordCasLoss(); + + StoreStatus structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + long observedIncarnation = Incarnation(observed); + int observedState = State(observed); + if (observedIncarnation == incarnation + && observedState is ReleasingState or RecoveringState) + { + StoreStatus recycle = TryRecycle( + index, + incarnation, + observed, + out _); + return recycle == StoreStatus.Success + ? StoreStatus.LeaseAlreadyReleased + : recycle; + } + + if ((observedState == FreeState && observedIncarnation == incarnation + 1) + || (observedState == RetiredState && observedIncarnation == incarnation)) + { + return StoreStatus.LeaseAlreadyReleased; + } + + if (observedIncarnation > incarnation) + { + return StoreStatus.InvalidLease; + } + + // Active with the exact token would have won the CAS. Every other + // same/older-incarnation word is a protocol regression, not a + // stale-handle outcome. Confirm the exact mapped word before + // poisoning; movement during confirmation retries the release. + long confirmed = AtomicControlWord.CompareExchange( + ref record.Control, + observed, + observed); + if (confirmed == observed) + { + return CorruptHere(); + } + + structure = ValidateControl(confirmed); + if (structure != StoreStatus.Success) + { + return structure; + } + } + + return StoreStatus.StoreBusy; + } + + /// + /// Stable double-read classification used by remove/reclamation. Claiming + /// records are not protection; only exact Active records count. + /// + internal StoreStatus ScanHasActiveLease( + ulong slotBinding, + in LockFreeOperationBudget budget, + out bool hasActiveLease) + { + hasActiveLease = false; + for (var index = 0; index < _layout.LeaseRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref var record = ref Record(index); + var control1 = AtomicControlWord.LoadAcquire(ref record.Control); + StoreStatus structure = ValidateControl(control1); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (State(control1) != ActiveState) + { + continue; + } + + var observedBinding = record.SlotBinding; + var control2 = AtomicControlWord.LoadAcquire(ref record.Control); + structure = ValidateControl(control2); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (control1 != control2) + { + continue; + } + + if (!IsValidSlotBinding(observedBinding)) + { + return CorruptFrom(nameof(LockFreeLeaseRegistry)); + } + + if (observedBinding == slotBinding) + { + hasActiveLease = true; + return StoreStatus.Success; + } + } + + return StoreStatus.Success; + } + + /// Releases every exact claim/lease owned by one closing participant. + internal int ReleaseParticipantLeases( + ulong participantToken, + LockFreeReclaimer reclaimer) + { + NoOpLockFreeCheckpoint checkpoint = default; + return ReleaseParticipantLeases(participantToken, reclaimer, ref checkpoint); + } + + internal int ReleaseParticipantLeases( + ulong participantToken, + LockFreeReclaimer reclaimer, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + _ = ReleaseParticipantLeases( + participantToken, + reclaimer, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint, + out int released); + return released; + } + + internal StoreStatus ReleaseParticipantLeases( + ulong participantToken, + LockFreeReclaimer reclaimer, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out int released) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + ArgumentNullException.ThrowIfNull(reclaimer); + released = 0; + for (var index = 0; index < _layout.LeaseRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref var record = ref Record(index); + long observed = AtomicControlWord.LoadAcquire(ref record.Control); + StoreStatus structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + int state = State(observed); + ulong slotBinding = 0; + if (state == ActiveState) + { + slotBinding = unchecked((ulong)Volatile.Read(ref record.SlotBinding)); + long confirmed = AtomicControlWord.LoadAcquire(ref record.Control); + structure = ValidateControl(confirmed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (confirmed != observed) + { + continue; + } + + if (!IsValidSlotBinding(slotBinding)) + { + return CorruptFrom(nameof(LockFreeLeaseRegistry)); + } + } + + if (Participant(observed) != participantToken + || state is not (ClaimingState or ActiveState)) + { + continue; + } + + long incarnation = Incarnation(observed); + long recovering = UnownedControl(RecoveringState, incarnation); + long releaseObservation = AtomicControlWord.CompareExchange( + ref record.Control, + recovering, + observed); + if (releaseObservation != observed) + { + _telemetry.RecordCasLoss(); + structure = ValidateControl(releaseObservation); + if (structure != StoreStatus.Success) + { + return structure; + } + + continue; + } + + // The exact participant-owned lease is now universally helpable. + // Do not spend another retry before publishing this checkpoint or + // recycling the exact incarnation. + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.ReleaseAfterOwnershipReleaseCas); + _ = TryRecycle(index, incarnation, recovering, out _); + released++; + + if (slotBinding != 0 && budget.Check() == StoreStatus.Success) + { + _ = reclaimer.TryReclaim(slotBinding, budget, ref checkpoint); + } + } + + return StoreStatus.Success; + } + + /// + /// Performs an explicit bounded scan. Owner-controlled records are changed + /// only by an exact participant-token/incarnation CAS; published unowned + /// release phases are safe for every caller to help. + /// + internal StoreStatus TryRecover( + LeaseRecoveryOptions options, + StoreWaitOptions waitOptions, + LockFreeReclaimer reclaimer, + out LeaseRecoveryReport report) + { + report = default; + if (!waitOptions.IsValid) + { + return StoreStatus.UnknownFailure; + } + + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions); + return TryRecover(options, budget, reclaimer, out report); + } + + internal StoreStatus TryRecover( + LeaseRecoveryOptions options, + in LockFreeOperationBudget budget, + LockFreeReclaimer reclaimer, + out LeaseRecoveryReport report) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryRecover(options, budget, reclaimer, ref checkpoint, out report); + } + + internal StoreStatus TryRecover( + LeaseRecoveryOptions options, + in LockFreeOperationBudget budget, + LockFreeReclaimer reclaimer, + ref TCheckpoint checkpoint, + out LeaseRecoveryReport report) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + ArgumentNullException.ThrowIfNull(reclaimer); + report = default; + + var scanned = 0; + var recovered = 0; + var active = 0; + var unsupported = 0; + var failed = 0; + LockFreeOperationBudget postOwnershipCleanup = default; + bool postOwnershipCleanupStarted = false; + var initialAttemptBudget = budget.IsNoWait + ? 1 + : MaxRecoveryAttemptsPerRecord; + + for (var index = 0; index < _layout.LeaseRecordCount; index++) + { + var bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return recovered > 0 ? StoreStatus.Success : bound; + } + + scanned++; + ref var record = ref Record(index); + var initial = AtomicControlWord.LoadAcquire(ref record.Control); + StoreStatus structure = ValidateControl(initial); + if (structure != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return structure; + } + + var targetIncarnation = Incarnation(initial); + var initialState = State(initial); + var targetParticipant = Participant(initial); + + if (initialState is FreeState or RetiredState) + { + continue; + } + + if (initialState is not (ClaimingState + or ActiveState + or ReleasingState + or RecoveringState)) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return CorruptHere(); + } + + var completed = false; + for (var attempt = 0; ; attempt++) + { + if (attempt >= initialAttemptBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return recovered > 0 ? StoreStatus.Success : terminal; + } + + bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return recovered > 0 ? StoreStatus.Success : bound; + } + + var observed = AtomicControlWord.LoadAcquire(ref record.Control); + structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return structure; + } + + if (Incarnation(observed) != targetIncarnation) + { + // The record was recycled and may already protect a new + // lease. One visit is permanently fenced to the incarnation + // first observed, so it never follows reuse. + completed = true; + break; + } + + var state = State(observed); + if (state is FreeState or RetiredState) + { + completed = true; + break; + } + + if (state is ReleasingState or RecoveringState) + { + structure = TryRecycle( + index, + targetIncarnation, + observed, + out bool recycled); + if (structure != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return structure; + } + + if (recycled) + { + _telemetry.RecordHelpedTransition(); + } + + completed = true; + break; + } + + if (state is not (ClaimingState or ActiveState) + || Participant(observed) != targetParticipant + || (initialState == ActiveState && state != ActiveState)) + { + failed++; + completed = true; + break; + } + + var slotBinding = record.SlotBinding; + long confirmed = AtomicControlWord.LoadAcquire(ref record.Control); + structure = ValidateControl(confirmed); + if (structure != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return structure; + } + + if (confirmed != observed) + { + continue; + } + + // Claiming may have stopped before initializing either ordinary + // field. Active must always carry a structurally valid exact + // binding before recovery is allowed to end its protection. + if ((state == ActiveState && !IsValidSlotBinding(slotBinding)) + || (state == ClaimingState + && slotBinding != 0 + && !IsValidSlotBinding(slotBinding))) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return CorruptHere(); + } + + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + bound = budget.Check(); + if (bound != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return recovered > 0 ? StoreStatus.Success : bound; + } + + var classification = _participants.ClassifyParticipant(targetParticipant); + if (_storeControl?.Validate() == StoreStatus.CorruptStore) + { + failed++; + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return CorruptHere(); + } + + confirmed = AtomicControlWord.LoadAcquire(ref record.Control); + structure = ValidateControl(confirmed); + if (structure != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return structure; + } + + if (confirmed != observed) + { + continue; + } + + var participantHandoffPublished = + classification.Incarnation.Token == targetParticipant + && classification.Kind is not ( + ParticipantClassificationKind.Changing or + ParticipantClassificationKind.Inconsistent) + && classification.Incarnation.State is + LayoutV2Constants.ParticipantClosing or + LayoutV2Constants.ParticipantRecovering; + var disposition = RecoveryDispositionFor( + state, + classification.Kind, + classification.Incarnation.State, + participantHandoffPublished, + options.RecoverCurrentProcessLeases); + switch (disposition) + { + case RecoveryDisposition.Retry: + continue; + case RecoveryDisposition.Failed: + failed++; + completed = true; + break; + case RecoveryDisposition.Unsupported: + unsupported++; + completed = true; + break; + case RecoveryDisposition.Active: + active++; + completed = true; + break; + case RecoveryDisposition.Recover: + bound = budget.Check(); + if (bound != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return recovered > 0 ? StoreStatus.Success : bound; + } + + var recovering = UnownedControl(RecoveringState, targetIncarnation); + long recoveryObservation = AtomicControlWord.CompareExchange( + ref record.Control, + recovering, + observed); + structure = ValidateControl(recoveryObservation); + if (structure != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return structure; + } + + if (recoveryObservation != observed) + { + _telemetry.RecordCasLoss(); + continue; + } + + // This exact CAS is the recovery/release point. From here + // cancellation cannot strand owner-controlled state. + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.RecoveryAfterExactRecoveryCas); + structure = TryRecycle( + index, + targetIncarnation, + recovering, + out _); + if (structure != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return structure; + } + + recovered++; + + if (slotBinding != 0) + { + StoreStatus removal = TryIsRemoveRequested( + slotBinding, + out bool isRemoveRequested); + if (removal != StoreStatus.Success) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return removal; + } + + if (isRemoveRequested) + { + if (!postOwnershipCleanupStarted) + { + postOwnershipCleanup = LockFreeOperationBudget.StartPostOwnershipCleanup(); + postOwnershipCleanupStarted = true; + } + + // TryReclaim performs the required fresh stable + // no-active-lease scan before changing slot control. + StoreStatus reclaim = reclaimer.TryReclaim( + slotBinding, + postOwnershipCleanup, + ref checkpoint); + if (reclaim == StoreStatus.CorruptStore) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return CorruptHere(); + } + } + } + + completed = true; + break; + default: + failed++; + completed = true; + break; + } + + if (completed) + { + break; + } + } + + if (!completed) + { + failed++; + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return StoreStatus.StoreBusy; + } + } + + var participantSweep = _participants.TryRecoverUnreferencedStaleParticipants( + budget, + ref checkpoint, + out _); + if (participantSweep == StoreStatus.CorruptStore) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return CorruptHere(); + } + + if (participantSweep != StoreStatus.Success && recovered == 0) + { + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return participantSweep; + } + + report = RecoveryReport(scanned, recovered, active, unsupported, failed); + return StoreStatus.Success; + } + + /// + /// Resolves only recovery authority; the caller still exact-CASes the lease + /// control observed before classification. A live current-process Claiming + /// record remains ineligible even with the test/shutdown override because + /// its claimant may still have ordinary initialization writes in flight. + /// Exact stable Closing or published Recovering participant control provides + /// unconditional quiescence/handoff; stale ownership is safe because its + /// process is gone. + /// + internal static RecoveryDisposition RecoveryDispositionFor( + int leaseState, + ParticipantClassificationKind classification, + int participantState, + bool participantHandoffPublished, + bool recoverCurrentProcessLeases) + { + if (leaseState is not (ClaimingState or ActiveState)) + { + return RecoveryDisposition.Failed; + } + + // Exact stable Closing/Recovering is the participant owner's durable + // quiescent handoff. It overrides live/current/unsupported liveness + // outcomes for both Claiming and Active; the lease mutation remains + // fenced by the exact control word observed before classification. + if (participantHandoffPublished + && participantState is LayoutV2Constants.ParticipantClosing + or LayoutV2Constants.ParticipantRecovering) + { + return RecoveryDisposition.Recover; + } + + return classification switch + { + ParticipantClassificationKind.Changing => RecoveryDisposition.Retry, + ParticipantClassificationKind.Inconsistent => RecoveryDisposition.Failed, + ParticipantClassificationKind.Unsupported => RecoveryDisposition.Unsupported, + ParticipantClassificationKind.Live => RecoveryDisposition.Active, + ParticipantClassificationKind.CurrentProcess + when leaseState == ClaimingState + && participantState != LayoutV2Constants.ParticipantClosing => RecoveryDisposition.Active, + ParticipantClassificationKind.CurrentProcess => + recoverCurrentProcessLeases + ? RecoveryDisposition.Recover + : RecoveryDisposition.Active, + ParticipantClassificationKind.Stale => RecoveryDisposition.Recover, + _ => RecoveryDisposition.Failed + }; + } + + /// Pure rollover transition used by layout and ABA tests. + internal static ulong AdvanceOrRetire(long incarnation) + { + if (incarnation is < 1 or > TerminalIncarnation) + { + throw new ArgumentOutOfRangeException(nameof(incarnation)); + } + + return incarnation == TerminalIncarnation + ? AtomicControlWord.EncodeLease(RetiredState, incarnation, participantToken: 0) + : AtomicControlWord.EncodeLease(FreeState, incarnation + 1, participantToken: 0); + } + + private StoreStatus TryRecycle( + int index, + long incarnation, + long expectedTransition, + out bool recycled) + { + recycled = false; + ref var record = ref Record(index); + // Do not write non-atomic fields while an unowned helper may be paused: + // after the generation CAS a stale helper must have no write left that + // could corrupt a newly claimed incarnation. The next Claiming owner + // overwrites both fields before activation; Free/Retired readers ignore + // their incarnation-fenced stale contents. + long terminal = unchecked((long)AdvanceOrRetire(incarnation)); + const int confirmationAttempts = 8; + for (var attempt = 0; attempt < confirmationAttempts; attempt++) + { + long observed = AtomicControlWord.CompareExchange( + ref record.Control, + terminal, + expectedTransition); + if (observed == expectedTransition) + { + recycled = true; + return StoreStatus.Success; + } + + _telemetry.RecordCasLoss(); + StoreStatus structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed == terminal || Incarnation(observed) > incarnation) + { + return StoreStatus.Success; + } + + // Releasing/Recovering has only one legal successor: the exact + // Free-next-generation or Retired word above. A stable lateral, + // same-generation reactivation, or generation regression is + // persistent structural corruption. If the source moved during + // confirmation, retry against the original exact transition. + long confirmed = AtomicControlWord.CompareExchange( + ref record.Control, + observed, + observed); + if (confirmed == observed) + { + return CorruptHere(); + } + + structure = ValidateControl(confirmed); + if (structure != StoreStatus.Success) + { + return structure; + } + } + + return StoreStatus.StoreBusy; + } + + private bool TryDecodeHandle(in LeaseHandle lease, out int recordIndex, out long incarnation) + { + recordIndex = -1; + incarnation = 0; + if (lease.StoreId != _storeId + || lease.ParticipantToken != _participant.Token + || lease.SlotBinding == 0 + || lease.LeaseToken == 0) + { + return false; + } + + var indexPlusOne = lease.LeaseToken & RecordIndexMask; + var rawIncarnation = lease.LeaseToken >> 31; + if (indexPlusOne == 0 || indexPlusOne > (ulong)_layout.LeaseRecordCount + || rawIncarnation is 0 or > IncarnationMask) + { + return false; + } + + recordIndex = checked((int)indexPlusOne - 1); + incarnation = checked((long)rawIncarnation); + return IsValidSlotBinding(lease.SlotBinding); + } + + private bool IsParticipantActive() + { + ref var record = ref *(ParticipantRecordV2*)( + _mappingBase + + _layout.ParticipantOffset + + ((long)_participant.RecordIndex * _layout.ParticipantStride)); + long control = AtomicControlWord.LoadAcquire(ref record.Control); + if (!LockFreeParticipantRegistry.IsStructuralControlValid( + control, + _layout.ParticipantGenerationMask)) + { + _ = CorruptHere(); + return false; + } + + return control == _participant.ActiveControl; + } + + private StoreStatus ParticipantUnavailableStatus() => + _storeControl?.Validate() == StoreStatus.CorruptStore + ? StoreStatus.CorruptStore + : StoreStatus.StoreDisposed; + + private bool IsValidSlotBinding(ulong binding) + { + return TryDecodeSlotBinding(binding, out _, out _); + } + + private StoreStatus TryIsRemoveRequested(ulong binding, out bool isRemoveRequested) + { + isRemoveRequested = false; + if (!TryDecodeSlotBinding(binding, out var slotIndex, out var generation)) + { + return CorruptHere(); + } + + ref var slot = ref *(ValueSlotMetadataV2*)( + _mappingBase + + _layout.SlotMetadataOffset + + ((long)slotIndex * _layout.SlotMetadataStride)); + var removeRequested = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.RemoveRequestedState, + generation, + participantToken: 0)); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + if (!LockFreeSlotTable.TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out _)) + { + return CorruptHere(); + } + + isRemoveRequested = control == removeRequested; + return StoreStatus.Success; + } + + private bool TryConfirmStructuralControl(ref long location, long expected) + { + long control = AtomicControlWord.LoadAcquire(ref location); + return ValidateControl(control) == StoreStatus.Success && control == expected; + } + + private bool TryDecodeSlotBinding(ulong binding, out int slotIndex, out long generation) + { + slotIndex = -1; + generation = 0; + try + { + var decoded = IndexBinding.Decode(binding); + if (decoded.SlotIndex >= _layout.SlotCount) + { + return false; + } + + slotIndex = decoded.SlotIndex; + generation = decoded.Generation; + return generation is >= 1 and <= TerminalIncarnation; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + catch (OverflowException) + { + return false; + } + } + + private static LeaseRecoveryReport RecoveryReport( + int scanned, + int recovered, + int active, + int unsupported, + int failed) => + new(scanned, recovered, active, unsupported, failed); + + private StoreStatus LeaseStatus(long observed, long expectedIncarnation) + { + StoreStatus structure = ValidateControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (Incarnation(observed) != expectedIncarnation) + { + return StoreStatus.InvalidLease; + } + + return State(observed) is FreeState or ReleasingState or RecoveringState + ? StoreStatus.LeaseAlreadyReleased + : StoreStatus.InvalidLease; + } + + internal StoreStatus ValidateStructuralControl(long control) => + ClassifyCapacityControl(control, out _); + + private StoreStatus ValidateControl(long control) => + ValidateStructuralControl(control); + + internal ref LeaseRecordV2 Record(int index) => + ref *(LeaseRecordV2*)( + _mappingBase + _layout.LeaseRegistryOffset + ((long)index * _layout.LeaseStride)); + + private static int State(long control) => (int)((ulong)control & 0x7UL); + + private static long Incarnation(long control) => + (long)(((ulong)control >> 3) & IncarnationMask); + + private static ulong Participant(long control) => ((ulong)control >> 36) & ParticipantMask; + + private StoreStatus ClassifyCapacityControl(long control, out bool occupied) + { + if (TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out occupied)) + { + return StoreStatus.Success; + } + + return CorruptFrom(nameof(LockFreeLeaseRegistry)); + } + + /// Pure canonical validation for a lease lifecycle word. + internal static bool TryClassifyStructuralControl( + long control, + int participantRecordCount, + out bool occupied) + { + occupied = true; + int state = State(control); + long incarnation = Incarnation(control); + ulong participant = Participant(control); + if (incarnation is < 1 or > TerminalIncarnation) + { + return false; + } + + switch (state) + { + case FreeState: + if (participant != 0) + { + return false; + } + + occupied = false; + return true; + + case ClaimingState: + case ActiveState: + return ParticipantToken.IsStructurallyValid( + participant, + participantRecordCount); + + case ReleasingState: + case RecoveringState: + return participant == 0; + + case RetiredState: + return participant == 0 && incarnation == TerminalIncarnation; + + default: + return false; + } + } + + private StoreStatus CorruptHere( + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) => + LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeLeaseRegistry), + member, + line); + + private StoreStatus CorruptFrom( + string component, + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) => + LockFreeStoreControl.ReportCorruption( + _storeControl, + component, + member, + line); + + private static long OwnedControl(int state, long incarnation, ulong participantToken) => + unchecked((long)AtomicControlWord.EncodeLease( + state, + incarnation, + checked((int)participantToken))); + + private static long UnownedControl(int state, long incarnation) => + unchecked((long)AtomicControlWord.EncodeLease(state, incarnation, participantToken: 0)); + + internal enum RecoveryDisposition + { + Recover, + Active, + Unsupported, + Failed, + Retry + } + +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeOperationBudget.cs b/src/SharedMemoryStore/LockFree/LockFreeOperationBudget.cs new file mode 100644 index 0000000..f747748 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeOperationBudget.cs @@ -0,0 +1,160 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace SharedMemoryStore.LockFree; + +/// +/// Allocation-free, operation-wide deadline and cancellation probe for layout-v2 +/// work. A single value is created at the public engine boundary and passed by +/// readonly reference through every variable-length scan and retry loop. +/// +internal readonly struct LockFreeOperationBudget +{ + // Clock/token reads on every mapped record are measurably expensive. Sixty-four + // records is still a very small cancellation quantum and keeps large-table + // scans comfortably inside the public limit-plus-250-ms qualification bound. + private const int ProbeMask = 63; + private static readonly StoreWaitOptions PostOwnershipCleanupOptions = + new(TimeSpan.FromMilliseconds(250)); + + private readonly StoreWaitOptions _options; + private readonly long _started; + private readonly bool _fullStructuralScan; + + private LockFreeOperationBudget( + in StoreWaitOptions options, + long started, + bool fullStructuralScan = false) + { + _options = options; + _started = started; + _fullStructuralScan = fullStructuralScan; + } + + internal static LockFreeOperationBudget Start(in StoreWaitOptions options) => + new(options, Stopwatch.GetTimestamp()); + + internal static LockFreeOperationBudget Start( + in StoreWaitOptions options, + long started) => new(options, started); + + /// + /// Once an exact CAS has removed caller ownership, public convenience + /// operations may spend at most the documented completion allowance on + /// physical unlink/recycle. Expiry leaves only universally helpable state. + /// + internal static LockFreeOperationBudget StartPostOwnershipCleanup() => + Start(PostOwnershipCleanupOptions); + + /// + /// Internal protocol/test calls without a public wait policy remain bounded by + /// their structural scan sizes but have no deadline or cancellation source. + /// + internal static LockFreeOperationBudget StructuralAttempt { get; } = + new(StoreWaitOptions.NoWait, started: 0, fullStructuralScan: true); + + internal static LockFreeOperationBudget UnboundedScan { get; } = + Start(StoreWaitOptions.Infinite, started: 0); + + internal bool IsInfinite => _options.IsInfinite; + + internal bool IsNoWait => _options.Timeout == TimeSpan.Zero; + + internal CancellationToken CancellationToken => _options.CancellationToken; + + internal StoreStatus TryGetRemainingWaitOptions(out StoreWaitOptions remaining) + { + remaining = default; + if (_options.CancellationToken.IsCancellationRequested) + { + return StoreStatus.OperationCanceled; + } + + if (_options.IsInfinite || _options.Timeout == TimeSpan.Zero) + { + remaining = _options; + return StoreStatus.Success; + } + + TimeSpan elapsed = Stopwatch.GetElapsedTime(_started); + if (elapsed >= _options.Timeout) + { + return StoreStatus.StoreBusy; + } + + remaining = new StoreWaitOptions( + _options.Timeout - elapsed, + _options.CancellationToken); + return StoreStatus.Success; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal StoreStatus Check() + { + if (_options.CancellationToken.IsCancellationRequested) + { + return StoreStatus.OperationCanceled; + } + + // NoWait permits one minimum safe structural attempt. It is bounded by + // the caller's scan/retry path rather than by an already-expired clock. + if (_options.IsInfinite || _options.Timeout == TimeSpan.Zero) + { + return StoreStatus.Success; + } + + return Stopwatch.GetElapsedTime(_started) >= _options.Timeout + ? StoreStatus.StoreBusy + : StoreStatus.Success; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal StoreStatus CheckPeriodic(int iteration) + { + if ((iteration & ProbeMask) != 0) + { + return StoreStatus.Success; + } + + StoreStatus status = Check(); + if (status != StoreStatus.Success) + { + return status; + } + + // NoWait gets one bounded probe chunk, not an arbitrarily large + // layout-sized scan. Nested scans receive the same rule, bounding a + // minimum safe attempt without requiring mutable/shared budget state. + return IsNoWait && !_fullStructuralScan && iteration != 0 + ? StoreStatus.StoreBusy + : StoreStatus.Success; + } + + /// + /// Decides whether a transient StoreBusy result should be retried. Infinite + /// waits keep helping until a non-transient result while still observing an + /// explicit cancellation token; NoWait never starts another attempt. + /// + internal bool TryContinueAfterContention(int attempt, out StoreStatus terminalStatus) + { + terminalStatus = Check(); + if (terminalStatus != StoreStatus.Success) + { + return false; + } + + if (IsNoWait) + { + terminalStatus = StoreStatus.StoreBusy; + return false; + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + if ((attempt & 63) == 63) + { + Thread.Yield(); + } + + return true; + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeParticipantRegistry.cs b/src/SharedMemoryStore/LockFree/LockFreeParticipantRegistry.cs new file mode 100644 index 0000000..cadf091 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeParticipantRegistry.cs @@ -0,0 +1,1810 @@ +using System.Buffers; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.Leasing; + +namespace SharedMemoryStore.LockFree; + +/// +/// Cold-path participant registration and record-local participant recovery for +/// one mapped layout-v2 store. Data operations use only the immutable token +/// returned at registration and never mutate this registry on their hot path. +/// +internal sealed unsafe class LockFreeParticipantRegistry +{ + private const ulong ParticipantMask = 0x0fff_ffffUL; + + private readonly byte* _mappingBase; + private readonly StoreLayoutV2 _layout; + private readonly LockFreeTelemetry _telemetry; + private readonly ulong _pidNamespaceId; + private readonly LockFreeStoreControl? _storeControl; + + internal LockFreeParticipantRegistry( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout) + : this(region, layout, new LockFreeTelemetry()) + { + } + + internal LockFreeParticipantRegistry( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + LockFreeTelemetry telemetry, + LockFreeStoreControl? storeControl = null) + { + ArgumentNullException.ThrowIfNull(region); + _mappingBase = region.Pointer; + _layout = layout; + _telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry)); + _storeControl = storeControl; + _pidNamespaceId = ((StoreHeaderV2*)_mappingBase)->PidNamespaceId; + } + + internal void InitializeRecords() + { + _ = InitializeRecords(LockFreeOperationBudget.UnboundedScan); + } + + internal StoreStatus InitializeRecords(in LockFreeOperationBudget budget) + { + long freeControl = ToSigned(AtomicControlWord.EncodeParticipant( + LayoutV2Constants.ParticipantFree, + incarnation: 1, + pid: 0)); + + for (var index = 0; index < _layout.ParticipantRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref ParticipantRecordV2 record = ref Record(index); + record.IdentityKind = LayoutV2Constants.IdentityUnknown; + record.Reserved = 0; + record.ProcessStartValue = 0; + record.OpenSequence = 0; + record.PidNamespaceId = 0; + AtomicControlWord.StoreRelease(ref record.Control, freeControl); + } + + return StoreStatus.Success; + } + + internal bool TryRegister(ref StoreHeaderV2 header, out Registration registration) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryRegister( + ref header, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint, + out registration) + == StoreOpenStatus.Success; + } + + internal StoreOpenStatus TryRegister( + ref StoreHeaderV2 header, + in LockFreeOperationBudget budget, + out Registration registration) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryRegister(ref header, budget, ref checkpoint, out registration); + } + + internal StoreOpenStatus TryRegister( + ref StoreHeaderV2 header, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out Registration registration) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + registration = default; + StoreOpenStatus storeState = ValidateStoreControlForOpen(); + if (storeState != StoreOpenStatus.Success) + { + return storeState; + } + + int pid = Environment.ProcessId; + bool capturedIdentity = LeaseOwnerClassifier.TryCaptureCurrentProcessIdentity( + out int identityKind, + out long processStartValue, + out ulong pidNamespaceId); + if (!capturedIdentity) + { + identityKind = LayoutV2Constants.IdentityUnknown; + processStartValue = 0; + } + + for (var index = 0; index < _layout.ParticipantRecordCount; index++) + { + storeState = ValidateStoreControlForOpen(); + if (storeState != StoreOpenStatus.Success) + { + return storeState; + } + + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound == StoreStatus.OperationCanceled + ? StoreOpenStatus.OperationCanceled + : StoreOpenStatus.StoreBusy; + } + + ref ParticipantRecordV2 record = ref Record(index); + long observed = AtomicControlWord.LoadAcquire(ref record.Control); + if (!ObserveParticipantControl(observed)) + { + return StoreOpenStatus.IncompatibleLayout; + } + + int state = DecodeState(observed); + int generation = DecodeIncarnation(observed); + + if (state == LayoutV2Constants.ParticipantReclaiming + && generation is >= 1 + && generation <= _layout.ParticipantGenerationMask) + { + _ = HelpReclaiming(index, generation); + observed = AtomicControlWord.LoadAcquire(ref record.Control); + if (!ObserveParticipantControl(observed)) + { + return StoreOpenStatus.IncompatibleLayout; + } + + state = DecodeState(observed); + generation = DecodeIncarnation(observed); + } + + if (state != LayoutV2Constants.ParticipantFree + || generation < 1 + || generation > _layout.ParticipantGenerationMask + || DecodeProcessId(observed) != 0) + { + continue; + } + + long registering = EncodeControl( + LayoutV2Constants.ParticipantRegistering, + generation, + pid); + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.ParticipantBeforeRegisteringCas); + long claimObservation = AtomicControlWord.CompareExchange( + ref record.Control, + registering, + observed); + if (claimObservation != observed) + { + _telemetry.RecordCasLoss(); + if (!ObserveParticipantControl(claimObservation)) + { + return StoreOpenStatus.IncompatibleLayout; + } + + continue; + } + + long active = EncodeControl(LayoutV2Constants.ParticipantActive, generation, pid); + try + { + // Registering grants exclusive initialization. Every ordinary + // field is overwritten before Active release-publication, so + // Free records may safely retain semantically dead fields from + // an older owner. + record.IdentityKind = identityKind; + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.ParticipantAfterIdentityKindWrite); + record.Reserved = 0; + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.ParticipantAfterReservedWrite); + record.ProcessStartValue = processStartValue; + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.ParticipantAfterProcessStartWrite); + record.PidNamespaceId = pidNamespaceId; + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.ParticipantAfterPidNamespaceWrite); + record.OpenSequence = Interlocked.Increment(ref header.Sequence); + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.ParticipantAfterOpenSequenceWrite); + + AtomicControlWord.StoreRelease(ref record.Control, active); + registration = new Registration( + index, + generation, + ParticipantToken.Encode(index, generation, _layout.ParticipantRecordCount), + active); + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.ParticipantAfterActivePublication); + return StoreOpenStatus.Success; + } + catch + { + // No token has escaped TryRegister, so neither Registering nor + // the just-published Active control can have a legitimate data + // reference. Retire the exact claim locally; never leave a live + // PID record for an observer/injected construction failure. + registration = default; + RetireUnescapedRegistrationClaim(index, generation, pid, registering, active); + throw; + } + } + + return StoreOpenStatus.ParticipantTableFull; + } + + /// + /// Closes and retires the local registration after its engine has stopped + /// local entry and relinquished exact resources. If references remain, the + /// record stays Closing and cannot be reused prematurely. + /// + internal void Unregister(in Registration registration) + { + ParticipantTransitionResult close = TryBeginClose(registration); + if (close is not (ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted)) + { + return; + } + + _ = TryRetireClosingRegistration( + registration, + LockFreeOperationBudget.StartPostOwnershipCleanup()); + } + + /// + /// Publishes the exact local Active-to-Closing handoff. The facade has + /// already stopped and drained local entry before the engine calls this, + /// so Closing is a durable proof that no owner-side ordinary write remains. + /// + internal ParticipantTransitionResult TryBeginClose(in Registration registration) + { + if (!registration.IsValid + || (uint)registration.RecordIndex >= (uint)_layout.ParticipantRecordCount + || registration.Generation > _layout.ParticipantGenerationMask) + { + return ParticipantTransitionResult.Inconsistent; + } + + int pid = DecodeProcessId(registration.ActiveControl); + if (pid <= 0) + { + return ParticipantTransitionResult.Inconsistent; + } + + ref ParticipantRecordV2 record = ref Record(registration.RecordIndex); + long closing = EncodeControl( + LayoutV2Constants.ParticipantClosing, + registration.Generation, + pid); + long observed = AtomicControlWord.CompareExchange( + ref record.Control, + closing, + registration.ActiveControl); + if (observed == registration.ActiveControl) + { + return ParticipantTransitionResult.Succeeded; + } + + if (!ObserveParticipantControl(observed)) + { + return ParticipantTransitionResult.Inconsistent; + } + + return observed == closing + ? ParticipantTransitionResult.AlreadyCompleted + : ParticipantTransitionResult.Changed; + } + + /// + /// Attempts the final exact Closing-to-Reclaiming retirement within the + /// caller's finite cleanup allowance. A timeout leaves Closing published + /// and therefore recoverable by every other handle. + /// + internal StoreStatus TryRetireClosingRegistration( + in Registration registration, + in LockFreeOperationBudget budget) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryRetireClosingRegistration(registration, budget, ref checkpoint); + } + + internal StoreStatus TryRetireClosingRegistration( + in Registration registration, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (!TryGetRegistrationControls( + registration, + out long closing, + out long reclaiming, + out long terminal)) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + } + + ref ParticipantRecordV2 record = ref Record(registration.RecordIndex); + long observed = AtomicControlWord.LoadAcquire(ref record.Control); + if (!ObserveParticipantControl(observed)) + { + return StoreStatus.CorruptStore; + } + + if (observed == reclaiming) + { + _ = HelpReclaiming( + registration.RecordIndex, + registration.Generation, + ref checkpoint); + return StoreStatus.Success; + } + + if (observed == terminal) + { + return StoreStatus.Success; + } + + if (observed != closing) + { + return StoreStatus.StoreBusy; + } + + StoreStatus references = HasParticipantReferences( + registration.Token, + budget, + out bool hasReferences); + if (references != StoreStatus.Success || hasReferences) + { + return references == StoreStatus.Success ? StoreStatus.StoreBusy : references; + } + + StoreStatus bound = budget.Check(); + if (bound != StoreStatus.Success) + { + return bound; + } + + ParticipantTransitionResult handoff = TryAdvanceClaimClosedControl( + ref record.Control, + closing, + reclaiming, + registration.Generation); + if (handoff == ParticipantTransitionResult.Inconsistent) + { + return StoreStatus.CorruptStore; + } + + if (handoff == ParticipantTransitionResult.Changed) + { + return StoreStatus.StoreBusy; + } + + observed = AtomicControlWord.LoadAcquire(ref record.Control); + if (!ObserveParticipantControl(observed)) + { + return StoreStatus.CorruptStore; + } + + if (observed == reclaiming) + { + _ = HelpReclaiming( + registration.RecordIndex, + registration.Generation, + ref checkpoint); + } + + return StoreStatus.Success; + } + + /// + /// Exception-path cleanup after registration succeeded but before an + /// engine escaped. No data claim can exist, so the construction owner can + /// publish Closing and retire without an O(S+L) reference scan. + /// + internal void RetireUnreferencedRegistration(in Registration registration) + { + ParticipantTransitionResult close = TryBeginClose(registration); + if (close is not (ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted)) + { + return; + } + + if (!TryGetRegistrationControls( + registration, + out long closing, + out long reclaiming, + out _)) + { + return; + } + + ref ParticipantRecordV2 record = ref Record(registration.RecordIndex); + ParticipantTransitionResult handoff = TryAdvanceClaimClosedControl( + ref record.Control, + closing, + reclaiming, + registration.Generation); + if (handoff is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted) + { + _ = HelpReclaiming(registration.RecordIndex, registration.Generation); + } + else if (handoff == ParticipantTransitionResult.Changed) + { + _telemetry.RecordCasLoss(); + } + } + + private void RetireUnescapedRegistrationClaim( + int recordIndex, + int generation, + int pid, + long registering, + long active) + { + ref ParticipantRecordV2 record = ref Record(recordIndex); + long reclaiming = EncodeControl( + LayoutV2Constants.ParticipantReclaiming, + generation, + pid: 0); + long terminal = AdvanceOrRetire(generation); + long observed = AtomicControlWord.CompareExchange( + ref record.Control, + reclaiming, + registering); + if (observed == registering || observed == reclaiming) + { + _ = HelpReclaiming(recordIndex, generation); + return; + } + + long closing = EncodeControl(LayoutV2Constants.ParticipantClosing, generation, pid); + if (observed == active) + { + observed = AtomicControlWord.CompareExchange(ref record.Control, closing, active); + } + + if (observed == active || observed == closing) + { + ParticipantTransitionResult handoff = TryAdvanceClaimClosedControl( + ref record.Control, + closing, + reclaiming, + generation); + if (handoff is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted) + { + _ = HelpReclaiming(recordIndex, generation); + return; + } + + observed = AtomicControlWord.LoadAcquire(ref record.Control); + } + + if (observed != terminal) + { + _ = ObserveParticipantControl(observed); + _telemetry.RecordCasLoss(); + } + } + + /// Stabilizes and conservatively classifies an exact compact token. + internal ParticipantClassification ClassifyParticipant(ulong participantToken) + { + ParticipantSnapshotStatus snapshot = ReadSnapshot( + participantToken, + out ParticipantIncarnation incarnation); + if (snapshot == ParticipantSnapshotStatus.Changing) + { + return ObserveClassification(new ParticipantClassification( + ParticipantClassificationKind.Changing, + incarnation)); + } + + if (snapshot == ParticipantSnapshotStatus.Inconsistent) + { + return ObserveClassification(new ParticipantClassification( + ParticipantClassificationKind.Inconsistent, + incarnation)); + } + + if (snapshot == ParticipantSnapshotStatus.Stale) + { + return ObserveClassification(new ParticipantClassification( + ParticipantClassificationKind.Stale, + incarnation)); + } + + LeaseOwnerClassification owner = ClassifySnapshotOwner( + incarnation, + _pidNamespaceId, + IsPidNamespaceRecoveryEnabled()); + // Registering ordinary fields are not a coherent identity snapshot: + // Free records deliberately retain the previous incarnation's values, + // and a new claimant may be paused between overwrites. PID presence is + // therefore the only safe live-owner signal until Active publishes all + // fields with release ordering. This may conservatively retain a + // Registering record after PID reuse, but can never reclaim a live + // opener from a mixed old/new identity. + return ObserveClassification(new ParticipantClassification(Map(owner.Kind), incarnation)); + } + + /// + /// Selects the only safe identity classifier for a stabilized participant + /// control. Registering ordinary fields can be a mixture of the previous + /// incarnation and the new claimant, so even apparently valid identity + /// fields must never be compared until Active release-publication. + /// + internal static LeaseOwnerClassification ClassifySnapshotOwner( + in ParticipantIncarnation incarnation, + ulong storePidNamespaceId, + bool presenceOnlyRecoveryEnabled = true) => + incarnation.State == LayoutV2Constants.ParticipantRegistering + ? presenceOnlyRecoveryEnabled + ? LeaseOwnerClassifier.ClassifyPresenceOnly( + incarnation.ProcessId, + storePidNamespaceId) + : new LeaseOwnerClassification( + LeaseOwnerKind.Unsupported, + incarnation.ProcessId) + : LeaseOwnerClassifier.Classify(incarnation); + + /// + /// Performs conservative stale-participant retirement after resource-level + /// recovery has removed every exact token reference. + /// + internal ParticipantTransitionResult TryRecoverParticipant(ulong participantToken) + { + return TryRecoverParticipantCore(participantToken, referencesKnownAbsent: false); + } + + private ParticipantTransitionResult TryRecoverParticipantCore( + ulong participantToken, + bool referencesKnownAbsent) + { + ParticipantSnapshotStatus snapshot = ReadSnapshot( + participantToken, + out ParticipantIncarnation incarnation); + if (snapshot == ParticipantSnapshotStatus.Changing) + { + return ParticipantTransitionResult.Changed; + } + + if (snapshot == ParticipantSnapshotStatus.Inconsistent) + { + return ParticipantTransitionResult.Inconsistent; + } + + if (!TokenMatchesSnapshot(incarnation)) + { + return ParticipantTransitionResult.AlreadyCompleted; + } + + if (incarnation.State == LayoutV2Constants.ParticipantReclaiming) + { + return HelpReclaiming(incarnation.RecordIndex, incarnation.Generation); + } + + if (incarnation.State is LayoutV2Constants.ParticipantFree + or LayoutV2Constants.ParticipantRetired) + { + return ParticipantTransitionResult.AlreadyCompleted; + } + + // Closing and Recovering are explicit claim-closed handoffs. Their + // owner may still be a perfectly live process paused in Dispose, so OS + // liveness classification is both unnecessary and actively harmful to + // progress. Exact token/control fencing plus the fresh absence proof is + // the complete retirement authority for these states. + if (incarnation.State is LayoutV2Constants.ParticipantClosing + or LayoutV2Constants.ParticipantRecovering) + { + if (!referencesKnownAbsent && HasParticipantReferences(participantToken)) + { + return ParticipantTransitionResult.ReferencesRemain; + } + + ParticipantTransitionResult handedOff = TryBeginReclaim( + incarnation, + referencesKnownAbsent); + return handedOff is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted + ? HelpReclaiming(incarnation.RecordIndex, incarnation.Generation) + : handedOff; + } + + ParticipantClassification classification; + if (snapshot == ParticipantSnapshotStatus.Stale) + { + classification = ObserveClassification(new ParticipantClassification( + ParticipantClassificationKind.Stale, + incarnation)); + } + else + { + LeaseOwnerClassification owner = ClassifySnapshotOwner( + incarnation, + _pidNamespaceId, + IsPidNamespaceRecoveryEnabled()); + classification = ObserveClassification(new ParticipantClassification( + Map(owner.Kind), + incarnation)); + } + + switch (classification.Kind) + { + case ParticipantClassificationKind.CurrentProcess: + case ParticipantClassificationKind.Live: + return ParticipantTransitionResult.LiveOwner; + case ParticipantClassificationKind.Unsupported: + return ParticipantTransitionResult.Unsupported; + case ParticipantClassificationKind.Inconsistent: + return ParticipantTransitionResult.Inconsistent; + case ParticipantClassificationKind.Changing: + return ParticipantTransitionResult.Changed; + case ParticipantClassificationKind.Stale: + break; + default: + return ParticipantTransitionResult.Inconsistent; + } + + if (!referencesKnownAbsent && HasParticipantReferences(participantToken)) + { + return ParticipantTransitionResult.ReferencesRemain; + } + + ParticipantTransitionResult transition; + if (incarnation.State == LayoutV2Constants.ParticipantRegistering) + { + transition = TryRecoverRegistering(incarnation, referencesKnownAbsent); + return transition is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted + ? HelpReclaiming(incarnation.RecordIndex, incarnation.Generation) + : transition; + } + + if (incarnation.State == LayoutV2Constants.ParticipantActive) + { + transition = TryBeginRecovery(incarnation); + if (transition != ParticipantTransitionResult.Succeeded) + { + return transition; + } + + if (ReadSnapshot(participantToken, out incarnation) != ParticipantSnapshotStatus.Stable + || incarnation.State != LayoutV2Constants.ParticipantRecovering) + { + return ParticipantTransitionResult.Changed; + } + } + else if (incarnation.State != LayoutV2Constants.ParticipantRecovering) + { + return ParticipantTransitionResult.Inconsistent; + } + + if (!referencesKnownAbsent && HasParticipantReferences(participantToken)) + { + return ParticipantTransitionResult.ReferencesRemain; + } + + transition = TryBeginReclaim(incarnation, referencesKnownAbsent); + return transition is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted + ? HelpReclaiming(incarnation.RecordIndex, incarnation.Generation) + : transition; + } + + /// Exact Active-to-Recovering CAS. The caller must have classified this snapshot stale. + internal ParticipantTransitionResult TryBeginRecovery(ParticipantIncarnation expected) + { + if (!IsExactOwnedSnapshot(expected, LayoutV2Constants.ParticipantActive)) + { + return ParticipantTransitionResult.Inconsistent; + } + + if (!IsPidNamespaceRecoveryEnabled()) + { + LeaseOwnerClassification owner = LeaseOwnerClassifier.Classify(expected); + if (owner.Kind != LeaseOwnerKind.StaleProcess) + { + return owner.Kind is LeaseOwnerKind.CurrentProcess or LeaseOwnerKind.OtherLiveProcess + ? ParticipantTransitionResult.LiveOwner + : owner.Kind == LeaseOwnerKind.UnsafeRecord + ? ParticipantTransitionResult.Inconsistent + : ParticipantTransitionResult.Unsupported; + } + } + + ref ParticipantRecordV2 record = ref Record(expected.RecordIndex); + long recovering = EncodeControl( + LayoutV2Constants.ParticipantRecovering, + expected.Generation, + expected.ProcessId); + long observed = AtomicControlWord.CompareExchange( + ref record.Control, + recovering, + expected.Control); + if (observed == expected.Control) + { + return ParticipantTransitionResult.Succeeded; + } + + if (!ObserveParticipantControl(observed)) + { + return ParticipantTransitionResult.Inconsistent; + } + + return observed == recovering + ? ParticipantTransitionResult.AlreadyCompleted + : ParticipantTransitionResult.Changed; + } + + /// + /// Exact recovery of an incomplete Registering claim. Registering cannot be + /// referenced by a data control, so definite owner absence permits direct + /// publication of the universally helpable Reclaiming state. + /// + internal ParticipantTransitionResult TryRecoverRegistering(ParticipantIncarnation expected) + { + return TryRecoverRegistering(expected, referencesKnownAbsent: false); + } + + private ParticipantTransitionResult TryRecoverRegistering( + ParticipantIncarnation expected, + bool referencesKnownAbsent) + { + if (!IsExactOwnedSnapshot(expected, LayoutV2Constants.ParticipantRegistering)) + { + return ParticipantTransitionResult.Inconsistent; + } + + if (!IsPidNamespaceRecoveryEnabled()) + { + return ParticipantTransitionResult.Unsupported; + } + + if (!referencesKnownAbsent && HasParticipantReferences(expected.Token)) + { + return ParticipantTransitionResult.Inconsistent; + } + + ref ParticipantRecordV2 record = ref Record(expected.RecordIndex); + long reclaiming = EncodeControl( + LayoutV2Constants.ParticipantReclaiming, + expected.Generation, + pid: 0); + long observed = AtomicControlWord.CompareExchange( + ref record.Control, + reclaiming, + expected.Control); + if (observed == expected.Control) + { + return ParticipantTransitionResult.Succeeded; + } + + if (!ObserveParticipantControl(observed)) + { + return ParticipantTransitionResult.Inconsistent; + } + + return observed == reclaiming + ? ParticipantTransitionResult.AlreadyCompleted + : ParticipantTransitionResult.Changed; + } + + /// + /// Exact Closing/Recovering-to-Reclaiming CAS after a zero-reference proof. + /// The transition clears PID ownership atomically. + /// + internal ParticipantTransitionResult TryBeginReclaim(ParticipantIncarnation expected) + { + return TryBeginReclaim(expected, referencesKnownAbsent: false); + } + + private ParticipantTransitionResult TryBeginReclaim( + ParticipantIncarnation expected, + bool referencesKnownAbsent) + { + if (!IsExactOwnedSnapshot(expected, LayoutV2Constants.ParticipantClosing) + && !IsExactOwnedSnapshot(expected, LayoutV2Constants.ParticipantRecovering)) + { + return ParticipantTransitionResult.Inconsistent; + } + + if (!referencesKnownAbsent && HasParticipantReferences(expected.Token)) + { + return ParticipantTransitionResult.ReferencesRemain; + } + + ref ParticipantRecordV2 record = ref Record(expected.RecordIndex); + long reclaiming = EncodeControl( + LayoutV2Constants.ParticipantReclaiming, + expected.Generation, + pid: 0); + return TryAdvanceClaimClosedControl( + ref record.Control, + expected.Control, + reclaiming, + expected.Generation); + } + + /// + /// Advances an exact Closing/Recovering record to universally helpable + /// Reclaiming. Once claim-closed, no same-generation owned state can be a + /// successor. Exact-confirm such regressions before latching corruption; + /// movement during confirmation remains an ordinary Changed race. + /// + private ParticipantTransitionResult TryAdvanceClaimClosedControl( + ref long control, + long expected, + long reclaiming, + int generation) + { + long terminal = AdvanceOrRetire(generation); + const int confirmationAttempts = 8; + for (var attempt = 0; attempt < confirmationAttempts; attempt++) + { + long observed = AtomicControlWord.CompareExchange( + ref control, + reclaiming, + expected); + if (observed == expected) + { + return ParticipantTransitionResult.Succeeded; + } + + _telemetry.RecordCasLoss(); + if (!ObserveParticipantControl(observed)) + { + return ParticipantTransitionResult.Inconsistent; + } + + int observedGeneration = DecodeIncarnation(observed); + if (observed == reclaiming + || observed == terminal + || observedGeneration > generation) + { + return ParticipantTransitionResult.AlreadyCompleted; + } + + long confirmed = AtomicControlWord.CompareExchange( + ref control, + observed, + observed); + if (confirmed == observed) + { + _ = LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + return ParticipantTransitionResult.Inconsistent; + } + + if (!ObserveParticipantControl(confirmed)) + { + return ParticipantTransitionResult.Inconsistent; + } + } + + return ParticipantTransitionResult.Changed; + } + + /// + /// Universally helps an unowned Reclaiming record advance or retire. No + /// ordinary identity write occurs here: a delayed helper therefore cannot + /// erase identity fields belonging to a later Active incarnation. + /// + internal ParticipantTransitionResult HelpReclaiming(int recordIndex, int generation) + { + NoOpLockFreeCheckpoint checkpoint = default; + return HelpReclaiming(recordIndex, generation, ref checkpoint); + } + + internal ParticipantTransitionResult HelpReclaiming( + int recordIndex, + int generation, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if ((uint)recordIndex >= (uint)_layout.ParticipantRecordCount + || generation < 1 + || generation > _layout.ParticipantGenerationMask) + { + return ParticipantTransitionResult.Inconsistent; + } + + ref ParticipantRecordV2 record = ref Record(recordIndex); + long reclaiming = EncodeControl( + LayoutV2Constants.ParticipantReclaiming, + generation, + pid: 0); + long terminal = AdvanceOrRetire(generation); + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.ParticipantBeforeReclaimGenerationAdvanceCas); + + const int confirmationAttempts = 8; + for (var attempt = 0; attempt < confirmationAttempts; attempt++) + { + long observed = AtomicControlWord.CompareExchange( + ref record.Control, + terminal, + reclaiming); + if (observed == reclaiming) + { + return ParticipantTransitionResult.Succeeded; + } + + _telemetry.RecordCasLoss(); + if (!ObserveParticipantControl(observed)) + { + return ParticipantTransitionResult.Inconsistent; + } + + int observedGeneration = DecodeIncarnation(observed); + if (observed == terminal || observedGeneration > generation) + { + return ParticipantTransitionResult.AlreadyCompleted; + } + + // A Reclaiming record has no legal same-generation transition + // except the exact terminal word above. Confirm the impossible + // regression against the mapped source word before latching; if it + // moved meanwhile, retry and classify the new successor. + long confirmed = AtomicControlWord.CompareExchange( + ref record.Control, + observed, + observed); + if (confirmed == observed) + { + _ = LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + return ParticipantTransitionResult.Inconsistent; + } + + if (!ObserveParticipantControl(confirmed)) + { + return ParticipantTransitionResult.Inconsistent; + } + } + + return ParticipantTransitionResult.Changed; + } + + /// Returns the exact next Free or terminal Retired participant control. + internal long AdvanceOrRetire(int generation) + { + if (generation < 1 || generation > _layout.ParticipantGenerationMask) + { + throw new ArgumentOutOfRangeException(nameof(generation)); + } + + return generation == _layout.ParticipantGenerationMask + ? EncodeControl(LayoutV2Constants.ParticipantRetired, generation, pid: 0) + : EncodeControl(LayoutV2Constants.ParticipantFree, generation + 1, pid: 0); + } + + /// Conservative full scan for an exact compact participant token. + internal bool HasParticipantReferences(ulong participantToken) + { + return HasParticipantReferences( + participantToken, + LockFreeOperationBudget.UnboundedScan, + out bool hasReferences) + != StoreStatus.Success + || hasReferences; + } + + internal StoreStatus HasParticipantReferences( + ulong participantToken, + in LockFreeOperationBudget budget, + out bool hasReferences) + { + hasReferences = true; + if (participantToken == 0 || participantToken > ParticipantMask) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + } + + for (var index = 0; index < _layout.SlotCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref ValueSlotMetadataV2 slot = ref SlotRecord(index); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + if (!LockFreeSlotTable.TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out _)) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + } + + if (DecodeOwnedParticipant(control) == participantToken) + { + return StoreStatus.Success; + } + } + + for (var index = 0; index < _layout.LeaseRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref LeaseRecordV2 lease = ref LeaseRecord(index); + long control = AtomicControlWord.LoadAcquire(ref lease.Control); + if (!LockFreeLeaseRegistry.TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out _)) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + } + + if (DecodeOwnedParticipant(control) == participantToken) + { + return StoreStatus.Success; + } + } + + hasReferences = false; + return StoreStatus.Success; + } + + /// + /// Bounded explicit-recovery sweep for stale participant incarnations that + /// own no remaining slot or lease reference. This closes the crash window + /// after Active participant publication but before the first resource claim. + /// Live/current/unsupported identities are conservatively preserved. + /// + internal StoreStatus TryRecoverUnreferencedStaleParticipants( + StoreWaitOptions waitOptions, + long recoveryStarted, + out int recoveredCount) + { + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions, recoveryStarted); + NoOpLockFreeCheckpoint checkpoint = default; + return TryRecoverUnreferencedStaleParticipants( + budget, + ref checkpoint, + out recoveredCount); + } + + internal StoreStatus TryRecoverUnreferencedStaleParticipants( + in LockFreeOperationBudget budget, + out int recoveredCount) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryRecoverUnreferencedStaleParticipants( + budget, + ref checkpoint, + out recoveredCount); + } + + internal StoreStatus TryRecoverUnreferencedStaleParticipants( + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out int recoveredCount) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + recoveredCount = 0; + StoreStatus initialBound = budget.Check(); + if (initialBound != StoreStatus.Success) + { + return initialBound; + } + + int wordCount = checked((_layout.ParticipantRecordCount + 63) / 64); + int candidateCount = _layout.ParticipantRecordCount; + ulong[] sweepState = ArrayPool.Shared.Rent(checked((candidateCount * 2) + wordCount)); + try + { + Span candidates = sweepState.AsSpan(0, candidateCount); + Span candidateControls = sweepState.AsSpan(candidateCount, candidateCount); + Span referenced = sweepState.AsSpan(candidateCount * 2, wordCount); + + StoreStatus bound; + for (var wordIndex = 0; wordIndex < wordCount; wordIndex++) + { + bound = budget.CheckPeriodic(wordIndex); + if (bound != StoreStatus.Success) + { + return bound; + } + + referenced[wordIndex] = 0; + } + + // Phase 1: publish claim-closed ownership before taking the absence + // proof. A resource CAS that completed before Active->Recovering is + // visible to the later scan; a live claim cannot begin afterward + // because the claimant's exact Active post-check fails. Closing is + // already locally gated and Recovering is already claim-closed. + for (var index = 0; index < _layout.ParticipantRecordCount; index++) + { + bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + candidates[index] = 0; + candidateControls[index] = 0; + + ref ParticipantRecordV2 record = ref Record(index); + long control = AtomicControlWord.LoadAcquire(ref record.Control); + if (!ObserveParticipantControl(control)) + { + return StoreStatus.CorruptStore; + } + + int state = DecodeState(control); + int generation = DecodeIncarnation(control); + + if (state == LayoutV2Constants.ParticipantReclaiming) + { + ParticipantTransitionResult helped = HelpReclaiming(index, generation); + if (HasCorruptStoreControl()) + { + return StoreStatus.CorruptStore; + } + + if (helped == ParticipantTransitionResult.Succeeded) + { + recoveredCount++; + } + + continue; + } + + if (state is LayoutV2Constants.ParticipantFree + or LayoutV2Constants.ParticipantRetired) + { + continue; + } + + ulong token; + try + { + token = ParticipantToken.Encode(index, generation, _layout.ParticipantRecordCount); + } + catch (ArgumentOutOfRangeException) + { + continue; + } + + // Closing and Recovering already prove that the owner has + // stopped all ordinary writes and that new claims fail their + // Active recheck. Preserve the exact captured control as a + // candidate without consulting process liveness: a live + // disposer may intentionally be paused here. + if (state is LayoutV2Constants.ParticipantClosing + or LayoutV2Constants.ParticipantRecovering) + { + candidates[index] = token; + candidateControls[index] = unchecked((ulong)control); + continue; + } + + if (state is not (LayoutV2Constants.ParticipantRegistering + or LayoutV2Constants.ParticipantActive)) + { + continue; + } + + ParticipantClassification classification = ClassifyParticipant(token); + if (HasCorruptStoreControl()) + { + return StoreStatus.CorruptStore; + } + + bound = budget.Check(); + if (bound != StoreStatus.Success) + { + return bound; + } + + ParticipantIncarnation incarnation = classification.Incarnation; + if (classification.Kind != ParticipantClassificationKind.Stale + || !TokenMatchesSnapshot(incarnation) + || incarnation.Control != control + || incarnation.State != state) + { + continue; + } + + if (state == LayoutV2Constants.ParticipantRegistering) + { + // Registering has never published an Active token, so the + // protocol forbids a slot or lease reference. Definite owner + // absence permits direct exact retirement of this partial open. + ParticipantTransitionResult registering = + TryRecoverRegistering(incarnation, referencesKnownAbsent: true); + if (HasCorruptStoreControl()) + { + return StoreStatus.CorruptStore; + } + + if (registering is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted) + { + ParticipantTransitionResult helped = HelpReclaiming(index, generation); + if (HasCorruptStoreControl()) + { + return StoreStatus.CorruptStore; + } + + if (helped == ParticipantTransitionResult.Succeeded) + { + recoveredCount++; + } + } + + continue; + } + + if (state == LayoutV2Constants.ParticipantActive) + { + ParticipantTransitionResult fenced = TryBeginRecovery(incarnation); + if (HasCorruptStoreControl()) + { + return StoreStatus.CorruptStore; + } + + if (fenced == ParticipantTransitionResult.Succeeded) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.ParticipantAfterRecoveryFenceBeforeReferenceScan); + } + + if (fenced is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted) + { + candidates[index] = token; + candidateControls[index] = unchecked((ulong)EncodeControl( + LayoutV2Constants.ParticipantRecovering, + generation, + incarnation.ProcessId)); + } + + continue; + } + + } + + // Phase 2: after every candidate is claim-closed, build one fresh + // exact-token reference index in O(S+L). Each bit is set only when + // the resource token equals the candidate captured for that record; + // unrelated records and incarnations cannot affect its proof. + for (var index = 0; index < _layout.SlotCount; index++) + { + bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref SlotRecord(index).Control); + if (!LockFreeSlotTable.TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out _)) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + } + + ulong token = DecodeOwnedParticipant(control); + MarkParticipantReference(candidates, referenced, token); + } + + for (var index = 0; index < _layout.LeaseRecordCount; index++) + { + bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref LeaseRecord(index).Control); + if (!LockFreeLeaseRegistry.TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out _)) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + } + + ulong token = DecodeOwnedParticipant(control); + MarkParticipantReference(candidates, referenced, token); + } + + // Phase 3: exact-token revalidation prevents a candidate bit from + // applying to a later incarnation if another helper completed the + // same retirement. Only absent, still claim-closed candidates may + // publish Reclaiming and become reusable. + for (var index = 0; index < _layout.ParticipantRecordCount; index++) + { + bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ulong token = candidates[index]; + if (token == 0 + || (referenced[index >> 6] & (1UL << (index & 63))) != 0) + { + continue; + } + + ParticipantToken decoded; + try + { + decoded = ParticipantToken.Decode(token, _layout.ParticipantRecordCount); + } + catch (ArgumentOutOfRangeException) + { + continue; + } + catch (OverflowException) + { + continue; + } + + ref ParticipantRecordV2 record = ref Record(index); + long control = AtomicControlWord.LoadAcquire(ref record.Control); + if (!ObserveParticipantControl(control)) + { + return StoreStatus.CorruptStore; + } + + int state = DecodeState(control); + int generation = DecodeIncarnation(control); + if (decoded.RecordIndex != index || decoded.Generation != generation) + { + continue; + } + + if (state == LayoutV2Constants.ParticipantReclaiming) + { + ParticipantTransitionResult helped = HelpReclaiming(index, generation); + if (HasCorruptStoreControl()) + { + return StoreStatus.CorruptStore; + } + + if (helped == ParticipantTransitionResult.Succeeded) + { + recoveredCount++; + } + + continue; + } + + if (unchecked((ulong)control) != candidateControls[index]) + { + continue; + } + + if (state is not (LayoutV2Constants.ParticipantClosing + or LayoutV2Constants.ParticipantRecovering) + || DecodeProcessId(control) <= 0 + || ((ulong)control >> 63) != 0) + { + continue; + } + + var incarnation = new ParticipantIncarnation( + index, + generation, + token, + state, + DecodeProcessId(control), + IdentityKind: 0, + ProcessStartValue: 0, + OpenSequence: 0, + PidNamespaceId: 0, + ReservedValue: 0, + Control: control); + ParticipantTransitionResult reclaim = + TryBeginReclaim(incarnation, referencesKnownAbsent: true); + if (HasCorruptStoreControl()) + { + return StoreStatus.CorruptStore; + } + + if (reclaim is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted) + { + ParticipantTransitionResult helped = HelpReclaiming(index, generation); + if (HasCorruptStoreControl()) + { + return StoreStatus.CorruptStore; + } + + if (helped == ParticipantTransitionResult.Succeeded) + { + recoveredCount++; + } + } + } + + return HasCorruptStoreControl() + ? StoreStatus.CorruptStore + : StoreStatus.Success; + } + finally + { + ArrayPool.Shared.Return(sweepState, clearArray: false); + } + } + + private void MarkParticipantReference( + ReadOnlySpan candidates, + Span referenced, + ulong participantToken) + { + if (participantToken == 0 || participantToken > ParticipantMask) + { + return; + } + + try + { + ParticipantToken decoded = ParticipantToken.Decode( + participantToken, + _layout.ParticipantRecordCount); + if (candidates[decoded.RecordIndex] == participantToken) + { + referenced[decoded.RecordIndex >> 6] |= 1UL << (decoded.RecordIndex & 63); + } + } + catch (ArgumentOutOfRangeException) + { + // Malformed owner controls are handled by their resource scanner. + // They cannot justify participant retirement here. + } + catch (OverflowException) + { + } + } + + private ParticipantClassification ObserveClassification(ParticipantClassification classification) + { + _telemetry.RecordOwnerClassification(classification.Kind); + return classification; + } + + private bool TryGetRegistrationControls( + in Registration registration, + out long closing, + out long reclaiming, + out long terminal) + { + closing = 0; + reclaiming = 0; + terminal = 0; + if (!registration.IsValid + || (uint)registration.RecordIndex >= (uint)_layout.ParticipantRecordCount + || registration.Generation > _layout.ParticipantGenerationMask) + { + return false; + } + + int pid = DecodeProcessId(registration.ActiveControl); + if (pid <= 0) + { + return false; + } + + closing = EncodeControl( + LayoutV2Constants.ParticipantClosing, + registration.Generation, + pid); + reclaiming = EncodeControl( + LayoutV2Constants.ParticipantReclaiming, + registration.Generation, + pid: 0); + terminal = AdvanceOrRetire(registration.Generation); + return true; + } + + private ParticipantSnapshotStatus ReadSnapshot( + ulong participantToken, + out ParticipantIncarnation incarnation) + { + incarnation = default; + if (participantToken == 0 || participantToken > ParticipantMask) + { + return ParticipantSnapshotStatus.Inconsistent; + } + + ParticipantToken decoded; + try + { + decoded = ParticipantToken.Decode(participantToken, _layout.ParticipantRecordCount); + } + catch (ArgumentOutOfRangeException) + { + return ParticipantSnapshotStatus.Inconsistent; + } + catch (OverflowException) + { + return ParticipantSnapshotStatus.Inconsistent; + } + + if (decoded.Generation > _layout.ParticipantGenerationMask) + { + return ParticipantSnapshotStatus.Inconsistent; + } + + ref ParticipantRecordV2 record = ref Record(decoded.RecordIndex); + long control1 = AtomicControlWord.LoadAcquire(ref record.Control); + int identityKind = record.IdentityKind; + int reserved = record.Reserved; + long processStartValue = record.ProcessStartValue; + long openSequence = record.OpenSequence; + ulong pidNamespaceId = record.PidNamespaceId; + long control2 = AtomicControlWord.LoadAcquire(ref record.Control); + + int state = DecodeState(control1); + int generation = DecodeIncarnation(control1); + int processId = DecodeProcessId(control1); + incarnation = new ParticipantIncarnation( + decoded.RecordIndex, + generation, + participantToken, + state, + processId, + identityKind, + processStartValue, + openSequence, + pidNamespaceId, + reserved, + control1); + + if (control1 != control2) + { + return ParticipantSnapshotStatus.Changing; + } + + if (!ObserveParticipantControl(control1)) + { + return ParticipantSnapshotStatus.Inconsistent; + } + + if (reserved != 0 + || identityKind is < LayoutV2Constants.IdentityUnknown + or > LayoutV2Constants.IdentityLinuxProcStartTicks + || processStartValue < 0) + { + _ = LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + return ParticipantSnapshotStatus.Inconsistent; + } + + bool ownedState = state is LayoutV2Constants.ParticipantRegistering + or LayoutV2Constants.ParticipantActive + or LayoutV2Constants.ParticipantClosing + or LayoutV2Constants.ParticipantRecovering; + if ((ownedState && processId <= 0) || (!ownedState && processId != 0)) + { + _ = LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + return ParticipantSnapshotStatus.Inconsistent; + } + + if (generation != decoded.Generation + || state is LayoutV2Constants.ParticipantFree + or LayoutV2Constants.ParticipantReclaiming + or LayoutV2Constants.ParticipantRetired) + { + return ParticipantSnapshotStatus.Stale; + } + + if ((state is LayoutV2Constants.ParticipantActive + or LayoutV2Constants.ParticipantClosing + or LayoutV2Constants.ParticipantRecovering) + && (openSequence <= 0 + || (identityKind != LayoutV2Constants.IdentityUnknown + && processStartValue == 0))) + { + _ = LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + return ParticipantSnapshotStatus.Inconsistent; + } + + return ParticipantSnapshotStatus.Stable; + } + + private bool IsExactOwnedSnapshot(ParticipantIncarnation snapshot, int requiredState) + { + if ((uint)snapshot.RecordIndex >= (uint)_layout.ParticipantRecordCount + || snapshot.Generation < 1 + || snapshot.Generation > _layout.ParticipantGenerationMask + || snapshot.Token == 0 + || snapshot.Token > ParticipantMask + || snapshot.State != requiredState + || snapshot.ProcessId <= 0) + { + return false; + } + + try + { + ParticipantToken decoded = ParticipantToken.Decode( + snapshot.Token, + _layout.ParticipantRecordCount); + return decoded.RecordIndex == snapshot.RecordIndex + && decoded.Generation == snapshot.Generation + && snapshot.Control == EncodeControl( + requiredState, + snapshot.Generation, + snapshot.ProcessId); + } + catch (ArgumentOutOfRangeException) + { + return false; + } + catch (OverflowException) + { + return false; + } + } + + private bool TokenMatchesSnapshot(ParticipantIncarnation snapshot) + { + if ((uint)snapshot.RecordIndex >= (uint)_layout.ParticipantRecordCount + || snapshot.Token == 0 + || snapshot.Token > ParticipantMask) + { + return false; + } + + try + { + ParticipantToken decoded = ParticipantToken.Decode( + snapshot.Token, + _layout.ParticipantRecordCount); + return decoded.RecordIndex == snapshot.RecordIndex + && decoded.Generation == snapshot.Generation; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + catch (OverflowException) + { + return false; + } + } + + internal ref ParticipantRecordV2 Record(int index) => + ref *(ParticipantRecordV2*)( + _mappingBase + _layout.ParticipantOffset + ((long)index * _layout.ParticipantStride)); + + private bool IsPidNamespaceRecoveryEnabled() => + AtomicControlWord.LoadAcquire(ref ((StoreHeaderV2*)_mappingBase)->PidNamespaceMode) + == LayoutV2Constants.PidNamespaceRecoveryEnabled; + + private ref ValueSlotMetadataV2 SlotRecord(int index) => + ref *(ValueSlotMetadataV2*)( + _mappingBase + _layout.SlotMetadataOffset + ((long)index * _layout.SlotMetadataStride)); + + private ref LeaseRecordV2 LeaseRecord(int index) => + ref *(LeaseRecordV2*)( + _mappingBase + _layout.LeaseRegistryOffset + ((long)index * _layout.LeaseStride)); + + private static ParticipantClassificationKind Map(LeaseOwnerKind ownerKind) => ownerKind switch + { + LeaseOwnerKind.CurrentProcess => ParticipantClassificationKind.CurrentProcess, + LeaseOwnerKind.OtherLiveProcess => ParticipantClassificationKind.Live, + LeaseOwnerKind.StaleProcess => ParticipantClassificationKind.Stale, + LeaseOwnerKind.Unsupported => ParticipantClassificationKind.Unsupported, + LeaseOwnerKind.UnsafeRecord => ParticipantClassificationKind.Inconsistent, + _ => ParticipantClassificationKind.Inconsistent + }; + + private static int DecodeState(long control) => (int)((ulong)control & 0x7UL); + + private static int DecodeIncarnation(long control) => + (int)(((ulong)control >> 3) & ParticipantMask); + + private static int DecodeProcessId(long control) => + unchecked((int)(((ulong)control >> 31) & 0xffff_ffffUL)); + + private static ulong DecodeOwnedParticipant(long control) => + ((ulong)control >> 36) & ParticipantMask; + + private static long EncodeControl(int state, int generation, int pid) => + ToSigned(AtomicControlWord.EncodeParticipant(state, generation, pid)); + + private bool ObserveParticipantControl(long control) + { + if (IsStructuralControlValid(control, _layout.ParticipantGenerationMask)) + { + return true; + } + + _ = LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeParticipantRegistry)); + return false; + } + + private bool HasCorruptStoreControl() => + _storeControl?.Validate() == StoreStatus.CorruptStore; + + /// Pure canonical validation for a participant lifecycle word. + internal static bool IsStructuralControlValid(long control, int generationMask) + { + int state = DecodeState(control); + int generation = DecodeIncarnation(control); + int processId = DecodeProcessId(control); + bool ownedState = state is LayoutV2Constants.ParticipantRegistering + or LayoutV2Constants.ParticipantActive + or LayoutV2Constants.ParticipantClosing + or LayoutV2Constants.ParticipantRecovering; + return ((ulong)control >> 63) == 0 + && state is >= LayoutV2Constants.ParticipantFree + and <= LayoutV2Constants.ParticipantRetired + && generation is >= 1 + && generation <= generationMask + && (ownedState ? processId > 0 : processId == 0) + && (state != LayoutV2Constants.ParticipantRetired + || generation == generationMask); + } + + private StoreOpenStatus ValidateStoreControlForOpen() + { + StoreStatus state = _storeControl?.Validate() ?? StoreStatus.Success; + return state switch + { + StoreStatus.Success => StoreOpenStatus.Success, + StoreStatus.UnsupportedPlatform => StoreOpenStatus.UnsupportedPlatform, + _ => StoreOpenStatus.IncompatibleLayout + }; + } + + private static long ToSigned(ulong value) => unchecked((long)value); + + internal readonly record struct Registration( + int RecordIndex, + int Generation, + ulong Token, + long ActiveControl) + { + internal bool IsValid => Generation > 0 && Token != 0; + } + + private enum ParticipantSnapshotStatus + { + Stable, + Stale, + Changing, + Inconsistent + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeReclaimer.cs b/src/SharedMemoryStore/LockFree/LockFreeReclaimer.cs new file mode 100644 index 0000000..dc6a488 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeReclaimer.cs @@ -0,0 +1,534 @@ +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Cooperative exact-generation removal and slot reuse. No helper owns global +/// progress: every transition is either an exact CAS or a fully described +/// directory unlink that another participant can finish. +/// +internal sealed class LockFreeReclaimer +{ + private readonly StoreLayoutV2 _layout; + private readonly LockFreeSlotTable _slots; + private readonly LockFreeKeyDirectory _directory; + private readonly LockFreeLeaseRegistry _leases; + private readonly LockFreeTelemetry _telemetry; + private readonly LockFreeStoreControl? _storeControl; + + internal LockFreeReclaimer( + StoreLayoutV2 layout, + LockFreeSlotTable slots, + LockFreeKeyDirectory directory, + LockFreeLeaseRegistry leases) + : this(layout, slots, directory, leases, new LockFreeTelemetry()) + { + } + + internal LockFreeReclaimer( + StoreLayoutV2 layout, + LockFreeSlotTable slots, + LockFreeKeyDirectory directory, + LockFreeLeaseRegistry leases, + LockFreeTelemetry telemetry, + LockFreeStoreControl? storeControl = null) + { + _layout = layout; + _slots = slots; + _directory = directory; + _leases = leases; + _telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry)); + _storeControl = storeControl; + } + + internal StoreStatus TryReclaim(ulong exactBinding) => + TryReclaim(exactBinding, LockFreeOperationBudget.StructuralAttempt); + + internal StoreStatus TryReclaim( + ulong exactBinding, + in LockFreeOperationBudget budget) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryReclaim(exactBinding, budget, ref checkpoint); + } + + internal StoreStatus TryReclaim( + ulong exactBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + bool reportRemoveClassification = false) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (!TryDecode(exactBinding, out int slotIndex, out long generation) + || (uint)slotIndex >= (uint)_layout.SlotCount) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeReclaimer)); + } + + ref ValueSlotMetadataV2 slot = ref _slots.Slot(slotIndex); + long removeRequested = Control(LockFreeSlotTable.RemoveRequestedState, generation); + long reclaiming = Control(LockFreeSlotTable.ReclaimingState, generation); + long observed = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed == removeRequested) + { + StoreStatus scan = _leases.ScanHasActiveLease( + exactBinding, + budget, + out bool hasActiveLease); + _ = ObserveStructuralStatus(scan); + if (scan != StoreStatus.Success) + { + return scan; + } + + if (hasActiveLease) + { + if (reportRemoveClassification) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.RemoveAfterLeaseClassification); + } + + return StoreStatus.RemovePending; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.ReclaimAfterLeaseScanBeforeOwnershipCas); + + // The lease scan establishes that claiming Reclaiming would be + // safe, but it does not authorize starting new caller-bounded work + // after the public deadline/cancellation point. Check after the + // instrumentable validation window as well as after the scan so a + // delayed participant cannot claim ownership with a stale budget. + // Leaving the slot in RemoveRequested keeps it universally + // helpable; the public remove facade normalizes this terminal + // budget status to RemovePending because logical removal already + // linearized. + StoreStatus ownershipBudget = budget.Check(); + if (ownershipBudget != StoreStatus.Success) + { + return ownershipBudget; + } + + structure = TryLifecycleTransition( + ref slot.Control, + removeRequested, + reclaiming, + generation, + out _); + if (structure != StoreStatus.Success) + { + return structure; + } + + observed = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + } + + if (observed != removeRequested && observed != reclaiming) + { + return HasAdvancedOrRetired(observed, generation) + ? StoreStatus.Success + : StoreStatus.NotFound; + } + + StoreStatus unlink = _directory.TryUnlink(exactBinding, budget, ref checkpoint); + _ = ObserveStructuralStatus(unlink); + if (unlink != StoreStatus.Success) + { + observed = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (HasAdvancedOrRetired(observed, generation)) + { + return StoreStatus.Success; + } + + return unlink; + } + + // TryUnlink owns exact directory-cell and descriptor cleanup. A + // reclaimer never performs delayed plain writes: another helper may + // already have advanced and reused the slot by the time this one runs. + if (AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation) != 0 + || AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation) != 0) + { + observed = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + return HasAdvancedOrRetired(observed, generation) + ? StoreStatus.Success + : StoreStatus.StoreBusy; + } + + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.ReclaimAfterMetadataValidation); + observed = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed != reclaiming + || AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation) != 0 + || AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation) != 0) + { + return HasAdvancedOrRetired(observed, generation) + ? StoreStatus.Success + : StoreStatus.StoreBusy; + } + + long reusable = unchecked((long)LockFreeSlotTable.AdvanceOrRetire(generation)); + structure = TryLifecycleTransition( + ref slot.Control, + reclaiming, + reusable, + generation, + out bool advanced); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (advanced) + { + LockFreeCheckpoint.ObserveSlotResource( + ref checkpoint, + generation == LockFreeSlotTable.TerminalGeneration + ? LockFreeSlotResourceEventKind.Retire + : LockFreeSlotResourceEventKind.Free, + slotIndex, + generation); + } + + observed = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed == reclaiming || HasAdvancedOrRetired(observed, generation)) + { + return StoreStatus.Success; + } + + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeReclaimer)); + } + + /// + /// Performs one exact lifecycle transition and accepts only its desired + /// control or a structurally valid later incarnation. A same/older + /// incarnation observation is impossible after the source state was + /// validated; an exact no-op CAS confirms it before poisoning the mapping. + /// Movement during confirmation is retried as an ordinary race. + /// + private StoreStatus TryLifecycleTransition( + ref long control, + long expected, + long desired, + long generation, + out bool transitioned) + { + transitioned = false; + const int confirmationAttempts = 8; + for (var attempt = 0; attempt < confirmationAttempts; attempt++) + { + long observed = AtomicControlWord.CompareExchange(ref control, desired, expected); + if (observed == expected) + { + transitioned = true; + return StoreStatus.Success; + } + + _telemetry.RecordCasLoss(); + StoreStatus structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed == desired || HasAdvancedOrRetired(observed, generation)) + { + return StoreStatus.Success; + } + + long confirmed = AtomicControlWord.CompareExchange(ref control, observed, observed); + if (confirmed == observed) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeReclaimer)); + } + + structure = _slots.ValidateStructuralControl(confirmed); + if (structure != StoreStatus.Success) + { + return structure; + } + } + + return StoreStatus.StoreBusy; + } + + internal StoreStatus TryLogicalRemove(ulong exactBinding, out bool alreadyRemoved) + { + alreadyRemoved = false; + if (!TryDecode(exactBinding, out int slotIndex, out long generation) + || (uint)slotIndex >= (uint)_layout.SlotCount) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeReclaimer)); + } + + ref ValueSlotMetadataV2 slot = ref _slots.Slot(slotIndex); + long published = Control(LockFreeSlotTable.PublishedState, generation); + long removeRequested = Control(LockFreeSlotTable.RemoveRequestedState, generation); + long observed = AtomicControlWord.CompareExchange(ref slot.Control, removeRequested, published); + StoreStatus structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + alreadyRemoved = observed == removeRequested; + return observed == published || alreadyRemoved + ? StoreStatus.Success + : StoreStatus.NotFound; + } + + internal int HelpReclaimableSlots() + { + _ = HelpReclaimableSlots(LockFreeOperationBudget.StructuralAttempt, out int reclaimed); + return reclaimed; + } + + internal StoreStatus HelpReclaimableSlots( + in LockFreeOperationBudget budget, + out int reclaimed) + { + NoOpLockFreeCheckpoint checkpoint = default; + return HelpReclaimableSlots(budget, ref checkpoint, out reclaimed); + } + + internal StoreStatus HelpReclaimableSlots( + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out int reclaimed) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + reclaimed = 0; + for (var index = 0; index < _layout.SlotCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref ValueSlotMetadataV2 slot = ref _slots.Slot(index); + long observedControl = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus structure = _slots.ValidateStructuralControl(observedControl); + if (structure != StoreStatus.Success) + { + return structure; + } + + ulong control = unchecked((ulong)observedControl); + int state = (int)(control & 0x7UL); + if (state == LockFreeSlotTable.AbortingState) + { + long abortingGeneration = (long)((control >> 3) & 0x1_ffff_ffffUL); + if (abortingGeneration is < 1 or > LockFreeSlotTable.TerminalGeneration) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeReclaimer)); + } + + ulong abortingBinding = IndexBinding.Encode(index, abortingGeneration); + for (var attempt = 0; ; attempt++) + { + StoreStatus unlink = _directory.TryUnlink( + abortingBinding, + budget, + ref checkpoint); + _ = ObserveStructuralStatus(unlink); + StoreStatus normalizedUnlink = NormalizeAbortingUnlinkOutcome(unlink); + if (normalizedUnlink == StoreStatus.Success) + { + StoreStatus completion = + _slots.TryCompleteRecoveryReclaim( + abortingBinding, + budget, + ref checkpoint); + _ = ObserveStructuralStatus(completion); + if (completion == StoreStatus.Success) + { + reclaimed++; + _telemetry.RecordHelpedTransition(); + break; + } + + normalizedUnlink = completion; + } + + if (normalizedUnlink != StoreStatus.StoreBusy) + { + return normalizedUnlink; + } + + if (!budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + if (terminal == StoreStatus.StoreBusy) + { + _telemetry.RecordContentionBudgetExhaustion(); + } + + return terminal; + } + } + + continue; + } + + if (state is not (LockFreeSlotTable.RemoveRequestedState or LockFreeSlotTable.ReclaimingState)) + { + continue; + } + + long generation = (long)((control >> 3) & 0x1_ffff_ffffUL); + if (generation is < 1 or > LockFreeSlotTable.TerminalGeneration) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeReclaimer)); + } + + // Control is the authoritative exact lifecycle. DirectoryBinding + // is ordinary metadata and may be stale while another helper is + // completing unlink; never use it to choose which generation to + // reclaim under allocation pressure. + ulong binding = IndexBinding.Encode(index, generation); + for (var attempt = 0; ; attempt++) + { + StoreStatus reclaim = TryReclaim(binding, budget, ref checkpoint); + if (reclaim == StoreStatus.Success) + { + reclaimed++; + _telemetry.RecordHelpedTransition(); + break; + } + + StoreStatus normalized = NormalizeObservedReclaimOutcome(reclaim); + if (normalized == StoreStatus.Success) + { + break; + } + + if (normalized != StoreStatus.StoreBusy) + { + return normalized; + } + + if (!budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + if (terminal == StoreStatus.StoreBusy) + { + _telemetry.RecordContentionBudgetExhaustion(); + } + + return terminal; + } + } + } + + return StoreStatus.Success; + } + + internal static StoreStatus NormalizeAbortingUnlinkOutcome(StoreStatus status) => + status is StoreStatus.Success or StoreStatus.NotFound + ? StoreStatus.Success + : status; + + internal static StoreStatus NormalizeObservedReclaimOutcome(StoreStatus status) => + status is StoreStatus.Success or StoreStatus.NotFound or StoreStatus.RemovePending + ? StoreStatus.Success + : status; + + internal static ulong LogicalRemoveControl(long generation) => + AtomicControlWord.EncodeSlot( + LockFreeSlotTable.RemoveRequestedState, + generation, + participantToken: 0); + + private static long Control(int state, long generation) => + unchecked((long)AtomicControlWord.EncodeSlot(state, generation, participantToken: 0)); + + private StoreStatus ObserveStructuralStatus(StoreStatus status) + { + if (status == StoreStatus.CorruptStore) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeReclaimer)); + } + + return status; + } + + private static bool HasAdvancedOrRetired(long control, long generation) + { + ulong raw = unchecked((ulong)control); + long observedGeneration = (long)((raw >> 3) & 0x1_ffff_ffffUL); + int state = (int)(raw & 0x7UL); + return observedGeneration > generation + || (observedGeneration == generation && state == LockFreeSlotTable.RetiredState); + } + + private static bool TryDecode(ulong binding, out int slotIndex, out long generation) + { + slotIndex = -1; + generation = 0; + try + { + IndexBinding decoded = IndexBinding.Decode(binding); + slotIndex = decoded.SlotIndex; + generation = decoded.Generation; + return true; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + catch (OverflowException) + { + return false; + } + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeRecovery.cs b/src/SharedMemoryStore/LockFree/LockFreeRecovery.cs new file mode 100644 index 0000000..100d2cc --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeRecovery.cs @@ -0,0 +1,1082 @@ +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Explicit, record-local recovery for layout-v2 resources. Owner +/// classification is kept separate from universally safe helping, and every +/// owner-controlled reservation is relinquished by one exact control-word CAS. +/// +internal sealed class LockFreeRecovery +{ + private const int ClassificationRetryBudget = 64; + private const int ClaimRetryBudget = 4; + private const int HelpRetryBudget = 32; + private const int IntentInsert = 1; + private const int IntentUnlink = 2; + private const int PhasePrepared = 1; + private const int PhaseTargetSelected = 2; + private const int PhaseBindingChanged = 3; + private const int PhaseRejected = 4; + private const int PhaseComplete = 5; + private const int TargetPrimary = 1; + private const int TargetOverflow = 2; + + private readonly StoreLayoutV2 _layout; + private readonly LockFreeSlotTable _slots; + private readonly LockFreeKeyDirectory _directory; + private readonly LockFreeParticipantRegistry _participants; + private readonly LockFreeTelemetry _telemetry; + private readonly LockFreeStoreControl? _storeControl; + + internal LockFreeRecovery( + StoreLayoutV2 layout, + LockFreeSlotTable slots, + LockFreeKeyDirectory directory, + LockFreeParticipantRegistry participants) + : this(layout, slots, directory, participants, new LockFreeTelemetry()) + { + } + + internal LockFreeRecovery( + StoreLayoutV2 layout, + LockFreeSlotTable slots, + LockFreeKeyDirectory directory, + LockFreeParticipantRegistry participants, + LockFreeTelemetry telemetry, + LockFreeStoreControl? storeControl = null) + { + ArgumentNullException.ThrowIfNull(slots); + ArgumentNullException.ThrowIfNull(directory); + ArgumentNullException.ThrowIfNull(participants); + + _layout = layout; + _slots = slots; + _directory = directory; + _participants = participants; + _telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry)); + _storeControl = storeControl; + } + + /// + /// Scans owner-controlled value-slot lifecycles, classifies their exact + /// participant incarnations (or consumes an exact claim-closed handoff), and + /// hands safely recoverable slots to the ordinary generation-fenced + /// abort/unlink helper. + /// + internal StoreStatus TryRecoverReservations( + in ReservationRecoveryOptions options, + in StoreWaitOptions waitOptions, + out ReservationRecoveryReport report) + { + report = default; + if (!waitOptions.IsValid) + { + return StoreStatus.UnknownFailure; + } + + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions); + return TryRecoverReservations(options, budget, out report); + } + + internal StoreStatus TryRecoverReservations( + in ReservationRecoveryOptions options, + in LockFreeOperationBudget budget, + out ReservationRecoveryReport report) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryRecoverReservations(options, budget, ref checkpoint, out report); + } + + internal StoreStatus TryRecoverReservations( + in ReservationRecoveryOptions options, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out ReservationRecoveryReport report) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + var counts = new ReservationRecoveryAccumulator(); + report = default; + LockFreeOperationBudget postOwnershipCleanup = default; + bool postOwnershipCleanupStarted = false; + + for (var slotIndex = 0; slotIndex < _layout.SlotCount; slotIndex++) + { + StoreStatus bound = budget.CheckPeriodic(slotIndex); + if (bound != StoreStatus.Success) + { + report = counts.ToReport(); + return counts.Recovered > 0 ? StoreStatus.Success : bound; + } + + ref ValueSlotMetadataV2 slot = ref _slots.Slot(slotIndex); + long observed = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus structure = _slots.ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + counts.Failed++; + report = counts.ToReport(); + return structure; + } + + int state = State(observed); + if (state is LockFreeSlotTable.AbortingState or LockFreeSlotTable.ReclaimingState) + { + StoreStatus metadata = ValidateReservationMetadata( + slotIndex, + observed, + ref slot, + budget, + out bool lifecycleStillCurrent, + out bool unreferencedPreMetadata); + _ = ObserveStructuralStatus(metadata); + if (metadata != StoreStatus.Success) + { + if (metadata == StoreStatus.CorruptStore) + { + counts.Failed++; + } + + report = counts.ToReport(); + return metadata == StoreStatus.CorruptStore || counts.Recovered == 0 + ? metadata + : StoreStatus.Success; + } + + if (!lifecycleStillCurrent) + { + continue; + } + + StoreStatus help = unreferencedPreMetadata + ? HelpUnreferencedReservationRecovery( + IndexBinding.Encode(slotIndex, Generation(observed)), + budget, + ref checkpoint) + : HelpPublishedReservationTransition( + slotIndex, + observed, + budget, + ref checkpoint); + if (help is StoreStatus.StoreBusy or StoreStatus.OperationCanceled) + { + report = counts.ToReport(); + return counts.Recovered > 0 ? StoreStatus.Success : help; + } + + if (help != StoreStatus.Success) + { + counts.Failed++; + if (help == StoreStatus.CorruptStore) + { + report = counts.ToReport(); + return CorruptHere(); + } + } + else + { + _telemetry.RecordHelpedTransition(); + } + + continue; + } + + if (state is not (LockFreeSlotTable.InitializingState + or LockFreeSlotTable.ReservedState)) + { + continue; + } + + counts.Scanned++; + ulong participantToken = Participant(observed); + long generation = Generation(observed); + StoreStatus metadataStatus = ValidateReservationMetadata( + slotIndex, + observed, + ref slot, + budget, + out bool ownedLifecycleStillCurrent, + out bool unreferencedOwnedPreMetadata); + _ = ObserveStructuralStatus(metadataStatus); + if (metadataStatus != StoreStatus.Success) + { + if (metadataStatus == StoreStatus.CorruptStore) + { + counts.Failed++; + } + + report = counts.ToReport(); + return metadataStatus == StoreStatus.CorruptStore || counts.Recovered == 0 + ? metadataStatus + : StoreStatus.Success; + } + + if (!ownedLifecycleStillCurrent) + { + continue; + } + + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + bound = budget.Check(); + if (bound != StoreStatus.Success) + { + report = counts.ToReport(); + return counts.Recovered > 0 ? StoreStatus.Success : bound; + } + + ParticipantClassification classification = default; + bool classified = false; + for (var attempt = 0; ; attempt++) + { + classification = ClassifyReservationParticipant(participantToken); + if (_storeControl?.Validate() == StoreStatus.CorruptStore) + { + counts.Failed++; + report = counts.ToReport(); + return CorruptHere(); + } + + if (classification.Kind != ParticipantClassificationKind.Changing) + { + classified = true; + break; + } + + long current = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(current); + if (structure != StoreStatus.Success) + { + counts.Failed++; + report = counts.ToReport(); + return structure; + } + + if (!IsSameOwnedLifecycle(current, observed)) + { + break; + } + + bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + report = counts.ToReport(); + return counts.Recovered > 0 ? StoreStatus.Success : bound; + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + + if (attempt + 1 >= ClassificationRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + report = counts.ToReport(); + return counts.Recovered > 0 ? StoreStatus.Success : terminal; + } + } + + if (!classified) + { + long current = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(current); + if (structure != StoreStatus.Success) + { + counts.Failed++; + report = counts.ToReport(); + return structure; + } + + if (IsSameOwnedLifecycle(current, observed)) + { + counts.Failed++; + } + + continue; + } + + bool recoverable = CanRecoverReservation( + state, + classification, + options.RecoverCurrentProcessReservations); + if (!recoverable) + { + long current = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(current); + if (structure != StoreStatus.Success) + { + counts.Failed++; + report = counts.ToReport(); + return structure; + } + + CountPreservedReservation( + classification.Kind, + current, + observed, + ref counts); + continue; + } + + long expected = observed; + bool currentUnreferencedPreMetadata = unreferencedOwnedPreMetadata; + bool claimCompleted = false; + for (var attempt = 0; ; attempt++) + { + // Check the public bound before the exact CAS. A pre-metadata + // candidate is then revalidated as the final shared-state + // observation; supported stale/quiescent ownership cannot + // publish ordinary metadata after that proof. Once the CAS + // wins, cancellation/deadline cannot strand the claimed abort. + bound = budget.Check(); + if (bound != StoreStatus.Success) + { + report = counts.ToReport(); + return counts.Recovered > 0 ? StoreStatus.Success : bound; + } + + if (currentUnreferencedPreMetadata) + { + StoreStatus revalidation = ValidateReservationMetadata( + slotIndex, + expected, + ref slot, + budget, + out bool preMetadataLifecycleStillCurrent, + out bool stillUnreferencedPreMetadata); + _ = ObserveStructuralStatus(revalidation); + if (revalidation != StoreStatus.Success) + { + if (revalidation == StoreStatus.CorruptStore) + { + counts.Failed++; + } + + report = counts.ToReport(); + return revalidation == StoreStatus.CorruptStore || counts.Recovered == 0 + ? revalidation + : StoreStatus.Success; + } + + if (!preMetadataLifecycleStillCurrent) + { + claimCompleted = true; + break; + } + + currentUnreferencedPreMetadata = stillUnreferencedPreMetadata; + } + + ReservationRecoveryClaim claim = _slots.TryBeginReservationRecovery( + slotIndex, + expected); + switch (claim.Kind) + { + case ReservationRecoveryClaimKind.Acquired: + { + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.RecoveryAfterExactRecoveryCas); + if (!postOwnershipCleanupStarted) + { + postOwnershipCleanup = LockFreeOperationBudget.StartPostOwnershipCleanup(); + postOwnershipCleanupStarted = true; + } + + StoreStatus help = HelpReservationRecovery( + claim.SlotBinding, + postOwnershipCleanup, + ref checkpoint, + currentUnreferencedPreMetadata); + _ = ObserveStructuralStatus(help); + if (help != StoreStatus.CorruptStore) + { + // The exact owner-to-Aborting CAS is the recovery + // ordering point. Deadline/cancellation may stop + // optional physical help, but cannot rewrite that + // durable, universally helpable recovery outcome. + counts.Recovered++; + } + else + { + counts.Failed++; + report = counts.ToReport(); + return CorruptHere(); + } + + claimCompleted = true; + break; + } + + case ReservationRecoveryClaimKind.HelpRequired: + { + StoreStatus help = HelpReservationRecovery( + claim.SlotBinding, + budget, + ref checkpoint, + currentUnreferencedPreMetadata); + _ = ObserveStructuralStatus(help); + if (help is StoreStatus.StoreBusy or StoreStatus.OperationCanceled) + { + report = counts.ToReport(); + return counts.Recovered > 0 ? StoreStatus.Success : help; + } + + if (help != StoreStatus.Success) + { + counts.Failed++; + if (help == StoreStatus.CorruptStore) + { + report = counts.ToReport(); + return CorruptHere(); + } + } + else + { + _telemetry.RecordHelpedTransition(); + } + + claimCompleted = true; + break; + } + + case ReservationRecoveryClaimKind.CompletedRace: + claimCompleted = true; + break; + + case ReservationRecoveryClaimKind.OwnerStateChanged: + { + expected = claim.ObservedControl; + StoreStatus revalidation = ValidateReservationMetadata( + slotIndex, + expected, + ref slot, + budget, + out bool changedLifecycleStillCurrent, + out bool changedUnreferencedPreMetadata); + _ = ObserveStructuralStatus(revalidation); + if (revalidation != StoreStatus.Success) + { + if (revalidation == StoreStatus.CorruptStore) + { + counts.Failed++; + } + + report = counts.ToReport(); + return revalidation == StoreStatus.CorruptStore || counts.Recovered == 0 + ? revalidation + : StoreStatus.Success; + } + + if (!changedLifecycleStillCurrent) + { + claimCompleted = true; + break; + } + + if (!CanRecoverReservation( + State(expected), + classification, + options.RecoverCurrentProcessReservations)) + { + long current = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = _slots.ValidateStructuralControl(current); + if (structure != StoreStatus.Success) + { + counts.Failed++; + report = counts.ToReport(); + return structure; + } + + CountPreservedReservation( + classification.Kind, + current, + expected, + ref counts); + claimCompleted = true; + break; + } + + currentUnreferencedPreMetadata = changedUnreferencedPreMetadata; + break; + } + + case ReservationRecoveryClaimKind.Inconsistent: + default: + counts.Failed++; + report = counts.ToReport(); + return CorruptHere(); + } + + if (claimCompleted) + { + break; + } + + if (attempt + 1 >= ClaimRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + report = counts.ToReport(); + return counts.Recovered > 0 ? StoreStatus.Success : terminal; + } + } + + if (!claimCompleted) + { + counts.Failed++; + } + } + + StoreStatus participantSweep = _participants.TryRecoverUnreferencedStaleParticipants( + budget, + ref checkpoint, + out _); + _ = ObserveStructuralStatus(participantSweep); + if (participantSweep == StoreStatus.CorruptStore) + { + report = counts.ToReport(); + return CorruptHere(); + } + + if (participantSweep != StoreStatus.Success && counts.Recovered == 0) + { + report = counts.ToReport(); + return participantSweep; + } + + report = counts.ToReport(); + return StoreStatus.Success; + } + + private StoreStatus ValidateReservationMetadata( + int slotIndex, + long expectedControl, + ref ValueSlotMetadataV2 slot, + in LockFreeOperationBudget budget, + out bool lifecycleStillCurrent, + out bool unreferencedPreMetadata) + { + lifecycleStillCurrent = false; + unreferencedPreMetadata = false; + long generation = Generation(expectedControl); + ulong exactBinding = IndexBinding.Encode(slotIndex, generation); + int state = State(expectedControl); + + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + ulong operationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + ulong locationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryLocation)); + ulong directoryBinding = slot.DirectoryBinding; + + if (operationRaw == 0) + { + StoreStatus referenceStatus = _directory.ContainsExactBindingReference( + exactBinding, + budget, + out bool hasDirectoryReference); + _ = ObserveStructuralStatus(referenceStatus); + if (referenceStatus != StoreStatus.Success) + { + return referenceStatus; + } + + long currentControl = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus structure = _slots.ValidateStructuralControl(currentControl); + if (structure != StoreStatus.Success) + { + return structure; + } + + ulong currentOperation = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + ulong currentLocation = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryLocation)); + ulong currentBinding = slot.DirectoryBinding; + if (currentControl != expectedControl) + { + return StoreStatus.Success; + } + + if (currentOperation != operationRaw + || currentLocation != locationRaw + || currentBinding != directoryBinding) + { + continue; + } + + LocationReferenceStatus locationStatus = ClassifyLocationReference( + locationRaw, + generation); + if (state == LockFreeSlotTable.ReservedState + || hasDirectoryReference + || locationStatus is LocationReferenceStatus.Current + or LocationReferenceStatus.Invalid) + { + lifecycleStillCurrent = true; + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeRecovery)); + } + + lifecycleStillCurrent = true; + unreferencedPreMetadata = true; + return StoreStatus.Success; + } + + bool operationValid = TryDecodeRecoveryOperation( + operationRaw, + generation, + state, + out DirectoryOperation operation); + bool locationValid = operationValid + && IsRecoveryOperationLocationValid(operation, locationRaw, state); + int publicationIntent = Volatile.Read(ref slot.PublicationIntent); + long control2 = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus controlStructure = _slots.ValidateStructuralControl(control2); + if (controlStructure != StoreStatus.Success) + { + return controlStructure; + } + + ulong operationRaw2 = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + ulong locationRaw2 = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryLocation)); + ulong directoryBinding2 = slot.DirectoryBinding; + int publicationIntent2 = Volatile.Read(ref slot.PublicationIntent); + if (control2 != expectedControl) + { + return StoreStatus.Success; + } + + if (operationRaw2 != operationRaw + || locationRaw2 != locationRaw + || directoryBinding2 != directoryBinding + || publicationIntent2 != publicationIntent) + { + continue; + } + + if (!operationValid + || !locationValid + || directoryBinding != exactBinding + || publicationIntent is not ( + (int)SlotPublicationIntent.ExplicitReservation + or (int)SlotPublicationIntent.AtomicPublication)) + { + lifecycleStillCurrent = true; + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeRecovery)); + } + + lifecycleStillCurrent = true; + return StoreStatus.Success; + } + } + + private bool TryDecodeRecoveryOperation( + ulong raw, + long generation, + int slotState, + out DirectoryOperation operation) + { + try + { + operation = DirectoryOperation.Decode(raw); + } + catch (ArgumentOutOfRangeException) + { + operation = default; + return false; + } + catch (OverflowException) + { + operation = default; + return false; + } + + bool owned = slotState is LockFreeSlotTable.InitializingState + or LockFreeSlotTable.ReservedState; + if (operation.Value != raw + || operation.Generation != generation + || operation.Intent is not (IntentInsert or IntentUnlink) + || (owned && operation.Intent != IntentInsert)) + { + return false; + } + + return operation.Phase switch + { + PhasePrepared => operation.Kind == 0 && operation.Index == 0, + PhaseRejected => operation.Intent == IntentInsert + && operation.Kind == 0 + && operation.Index == 0, + PhaseTargetSelected or PhaseBindingChanged => + IsDirectoryTargetInBounds(operation.Kind, operation.Index), + PhaseComplete when operation.Intent == IntentUnlink && operation.Kind == 0 => + operation.Index == 0, + PhaseComplete => IsDirectoryTargetInBounds(operation.Kind, operation.Index), + _ => false, + }; + } + + private bool IsDirectoryTargetInBounds(int kind, long index) => + kind switch + { + TargetPrimary => (ulong)index < (ulong)_layout.PrimaryLaneCount, + TargetOverflow => (ulong)index < (ulong)_layout.SlotCount, + _ => false, + }; + + private bool IsRecoveryOperationLocationValid( + DirectoryOperation operation, + ulong locationRaw, + int slotState) + { + if (locationRaw == 0) + { + if (operation.Intent == IntentInsert) + { + return operation.Phase is PhasePrepared + or PhaseTargetSelected + or PhaseRejected + || ((slotState is LockFreeSlotTable.AbortingState + or LockFreeSlotTable.ReclaimingState) + && (operation.Phase is PhaseBindingChanged or PhaseComplete)); + } + + // Unlink helpers may reconstruct a missing location before target + // selection, and clear it before publishing BindingChanged or + // Complete. Zero is therefore valid throughout the unlink phases. + return operation.Intent == IntentUnlink; + } + + LocationReferenceStatus status = ClassifyLocationReference( + locationRaw, + operation.Generation); + if (status == LocationReferenceStatus.Older) + { + // Older residue is tolerated only before a helper has ordered an + // exact current-generation location. The publication helper owns + // its exact-value cleanup. + return operation.Intent == IntentInsert + && operation.Phase is PhasePrepared or PhaseTargetSelected; + } + + if (status != LocationReferenceStatus.Current) + { + return false; + } + + if (operation.Phase == PhasePrepared) + { + // A prepared unlink starts from the location published by the + // completed insert and has not copied that target into its own + // descriptor yet. + return operation.Intent == IntentUnlink; + } + + if (operation.Phase == PhaseRejected + || operation.Kind is not (TargetPrimary or TargetOverflow)) + { + return false; + } + + return locationRaw == DirectoryLocation.Encode( + operation.Kind, + operation.Index, + operation.Generation); + } + + private LocationReferenceStatus ClassifyLocationReference(ulong raw, long generation) + { + if (raw == 0) + { + return LocationReferenceStatus.None; + } + + try + { + DirectoryLocation location = DirectoryLocation.Decode(raw); + if (location.Generation < generation) + { + return LocationReferenceStatus.Older; + } + + if (location.Generation > generation + || !IsDirectoryTargetInBounds(location.Kind, location.Index)) + { + return LocationReferenceStatus.Invalid; + } + + return LocationReferenceStatus.Current; + } + catch (ArgumentOutOfRangeException) + { + return LocationReferenceStatus.Invalid; + } + catch (OverflowException) + { + return LocationReferenceStatus.Invalid; + } + } + + /// Stabilizes and classifies the participant named by a slot control. + private ParticipantClassification ClassifyReservationParticipant(ulong participantToken) => + _participants.ClassifyParticipant(participantToken); + + /// + /// Performs the optional post-resource zero-reference participant retirement + /// pass. Current/live participants are preserved by the registry contract. + /// + private ParticipantTransitionResult TryReclaimReservationParticipant(ulong participantToken) => + _participants.TryRecoverParticipant(participantToken); + + /// Helps an already published abort/reclaim phase without classifying an owner. + private StoreStatus HelpPublishedReservationTransition( + int slotIndex, + long observedControl, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + long generation = Generation(observedControl); + if (generation is < 1 or > LockFreeSlotTable.TerminalGeneration + || Participant(observedControl) != 0) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeRecovery)); + } + + return HelpReservationRecovery( + IndexBinding.Encode(slotIndex, generation), + budget, + ref checkpoint, + unreferencedPreMetadata: false); + } + + /// + /// Uses the ordinary directory unlink protocol and slot-generation advance; + /// no recovery-specific descriptor clear or metadata reset exists. + /// + private StoreStatus HelpReservationRecovery( + ulong exactBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + bool unreferencedPreMetadata) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (unreferencedPreMetadata) + { + return HelpUnreferencedReservationRecovery( + exactBinding, + budget, + ref checkpoint); + } + + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + StoreStatus unlink = _directory.TryUnlink(exactBinding, budget, ref checkpoint); + _ = ObserveStructuralStatus(unlink); + if (unlink is not (StoreStatus.Success or StoreStatus.NotFound)) + { + if (unlink != StoreStatus.StoreBusy) + { + return unlink; + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + continue; + } + + StoreStatus completion = _slots.TryCompleteRecoveryReclaim( + exactBinding, + budget, + ref checkpoint); + _ = ObserveStructuralStatus(completion); + if (completion == StoreStatus.Success) + { + return StoreStatus.Success; + } + + if (completion != StoreStatus.StoreBusy) + { + return completion; + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + + if (attempt + 1 >= HelpRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + } + } + + private StoreStatus HelpUnreferencedReservationRecovery( + ulong exactBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + StoreStatus completion = _slots.TryCompleteRecoveryReclaim( + exactBinding, + budget, + ref checkpoint); + _ = ObserveStructuralStatus(completion); + if (completion != StoreStatus.StoreBusy) + { + return completion; + } + + Thread.SpinWait(4 << Math.Min(attempt, 10)); + if (attempt + 1 >= HelpRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + } + } + + private StoreStatus ObserveStructuralStatus(StoreStatus status) + { + if (status == StoreStatus.CorruptStore) + { + return LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeRecovery)); + } + + return status; + } + + private StoreStatus CorruptHere() => + LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeRecovery)); + + private static void CountPreservedReservation( + ParticipantClassificationKind classification, + long currentControl, + long classifiedControl, + ref ReservationRecoveryAccumulator counts) + { + if (!IsSameOwnedLifecycle(currentControl, classifiedControl)) + { + return; + } + + switch (classification) + { + case ParticipantClassificationKind.CurrentProcess: + case ParticipantClassificationKind.Live: + counts.Active++; + break; + case ParticipantClassificationKind.Unsupported: + counts.Unsupported++; + break; + case ParticipantClassificationKind.Inconsistent: + case ParticipantClassificationKind.Changing: + case ParticipantClassificationKind.Stale: + default: + counts.Failed++; + break; + } + } + + /// + /// Initializing still permits owner-only ordinary metadata writes. A live + /// current-process writer therefore cannot be reclaimed merely because a + /// caller selected the controlled-shutdown override: the delayed writer + /// could otherwise overwrite a reused generation. Closing/Recovering is the + /// explicit quiescent handoff. Reserved has finished all ordinary metadata + /// initialization and retains the documented explicit override. + /// + internal static bool CanRecoverReservation( + int slotState, + in ParticipantClassification classification, + bool recoverCurrentProcessReservations) + { + if (classification.Kind == ParticipantClassificationKind.Stale) + { + return true; + } + + bool handedOff = classification.Kind is not ( + ParticipantClassificationKind.Changing or + ParticipantClassificationKind.Inconsistent) + && classification.Incarnation.Token != 0 + && classification.Incarnation.State is + LayoutV2Constants.ParticipantClosing or + LayoutV2Constants.ParticipantRecovering; + if (slotState == LockFreeSlotTable.InitializingState) + { + return handedOff; + } + + return slotState == LockFreeSlotTable.ReservedState + && (handedOff + || (recoverCurrentProcessReservations + && classification.Kind == ParticipantClassificationKind.CurrentProcess)); + } + + private static bool IsSameOwnedLifecycle(long current, long classified) + { + int state = State(current); + return state is LockFreeSlotTable.InitializingState or LockFreeSlotTable.ReservedState + && Generation(current) == Generation(classified) + && Participant(current) == Participant(classified) + && Participant(current) != 0; + } + + private static int State(long control) => (int)((ulong)control & 0x7UL); + + private static long Generation(long control) => + (long)(((ulong)control >> 3) & 0x1_ffff_ffffUL); + + private static ulong Participant(long control) => ((ulong)control >> 36) & 0x0fff_ffffUL; + + private enum LocationReferenceStatus + { + None, + Older, + Current, + Invalid, + } + + private struct ReservationRecoveryAccumulator + { + internal int Scanned; + internal int Recovered; + internal int Active; + internal int Unsupported; + internal int Failed; + + internal readonly ReservationRecoveryReport ToReport() => new( + Scanned, + Recovered, + Active, + Unsupported, + Failed); + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeReservationMemory.cs b/src/SharedMemoryStore/LockFree/LockFreeReservationMemory.cs new file mode 100644 index 0000000..dd5c9e5 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeReservationMemory.cs @@ -0,0 +1,205 @@ +using System.Buffers; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Sparse, warmed per-slot memory managers for retained-capable direct ingest. +/// Span-only users allocate no manager table; DangerousGetMemory creates one +/// small page/manager on first use and reuses it for later slot generations. +/// +internal sealed unsafe class LockFreeReservationMemory : IDisposable +{ + private const int ManagerPageShift = 8; + private const int ManagerPageSize = 1 << ManagerPageShift; + private const int ManagerPageMask = ManagerPageSize - 1; + + private readonly byte* _payloadStorage; + private readonly int _payloadStride; + private readonly int _maxValueBytes; + private readonly int _slotCount; + private readonly LockFreeSlotTable _slots; + private SlotPayloadMemoryManager?[][]? _managerPages; + private int _disposed; + + internal LockFreeReservationMemory( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + LockFreeSlotTable slots) + { + ArgumentNullException.ThrowIfNull(region); + ArgumentNullException.ThrowIfNull(slots); + _payloadStorage = region.Pointer + layout.PayloadStorageOffset; + _payloadStride = layout.PayloadStride; + _maxValueBytes = layout.MaxValueBytes; + _slotCount = layout.SlotCount; + _slots = slots; + } + + internal Span GetSpan(ReservationHandle reservation, int sizeHint) + { + if (Volatile.Read(ref _disposed) != 0 + || !_slots.TryGetWritableRange( + reservation, + sizeHint, + out int slotIndex, + out int offset, + out int length)) + { + return Span.Empty; + } + + return new Span( + _payloadStorage + ((long)slotIndex * _payloadStride) + offset, + length); + } + + internal Memory GetMemory(in ReservationHandle reservation, int sizeHint) + { + if (Volatile.Read(ref _disposed) != 0 + || !_slots.TryGetWritableRange( + reservation, + sizeHint, + out int slotIndex, + out int offset, + out int length)) + { + return Memory.Empty; + } + + SlotPayloadMemoryManager manager = GetOrCreateManager(slotIndex); + manager.Activate(reservation); + Memory memory = manager.Memory; + return memory.Length >= offset + length + ? memory.Slice(offset, length) + : Memory.Empty; + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + SlotPayloadMemoryManager?[][]? pages = Interlocked.Exchange(ref _managerPages, null); + if (pages is null) + { + return; + } + + foreach (SlotPayloadMemoryManager?[]? page in pages) + { + if (page is null) + { + continue; + } + + foreach (SlotPayloadMemoryManager? manager in page) + { + if (manager is not null) + { + ((IDisposable)manager).Dispose(); + } + } + } + } + + private SlotPayloadMemoryManager GetOrCreateManager(int slotIndex) + { + SlotPayloadMemoryManager?[][]? pages = Volatile.Read(ref _managerPages); + if (pages is null) + { + var created = new SlotPayloadMemoryManager?[ + (_slotCount + ManagerPageSize - 1) >> ManagerPageShift][]; + pages = Interlocked.CompareExchange(ref _managerPages, created, null) ?? created; + } + + int pageIndex = slotIndex >> ManagerPageShift; + SlotPayloadMemoryManager?[]? page = Volatile.Read(ref pages[pageIndex]); + if (page is null) + { + var created = new SlotPayloadMemoryManager?[ManagerPageSize]; + page = Interlocked.CompareExchange(ref pages[pageIndex], created, null) ?? created; + } + + int entryIndex = slotIndex & ManagerPageMask; + SlotPayloadMemoryManager? manager = Volatile.Read(ref page[entryIndex]); + if (manager is not null) + { + return manager; + } + + var candidate = new SlotPayloadMemoryManager( + _payloadStorage + ((long)slotIndex * _payloadStride), + _maxValueBytes, + _slots); + manager = Interlocked.CompareExchange(ref page[entryIndex], candidate, null); + if (manager is null) + { + return candidate; + } + + ((IDisposable)candidate).Dispose(); + return manager; + } + + private sealed unsafe class SlotPayloadMemoryManager : MemoryManager + { + private readonly byte* _pointer; + private readonly int _length; + private readonly LockFreeSlotTable _slots; + private ReservationHandle _reservation; + private int _disposed; + + internal SlotPayloadMemoryManager( + byte* pointer, + int length, + LockFreeSlotTable slots) + { + _pointer = pointer; + _length = length; + _slots = slots; + } + + internal void Activate(in ReservationHandle reservation) + { + _reservation = reservation; + } + + public override Span GetSpan() + { + return Volatile.Read(ref _disposed) != 0 || !_slots.IsReservationPending(_reservation) + ? Span.Empty + : new Span(_pointer, _length); + } + + public override MemoryHandle Pin(int elementIndex = 0) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + if (!_slots.IsReservationPending(_reservation)) + { + throw new InvalidOperationException("The reservation is no longer writable."); + } + + if ((uint)elementIndex > (uint)_length) + { + throw new ArgumentOutOfRangeException(nameof(elementIndex)); + } + + return new MemoryHandle(_pointer + elementIndex); + } + + public override void Unpin() + { + } + + protected override void Dispose(bool disposing) + { + Volatile.Write(ref _disposed, 1); + _reservation = default; + } + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeSlotTable.cs b/src/SharedMemoryStore/LockFree/LockFreeSlotTable.cs new file mode 100644 index 0000000..a4af284 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeSlotTable.cs @@ -0,0 +1,2141 @@ +using System.Runtime.CompilerServices; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Result of attempting the exact owner-controlled slot transition used by +/// reservation recovery. A failed compare/exchange is not itself an error: +/// the observed state determines whether the owner completed, another helper +/// already took over, or the slot must be classified again. +/// +internal enum ReservationRecoveryClaimKind +{ + Acquired, + HelpRequired, + CompletedRace, + OwnerStateChanged, + Inconsistent +} + +/// Exact slot observation returned by a reservation recovery CAS. +internal readonly record struct ReservationRecoveryClaim( + ReservationRecoveryClaimKind Kind, + ulong SlotBinding, + long ObservedControl); + +/// +/// Generation-fenced value-slot state transitions for layout 2.0. +/// Directory placement/unlink remains the responsibility of +/// LockFreeKeyDirectory. +/// +internal sealed unsafe class LockFreeSlotTable +{ + internal const int FreeState = 0; + internal const int InitializingState = 1; + internal const int ReservedState = 2; + internal const int PublishedState = 3; + internal const int RemoveRequestedState = 4; + internal const int AbortingState = 5; + internal const int ReclaimingState = 6; + internal const int RetiredState = 7; + internal const long TerminalGeneration = 0x1_ffff_ffffL; + + private const ulong SlotIndexMask = 0x7fff_ffffUL; + private const ulong SlotGenerationMask = 0x1_ffff_ffffUL; + private const ulong ParticipantMask = 0x0fff_ffffUL; + private const int ResidueCleanupRetryBudget = 128; + private const int AdvanceRetryBudget = 128; + + private readonly byte* _mappingBase; + private readonly StoreLayoutV2 _layout; + private readonly LockFreeParticipantRegistry.Registration _participant; + private readonly LockFreeTelemetry _telemetry; + private readonly LockFreeStoreControl? _storeControl; + private readonly ulong _storeId; + private readonly long[] _storeFullSnapshot; + private int _nextSlot; + private int _storeFullProofGate; + + internal LockFreeSlotTable( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + in LockFreeParticipantRegistry.Registration participant) + : this(region, layout, participant, new LockFreeTelemetry()) + { + } + + internal LockFreeSlotTable( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + in LockFreeParticipantRegistry.Registration participant, + LockFreeTelemetry telemetry, + LockFreeStoreControl? storeControl = null) + { + ArgumentNullException.ThrowIfNull(region); + if (!participant.IsValid || participant.Token > ParticipantMask) + { + throw new ArgumentOutOfRangeException(nameof(participant)); + } + + _mappingBase = region.Pointer; + _layout = layout; + _participant = participant; + _telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry)); + _storeControl = storeControl; + // A failed allocation scan is not by itself a linearizable StoreFull + // witness: a reusable slot can rotate behind a sequential scanner. The + // per-open, process-local buffer supports an exact double collect only + // on that rare slow path, with no per-operation allocation or shared + // cache line. Maximum configured cost is eight bytes per value slot. + _storeFullSnapshot = GC.AllocateUninitializedArray(layout.SlotCount); + // Spread the first local claim of concurrently opened publishers over + // the bounded slot table. Later claims remain a local round-robin scan. + _nextSlot = (participant.RecordIndex % layout.SlotCount) - 1; + _storeId = ((StoreHeaderV2*)_mappingBase)->StoreId; + if (_storeId == 0) + { + throw new ArgumentException("The layout-v2 mapping has no store incarnation.", nameof(region)); + } + } + + internal StoreStatus TryClaimReservation( + ulong keyHash, + int keyLength, + int descriptorLength, + int payloadLength, + out ReservationHandle reservation) => + TryClaimReservationNoCheckpoint( + keyHash, + keyLength, + descriptorLength, + payloadLength, + SlotPublicationIntent.ExplicitReservation, + LockFreeOperationBudget.StructuralAttempt, + out reservation); + + internal StoreStatus TryClaimReservation( + ulong keyHash, + int keyLength, + int descriptorLength, + int payloadLength, + in LockFreeOperationBudget budget, + out ReservationHandle reservation) + { + return TryClaimReservationNoCheckpoint( + keyHash, + keyLength, + descriptorLength, + payloadLength, + SlotPublicationIntent.ExplicitReservation, + budget, + out reservation); + } + + private StoreStatus TryClaimReservationNoCheckpoint( + ulong keyHash, + int keyLength, + int descriptorLength, + int payloadLength, + SlotPublicationIntent publicationIntent, + in LockFreeOperationBudget budget, + out ReservationHandle reservation) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryClaimReservation( + keyHash, + keyLength, + descriptorLength, + payloadLength, + publicationIntent, + budget, + ref checkpoint, + out reservation); + } + + internal StoreStatus TryClaimReservation( + ulong keyHash, + int keyLength, + int descriptorLength, + int payloadLength, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out ReservationHandle reservation) + where TCheckpoint : struct, ILockFreeCheckpointStrategy => + TryClaimReservation( + keyHash, + keyLength, + descriptorLength, + payloadLength, + SlotPublicationIntent.ExplicitReservation, + budget, + ref checkpoint, + out reservation); + + internal StoreStatus TryClaimReservation( + ulong keyHash, + int keyLength, + int descriptorLength, + int payloadLength, + SlotPublicationIntent publicationIntent, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out ReservationHandle reservation) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + reservation = default; + if (!IsParticipantActive()) + { + return ParticipantUnavailableStatus(); + } + + if (publicationIntent is not ( + SlotPublicationIntent.ExplicitReservation + or SlotPublicationIntent.AtomicPublication) + || keyLength <= 0 || keyLength > _layout.MaxKeyBytes + || descriptorLength < 0 || descriptorLength > _layout.MaxDescriptorBytes + || payloadLength < 0 || payloadLength > _layout.MaxValueBytes) + { + return StoreStatus.InvalidReservation; + } + + int start = (int)((uint)Interlocked.Increment(ref _nextSlot) % (uint)_layout.SlotCount); + for (var visited = 0; visited < _layout.SlotCount; visited++) + { + StoreStatus bound = budget.CheckPeriodic(visited); + if (bound != StoreStatus.Success) + { + return bound; + } + + int slotIndex = start + visited; + if (slotIndex >= _layout.SlotCount) + { + slotIndex -= _layout.SlotCount; + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long observed = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus structure = ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (State(observed) != FreeState) + { + continue; + } + + long generation = Generation(observed); + long initializing = Signed(AtomicControlWord.EncodeSlot( + InitializingState, + generation, + checked((int)_participant.Token))); + long claimObservation = AtomicControlWord.CompareExchange( + ref slot.Control, + initializing, + observed); + if (claimObservation != observed) + { + _telemetry.RecordCasLoss(); + structure = ValidateStructuralControl(claimObservation); + if (structure != StoreStatus.Success) + { + return structure; + } + + continue; + } + + LockFreeCheckpoint.ObserveSlotResource( + ref checkpoint, + LockFreeSlotResourceEventKind.Claim, + slotIndex, + generation); + + ulong binding = IndexBinding.Encode(slotIndex, generation); + reservation = new ReservationHandle( + _storeId, + _participant.Token, + binding, + payloadLength); + StoreStatus residueStatus = SanitizeOlderDirectoryResidue(ref slot, generation, budget); + if (residueStatus != StoreStatus.Success) + { + // Never roll Initializing(g) back to Free(g). Reusing the same + // control word would introduce ABA into slot snapshots and let + // a delayed observer confuse two claims. Relinquish through the + // normal helpable lifecycle so every reuse advances generation. + StoreStatus beginAbort = TryBeginAbort(reservation); + if (residueStatus == StoreStatus.CorruptStore) + { + reservation = default; + return CorruptFrom(nameof(LockFreeSlotTable)); + } + + StoreStatus cleanup = TryCompleteRecoveryReclaim( + binding, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint); + reservation = default; + if (beginAbort is not (StoreStatus.Success or StoreStatus.InvalidReservation) + || cleanup == StoreStatus.CorruptStore) + { + return CorruptFrom(nameof(LockFreeSlotTable)); + } + + return residueStatus; + } + + // The first claim CAS already contains the complete token. Its exact + // participant control is acquire-revalidated before metadata becomes + // usable, closing the crash/participant-retirement window. + if (!IsParticipantActive()) + { + StoreStatus unavailable = ParticipantUnavailableStatus(); + if (unavailable != StoreStatus.CorruptStore) + { + _ = TryBeginAbort(reservation); + _ = TryCompleteReclaim( + reservation, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint); + } + + reservation = default; + return unavailable; + } + + LockFreeCheckpoint.Reach(ref checkpoint, LockFreeCheckpointId.SlotClaimAfterParticipantRecheck); + bound = budget.Check(); + if (bound != StoreStatus.Success) + { + _ = TryBeginAbort(reservation); + _ = TryCompleteReclaim(reservation, LockFreeOperationBudget.StructuralAttempt, ref checkpoint); + reservation = default; + return bound; + } + + // This ordinary field is immutable for the claimed generation and + // is published by the later directory-operation/cell release chain. + // A bare Initializing control predates it and is not a read witness. + Volatile.Write(ref slot.PublicationIntent, (int)publicationIntent); + slot.DirectoryBinding = binding; + slot.KeyHash = keyHash; + slot.KeyLength = keyLength; + slot.DescriptorLength = descriptorLength; + slot.ValueLength = payloadLength; + AtomicControlWord.StoreRelease(ref slot.BytesAdvanced, 0); + slot.CommitSequence = 0; + return StoreStatus.Success; + } + + return StoreStatus.StoreFull; + } + + /// + /// Converts a scan-exhaustion candidate into an exact physical-capacity + /// result. Equal all-occupied collects in the same order prove a common + /// point between the two collects at which every slot was unavailable. + /// The method distinguishes an unconfirmed transient from a caller-budget + /// terminal so finite and infinite callers can apply their wait policy. + /// + internal StoreStatus TryProveStoreFull( + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint, + out bool provenFull) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + provenFull = false; + if (Interlocked.CompareExchange(ref _storeFullProofGate, 1, 0) != 0) + { + return StoreStatus.Success; + } + + long proofToken = 0; + var candidateObserved = false; + var proofConfirmed = false; + try + { + for (var slotIndex = 0; slotIndex < _layout.SlotCount; slotIndex++) + { + StoreStatus bound = budget.CheckPeriodic(slotIndex); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref Slot(slotIndex).Control); + StoreStatus classification = ClassifyFullnessControl(control, out bool occupied); + if (classification != StoreStatus.Success) + { + return classification; + } + + if (!occupied) + { + return StoreStatus.Success; + } + + _storeFullSnapshot[slotIndex] = control; + } + + proofToken = LockFreeCheckpoint.BeginStoreFullProof( + ref checkpoint, + _layout.SlotCount); + candidateObserved = true; + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification); + + for (var slotIndex = 0; slotIndex < _layout.SlotCount; slotIndex++) + { + StoreStatus bound = budget.CheckPeriodic(slotIndex); + if (bound != StoreStatus.Success) + { + return bound; + } + + long control = AtomicControlWord.LoadAcquire(ref Slot(slotIndex).Control); + StoreStatus classification = ClassifyFullnessControl(control, out bool occupied); + if (classification != StoreStatus.Success) + { + return classification; + } + + if (!occupied || control != _storeFullSnapshot[slotIndex]) + { + return StoreStatus.Success; + } + } + + proofConfirmed = true; + LockFreeCheckpoint.CompleteStoreFullProof( + ref checkpoint, + proofToken, + confirmed: true); + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.StoreFullAfterExactDoubleCollect); + provenFull = true; + return StoreStatus.Success; + } + finally + { + if (candidateObserved && !proofConfirmed) + { + LockFreeCheckpoint.CompleteStoreFullProof( + ref checkpoint, + proofToken, + confirmed: false); + } + + Volatile.Write(ref _storeFullProofGate, 0); + } + } + + internal Span GetInitializingKeySpan(in ReservationHandle reservation) + { + return TryReadInitializingProjection( + reservation, + out int slotIndex, + out int keyLength, + out _) + ? new Span( + _mappingBase + _layout.KeyStorageOffset + ((long)slotIndex * _layout.KeyStride), + keyLength) + : Span.Empty; + } + + internal Span GetInitializingDescriptorSpan(in ReservationHandle reservation) + { + return TryReadInitializingProjection( + reservation, + out int slotIndex, + out _, + out int descriptorLength) + ? new Span( + _mappingBase + _layout.DescriptorStorageOffset + + ((long)slotIndex * _layout.DescriptorStride), + descriptorLength) + : Span.Empty; + } + + private bool TryReadInitializingProjection( + in ReservationHandle reservation, + out int slotIndex, + out int keyLength, + out int descriptorLength) + { + slotIndex = -1; + keyLength = 0; + descriptorLength = 0; + if (_storeControl is not null && !_storeControl.IsReady) + { + return false; + } + + if (!TryDecodeHandle(reservation, out slotIndex, out long generation)) + { + return false; + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long expected = OwnedControl(InitializingState, generation, reservation.ParticipantToken); + long control1 = AtomicControlWord.LoadAcquire(ref slot.Control); + if (ValidateStructuralControl(control1) != StoreStatus.Success + || control1 != expected + || !IsParticipantActive()) + { + return false; + } + + ulong directoryBinding = Volatile.Read(ref slot.DirectoryBinding); + int observedKeyLength = Volatile.Read(ref slot.KeyLength); + int observedDescriptorLength = Volatile.Read(ref slot.DescriptorLength); + int observedValueLength = Volatile.Read(ref slot.ValueLength); + int publicationIntent = Volatile.Read(ref slot.PublicationIntent); + long advanced = AtomicControlWord.LoadAcquire(ref slot.BytesAdvanced); + long keyOffset = Volatile.Read(ref slot.KeyOffset); + long descriptorOffset = Volatile.Read(ref slot.DescriptorOffset); + long payloadOffset = Volatile.Read(ref slot.PayloadOffset); + long control2 = AtomicControlWord.LoadAcquire(ref slot.Control); + if (control2 != control1 || !IsParticipantActive()) + { + _ = ValidateStructuralControl(control2); + return false; + } + + long expectedKeyOffset = _layout.KeyStorageOffset + ((long)slotIndex * _layout.KeyStride); + long expectedDescriptorOffset = + _layout.DescriptorStorageOffset + ((long)slotIndex * _layout.DescriptorStride); + long expectedPayloadOffset = + _layout.PayloadStorageOffset + ((long)slotIndex * _layout.PayloadStride); + if (directoryBinding != reservation.SlotBinding + || observedKeyLength is < 1 || observedKeyLength > _layout.MaxKeyBytes + || observedDescriptorLength < 0 + || observedDescriptorLength > _layout.MaxDescriptorBytes + || observedValueLength < 0 || observedValueLength > _layout.MaxValueBytes + || observedValueLength != reservation.PayloadLength + || publicationIntent is not ( + (int)SlotPublicationIntent.ExplicitReservation + or (int)SlotPublicationIntent.AtomicPublication) + || advanced != 0 + || keyOffset != expectedKeyOffset + || descriptorOffset != expectedDescriptorOffset + || payloadOffset != expectedPayloadOffset) + { + _ = CorruptHere(); + return false; + } + + if (_storeControl is not null && !_storeControl.IsReady) + { + return false; + } + + keyLength = observedKeyLength; + descriptorLength = observedDescriptorLength; + return true; + } + + internal StoreStatus TryMarkReserved(in ReservationHandle reservation) + { + if (!TryDecodeHandle(reservation, out int slotIndex, out long generation)) + { + return StoreStatus.InvalidReservation; + } + + if (!IsParticipantActive()) + { + return ParticipantUnavailableStatus(); + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long initializing = OwnedControl(InitializingState, generation, reservation.ParticipantToken); + long reserved = OwnedControl(ReservedState, generation, reservation.ParticipantToken); + long observed = AtomicControlWord.CompareExchange(ref slot.Control, reserved, initializing); + return observed == initializing + ? StoreStatus.Success + : ReservationStatus(observed, generation); + } + + internal bool IsReservationPending(in ReservationHandle reservation) + { + return TryReadReservationProjection( + reservation, + out _, + out _, + out _, + out _); + } + + internal StoreStatus ClassifyDirectoryBinding( + ulong exactBinding, + out int state, + out SlotPublicationIntent publicationIntent) + { + state = FreeState; + publicationIntent = SlotPublicationIntent.None; + if (!TryDecodeSlotBinding(exactBinding, out int slotIndex, out long generation)) + { + return StoreStatus.CorruptStore; + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + for (var attempt = 0; attempt < 8; attempt++) + { + long control1 = AtomicControlWord.LoadAcquire(ref slot.Control); + if (!TryClassifyStructuralControl( + control1, + _layout.ParticipantRecordCount, + out _)) + { + return StoreStatus.CorruptStore; + } + + long snapshotGeneration = Generation(control1); + if (snapshotGeneration > generation) + { + return StoreStatus.NotFound; + } + + if (snapshotGeneration < generation) + { + return StoreStatus.CorruptStore; + } + + int observedState = State(control1); + if (observedState == RetiredState) + { + return generation == TerminalGeneration && Participant(control1) == 0 + ? StoreStatus.NotFound + : StoreStatus.CorruptStore; + } + + ulong observedBinding = Volatile.Read(ref slot.DirectoryBinding); + ulong observedOperation = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + int rawIntent = Volatile.Read(ref slot.PublicationIntent); + long control2 = AtomicControlWord.LoadAcquire(ref slot.Control); + ulong confirmedBinding = Volatile.Read(ref slot.DirectoryBinding); + ulong confirmedOperation = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + if (control1 != control2 + || observedBinding != confirmedBinding + || observedOperation != confirmedOperation) + { + continue; + } + + if (observedState == FreeState + || observedBinding != exactBinding + || !IsCurrentReferencedOperationValid( + observedOperation, + generation, + observedState)) + { + return StoreStatus.CorruptStore; + } + + bool owned = observedState is InitializingState or ReservedState; + ulong participant = Participant(control1); + if (owned + ? !ParticipantToken.IsStructurallyValid( + participant, + _layout.ParticipantRecordCount) + : participant != 0) + { + return StoreStatus.CorruptStore; + } + + publicationIntent = (SlotPublicationIntent)rawIntent; + if (publicationIntent is not ( + SlotPublicationIntent.ExplicitReservation + or SlotPublicationIntent.AtomicPublication)) + { + publicationIntent = SlotPublicationIntent.None; + return StoreStatus.CorruptStore; + } + + state = observedState; + return StoreStatus.Success; + } + + return StoreStatus.StoreBusy; + } + + /// + /// Classifies the exact lifecycle observed after the directory has reported + /// a completed insert. Normal recovery preserves a live Active owner, but + /// an exact adversarial cancellation or correctly quiesced administrative + /// recovery can follow this boundary. Generation advancement is therefore + /// an ordinary invalid-reservation observation, not evidence of corruption. + /// + internal StoreStatus ClassifyReservationAfterDirectoryInsert( + in ReservationHandle reservation) + { + if (!TryDecodeHandle(reservation, out int slotIndex, out long generation)) + { + // The engine created this handle in the same operation. Failure to + // decode it cannot be caused by cross-process recovery. + return CorruptHere(); + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + for (var attempt = 0; attempt < 8; attempt++) + { + long control1 = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus structure = ValidateStructuralControl(control1); + if (structure != StoreStatus.Success) + { + return structure; + } + + long observedGeneration = Generation(control1); + if (observedGeneration > generation) + { + return StoreStatus.InvalidReservation; + } + + if (observedGeneration < generation) + { + return CorruptHere(); + } + + ulong observedBinding = Volatile.Read(ref slot.DirectoryBinding); + SlotPublicationIntent publicationIntent = + (SlotPublicationIntent)Volatile.Read(ref slot.PublicationIntent); + long control2 = AtomicControlWord.LoadAcquire(ref slot.Control); + ulong confirmedBinding = Volatile.Read(ref slot.DirectoryBinding); + if (control1 != control2 || observedBinding != confirmedBinding) + { + continue; + } + + if (observedBinding != reservation.SlotBinding + || publicationIntent is not ( + SlotPublicationIntent.ExplicitReservation + or SlotPublicationIntent.AtomicPublication)) + { + return CorruptHere(); + } + + int state = State(control1); + ulong participant = Participant(control1); + if (state == ReservedState) + { + if (participant != reservation.ParticipantToken) + { + return CorruptHere(); + } + + // Participant retirement may precede the recovery CAS that + // changes Reserved to Aborting. Treat that legal interval as + // cancellation rather than accepting a dead owner. + if (IsParticipantActive()) + { + return StoreStatus.Success; + } + + return ParticipantUnavailableStatus() == StoreStatus.CorruptStore + ? StoreStatus.CorruptStore + : StoreStatus.InvalidReservation; + } + + if (participant != 0) + { + return CorruptHere(); + } + + return state switch + { + AbortingState or ReclaimingState + when publicationIntent == SlotPublicationIntent.ExplicitReservation + && HasCompletedInsertWitness(ref slot, generation) => + StoreStatus.ReservationAlreadyCompleted, + AbortingState or ReclaimingState => StoreStatus.InvalidReservation, + RetiredState when generation == TerminalGeneration => + StoreStatus.InvalidReservation, + _ => StoreStatus.CorruptStore, + }; + } + + return StoreStatus.StoreBusy; + } + + internal TentativeReservationAbortResult TryBeginTentativeAbort( + in ReservationHandle reservation) + { + if (!TryDecodeHandle(reservation, out int slotIndex, out long generation)) + { + return TentativeReservationAbortResult.Invalid; + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long initializing = OwnedControl( + InitializingState, + generation, + reservation.ParticipantToken); + long aborting = UnownedControl(AbortingState, generation); + long observed = AtomicControlWord.CompareExchange( + ref slot.Control, + aborting, + initializing); + if (ValidateStructuralControl(observed) != StoreStatus.Success) + { + return TentativeReservationAbortResult.Corrupt; + } + + if (observed == initializing || observed == aborting) + { + return TentativeReservationAbortResult.Aborted; + } + + if (observed == OwnedControl( + ReservedState, + generation, + reservation.ParticipantToken)) + { + return TentativeReservationAbortResult.Ordered; + } + + long observedGeneration = Generation(observed); + if (observedGeneration > generation) + { + return TentativeReservationAbortResult.Invalid; + } + + if (observedGeneration < generation) + { + return TentativeReservationAbortResult.Corrupt; + } + + int observedState = State(observed); + if (observedState == ReclaimingState + || (observedState == RetiredState && generation == TerminalGeneration)) + { + return TentativeReservationAbortResult.Invalid; + } + + // Aborting was accepted above. Every other same-generation state is + // impossible for this private candidate: ownership cannot transfer, + // no public handle can commit it, Free advances generation, and only + // terminal generation may retire. + return TentativeReservationAbortResult.Corrupt; + } + + /// + /// Aborts an atomic convenience-publication candidate that has not escaped + /// the reserving engine. Unlike an explicit reservation, Reserved remains + /// private staging and is abortable; it is never an ordered public result. + /// Impossible same-generation observations fail closed instead of being + /// normalized through the public reservation-handle status mapping. + /// + internal TentativeReservationAbortResult TryBeginAtomicCandidateAbort( + in ReservationHandle reservation) + { + if (!TryDecodeHandle(reservation, out int slotIndex, out long generation)) + { + return TentativeReservationAbortResult.Corrupt; + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + var metadataValidated = false; + for (var attempt = 0; attempt < 8; attempt++) + { + long control1 = AtomicControlWord.LoadAcquire(ref slot.Control); + if (ValidateStructuralControl(control1) != StoreStatus.Success) + { + return TentativeReservationAbortResult.Corrupt; + } + + long snapshotGeneration = Generation(control1); + if (snapshotGeneration > generation) + { + return TentativeReservationAbortResult.Invalid; + } + + if (snapshotGeneration < generation) + { + return TentativeReservationAbortResult.Corrupt; + } + + ulong binding = Volatile.Read(ref slot.DirectoryBinding); + int intent = Volatile.Read(ref slot.PublicationIntent); + long control2 = AtomicControlWord.LoadAcquire(ref slot.Control); + if (control1 != control2) + { + if (ValidateStructuralControl(control2) != StoreStatus.Success) + { + return TentativeReservationAbortResult.Corrupt; + } + + continue; + } + + if (binding != reservation.SlotBinding + || intent != (int)SlotPublicationIntent.AtomicPublication) + { + return TentativeReservationAbortResult.Corrupt; + } + + metadataValidated = true; + break; + } + + if (!metadataValidated) + { + // Persistent movement can only be concurrent recovery of this + // private lifecycle; no public outcome has been ordered. + return TentativeReservationAbortResult.Invalid; + } + + long aborting = UnownedControl(AbortingState, generation); + long initializing = OwnedControl( + InitializingState, + generation, + reservation.ParticipantToken); + long observed = AtomicControlWord.CompareExchange( + ref slot.Control, + aborting, + initializing); + if (ValidateStructuralControl(observed) != StoreStatus.Success) + { + return TentativeReservationAbortResult.Corrupt; + } + + if (observed == initializing || observed == aborting) + { + return TentativeReservationAbortResult.Aborted; + } + + long reserved = OwnedControl( + ReservedState, + generation, + reservation.ParticipantToken); + if (observed == reserved) + { + observed = AtomicControlWord.CompareExchange( + ref slot.Control, + aborting, + reserved); + if (ValidateStructuralControl(observed) != StoreStatus.Success) + { + return TentativeReservationAbortResult.Corrupt; + } + + if (observed == reserved || observed == aborting) + { + return TentativeReservationAbortResult.Aborted; + } + } + + long observedGeneration = Generation(observed); + if (observedGeneration > generation) + { + return TentativeReservationAbortResult.Invalid; + } + + if (observedGeneration < generation) + { + return TentativeReservationAbortResult.Corrupt; + } + + int observedState = State(observed); + ulong observedParticipant = Participant(observed); + if (observedState == AbortingState && observedParticipant == 0) + { + return TentativeReservationAbortResult.Aborted; + } + + if ((observedState == ReclaimingState && observedParticipant == 0) + || (observedState == RetiredState + && generation == TerminalGeneration + && observedParticipant == 0)) + { + return TentativeReservationAbortResult.Invalid; + } + + // Published/RemoveRequested, same-generation Free, wrong-owner owned + // states, malformed ownership, and nonterminal Retired are impossible + // for this private candidate. + return TentativeReservationAbortResult.Corrupt; + } + + internal int GetBytesAdvanced(in ReservationHandle reservation) + { + if (!TryReadReservationProjection( + reservation, + out _, + out _, + out long advanced, + out _)) + { + return 0; + } + + return (int)advanced; + } + + internal bool TryGetWritableRange( + in ReservationHandle reservation, + int sizeHint, + out int slotIndex, + out int offset, + out int length) + { + slotIndex = -1; + offset = 0; + length = 0; + if (sizeHint < 0 + || !TryReadReservationProjection( + reservation, + out slotIndex, + out int valueLength, + out long advanced, + out _)) + { + return false; + } + + int remaining = valueLength - (int)advanced; + if (remaining <= 0 || sizeHint > remaining) + { + return false; + } + + offset = (int)advanced; + length = remaining; + return true; + } + + private bool TryReadReservationProjection( + in ReservationHandle reservation, + out int slotIndex, + out int valueLength, + out long advanced, + out StoreStatus failure) + { + slotIndex = -1; + valueLength = 0; + advanced = 0; + failure = StoreStatus.Success; + StoreStatus storeState = _storeControl?.Validate() ?? StoreStatus.Success; + if (storeState != StoreStatus.Success) + { + failure = storeState; + return false; + } + + if (!TryDecodeHandle(reservation, out slotIndex, out long generation)) + { + failure = StoreStatus.InvalidReservation; + return false; + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long expected = OwnedControl(ReservedState, generation, reservation.ParticipantToken); + long control1 = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus structure = ValidateStructuralControl(control1); + if (structure != StoreStatus.Success) + { + failure = structure; + return false; + } + + if (control1 != expected) + { + failure = ReservationStatus(control1, generation); + return false; + } + + if (!IsParticipantActive()) + { + failure = ParticipantUnavailableStatus(); + return false; + } + + ulong directoryBinding = Volatile.Read(ref slot.DirectoryBinding); + int keyLength = Volatile.Read(ref slot.KeyLength); + int descriptorLength = Volatile.Read(ref slot.DescriptorLength); + int observedValueLength = Volatile.Read(ref slot.ValueLength); + int publicationIntent = Volatile.Read(ref slot.PublicationIntent); + long observedAdvanced = AtomicControlWord.LoadAcquire(ref slot.BytesAdvanced); + long keyOffset = Volatile.Read(ref slot.KeyOffset); + long descriptorOffset = Volatile.Read(ref slot.DescriptorOffset); + long payloadOffset = Volatile.Read(ref slot.PayloadOffset); + long control2 = AtomicControlWord.LoadAcquire(ref slot.Control); + if (control2 != control1) + { + failure = ValidateStructuralControl(control2); + if (failure == StoreStatus.Success) + { + failure = ReservationStatus(control2, generation); + } + + return false; + } + + + if (!IsParticipantActive()) + { + failure = ParticipantUnavailableStatus(); + return false; + } + + long expectedKeyOffset = _layout.KeyStorageOffset + ((long)slotIndex * _layout.KeyStride); + long expectedDescriptorOffset = + _layout.DescriptorStorageOffset + ((long)slotIndex * _layout.DescriptorStride); + long expectedPayloadOffset = + _layout.PayloadStorageOffset + ((long)slotIndex * _layout.PayloadStride); + if (directoryBinding != reservation.SlotBinding + || keyLength is < 1 || keyLength > _layout.MaxKeyBytes + || descriptorLength < 0 || descriptorLength > _layout.MaxDescriptorBytes + || observedValueLength < 0 || observedValueLength > _layout.MaxValueBytes + || observedValueLength != reservation.PayloadLength + || publicationIntent is not ( + (int)SlotPublicationIntent.ExplicitReservation + or (int)SlotPublicationIntent.AtomicPublication) + || observedAdvanced < 0 || observedAdvanced > observedValueLength + || keyOffset != expectedKeyOffset + || descriptorOffset != expectedDescriptorOffset + || payloadOffset != expectedPayloadOffset) + { + _ = CorruptHere(); + failure = StoreStatus.CorruptStore; + return false; + } + + storeState = _storeControl?.Validate() ?? StoreStatus.Success; + if (storeState != StoreStatus.Success) + { + failure = storeState; + return false; + } + + valueLength = observedValueLength; + advanced = observedAdvanced; + return true; + } + + internal StoreStatus AdvanceReservation(in ReservationHandle reservation, int byteCount) + { + return AdvanceReservation( + reservation, + byteCount, + LockFreeOperationBudget.StructuralAttempt); + } + + internal StoreStatus AdvanceReservation( + in ReservationHandle reservation, + int byteCount, + in LockFreeOperationBudget budget) + { + NoOpLockFreeCheckpoint checkpoint = default; + return AdvanceReservation(reservation, byteCount, budget, ref checkpoint); + } + + internal StoreStatus AdvanceReservation( + in ReservationHandle reservation, + int byteCount, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (!TryReadReservationProjection( + reservation, + out int slotIndex, + out int valueLength, + out _, + out StoreStatus validation)) + { + return validation; + } + + _ = TryDecodeHandle(reservation, out _, out long generation); + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long expected = OwnedControl(ReservedState, generation, reservation.ParticipantToken); + + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + long observed = AtomicControlWord.LoadAcquire(ref slot.BytesAdvanced); + if (observed < 0 + || observed > valueLength + || Volatile.Read(ref slot.ValueLength) != valueLength) + { + return CorruptHere(); + } + + if (byteCount < 0 || byteCount > valueLength - observed) + { + return StoreStatus.ReservationWriteOutOfRange; + } + + long next = observed + byteCount; + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.AdvanceBeforeBytesAdvancedCas); + bound = budget.Check(); + if (bound != StoreStatus.Success) + { + return bound; + } + + long observedControl = AtomicControlWord.LoadAcquire(ref slot.Control); + if (observedControl != expected) + { + return ReservationStatus(observedControl, generation); + } + + if (AtomicControlWord.CompareExchange(ref slot.BytesAdvanced, next, observed) == observed) + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.AdvanceAfterBytesAdvancedCas); + observedControl = AtomicControlWord.LoadAcquire(ref slot.Control); + return observedControl == expected + ? StoreStatus.Success + : ReservationStatus(observedControl, generation); + } + + _telemetry.RecordCasLoss(); + + if (AtomicControlWord.LoadAcquire(ref slot.Control) != expected) + { + return ReservationStatus(AtomicControlWord.LoadAcquire(ref slot.Control), generation); + } + + if (attempt + 1 >= AdvanceRetryBudget + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + } + } + + internal StoreStatus CommitReservation(in ReservationHandle reservation, long commitSequence) + { + if (!TryReadReservationProjection( + reservation, + out int slotIndex, + out int valueLength, + out _, + out StoreStatus validation)) + { + return validation; + } + + _ = TryDecodeHandle(reservation, out _, out long generation); + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long reserved = OwnedControl(ReservedState, generation, reservation.ParticipantToken); + long observedControl = AtomicControlWord.LoadAcquire(ref slot.Control); + StoreStatus structure = ValidateStructuralControl(observedControl); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observedControl != reserved) + { + return ReservationStatus(observedControl, generation); + } + + if (!IsParticipantActive()) + { + return ParticipantUnavailableStatus(); + } + + if (Volatile.Read(ref slot.ValueLength) != valueLength) + { + return CorruptHere(); + } + + long currentAdvanced = AtomicControlWord.LoadAcquire(ref slot.BytesAdvanced); + if (currentAdvanced < 0 || currentAdvanced > valueLength) + { + return CorruptHere(); + } + + if (currentAdvanced != valueLength) + { + return StoreStatus.ReservationIncomplete; + } + + slot.CommitSequence = commitSequence; + long published = UnownedControl(PublishedState, generation); + observedControl = AtomicControlWord.CompareExchange(ref slot.Control, published, reserved); + return observedControl == reserved + ? StoreStatus.Success + : ReservationStatus(observedControl, generation); + } + + internal StoreStatus TryBeginAbort(in ReservationHandle reservation) + { + if (!TryDecodeHandle(reservation, out int slotIndex, out long generation)) + { + return StoreStatus.InvalidReservation; + } + + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long aborting = UnownedControl(AbortingState, generation); + long initializing = OwnedControl(InitializingState, generation, reservation.ParticipantToken); + long observed = AtomicControlWord.CompareExchange(ref slot.Control, aborting, initializing); + if (observed == initializing || observed == aborting) + { + return StoreStatus.Success; + } + + long reserved = OwnedControl(ReservedState, generation, reservation.ParticipantToken); + observed = AtomicControlWord.CompareExchange(ref slot.Control, aborting, reserved); + return observed == reserved || observed == aborting + ? StoreStatus.Success + : ReservationStatus(observed, generation); + } + + internal StoreStatus AbortUnboundReservation(in ReservationHandle reservation) + { + NoOpLockFreeCheckpoint checkpoint = default; + return AbortUnboundReservation(reservation, ref checkpoint); + } + + internal StoreStatus AbortUnboundReservation( + in ReservationHandle reservation, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + StoreStatus begin = TryBeginAbort(reservation); + if (begin != StoreStatus.Success) + { + return begin; + } + + return TryCompleteReclaim( + reservation, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint) + ? StoreStatus.Success + : StoreStatus.StoreBusy; + } + + internal bool TryBeginRecoveryAbort( + int slotIndex, + long expectedOwnedControl, + out ReservationHandle reservation) + { + reservation = default; + ReservationRecoveryClaim claim = TryBeginReservationRecovery( + slotIndex, + expectedOwnedControl); + if (claim.Kind != ReservationRecoveryClaimKind.Acquired) + { + return false; + } + + long generation = Generation(expectedOwnedControl); + ulong participant = Participant(expectedOwnedControl); + reservation = new ReservationHandle( + _storeId, + participant, + claim.SlotBinding, + Slot(slotIndex).ValueLength); + return true; + } + + /// + /// Changes one exact Initializing/Reserved lifecycle to the + /// unowned helpable Aborting state. The returned observation makes + /// compare/exchange losses explicit so recovery can report benign commit, + /// abort, and generation-advance races without treating them as failures. + /// + internal ReservationRecoveryClaim TryBeginReservationRecovery( + int slotIndex, + long expectedOwnedControl) + { + if ((uint)slotIndex >= (uint)_layout.SlotCount) + { + return new ReservationRecoveryClaim( + ReservationRecoveryClaimKind.Inconsistent, + 0, + expectedOwnedControl); + } + + int expectedState = State(expectedOwnedControl); + long expectedGeneration = Generation(expectedOwnedControl); + ulong expectedParticipant = Participant(expectedOwnedControl); + if (ValidateStructuralControl(expectedOwnedControl) != StoreStatus.Success + || expectedState is not (InitializingState or ReservedState) + || expectedParticipant == 0) + { + return new ReservationRecoveryClaim( + ReservationRecoveryClaimKind.Inconsistent, + 0, + expectedOwnedControl); + } + + ulong binding = IndexBinding.Encode(slotIndex, expectedGeneration); + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long aborting = UnownedControl(AbortingState, expectedGeneration); + long observed = AtomicControlWord.CompareExchange( + ref slot.Control, + aborting, + expectedOwnedControl); + if (observed == expectedOwnedControl) + { + return new ReservationRecoveryClaim( + ReservationRecoveryClaimKind.Acquired, + binding, + aborting); + } + + _telemetry.RecordCasLoss(); + + if (ValidateStructuralControl(observed) != StoreStatus.Success) + { + return new ReservationRecoveryClaim( + ReservationRecoveryClaimKind.Inconsistent, + binding, + observed); + } + + long observedGeneration = Generation(observed); + int observedState = State(observed); + ulong observedParticipant = Participant(observed); + bool observedOwned = observedState is InitializingState or ReservedState; + if (observedGeneration is < 1 or > TerminalGeneration + || (observedOwned ? observedParticipant == 0 : observedParticipant != 0) + || (observedState == RetiredState && observedGeneration != TerminalGeneration)) + { + return new ReservationRecoveryClaim( + ReservationRecoveryClaimKind.Inconsistent, + binding, + observed); + } + + if (observedGeneration > expectedGeneration) + { + return new ReservationRecoveryClaim( + ReservationRecoveryClaimKind.CompletedRace, + binding, + observed); + } + + if (observedGeneration < expectedGeneration) + { + return new ReservationRecoveryClaim( + ReservationRecoveryClaimKind.Inconsistent, + binding, + observed); + } + + if (observedOwned) + { + return new ReservationRecoveryClaim( + observedParticipant == expectedParticipant + ? ReservationRecoveryClaimKind.OwnerStateChanged + : ReservationRecoveryClaimKind.Inconsistent, + binding, + observed); + } + + ReservationRecoveryClaimKind kind = observedState switch + { + AbortingState or ReclaimingState => ReservationRecoveryClaimKind.HelpRequired, + PublishedState or RemoveRequestedState => + ReservationRecoveryClaimKind.CompletedRace, + RetiredState when expectedGeneration == TerminalGeneration => + ReservationRecoveryClaimKind.CompletedRace, + _ => ReservationRecoveryClaimKind.Inconsistent, + }; + if (kind == ReservationRecoveryClaimKind.Inconsistent) + { + _ = CorruptHere(); + } + + return new ReservationRecoveryClaim(kind, binding, observed); + } + + internal bool TryCompleteReclaim(in ReservationHandle reservation) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryCompleteReclaim( + reservation, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint); + } + + internal bool TryCompleteReclaim( + in ReservationHandle reservation, + in LockFreeOperationBudget budget) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryCompleteReclaim(reservation, budget, ref checkpoint); + } + + internal bool TryCompleteReclaim( + in ReservationHandle reservation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + if (!TryDecodeHandle(reservation, out int slotIndex, out long generation)) + { + return false; + } + + return TryCompleteReclaim(slotIndex, generation, budget, ref checkpoint); + } + + /// + /// Completes an already unowned abort/reclaim lifecycle identified only by + /// its exact slot binding. This is the cross-participant recovery entry; + /// ordinary reservation actions still validate their local participant + /// token before reaching the shared completion routine. + /// + internal StoreStatus TryCompleteRecoveryReclaim(ulong exactBinding) + { + return TryCompleteRecoveryReclaim( + exactBinding, + LockFreeOperationBudget.StructuralAttempt); + } + + internal StoreStatus TryCompleteRecoveryReclaim( + ulong exactBinding, + in LockFreeOperationBudget budget) + { + NoOpLockFreeCheckpoint checkpoint = default; + return TryCompleteRecoveryReclaim(exactBinding, budget, ref checkpoint); + } + + internal StoreStatus TryCompleteRecoveryReclaim( + ulong exactBinding, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + return TryDecodeSlotBinding(exactBinding, out int slotIndex, out long generation) + ? TryCompleteReclaimStatus(slotIndex, generation, budget, ref checkpoint) + : CorruptHere(); + } + + private bool TryCompleteReclaim( + int slotIndex, + long generation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy => + TryCompleteReclaimStatus(slotIndex, generation, budget, ref checkpoint) == StoreStatus.Success; + + private StoreStatus TryCompleteReclaimStatus( + int slotIndex, + long generation, + in LockFreeOperationBudget budget, + ref TCheckpoint checkpoint) + where TCheckpoint : struct, ILockFreeCheckpointStrategy + { + ref ValueSlotMetadataV2 slot = ref Slot(slotIndex); + long aborting = UnownedControl(AbortingState, generation); + long reclaiming = UnownedControl(ReclaimingState, generation); + StoreStatus structure = TryLifecycleTransition( + ref slot.Control, + aborting, + reclaiming, + generation, + out _); + if (structure != StoreStatus.Success) + { + return structure; + } + + long observed = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed != reclaiming) + { + return HasAdvancedOrRetired(observed, generation) + ? StoreStatus.Success + : StoreStatus.StoreBusy; + } + + // Directory code must clear the exact cell and generation-tagged + // descriptors before storage can be made reusable. Free/Retired metadata + // is deliberately ignored: a delayed helper must have no plain write it + // can resume after another helper advances and reuses the generation. + StoreStatus residue = SanitizeOlderDirectoryResidue(ref slot, generation, budget); + if (residue != StoreStatus.Success) + { + return residue == StoreStatus.CorruptStore + ? CorruptHere() + : residue; + } + + if (AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation) != 0 + || AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation) != 0) + { + return StoreStatus.StoreBusy; + } + + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.ReclaimAfterMetadataValidation); + long terminal = Signed(AdvanceOrRetire(generation)); + structure = TryLifecycleTransition( + ref slot.Control, + reclaiming, + terminal, + generation, + out bool advanced); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (advanced) + { + LockFreeCheckpoint.ObserveSlotResource( + ref checkpoint, + generation == TerminalGeneration + ? LockFreeSlotResourceEventKind.Retire + : LockFreeSlotResourceEventKind.Free, + slotIndex, + generation); + } + + observed = AtomicControlWord.LoadAcquire(ref slot.Control); + structure = ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + return observed == reclaiming || HasAdvancedOrRetired(observed, generation) + ? StoreStatus.Success + : CorruptHere(); + } + + /// + /// Completes an exact unowned slot lifecycle transition. Only the desired + /// word and a structurally valid later generation can supersede it. A + /// stable same/older-generation observation is confirmed by an exact no-op + /// CAS before the store-wide corruption latch is published. + /// + private StoreStatus TryLifecycleTransition( + ref long control, + long expected, + long desired, + long generation, + out bool transitioned) + { + transitioned = false; + const int confirmationAttempts = 8; + for (var attempt = 0; attempt < confirmationAttempts; attempt++) + { + long observed = AtomicControlWord.CompareExchange(ref control, desired, expected); + if (observed == expected) + { + transitioned = true; + return StoreStatus.Success; + } + + _telemetry.RecordCasLoss(); + StoreStatus structure = ValidateStructuralControl(observed); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (observed == desired || HasAdvancedOrRetired(observed, generation)) + { + return StoreStatus.Success; + } + + long confirmed = AtomicControlWord.CompareExchange(ref control, observed, observed); + if (confirmed == observed) + { + return CorruptHere(); + } + + structure = ValidateStructuralControl(confirmed); + if (structure != StoreStatus.Success) + { + return structure; + } + } + + return StoreStatus.StoreBusy; + } + + internal ref ValueSlotMetadataV2 Slot(int slotIndex) + { + if ((uint)slotIndex >= (uint)_layout.SlotCount) + { + throw new ArgumentOutOfRangeException(nameof(slotIndex)); + } + + return ref *(ValueSlotMetadataV2*)( + _mappingBase + _layout.SlotMetadataOffset + ((long)slotIndex * _layout.SlotMetadataStride)); + } + + internal static ulong AdvanceOrRetire(long generation) + { + if (generation is < 1 or > TerminalGeneration) + { + throw new ArgumentOutOfRangeException(nameof(generation)); + } + + return generation == TerminalGeneration + ? AtomicControlWord.EncodeSlot(RetiredState, generation, participantToken: 0) + : AtomicControlWord.EncodeSlot(FreeState, generation + 1, participantToken: 0); + } + + private bool TryGetOwnedSlot( + in ReservationHandle reservation, + int requiredState, + out int slotIndex, + out ValueSlotMetadataV2* slot) + { + slotIndex = -1; + slot = null; + if (!TryDecodeHandle(reservation, out slotIndex, out long generation)) + { + return false; + } + + slot = (ValueSlotMetadataV2*)( + _mappingBase + _layout.SlotMetadataOffset + ((long)slotIndex * _layout.SlotMetadataStride)); + long control = AtomicControlWord.LoadAcquire(ref slot->Control); + return ValidateStructuralControl(control) == StoreStatus.Success + && control == OwnedControl(requiredState, generation, reservation.ParticipantToken) + && IsParticipantActive(); + } + + private bool TryDecodeHandle( + in ReservationHandle reservation, + out int slotIndex, + out long generation) + { + slotIndex = -1; + generation = 0; + if (reservation.StoreId != _storeId + || reservation.ParticipantToken != _participant.Token + || reservation.SlotBinding == 0) + { + return false; + } + + return TryDecodeSlotBinding(reservation.SlotBinding, out slotIndex, out generation); + } + + private bool TryDecodeSlotBinding( + ulong slotBinding, + out int slotIndex, + out long generation) + { + slotIndex = -1; + generation = 0; + + ulong indexPlusOne = slotBinding & SlotIndexMask; + ulong rawGeneration = slotBinding >> 31; + if (indexPlusOne == 0 || indexPlusOne > (ulong)_layout.SlotCount + || rawGeneration is 0 or > SlotGenerationMask) + { + return false; + } + + slotIndex = checked((int)indexPlusOne - 1); + generation = checked((long)rawGeneration); + return true; + } + + private bool IsParticipantActive() + { + ref ParticipantRecordV2 record = ref *(ParticipantRecordV2*)( + _mappingBase + _layout.ParticipantOffset + + ((long)_participant.RecordIndex * _layout.ParticipantStride)); + long control = AtomicControlWord.LoadAcquire(ref record.Control); + if (!LockFreeParticipantRegistry.IsStructuralControlValid( + control, + _layout.ParticipantGenerationMask)) + { + _ = LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeSlotTable)); + return false; + } + + return control == _participant.ActiveControl; + } + + private StoreStatus ParticipantUnavailableStatus() => + _storeControl?.Validate() == StoreStatus.CorruptStore + ? StoreStatus.CorruptStore + : StoreStatus.StoreDisposed; + + private static int State(long control) => (int)((ulong)control & 0x7UL); + + private static long Generation(long control) => + (long)(((ulong)control >> 3) & SlotGenerationMask); + + private static ulong Participant(long control) => ((ulong)control >> 36) & ParticipantMask; + + private StoreStatus ClassifyFullnessControl(long control, out bool occupied) + { + if (TryClassifyStructuralControl( + control, + _layout.ParticipantRecordCount, + out occupied)) + { + return StoreStatus.Success; + } + + return CorruptFrom(nameof(LockFreeSlotTable)); + } + + /// + /// Pure canonical validation for a slot lifecycle word. Callers that observe + /// false must report the persistent mapped corruption through their own + /// store-control boundary before returning it. + /// + internal static bool TryClassifyStructuralControl( + long control, + int participantRecordCount, + out bool occupied) + { + occupied = true; + int state = State(control); + long generation = Generation(control); + ulong participant = Participant(control); + if (generation is < 1 or > TerminalGeneration) + { + return false; + } + + switch (state) + { + case FreeState: + if (participant != 0) + { + return false; + } + + occupied = false; + return true; + + case InitializingState: + case ReservedState: + return ParticipantToken.IsStructurallyValid( + participant, + participantRecordCount); + + case PublishedState: + case RemoveRequestedState: + case AbortingState: + case ReclaimingState: + return participant == 0; + + case RetiredState: + return participant == 0 && generation == TerminalGeneration; + + default: + return false; + } + } + + internal StoreStatus ValidateStructuralControl(long control) => + ClassifyFullnessControl(control, out _); + + private static StoreStatus SanitizeOlderDirectoryResidue( + ref ValueSlotMetadataV2 slot, + long claimedGeneration, + in LockFreeOperationBudget budget) + { + StoreStatus location = SanitizeOlderLocation( + ref slot.DirectoryLocation, + claimedGeneration, + budget); + return location == StoreStatus.Success + ? SanitizeOlderOperation(ref slot.DirectoryOperation, claimedGeneration, budget) + : location; + } + + private static StoreStatus SanitizeOlderLocation( + ref long word, + long claimedGeneration, + in LockFreeOperationBudget budget) + { + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + ulong raw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref word)); + if (raw == 0) + { + return StoreStatus.Success; + } + + DirectoryLocation location; + try + { + location = DirectoryLocation.Decode(raw); + } + catch (ArgumentOutOfRangeException) + { + return LockFreeCorruptionTrace.Corrupt(nameof(LockFreeSlotTable)); + } + catch (OverflowException) + { + return LockFreeCorruptionTrace.Corrupt(nameof(LockFreeSlotTable)); + } + + if (location.Generation >= claimedGeneration) + { + return LockFreeCorruptionTrace.Corrupt(nameof(LockFreeSlotTable)); + } + + AtomicControlWord.CompareExchange(ref word, 0, unchecked((long)raw)); + + if ((attempt + 1) % ResidueCleanupRetryBudget == 0 + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + } + } + + private static StoreStatus SanitizeOlderOperation( + ref long word, + long claimedGeneration, + in LockFreeOperationBudget budget) + { + for (var attempt = 0; ; attempt++) + { + StoreStatus bound = budget.CheckPeriodic(attempt); + if (bound != StoreStatus.Success) + { + return bound; + } + + ulong raw = unchecked((ulong)AtomicControlWord.LoadAcquire(ref word)); + if (raw == 0) + { + return StoreStatus.Success; + } + + DirectoryOperation operation; + try + { + operation = DirectoryOperation.Decode(raw); + } + catch (ArgumentOutOfRangeException) + { + return LockFreeCorruptionTrace.Corrupt(nameof(LockFreeSlotTable)); + } + catch (OverflowException) + { + return LockFreeCorruptionTrace.Corrupt(nameof(LockFreeSlotTable)); + } + + if (operation.Generation >= claimedGeneration) + { + return LockFreeCorruptionTrace.Corrupt(nameof(LockFreeSlotTable)); + } + + AtomicControlWord.CompareExchange(ref word, 0, unchecked((long)raw)); + + if ((attempt + 1) % ResidueCleanupRetryBudget == 0 + && !budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + return terminal; + } + } + } + + private static bool HasAdvancedOrRetired(long control, long generation) + { + long observedGeneration = Generation(control); + return observedGeneration > generation + || (observedGeneration == generation && State(control) == RetiredState); + } + + private static long OwnedControl(int state, long generation, ulong participantToken) => + Signed(AtomicControlWord.EncodeSlot(state, generation, checked((int)participantToken))); + + private static long UnownedControl(int state, long generation) => + Signed(AtomicControlWord.EncodeSlot(state, generation, participantToken: 0)); + + private StoreStatus ReservationStatus(long observedControl, long expectedGeneration) + { + StoreStatus structure = ClassifyFullnessControl(observedControl, out _); + if (structure != StoreStatus.Success) + { + return structure; + } + + if (Generation(observedControl) != expectedGeneration) + { + return StoreStatus.InvalidReservation; + } + + return State(observedControl) == PublishedState + ? StoreStatus.ReservationAlreadyCompleted + : StoreStatus.InvalidReservation; + } + + private bool IsCurrentReferencedOperationValid( + ulong raw, + long generation, + int slotState) + { + DirectoryOperation operation; + try + { + operation = DirectoryOperation.Decode(raw); + } + catch (ArgumentOutOfRangeException) + { + return false; + } + catch (OverflowException) + { + return false; + } + + const int insertIntent = 1; + const int unlinkIntent = 2; + if (operation.Value != raw + || operation.Generation != generation + || (operation.Intent != insertIntent + && !(operation.Intent == unlinkIntent + && slotState is AbortingState or ReclaimingState))) + { + return false; + } + + bool shapeValid = operation.Phase switch + { + 1 => operation.Kind == 0 && operation.Index == 0, + 2 or 3 => IsDirectoryTargetInBounds(operation.Kind, operation.Index), + 4 => operation.Intent == insertIntent + && operation.Kind == 0 + && operation.Index == 0, + 5 when operation.Intent == unlinkIntent && operation.Kind == 0 => + operation.Index == 0, + 5 => IsDirectoryTargetInBounds(operation.Kind, operation.Index), + _ => false, + }; + if (!shapeValid) + { + return false; + } + + return slotState switch + { + InitializingState => operation.Intent == insertIntent + && operation.Phase is 2 or 3, + ReservedState => operation.Intent == insertIntent + && operation.Phase is 3 or 5, + PublishedState or RemoveRequestedState => + operation.Intent == insertIntent && operation.Phase == 5, + AbortingState or ReclaimingState => true, + _ => false, + }; + } + + private bool IsDirectoryTargetInBounds(int kind, long index) => + kind switch + { + 1 => (ulong)index < (ulong)_layout.PrimaryLaneCount, + 2 => (ulong)index < (ulong)_layout.SlotCount, + _ => false, + }; + + private bool HasCompletedInsertWitness( + ref ValueSlotMetadataV2 slot, + long generation) + { + try + { + ulong operationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + DirectoryOperation operation = DirectoryOperation.Decode(operationRaw); + if (operation.Value != operationRaw + || operation.Intent != 1 + || operation.Phase != 5 + || operation.Generation != generation + || operation.Kind is < 1 or > 2) + { + return false; + } + + long limit = operation.Kind == 1 + ? _layout.PrimaryLaneCount + : _layout.SlotCount; + if (operation.Index < 0 || operation.Index >= limit) + { + return false; + } + + ulong locationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryLocation)); + DirectoryLocation location = DirectoryLocation.Decode(locationRaw); + return location.Value == locationRaw + && location.Kind == operation.Kind + && location.Index == operation.Index + && location.Generation == generation; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + catch (OverflowException) + { + return false; + } + } + + private StoreStatus CorruptHere( + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) => + LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeSlotTable), + member, + line); + + private StoreStatus CorruptFrom( + string component, + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) => + LockFreeStoreControl.ReportCorruption( + _storeControl, + component, + member, + line); + + private static long Signed(ulong value) => unchecked((long)value); +} + +internal enum TentativeReservationAbortResult +{ + Aborted, + Ordered, + Invalid, + Corrupt, +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeStoreControl.cs b/src/SharedMemoryStore/LockFree/LockFreeStoreControl.cs new file mode 100644 index 0000000..a45d720 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeStoreControl.cs @@ -0,0 +1,90 @@ +using System.Runtime.CompilerServices; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Process-local access to the mapped store-wide control word. The control +/// word is the fail-closed boundary for persistent structural corruption; it +/// is never used for ordinary data-path mutual exclusion. +/// +internal sealed unsafe class LockFreeStoreControl : IDisposable +{ + private readonly long* _control; + private int _disposed; + + internal LockFreeStoreControl(MemoryMappedStoreRegion region) + { + ArgumentNullException.ThrowIfNull(region); + if (region.Capacity < LayoutV2Constants.HeaderLength) + { + throw new ArgumentOutOfRangeException(nameof(region)); + } + + _control = &((StoreHeaderV2*)region.Pointer)->Control; + } + + internal bool IsReady => Validate() == StoreStatus.Success; + + /// + /// Performs an acquire read of the mapped control state. Disposal is + /// checked first so this method never dereferences an unmapped view. + /// + internal StoreStatus Validate() + { + if (Volatile.Read(ref _disposed) != 0) + { + return StoreStatus.StoreDisposed; + } + + long observed = AtomicControlWord.LoadAcquire(ref *_control); + return observed switch + { + LayoutV2Constants.StoreReady => StoreStatus.Success, + LayoutV2Constants.StoreUnsupported => StoreStatus.UnsupportedPlatform, + LayoutV2Constants.StoreCorrupt => StoreStatus.CorruptStore, + _ => StoreStatus.CorruptStore + }; + } + + /// + /// Irreversibly publishes persistent mapped corruption. Only Ready may be + /// changed; Unsupported and unknown future states are never overwritten. + /// + internal void MarkCorrupt() + { + if (Volatile.Read(ref _disposed) != 0) + { + return; + } + + long observed = AtomicControlWord.LoadAcquire(ref *_control); + while (observed == LayoutV2Constants.StoreReady) + { + long exchanged = AtomicControlWord.CompareExchange( + ref *_control, + LayoutV2Constants.StoreCorrupt, + LayoutV2Constants.StoreReady); + if (exchanged == LayoutV2Constants.StoreReady) + { + return; + } + + observed = exchanged; + } + } + + internal static StoreStatus ReportCorruption( + LockFreeStoreControl? control, + string component, + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) + { + _ = LockFreeCorruptionTrace.Corrupt(component, member, line); + control?.MarkCorrupt(); + return StoreStatus.CorruptStore; + } + + public void Dispose() => Volatile.Write(ref _disposed, 1); +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs b/src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs new file mode 100644 index 0000000..e45922d --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeStoreEngine.cs @@ -0,0 +1,2676 @@ +using System.Buffers; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Non-generic construction and pure-policy facade. Ordinary construction is +/// permanently closed over the empty checkpoint strategy; friend-only +/// instrumented construction selects its own closed engine explicitly. +/// +internal static class LockFreeStoreEngine +{ + internal static StoreOpenStatus TryCreateOrOpenUnderColdGate( + SharedMemoryStoreOptions options, + StoreWaitOptions waitOptions, + long waitStartTimestamp, + MemoryMappedStoreRegion region, + ISharedStoreSynchronization coldSynchronization, + RegionOpenDisposition disposition, + out IStoreEngine? engine) + { + NoOpLockFreeCheckpoint checkpoint = default; + StoreOpenStatus status = LockFreeStoreEngine.TryCreateOrOpenUnderColdGate( + options, + waitOptions, + waitStartTimestamp, + region, + coldSynchronization, + disposition, + checkpoint, + out LockFreeStoreEngine? concrete); + engine = concrete; + return status; + } + + internal static StoreStatus NormalizePostLogicalRemoveOutcome(StoreStatus reclaimStatus) => + reclaimStatus switch + { + StoreStatus.Success => StoreStatus.Success, + StoreStatus.CorruptStore => StoreStatus.CorruptStore, + _ => StoreStatus.RemovePending + }; + + /// + /// A generation that is still being removed remains a duplicate from a + /// publisher's point of view. Operational and structural failures are not + /// lifecycle states and must retain their exact status. + /// + internal static StoreStatus NormalizeExistingGenerationReclaimOutcome(StoreStatus reclaimStatus) => + reclaimStatus switch + { + StoreStatus.Success => StoreStatus.Success, + StoreStatus.RemovePending or StoreStatus.NotFound => StoreStatus.DuplicateKey, + _ => reclaimStatus + }; +} + +/// +/// Layout-v2 engine foundation. This slice owns header compatibility and +/// participant lifetime; data operations are added by the following story +/// phases without changing the public facade. +/// +internal sealed unsafe class LockFreeStoreEngine : IStoreEngine, ILockFreeCheckpointEmitter + where TCheckpoint : struct, ILockFreeCheckpointStrategy +{ + private readonly MemoryMappedStoreRegion _region; + private readonly ISharedStoreSynchronization _coldSynchronization; + private readonly StoreLayoutV2 _layout; + private readonly LockFreeStoreControl _storeControl; + private readonly LockFreeParticipantRegistry _participants; + private readonly LockFreeParticipantRegistry.Registration _registration; + private readonly LockFreeSlotTable _slots; + private readonly LockFreeKeyDirectory _directory; + private readonly LockFreeLeaseRegistry _leases; + private readonly LockFreeReclaimer _reclaimer; + private readonly LockFreeRecovery _recovery; + private readonly LockFreeDiagnostics _diagnostics; + private readonly LockFreeReservationMemory _reservationMemory; + private readonly StoreProtocolInfo _protocolInfo; + private readonly bool _recoveryEnabled; + private TCheckpoint _checkpoint; + private int _disposed; + + private LockFreeStoreEngine( + MemoryMappedStoreRegion region, + ISharedStoreSynchronization coldSynchronization, + StoreLayoutV2 layout, + LockFreeStoreControl storeControl, + LockFreeParticipantRegistry participants, + LockFreeParticipantRegistry.Registration registration, + ulong requiredFeatures, + ulong optionalFeatures, + bool recoveryEnabled, + LockFreeTelemetry telemetry, + TCheckpoint checkpoint) + { + _region = region; + _coldSynchronization = coldSynchronization; + _layout = layout; + _storeControl = storeControl; + _participants = participants; + _registration = registration; + _checkpoint = checkpoint; + _slots = new LockFreeSlotTable(region, layout, registration, telemetry, storeControl); + _directory = new LockFreeKeyDirectory(region, layout, telemetry, storeControl); + _leases = new LockFreeLeaseRegistry( + region, + layout, + registration, + participants, + telemetry, + storeControl); + _reclaimer = new LockFreeReclaimer( + layout, + _slots, + _directory, + _leases, + telemetry, + storeControl); + _recovery = new LockFreeRecovery( + layout, + _slots, + _directory, + participants, + telemetry, + storeControl); + _reservationMemory = new LockFreeReservationMemory(region, layout, _slots); + _recoveryEnabled = recoveryEnabled; + _protocolInfo = new StoreProtocolInfo( + StoreProfile.LockFree, + LayoutV2Constants.LayoutMajorVersion, + LayoutV2Constants.LayoutMinorVersion, + LayoutV2Constants.ResourceProtocolVersion, + requiredFeatures, + optionalFeatures); + _diagnostics = new LockFreeDiagnostics( + region, + layout, + _protocolInfo, + telemetry, + storeControl); + } + + public StoreProfile Profile => StoreProfile.LockFree; + + public StoreProtocolInfo ProtocolInfo => _protocolInfo; + + public StoreStatus RecordFacadeStatus(StoreStatus status) => + _diagnostics.RecordStatus(status); + + public DiagnosticsSnapshot CreateDisposedDiagnosticsSnapshot() => + _diagnostics.CreateDisposedSnapshot(); + + void ILockFreeCheckpointEmitter.ReachCheckpoint(LockFreeCheckpointId checkpoint) => Reach(checkpoint); + + internal static StoreOpenStatus TryCreateOrOpenUnderColdGate( + SharedMemoryStoreOptions options, + StoreWaitOptions waitOptions, + long waitStartTimestamp, + MemoryMappedStoreRegion region, + ISharedStoreSynchronization coldSynchronization, + RegionOpenDisposition disposition, + TCheckpoint checkpoint, + out LockFreeStoreEngine? engine) + { + engine = null; + LockFreeOperationBudget operationBudget = LockFreeOperationBudget.Start( + waitOptions, + waitStartTimestamp); + if (!LayoutV2Constants.IsSupportedArchitecture(RuntimeInformation.ProcessArchitecture)) + { + return StoreOpenStatus.UnsupportedPlatform; + } + + if (!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + { + return StoreOpenStatus.UnsupportedPlatform; + } + + ulong pidNamespaceId = CaptureStorePidNamespaceId(); + + StoreLayoutV2 layout; + try + { + layout = StoreLayoutV2.FromOptions(options); + } + catch (ArgumentOutOfRangeException) + { + return StoreOpenStatus.InvalidOptions; + } + catch (OverflowException) + { + return StoreOpenStatus.InvalidOptions; + } + + StoreStatus remainingStatus = operationBudget.TryGetRemainingWaitOptions(out _); + if (remainingStatus != StoreStatus.Success) + { + return ToOpenStatus(remainingStatus); + } + + try + { + if (region.Capacity < LayoutV2Constants.HeaderLength) + { + return StoreOpenStatus.IncompatibleLayout; + } + + ref StoreHeaderV2 header = ref Header(region); + uint observedMagic = header.Magic; + bool initialize = disposition == RegionOpenDisposition.CreatedNew; + + if (initialize && options.OpenMode == OpenMode.OpenExisting) + { + return StoreOpenStatus.IncompatibleLayout; + } + + if (!initialize && options.OpenMode == OpenMode.CreateNew) + { + return StoreOpenStatus.AlreadyExists; + } + + if (!initialize && observedMagic == 0) + { + // The existing unpublished region may still be owned by an + // older creator that mapped before taking the named gate. This + // process cannot prove initialization ownership and therefore + // must never clear or publish the header. + return options.OpenMode == OpenMode.CreateOrOpen + ? StoreOpenStatus.StoreBusy + : StoreOpenStatus.IncompatibleLayout; + } + + if (initialize) + { + if (region.Capacity < layout.TotalBytes || !layout.FitsWithinTotalBytes()) + { + return StoreOpenStatus.IncompatibleLayout; + } + + StoreStatus initialized = InitializeMapping( + region, + layout, + pidNamespaceId, + operationBudget); + if (initialized != StoreStatus.Success) + { + return ToOpenStatus(initialized); + } + } + else + { + if (observedMagic != LayoutV2Constants.Magic + || region.Capacity < layout.TotalBytes + || !layout.MatchesHeader(header)) + { + return StoreOpenStatus.IncompatibleLayout; + } + + long storeControl = Volatile.Read(ref header.Control); + if (storeControl == LayoutV2Constants.StoreUnsupported) + { + return StoreOpenStatus.UnsupportedPlatform; + } + + if (storeControl != LayoutV2Constants.StoreReady) + { + return StoreOpenStatus.IncompatibleLayout; + } + + // Publish the irreversible recovery downgrade only after the + // complete header and Ready state validate, but before the + // first Registering CAS. Ordinary KV access remains supported + // across namespace views. + StoreOpenStatus namespaceStatus = AdmitPidNamespace( + ref header, + pidNamespaceId); + if (namespaceStatus != StoreOpenStatus.Success) + { + return namespaceStatus; + } + } + + var controlLatch = new LockFreeStoreControl(region); + StoreStatus attachedState = controlLatch.Validate(); + if (attachedState != StoreStatus.Success) + { + controlLatch.Dispose(); + return attachedState == StoreStatus.UnsupportedPlatform + ? StoreOpenStatus.UnsupportedPlatform + : StoreOpenStatus.IncompatibleLayout; + } + + var telemetry = new LockFreeTelemetry(); + var participants = new LockFreeParticipantRegistry( + region, + layout, + telemetry, + controlLatch); + StoreOpenStatus registerStatus = participants.TryRegister( + ref header, + operationBudget, + ref checkpoint, + out var registration); + if (registerStatus != StoreOpenStatus.Success) + { + controlLatch.Dispose(); + return registerStatus; + } + + attachedState = controlLatch.Validate(); + if (attachedState != StoreStatus.Success) + { + participants.RetireUnreferencedRegistration(registration); + controlLatch.Dispose(); + return attachedState == StoreStatus.UnsupportedPlatform + ? StoreOpenStatus.UnsupportedPlatform + : StoreOpenStatus.IncompatibleLayout; + } + + try + { + LockFreeCheckpoint.Reach( + ref checkpoint, + LockFreeCheckpointId.ParticipantAfterRegistrationBeforeEngineConstruction); + engine = new LockFreeStoreEngine( + region, + coldSynchronization, + layout, + controlLatch, + participants, + registration, + header.RequiredFeatures, + header.OptionalFeatures, + options.EnableLeaseRecovery, + telemetry, + checkpoint); + return StoreOpenStatus.Success; + } + catch + { + // Registration has published Active but no engine escaped and + // therefore no slot/lease claim can reference its token. Close + // and retire that exact incarnation before the outer owner + // disposes the mapping. + participants.RetireUnreferencedRegistration(registration); + controlLatch.Dispose(); + throw; + } + } + catch (UnauthorizedAccessException) + { + return StoreOpenStatus.AccessDenied; + } + catch (Exception) + { + return StoreOpenStatus.MappingFailed; + } + } + + public StoreStatus TryPublish( + ReadOnlySpan key, + ReadOnlySpan value, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions) + { + return RecordOperationStatus(TryPublishCore(key, value, descriptor, waitOptions)); + } + + private StoreStatus TryPublishCore( + ReadOnlySpan key, + ReadOnlySpan value, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions) + { + long started = Stopwatch.GetTimestamp(); + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions, started); + Reach(LockFreeCheckpointId.PublishBeforeSlotClaim); + StoreStatus status = TryReserveCore( + key, + value.Length, + descriptor, + waitOptions, + SlotPublicationIntent.AtomicPublication, + out var reservation, + started); + if (status != StoreStatus.Success) + { + return status; + } + + try + { + Span destination = _reservationMemory.GetSpan(reservation, value.Length); + if (value.Length != 0 && destination.Length < value.Length) + { + _ = AbortReservationCore(reservation); + return CorruptFrom(nameof(LockFreeStoreEngine)); + } + + StoreStatus copy = LockFreeByteOperations.TryCopy(value, destination, budget); + if (copy != StoreStatus.Success) + { + _ = AbortReservationCore(reservation); + return copy; + } + + status = _slots.AdvanceReservation(reservation, value.Length, budget); + if (status != StoreStatus.Success) + { + _ = AbortReservationCore(reservation); + return status; + } + + status = CommitReservationCore(reservation, waitOptions, started); + if (status == StoreStatus.Success) + { + Reach(LockFreeCheckpointId.PublishAfterCommitPublication); + } + else + { + // TryPublish owns this reservation and never exposes its token. + // A bounded commit may lose its budget immediately before the + // publication CAS. Publish unowned Aborting, then spend only + // the post-ownership completion allowance on physical cleanup + // while preserving the caller-visible timeout/cancel result. + _ = AbortReservationCore(reservation); + } + + return status; + } + catch + { + _ = AbortReservationCore(reservation); + return StoreStatus.UnknownFailure; + } + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public StoreStatus TryReserve( + ReadOnlySpan key, + int payloadLength, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions, + out ReservationHandle reservation) + { + return RecordOperationStatus( + TryReserveCore( + key, + payloadLength, + descriptor, + waitOptions, + SlotPublicationIntent.ExplicitReservation, + out reservation)); + } + + public StoreStatus TryPublishSegments( + ReadOnlySpan key, + ReadOnlySequence payload, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions, + out long copiedBytes) + { + return RecordOperationStatus( + TryPublishSegmentsCore(key, payload, descriptor, waitOptions, out copiedBytes)); + } + + private StoreStatus TryPublishSegmentsCore( + ReadOnlySpan key, + ReadOnlySequence payload, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions, + out long copiedBytes) + { + long started = Stopwatch.GetTimestamp(); + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions, started); + copiedBytes = 0; + long advertisedLength; + try + { + advertisedLength = payload.Length; + } + catch (ArgumentOutOfRangeException) + { + return StoreStatus.UnknownFailure; + } + catch (OverflowException) + { + return StoreStatus.UnknownFailure; + } + + if (advertisedLength < 0) + { + return StoreStatus.UnknownFailure; + } + + if (advertisedLength > int.MaxValue) + { + return StoreStatus.ValueTooLarge; + } + + int payloadLength = (int)advertisedLength; + + StoreStatus status = TryReserveCore( + key, + payloadLength, + descriptor, + waitOptions, + SlotPublicationIntent.AtomicPublication, + out var reservation, + started); + if (status != StoreStatus.Success) + { + return status; + } + + try + { + Span destination = _reservationMemory.GetSpan(reservation, payloadLength); + if (destination.Length < advertisedLength) + { + _ = AbortReservationCore(reservation); + return CorruptHere(); + } + + var segmentIndex = 0; + foreach (ReadOnlyMemory segment in payload) + { + StoreStatus segmentBound = budget.CheckPeriodic(segmentIndex++); + if (segmentBound != StoreStatus.Success) + { + _ = AbortReservationCore(reservation); + return segmentBound; + } + + // ReadOnlySequence is extensible, so a caller can supply a + // malformed sequence whose enumerated segment lengths exceed + // its advertised Length. That is invalid caller input, not + // evidence that the shared mapping is corrupt. + if (copiedBytes > destination.Length || + segment.Length > destination.Length - copiedBytes) + { + _ = AbortReservationCore(reservation); + return StoreStatus.UnknownFailure; + } + + StoreStatus copy = LockFreeByteOperations.TryCopy( + segment.Span, + destination[(int)copiedBytes..], + budget, + out int segmentCopiedBytes); + copiedBytes += segmentCopiedBytes; + if (copy != StoreStatus.Success) + { + _ = AbortReservationCore(reservation); + return copy; + } + } + + // The inverse malformed-sequence shape can advertise a Length + // larger than the bytes its segment chain actually enumerates. + // Preserve the exact copied prefix and keep this caller failure + // distinct from persistent mapped corruption. + if (copiedBytes != advertisedLength) + { + _ = AbortReservationCore(reservation); + return StoreStatus.UnknownFailure; + } + + status = _slots.AdvanceReservation(reservation, payloadLength, budget); + if (status != StoreStatus.Success) + { + _ = AbortReservationCore(reservation); + return status; + } + + status = CommitReservationCore(reservation, waitOptions, started); + if (status != StoreStatus.Success) + { + // The segmented convenience operation owns the reservation; + // callers cannot clean it after a bounded commit failure. + _ = AbortReservationCore(reservation); + } + + return status; + } + catch + { + _ = AbortReservationCore(reservation); + return StoreStatus.UnknownFailure; + } + } + + public StoreStatus TryAcquire( + ReadOnlySpan key, + StoreWaitOptions waitOptions, + out LeaseHandle lease) + { + return RecordOperationStatus(TryAcquireCore(key, waitOptions, out lease)); + } + + private StoreStatus TryAcquireCore( + ReadOnlySpan key, + StoreWaitOptions waitOptions, + out LeaseHandle lease) + { + long started = Stopwatch.GetTimestamp(); + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions, started); + lease = default; + StoreStatus ready = ValidateOperation(waitOptions); + if (ready != StoreStatus.Success) + { + return ready; + } + + StoreStatus keyStatus = StoreKey.Validate(key, _layout.MaxKeyBytes); + if (keyStatus != StoreStatus.Success) + { + return keyStatus; + } + + StoreStatus hashStatus = LockFreeByteOperations.TryHash(key, budget, out ulong hash); + if (hashStatus != StoreStatus.Success) + { + return hashStatus; + } + + StoreStatus lookup = _directory.TryLookup(key, hash, budget, out ulong slotBinding, out _); + if (lookup != StoreStatus.Success) + { + return lookup; + } + + if (!TryDecodeSlotBinding(slotBinding, out int slotIndex, out long generation)) + { + return CorruptHere(); + } + + StoreStatus publication = TryIsSlotPublished(slotIndex, generation, out bool isPublished); + if (publication != StoreStatus.Success) + { + return publication; + } + + if (!isPublished) + { + return StoreStatus.NotFound; + } + + Reach(LockFreeCheckpointId.AcquireBeforeLeaseClaimCas); + StoreStatus bound = CheckOperationBound(waitOptions, started); + if (bound != StoreStatus.Success) + { + return bound; + } + + // AcquireSequence is observational metadata, not an ordering or + // reclamation dependency. Reuse the process-wide monotonic timestamp + // captured at operation entry instead of bouncing one shared header + // cache line across every reader process. + long acquireSequence = Math.Max(1, started); + StoreStatus claim = _leases.TryClaimAndActivate( + slotBinding, + acquireSequence, + budget, + ref _checkpoint, + out lease); + if (claim != StoreStatus.Success) + { + lease = default; + if (claim != StoreStatus.LeaseTableFull) + { + return claim; + } + + // The lease proof identifies an exact full-table instant, but an + // acquire capacity result also requires the requested generation + // to exist at that instant. Revalidate after confirmation. If the + // exact binding is still Published, its monotonic lifecycle proves + // that it remained Published through the earlier candidate. A + // missing/changed generation instead keeps ordinary acquire + // semantics and cannot leak a stale LeaseTableFull result. + lookup = _directory.TryLookup( + key, + hash, + budget, + out ulong fullRevalidatedBinding, + out _); + if (lookup != StoreStatus.Success) + { + return lookup; + } + + if (fullRevalidatedBinding != slotBinding) + { + return StoreStatus.NotFound; + } + + publication = TryIsSlotPublished(slotIndex, generation, out isPublished); + if (publication != StoreStatus.Success) + { + return publication; + } + + return isPublished ? StoreStatus.LeaseTableFull : StoreStatus.NotFound; + } + + Reach(LockFreeCheckpointId.AcquireAfterLeaseActivationBeforeFinalLookup); + bound = budget.Check(); + if (bound != StoreStatus.Success) + { + _ = _leases.TryRelease(lease, ref _checkpoint); + lease = default; + return bound; + } + + lookup = _directory.TryLookup(key, hash, budget, out ulong revalidatedBinding, out _); + if (lookup != StoreStatus.Success) + { + _ = _leases.TryRelease(lease, ref _checkpoint); + lease = default; + return lookup; + } + + if (revalidatedBinding != slotBinding) + { + _ = _leases.TryRelease(lease, ref _checkpoint); + lease = default; + return StoreStatus.NotFound; + } + + publication = TryIsSlotPublished(slotIndex, generation, out isPublished); + if (publication != StoreStatus.Success) + { + _ = _leases.TryRelease(lease, ref _checkpoint); + lease = default; + return publication; + } + + if (!isPublished) + { + _ = _leases.TryRelease(lease, ref _checkpoint); + lease = default; + return StoreStatus.NotFound; + } + + Reach(LockFreeCheckpointId.AcquireAfterPublishedRevalidation); + return StoreStatus.Success; + } + + public StoreStatus TryRemove(ReadOnlySpan key, StoreWaitOptions waitOptions) + { + return RecordOperationStatus(TryRemoveCore(key, waitOptions)); + } + + private StoreStatus TryRemoveCore(ReadOnlySpan key, StoreWaitOptions waitOptions) + { + long started = Stopwatch.GetTimestamp(); + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions, started); + StoreStatus ready = ValidateOperation(waitOptions); + if (ready != StoreStatus.Success) + { + return ready; + } + + StoreStatus keyStatus = StoreKey.Validate(key, _layout.MaxKeyBytes); + if (keyStatus != StoreStatus.Success) + { + return keyStatus; + } + + StoreStatus hashStatus = LockFreeByteOperations.TryHash(key, budget, out ulong hash); + if (hashStatus != StoreStatus.Success) + { + return hashStatus; + } + + StoreStatus lookup = _directory.TryLookup(key, hash, budget, out ulong binding, out _); + if (lookup != StoreStatus.Success) + { + return lookup; + } + + if (!TryDecodeSlotBinding(binding, out int slotIndex, out long generation)) + { + return CorruptHere(); + } + + Reach(LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas); + if (waitOptions.CancellationToken.IsCancellationRequested) + { + return StoreStatus.OperationCanceled; + } + + if (HasExpired(waitOptions, started)) + { + return StoreStatus.StoreBusy; + } + + StoreStatus logicalRemove = _reclaimer.TryLogicalRemove(binding, out _); + if (logicalRemove != StoreStatus.Success) + { + return logicalRemove; + } + + // NoWait performs the logical ordering point but conservatively leaves + // the bounded classification/reclaim scan to a helper. + if (waitOptions.Timeout == TimeSpan.Zero) + { + return StoreStatus.RemovePending; + } + + if (HasExpired(waitOptions, started)) + { + return StoreStatus.RemovePending; + } + + Reach(LockFreeCheckpointId.ReclaimBeforeOwnershipCas); + StoreStatus reclaimed = _reclaimer.TryReclaim( + binding, + budget, + ref _checkpoint, + reportRemoveClassification: true); + for (var attempt = 0; + reclaimed == StoreStatus.StoreBusy && budget.IsInfinite; + attempt++) + { + if (!budget.TryContinueAfterContention(attempt, out StoreStatus terminal)) + { + reclaimed = terminal; + break; + } + + reclaimed = _reclaimer.TryReclaim( + binding, + budget, + ref _checkpoint, + reportRemoveClassification: true); + } + + if (reclaimed == StoreStatus.Success) + { + Reach(LockFreeCheckpointId.ReclaimAfterGenerationAdvance); + Reach(LockFreeCheckpointId.RemoveAfterLeaseClassification); + } + + return LockFreeStoreEngine.NormalizePostLogicalRemoveOutcome(reclaimed); + } + + public StoreStatus TryRecoverLeases( + LeaseRecoveryOptions options, + StoreWaitOptions waitOptions, + out LeaseRecoveryReport report) + { + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions); + report = default; + StoreStatus ready = ValidateOperation(waitOptions); + if (ready != StoreStatus.Success) + { + return RecordOperationStatus(ready); + } + + StoreStatus status = _recoveryEnabled + ? _leases.TryRecover(options, budget, _reclaimer, ref _checkpoint, out report) + : StoreStatus.UnsupportedPlatform; + if (status == StoreStatus.Success) + { + _diagnostics.RecordLeaseRecoveryResults(report); + } + + return RecordOperationStatus(status); + } + + public StoreStatus TryRecoverReservations( + ReservationRecoveryOptions options, + StoreWaitOptions waitOptions, + out ReservationRecoveryReport report) + { + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions); + report = default; + StoreStatus ready = ValidateOperation(waitOptions); + if (ready != StoreStatus.Success) + { + return RecordOperationStatus(ready); + } + + if (!_recoveryEnabled) + { + return RecordOperationStatus(StoreStatus.UnsupportedPlatform); + } + + StoreStatus status = _recovery.TryRecoverReservations( + options, + budget, + ref _checkpoint, + out report); + if (status == StoreStatus.Success) + { + _diagnostics.RecordReservationRecoveryResults(report); + } + + return RecordOperationStatus(status); + } + + public StoreStatus TryGetMetrics(StoreWaitOptions waitOptions, out EngineMetrics metrics) + { + long started = Stopwatch.GetTimestamp(); + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions, started); + metrics = default; + StoreStatus ready = ValidateOperation(waitOptions); + if (ready != StoreStatus.Success) + { + return RecordOperationStatus(ready); + } + + _diagnostics.ReachBeforeBoundedScan(ref _checkpoint); + StoreStatus bound = CheckOperationBound(waitOptions, started); + if (bound != StoreStatus.Success) + { + return RecordOperationStatus(bound); + } + + StoreStatus status = _diagnostics.TryScanMetricsAfterBoundedPrecheck( + budget, + ref _checkpoint, + out metrics); + return RecordOperationStatus(status); + } + + public StoreStatus TryGetDiagnostics(StoreWaitOptions waitOptions, out DiagnosticsSnapshot snapshot) + { + long started = Stopwatch.GetTimestamp(); + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions, started); + snapshot = default; + StoreStatus ready = ValidateOperation(waitOptions); + if (ready != StoreStatus.Success) + { + return RecordOperationStatus(ready); + } + + _diagnostics.ReachBeforeBoundedScan(ref _checkpoint); + StoreStatus bound = CheckOperationBound(waitOptions, started); + if (bound != StoreStatus.Success) + { + return RecordOperationStatus(bound); + } + + StoreStatus status = _diagnostics.TryCreateSnapshotAfterBoundedPrecheck( + budget, + ref _checkpoint, + out snapshot); + return RecordOperationStatus(status); + } + + public bool IsReservationPending(ReservationHandle reservation) => + CanProjectMappedState && _slots.IsReservationPending(reservation); + + public int GetReservationBytesWritten(ReservationHandle reservation) => + CanProjectMappedState ? _slots.GetBytesAdvanced(reservation) : 0; + + public Span GetReservationSpan(ReservationHandle reservation, int sizeHint) + { + Reach(LockFreeCheckpointId.ProjectBeforeHandleValidation); + Span span = CanProjectMappedState + ? _reservationMemory.GetSpan(reservation, sizeHint) + : Span.Empty; + Reach(LockFreeCheckpointId.ProjectAfterSpanProjection); + return span; + } + + public Memory DangerousGetReservationMemory(ReservationHandle reservation, int sizeHint) => + CanProjectMappedState + ? _reservationMemory.GetMemory(reservation, sizeHint) + : Memory.Empty; + + public StoreStatus AdvanceReservation( + ReservationHandle reservation, + int byteCount, + StoreWaitOptions waitOptions) + { + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions); + StoreStatus ready = ValidateOperation(waitOptions); + StoreStatus status = ready == StoreStatus.Success + ? _slots.AdvanceReservation(reservation, byteCount, budget, ref _checkpoint) + : ready; + return RecordOperationStatus(status); + } + + public StoreStatus CommitReservation(ReservationHandle reservation, StoreWaitOptions waitOptions) + { + long started = Stopwatch.GetTimestamp(); + StoreStatus ready = ValidateOperation(waitOptions); + StoreStatus status = ready == StoreStatus.Success + ? CommitReservationCore(reservation, waitOptions, started) + : ready; + return RecordOperationStatus(status); + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public StoreStatus AbortReservation(ReservationHandle reservation, StoreWaitOptions waitOptions) + { + long started = Stopwatch.GetTimestamp(); + StoreStatus ready = ValidateOperation(waitOptions); + StoreStatus status = ready == StoreStatus.Success + ? AbortReservationCore(reservation, waitOptions, started) + : ready; + if (status == StoreStatus.Success) + { + _diagnostics.RecordReservationAbort(); + } + + return RecordOperationStatus(status); + } + + public bool IsLeaseActive(LeaseHandle lease) => + TryValidateLeaseProjection(lease, out _, out _, out _, out _, out _); + + public int GetValueLength(LeaseHandle lease) => + TryValidateLeaseProjection(lease, out _, out int valueLength, out _, out _, out _) + ? valueLength + : 0; + + public int GetDescriptorLength(LeaseHandle lease) => + TryValidateLeaseProjection(lease, out _, out _, out int descriptorLength, out _, out _) + ? descriptorLength + : 0; + + public ReadOnlySpan GetValueSpan(LeaseHandle lease) + { + Reach(LockFreeCheckpointId.ProjectBeforeHandleValidation); + if (!TryValidateLeaseProjection( + lease, + out _, + out int valueLength, + out _, + out long payloadOffset, + out _)) + { + return ReadOnlySpan.Empty; + } + + ReadOnlySpan span = new(_region.Pointer + payloadOffset, valueLength); + Reach(LockFreeCheckpointId.ProjectAfterSpanProjection); + return span; + } + + public ReadOnlySpan GetDescriptorSpan(LeaseHandle lease) + { + if (!TryValidateLeaseProjection( + lease, + out _, + out _, + out int descriptorLength, + out _, + out long descriptorOffset)) + { + return ReadOnlySpan.Empty; + } + + return new ReadOnlySpan(_region.Pointer + descriptorOffset, descriptorLength); + } + + public StoreStatus ReleaseLease(LeaseHandle lease, StoreWaitOptions waitOptions) + { + long started = Stopwatch.GetTimestamp(); + StoreStatus ready = ValidateOperation(waitOptions); + if (ready != StoreStatus.Success) + { + return RecordOperationStatus(ready); + } + + Reach(LockFreeCheckpointId.ReleaseBeforeActiveReleaseCas); + StoreStatus bound = CheckOperationBound(waitOptions, started); + if (bound != StoreStatus.Success) + { + return RecordOperationStatus(bound); + } + + StoreStatus status = _leases.TryRelease(lease, ref _checkpoint); + if (status == StoreStatus.Success) + { + _ = _reclaimer.TryReclaim( + lease.SlotBinding, + LockFreeOperationBudget.StartPostOwnershipCleanup(), + ref _checkpoint); + Reach(LockFreeCheckpointId.ReleaseAfterRecordRecycle); + } + + return RecordOperationStatus(status); + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + try + { + if (_storeControl.Validate() != StoreStatus.Success) + { + _reservationMemory.Dispose(); + return; + } + + ParticipantTransitionResult close = _participants.TryBeginClose(_registration); + if (close is ParticipantTransitionResult.Succeeded + or ParticipantTransitionResult.AlreadyCompleted) + { + LockFreeOperationBudget cleanupBudget = + LockFreeOperationBudget.StartPostOwnershipCleanup(); + try + { + // Closing is published only after the facade lifecycle gate + // has drained every entered callback. Other handles can now + // recover exact resources even if this disposer pauses. + Reach(LockFreeCheckpointId.DisposalAfterParticipantClosingPublication); + _reservationMemory.Dispose(); + CleanupParticipantResources(cleanupBudget); + } + finally + { + // Retirement is attempted even if best-effort cleanup or a + // test checkpoint throws. A remaining reference/bound leaves + // exact Closing for an unrelated recovery caller to finish. + _ = _participants.TryRetireClosingRegistration( + _registration, + cleanupBudget, + ref _checkpoint); + Reach(LockFreeCheckpointId.DisposalAfterParticipantRelease); + } + } + else + { + _reservationMemory.Dispose(); + } + } + finally + { + try + { + _storeControl.Dispose(); + } + finally + { + try + { + // Linux owner cleanup may enter .lifecycle and retire the + // final pathname generation. Close the ordinary lock + // descriptor before region cleanup can reach that point. + _coldSynchronization.Dispose(); + } + finally + { + _region.Dispose(); + } + } + } + } + + private bool IsDisposed => Volatile.Read(ref _disposed) != 0; + + private bool CanProjectMappedState => !IsDisposed && _storeControl.IsReady; + + private void CleanupParticipantResources(in LockFreeOperationBudget budget) + { + _ = _leases.ReleaseParticipantLeases( + _registration.Token, + _reclaimer, + budget, + ref _checkpoint, + out _); + for (var index = 0; index < _layout.SlotCount; index++) + { + if (budget.CheckPeriodic(index) != StoreStatus.Success) + { + break; + } + + ref ValueSlotMetadataV2 slot = ref _slots.Slot(index); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + if (_slots.ValidateStructuralControl(control) != StoreStatus.Success) + { + break; + } + + if (ControlParticipant(control) != _registration.Token + || ControlState(control) is not ( + LockFreeSlotTable.InitializingState or LockFreeSlotTable.ReservedState)) + { + continue; + } + + if (_slots.TryBeginRecoveryAbort(index, control, out var reservation)) + { + Reach(LockFreeCheckpointId.AbortAfterOwnershipReleaseCas); + _ = CompleteAbortingReservation( + reservation, + budget); + } + } + } + + private StoreStatus Unsupported(StoreWaitOptions waitOptions) + { + if (IsDisposed) + { + return StoreStatus.StoreDisposed; + } + + StoreStatus storeState = _storeControl.Validate(); + if (storeState != StoreStatus.Success) + { + return storeState; + } + + if (!waitOptions.IsValid) + { + return StoreStatus.UnknownFailure; + } + + return waitOptions.CancellationToken.IsCancellationRequested + ? StoreStatus.OperationCanceled + : StoreStatus.UnsupportedPlatform; + } + + private StoreStatus InvalidReservationOrDisposed(StoreWaitOptions waitOptions) + { + if (IsDisposed) + { + return StoreStatus.StoreDisposed; + } + + StoreStatus storeState = _storeControl.Validate(); + if (storeState != StoreStatus.Success) + { + return storeState; + } + + return waitOptions.CancellationToken.IsCancellationRequested + ? StoreStatus.OperationCanceled + : StoreStatus.InvalidReservation; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private StoreStatus TryReserveCore( + ReadOnlySpan key, + int payloadLength, + ReadOnlySpan descriptor, + StoreWaitOptions waitOptions, + SlotPublicationIntent publicationIntent, + out ReservationHandle reservation, + long? operationStarted = null) + { + reservation = default; + long started = operationStarted ?? Stopwatch.GetTimestamp(); + LockFreeOperationBudget budget = LockFreeOperationBudget.Start(waitOptions, started); + StoreStatus ready = ValidateOperation(waitOptions); + if (ready != StoreStatus.Success) + { + return ready; + } + + StoreStatus input = StoreKey.Validate(key, _layout.MaxKeyBytes); + if (input != StoreStatus.Success) + { + return input; + } + + if (payloadLength < 0 || payloadLength > _layout.MaxValueBytes) + { + return StoreStatus.ValueTooLarge; + } + + if (descriptor.Length > _layout.MaxDescriptorBytes) + { + return StoreStatus.DescriptorTooLarge; + } + + StoreStatus hashStatus = LockFreeByteOperations.TryHash(key, budget, out ulong hash); + if (hashStatus != StoreStatus.Success) + { + return hashStatus; + } + + var rejectedCandidateRetry = false; + var candidateRetryAttempt = 0; + var capacityRetryAttempt = 0; + RetryReservation: + reservation = default; + StoreStatus lookup = ResolveCreateConflict(key, hash, budget); + if (lookup != StoreStatus.NotFound) + { + return lookup; + } + + if (rejectedCandidateRetry) + { + // A rejected candidate earns one fresh stable conflict + // resolution, so a real explicit/published winner may still + // justify DuplicateKey. Absence does not give NoWait an unbounded + // sequence of new slot claims; every further candidate is an + // operation-wide contention retry. + if (!budget.TryContinueAfterContention( + candidateRetryAttempt++, + out StoreStatus retryTerminal)) + { + return retryTerminal; + } + + rejectedCandidateRetry = false; + } + + Reach(LockFreeCheckpointId.ReserveBeforeSlotClaim); + StoreStatus claim = _slots.TryClaimReservation( + hash, + key.Length, + descriptor.Length, + payloadLength, + publicationIntent, + budget, + ref _checkpoint, + out reservation); + if (claim == StoreStatus.StoreFull) + { + StoreStatus help = _reclaimer.HelpReclaimableSlots( + budget, + ref _checkpoint, + out _); + if (help != StoreStatus.Success) + { + return help; + } + + // A concurrent reclaimer can advance the last unavailable slot + // after the claim scan but before the helper scan observes it. In + // that case reclaimed is zero even though capacity is now free. + // Always re-probe once after a successful helping pass so that a + // same-key release/re-publish race cannot leak a transient + // StoreFull result from the unlink-to-reusable window. + claim = _slots.TryClaimReservation( + hash, + key.Length, + descriptor.Length, + payloadLength, + publicationIntent, + budget, + ref _checkpoint, + out reservation); + + if (claim == StoreStatus.StoreFull) + { + // Two exhausted allocation scans plus a helping pass still do + // not prove simultaneous physical fullness: a free slot can + // rotate behind both sequential scans. Only the exact local + // double collect may expose StoreFull publicly. Movement or a + // competing local proof is ordinary bounded contention. + StoreStatus proof = _slots.TryProveStoreFull( + budget, + ref _checkpoint, + out bool provenFull); + if (proof != StoreStatus.Success) + { + return proof; + } + + if (provenFull) + { + claim = StoreStatus.StoreFull; + } + else + { + // A free slot, changing control word, or another local + // proof attempt is transient. NoWait terminates here; + // finite and infinite callers retry from a fresh key + // lookup under their operation-wide budget. + if (!budget.TryContinueAfterContention( + capacityRetryAttempt++, + out StoreStatus capacityTerminal)) + { + return capacityTerminal; + } + + goto RetryReservation; + } + } + } + if (claim != StoreStatus.Success) + { + return claim; + } + + try + { + StoreStatus keyCopy = LockFreeByteOperations.TryCopy( + key, + _slots.GetInitializingKeySpan(reservation), + budget); + if (keyCopy != StoreStatus.Success) + { + _ = _slots.AbortUnboundReservation(reservation, ref _checkpoint); + reservation = default; + return keyCopy == StoreStatus.CorruptStore + ? CorruptHere() + : keyCopy; + } + + StoreStatus descriptorCopy = LockFreeByteOperations.TryCopy( + descriptor, + _slots.GetInitializingDescriptorSpan(reservation), + budget); + if (descriptorCopy != StoreStatus.Success) + { + _ = _slots.AbortUnboundReservation(reservation, ref _checkpoint); + reservation = default; + return descriptorCopy == StoreStatus.CorruptStore + ? CorruptHere() + : descriptorCopy; + } + + Reach(LockFreeCheckpointId.DirectoryBeforeDescriptorPublication); + + if (waitOptions.CancellationToken.IsCancellationRequested) + { + _ = _slots.AbortUnboundReservation(reservation, ref _checkpoint); + reservation = default; + return StoreStatus.OperationCanceled; + } + + if (HasExpired(waitOptions, started)) + { + _ = _slots.AbortUnboundReservation(reservation, ref _checkpoint); + reservation = default; + return StoreStatus.StoreBusy; + } + + StoreStatus inserted = _directory.TryInsert( + key, + hash, + reservation.SlotBinding, + budget, + ref _checkpoint, + out _); + if (inserted != StoreStatus.Success) + { + StoreStatus resolvedFailure = ResolveFailedDirectoryInsert( + reservation, + publicationIntent, + inserted, + out bool ordered); + if (ordered) + { + ReachSuccessfulReservationReturn(); + return StoreStatus.Success; + } + + reservation = default; + if (resolvedFailure == StoreStatus.CorruptStore) + { + return CorruptFrom(nameof(LockFreeStoreEngine)); + } + + // PhaseRejected proves only that this candidate lost its + // insertion attempt. The winner may still be tentative (or + // may already have aborted), so only a fresh stable resolver + // is allowed to turn that observation into DuplicateKey. + if (inserted == StoreStatus.DuplicateKey + && resolvedFailure == StoreStatus.DuplicateKey) + { + rejectedCandidateRetry = true; + goto RetryReservation; + } + + return resolvedFailure; + } + + Reach(LockFreeCheckpointId.ReserveAfterDirectoryInsertBeforePendingClassification); + StoreStatus reservationState = + _slots.ClassifyReservationAfterDirectoryInsert(reservation); + if (reservationState != StoreStatus.Success) + { + if (reservationState == StoreStatus.CorruptStore) + { + reservation = default; + return CorruptFrom(nameof(LockFreeStoreEngine)); + } + + if (publicationIntent == SlotPublicationIntent.ExplicitReservation) + { + // TryInsert returning Success is itself a witness that a + // helper completed Initializing -> Reserved. Normal + // recovery preserves a live Active owner; this defensive + // branch covers an exact adversarial CAS or a correctly + // quiesced administrative recovery that follows ordering. + // Neither may turn that ordered reserve into a failure with + // no reservation token. + if (reservationState is StoreStatus.InvalidReservation + or StoreStatus.ReservationAlreadyCompleted) + { + StoreStatus cleanup = + CompleteOrderedReservationRecovery(reservation); + if (cleanup == StoreStatus.CorruptStore) + { + reservation = default; + return CorruptFrom( + nameof(LockFreeStoreEngine)); + } + } + + ReachSuccessfulReservationReturn(); + return StoreStatus.Success; + } + + _ = _slots.TryBeginAbort(reservation); + StoreStatus failedCleanup = CompleteAbortingReservation( + reservation, + LockFreeOperationBudget.StartPostOwnershipCleanup()); + reservation = default; + return failedCleanup == StoreStatus.CorruptStore + ? CorruptFrom(nameof(LockFreeStoreEngine)) + : reservationState; + } + + ReachSuccessfulReservationReturn(); + return StoreStatus.Success; + } + catch + { + StoreStatus resolvedFailure = ResolveFailedDirectoryInsert( + reservation, + publicationIntent, + StoreStatus.UnknownFailure, + out bool ordered); + if (ordered) + { + ReachSuccessfulReservationReturn(); + return StoreStatus.Success; + } + + reservation = default; + return resolvedFailure == StoreStatus.CorruptStore + ? CorruptFrom(nameof(LockFreeStoreEngine)) + : StoreStatus.UnknownFailure; + } + } + + /// + /// Resolves one structurally visible key binding into a public create + /// outcome. Directory lookup deliberately remains structural: slot state + /// plus immutable publication intent decide whether the binding is public + /// ownership, private staging, or cleanup work that any participant may + /// finish. + /// + private StoreStatus ResolveCreateConflict( + ReadOnlySpan key, + ulong keyHash, + in LockFreeOperationBudget budget) + { + var contentionAttempt = 0; + for (; ; ) + { + StoreStatus lookup = _directory.TryLookup( + key, + keyHash, + budget, + out ulong exactBinding, + out DirectoryLocation exactLocation); + if (lookup == StoreStatus.NotFound) + { + return StoreStatus.NotFound; + } + + if (lookup != StoreStatus.Success) + { + if (lookup != StoreStatus.StoreBusy) + { + return lookup; + } + + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus lookupTerminal)) + { + return lookupTerminal; + } + + continue; + } + + Reach(LockFreeCheckpointId.ReserveAfterExistingLookup); + for (; ; ) + { + StoreStatus classification = ClassifyDirectoryBindingWithSourceRevalidation( + exactBinding, + exactLocation, + out bool sourceChanged, + out int state, + out SlotPublicationIntent publicationIntent); + if (sourceChanged) + { + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus sourceChangedTerminal)) + { + return sourceChangedTerminal; + } + + break; + } + + if (classification == StoreStatus.NotFound) + { + // Generation advance makes this cell removable residue. + // Re-enter structural lookup so its exact-value CAS can + // clear the stale binding before a new claim proceeds. + break; + } + + if (classification != StoreStatus.Success) + { + if (classification != StoreStatus.StoreBusy) + { + return classification; + } + + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus classificationTerminal)) + { + return classificationTerminal; + } + + continue; + } + + switch (state) + { + case LockFreeSlotTable.InitializingState: + { + // Initializing is tentative for both public APIs. Help + // the canonical insertion before checking the caller's + // contention budget again; the help may establish the + // explicit reserve ordering point. + StoreStatus help = _directory.HelpMutationForKeyHash( + keyHash, + budget, + ref _checkpoint, + maxSteps: 8); + if (help is not (StoreStatus.Success or StoreStatus.StoreBusy)) + { + return help; + } + + StoreStatus postHelp = ClassifyDirectoryBindingWithSourceRevalidation( + exactBinding, + exactLocation, + out bool postHelpSourceChanged, + out int postHelpState, + out _); + if (postHelpSourceChanged) + { + break; + } + + if (postHelp == StoreStatus.NotFound) + { + break; + } + + if (postHelp != StoreStatus.Success + && postHelp != StoreStatus.StoreBusy) + { + return postHelp; + } + + if (postHelp == StoreStatus.Success + && postHelpState != LockFreeSlotTable.InitializingState) + { + // Reclassify the newly reached terminal state before + // consulting the wait budget. In particular, an + // explicit helper-won Reserved state must dominate + // a deadline observed immediately afterward. + continue; + } + + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus initializingTerminal)) + { + return initializingTerminal; + } + + continue; + } + + case LockFreeSlotTable.ReservedState: + if (publicationIntent == SlotPublicationIntent.ExplicitReservation) + { + return StoreStatus.DuplicateKey; + } + + // Atomic convenience publication owns a private + // reservation. Helpers may complete directory metadata + // but must never expose Reserved as public key ownership + // or commit another process's payload. + StoreStatus reservedHelp = _directory.HelpMutationForKeyHash( + keyHash, + budget, + ref _checkpoint, + maxSteps: 8); + if (reservedHelp is not (StoreStatus.Success or StoreStatus.StoreBusy)) + { + return reservedHelp; + } + + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus reservedTerminal)) + { + return reservedTerminal; + } + + continue; + + case LockFreeSlotTable.PublishedState: + return StoreStatus.DuplicateKey; + + case LockFreeSlotTable.RemoveRequestedState: + { + StoreStatus reclaim = _reclaimer.TryReclaim( + exactBinding, + budget, + ref _checkpoint); + StoreStatus normalized = + LockFreeStoreEngine.NormalizeExistingGenerationReclaimOutcome(reclaim); + if (normalized == StoreStatus.Success) + { + break; + } + + if (normalized != StoreStatus.StoreBusy) + { + return normalized; + } + + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus reclaimTerminal)) + { + return reclaimTerminal; + } + + continue; + } + + case LockFreeSlotTable.AbortingState: + { + StoreStatus cleanup = CompleteAbortingBinding(exactBinding, budget); + if (cleanup == StoreStatus.Success) + { + break; + } + + if (cleanup != StoreStatus.StoreBusy) + { + return cleanup; + } + + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus abortTerminal)) + { + return abortTerminal; + } + + continue; + } + + case LockFreeSlotTable.ReclaimingState: + { + StoreStatus reclaim = _reclaimer.TryReclaim( + exactBinding, + budget, + ref _checkpoint); + if (reclaim == StoreStatus.Success) + { + break; + } + + if (reclaim != StoreStatus.StoreBusy) + { + return reclaim; + } + + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus reclaimTerminal)) + { + return reclaimTerminal; + } + + continue; + } + + default: + return CorruptFrom( + nameof(LockFreeStoreEngine)); + } + + // The exact generation was cleaned or advanced. A fresh + // structural lookup decides whether another contender won. + // Successful cleanup is still operation-wide contention: an + // adversary can otherwise replace the same key with an + // unbounded succession of removable generations while one + // NoWait call keeps making progress forever. + if (!budget.TryContinueAfterContention( + contentionAttempt++, + out StoreStatus freshLookupTerminal)) + { + return freshLookupTerminal; + } + + break; + } + } + } + + /// + /// A binding returned by lookup is only a cached witness. If its slot + /// classification would be corruption, jointly revalidate the exact + /// source cell around a fresh stable slot snapshot. A changed source means + /// that unlink/reuse overtook the cached lookup and is ordinary + /// contention; only an unchanged exact source plus a repeated invalid slot + /// shape may fail closed. + /// + private StoreStatus ClassifyDirectoryBindingWithSourceRevalidation( + ulong exactBinding, + DirectoryLocation exactLocation, + out bool sourceChanged, + out int state, + out SlotPublicationIntent publicationIntent) + { + sourceChanged = false; + StoreStatus classification = _slots.ClassifyDirectoryBinding( + exactBinding, + out state, + out publicationIntent); + if (classification != StoreStatus.CorruptStore) + { + return classification; + } + + StoreStatus sourceStatus = _directory.TryConfirmExactLookupReference( + exactLocation, + exactBinding, + out bool sourceBefore); + if (sourceStatus != StoreStatus.Success) + { + return sourceStatus; + } + + if (!sourceBefore) + { + sourceChanged = true; + return StoreStatus.Success; + } + + classification = _slots.ClassifyDirectoryBinding( + exactBinding, + out state, + out publicationIntent); + + sourceStatus = _directory.TryConfirmExactLookupReference( + exactLocation, + exactBinding, + out bool sourceAfter); + if (sourceStatus != StoreStatus.Success) + { + return sourceStatus; + } + + if (!sourceAfter) + { + sourceChanged = true; + return StoreStatus.Success; + } + + return classification == StoreStatus.CorruptStore + ? CorruptHere() + : classification; + } + + private StoreStatus ResolveFailedDirectoryInsert( + in ReservationHandle reservation, + SlotPublicationIntent publicationIntent, + StoreStatus failure, + out bool ordered) + { + ordered = false; + if (publicationIntent == SlotPublicationIntent.ExplicitReservation) + { + TentativeReservationAbortResult abort = + _slots.TryBeginTentativeAbort(reservation); + if (abort == TentativeReservationAbortResult.Ordered) + { + // A rejected insertion cannot subsequently become Reserved; + // accepting that combination would hide a broken directory + // serialization invariant. + if (failure is StoreStatus.DuplicateKey or StoreStatus.CorruptStore) + { + return CorruptHere(); + } + + ordered = true; + return StoreStatus.Success; + } + + if (abort == TentativeReservationAbortResult.Corrupt) + { + return CorruptHere(); + } + + if (abort == TentativeReservationAbortResult.Aborted) + { + StoreStatus cleanup = CompleteAbortingReservation( + reservation, + LockFreeOperationBudget.StartPostOwnershipCleanup()); + return cleanup == StoreStatus.CorruptStore + ? cleanup + : failure; + } + + return failure; + } + + TentativeReservationAbortResult atomicAbort = + _slots.TryBeginAtomicCandidateAbort(reservation); + if (atomicAbort == TentativeReservationAbortResult.Corrupt + || atomicAbort == TentativeReservationAbortResult.Ordered) + { + return CorruptHere(); + } + + if (atomicAbort == TentativeReservationAbortResult.Aborted) + { + StoreStatus cleanup = CompleteAbortingReservation( + reservation, + LockFreeOperationBudget.StartPostOwnershipCleanup()); + if (cleanup == StoreStatus.CorruptStore) + { + return cleanup; + } + } + + return failure; + } + + private StoreStatus CompleteOrderedReservationRecovery( + in ReservationHandle reservation) + { + StoreStatus begin = _slots.TryBeginAbort(reservation); + if (begin == StoreStatus.Success) + { + StoreStatus cleanup = CompleteAbortingReservation( + reservation, + LockFreeOperationBudget.StartPostOwnershipCleanup()); + if (cleanup == StoreStatus.CorruptStore) + { + return cleanup; + } + } + + // Invalid/advanced means another helper already completed physical + // recovery. The caller still receives the exact ordered handle; using + // it subsequently reports the ordinary stale-reservation status. + return StoreStatus.Success; + } + + private StoreStatus CompleteAbortingBinding( + ulong exactBinding, + in LockFreeOperationBudget budget) + { + StoreStatus unlink = _directory.TryUnlink( + exactBinding, + budget, + ref _checkpoint); + if (unlink is not (StoreStatus.Success or StoreStatus.NotFound)) + { + return unlink; + } + + return _slots.TryCompleteRecoveryReclaim(exactBinding, budget, ref _checkpoint); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ReachSuccessfulReservationReturn() + { + Reach(LockFreeCheckpointId.DirectoryAfterDescriptorClear); + Reach(LockFreeCheckpointId.ReserveAfterReservationPublication); + } + + private StoreStatus CommitReservationCore(in ReservationHandle reservation) + { + Reach(LockFreeCheckpointId.CommitBeforePublicationCas); + return CommitReservationAfterCheckpoint(reservation); + } + + private StoreStatus CommitReservationCore( + in ReservationHandle reservation, + StoreWaitOptions waitOptions, + long started) + { + Reach(LockFreeCheckpointId.CommitBeforePublicationCas); + StoreStatus bound = CheckOperationBound(waitOptions, started); + return bound == StoreStatus.Success + ? CommitReservationAfterCheckpoint(reservation) + : bound; + } + + private StoreStatus CommitReservationAfterCheckpoint(in ReservationHandle reservation) + { + // CommitSequence is diagnostic metadata. The slot-control CAS below is + // the publication ordering point, so a monotonic timestamp avoids a + // store-wide hot counter without weakening the protocol. + long commitSequence = Math.Max(1, Stopwatch.GetTimestamp()); + StoreStatus status = _slots.CommitReservation(reservation, commitSequence); + if (status == StoreStatus.Success) + { + Reach(LockFreeCheckpointId.CommitAfterPublicationCas); + } + + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private StoreStatus AbortReservationCore(in ReservationHandle reservation) + { + Reach(LockFreeCheckpointId.AbortBeforeAbortCas); + return AbortReservationAfterCheckpoint(reservation); + } + + private StoreStatus AbortReservationCore( + in ReservationHandle reservation, + StoreWaitOptions waitOptions, + long started) + { + Reach(LockFreeCheckpointId.AbortBeforeAbortCas); + StoreStatus bound = CheckOperationBound(waitOptions, started); + return bound == StoreStatus.Success + ? AbortReservationAfterCheckpoint(reservation) + : bound; + } + + private StoreStatus AbortReservationAfterCheckpoint( + in ReservationHandle reservation) + { + StoreStatus begin = _slots.TryBeginAbort(reservation); + if (begin != StoreStatus.Success) + { + return begin; + } + + Reach(LockFreeCheckpointId.AbortAfterOwnershipReleaseCas); + + StoreStatus cleanup = CompleteAbortingReservation( + reservation, + LockFreeOperationBudget.StartPostOwnershipCleanup()); + if (cleanup == StoreStatus.CorruptStore) + { + return cleanup; + } + + if (cleanup == StoreStatus.Success) + { + Reach(LockFreeCheckpointId.AbortAfterUnlinkCompletion); + } + + // The ownership-release CAS is the public abort ordering point. Any + // ordinary incomplete unlink/reclaim result is now universally + // helpable and must not be reported as a pre-order StoreBusy/cancel. + return StoreStatus.Success; + } + + private StoreStatus CompleteAbortingReservation( + in ReservationHandle reservation, + in LockFreeOperationBudget budget) + { + if (!TryDecodeSlotBinding(reservation.SlotBinding, out int slotIndex, out _)) + { + return CorruptHere(); + } + + StoreStatus unlink = _directory.TryUnlink( + reservation.SlotBinding, + budget, + ref _checkpoint); + if (unlink is not (StoreStatus.Success or StoreStatus.NotFound)) + { + return unlink; + } + + return _slots.TryCompleteReclaim(reservation, budget, ref _checkpoint) + ? StoreStatus.Success + : StoreStatus.StoreBusy; + } + + private StoreStatus ValidateOperation(StoreWaitOptions waitOptions) + { + if (IsDisposed) + { + return StoreStatus.StoreDisposed; + } + + StoreStatus storeState = _storeControl.Validate(); + if (storeState != StoreStatus.Success) + { + return storeState; + } + + if (!waitOptions.IsValid) + { + return StoreStatus.UnknownFailure; + } + + return waitOptions.CancellationToken.IsCancellationRequested + ? StoreStatus.OperationCanceled + : StoreStatus.Success; + } + + private StoreStatus RecordOperationStatus(StoreStatus status) + { + // Every public engine path has completed its exact source/state + // revalidation before reaching this boundary. Caller-input failures + // use their own statuses, so a remaining CorruptStore is persistent + // mapped structural corruption and may safely poison the mapping. + if (status == StoreStatus.CorruptStore) + { + _storeControl.MarkCorrupt(); + } + + if (status is StoreStatus.InvalidLease or StoreStatus.InvalidReservation) + { + _diagnostics.RecordInvalidToken(stale: false); + } + else if (status is StoreStatus.LeaseAlreadyReleased or StoreStatus.ReservationAlreadyCompleted) + { + _diagnostics.RecordInvalidToken(stale: true); + } + + return _diagnostics.RecordStatus(status); + } + + private StoreStatus CorruptHere( + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) => + LockFreeStoreControl.ReportCorruption( + _storeControl, + nameof(LockFreeStoreEngine), + member, + line); + + private StoreStatus CorruptFrom( + string component, + [CallerMemberName] string member = "", + [CallerLineNumber] int line = 0) => + LockFreeStoreControl.ReportCorruption( + _storeControl, + component, + member, + line); + + private static bool HasExpired(StoreWaitOptions waitOptions, long started) + { + return !waitOptions.IsInfinite + && waitOptions.Timeout > TimeSpan.Zero + && Stopwatch.GetElapsedTime(started) >= waitOptions.Timeout; + } + + private static StoreStatus CheckOperationBound(StoreWaitOptions waitOptions, long started) + { + if (waitOptions.CancellationToken.IsCancellationRequested) + { + return StoreStatus.OperationCanceled; + } + + return HasExpired(waitOptions, started) + ? StoreStatus.StoreBusy + : StoreStatus.Success; + } + + private static bool TryDecodeSlotBinding(ulong binding, out int slotIndex, out long generation) + { + slotIndex = -1; + generation = 0; + try + { + IndexBinding decoded = IndexBinding.Decode(binding); + slotIndex = decoded.SlotIndex; + generation = decoded.Generation; + return true; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + catch (OverflowException) + { + return false; + } + } + + private StoreStatus TryIsSlotPublished( + int slotIndex, + long generation, + out bool isPublished) + { + isPublished = false; + if ((uint)slotIndex >= (uint)_layout.SlotCount) + { + return CorruptHere(); + } + + long control = AtomicControlWord.LoadAcquire(ref _slots.Slot(slotIndex).Control); + StoreStatus structure = _slots.ValidateStructuralControl(control); + if (structure != StoreStatus.Success) + { + return structure; + } + + ulong raw = unchecked((ulong)control); + isPublished = (raw & 0x7UL) == LockFreeSlotTable.PublishedState + && ((raw >> 3) & 0x1_ffff_ffffUL) == (ulong)generation + && (raw >> 36) == 0; + return StoreStatus.Success; + } + + private bool TryValidateLease(in LeaseHandle lease, out int slotIndex, out long generation) + { + slotIndex = -1; + generation = 0; + const int maximumSnapshotAttempts = 2; + for (var attempt = 0; attempt < maximumSnapshotAttempts; attempt++) + { + if (IsDisposed + || !_storeControl.IsReady + || !_leases.TryGetActiveSlotBinding(lease, out ulong slotBinding) + || slotBinding != lease.SlotBinding) + { + return false; + } + + if (!TryDecodeSlotBinding(slotBinding, out slotIndex, out generation) + || (uint)slotIndex >= (uint)_layout.SlotCount) + { + _ = CorruptHere(); + return false; + } + + ref ValueSlotMetadataV2 slot = ref _slots.Slot(slotIndex); + long observed = AtomicControlWord.LoadAcquire(ref slot.Control); + if (_slots.ValidateStructuralControl(observed) != StoreStatus.Success) + { + return false; + } + + ulong control = unchecked((ulong)observed); + int state = (int)(control & 0x7UL); + if (state is LockFreeSlotTable.PublishedState or LockFreeSlotTable.RemoveRequestedState + && ((control >> 3) & 0x1_ffff_ffffUL) == (ulong)generation) + { + return true; + } + + // A copied-token release/reclaim race invalidates the active-record + // proof before it can move this slot. Re-prove the lease first so + // that legal expiry returns false, while a stable impossible slot + // lifecycle paired with the exact Active lease poisons the store. + if (!_storeControl.IsReady + || !_leases.TryGetActiveSlotBinding(lease, out ulong confirmedBinding) + || confirmedBinding != lease.SlotBinding) + { + return false; + } + + long confirmed = AtomicControlWord.LoadAcquire(ref slot.Control); + if (_slots.ValidateStructuralControl(confirmed) != StoreStatus.Success) + { + return false; + } + + if (confirmed != observed) + { + continue; + } + + _ = CorruptHere(); + return false; + } + + return false; + } + + private bool TryValidateLeaseProjection( + in LeaseHandle lease, + out int slotIndex, + out int valueLength, + out int descriptorLength, + out long payloadOffset, + out long descriptorOffset) + { + slotIndex = -1; + valueLength = 0; + descriptorLength = 0; + payloadOffset = 0; + descriptorOffset = 0; + // A logical remove is allowed to change Published(g) to + // RemoveRequested(g) while an exact active lease continues protecting + // the generation. Do not turn that single legal transition into a + // transient empty projection. The active lease prevents reclamation, + // so after retrying the moving snapshot the metadata is stable in one + // of the two projectable states. + const int maximumSnapshotAttempts = 2; + for (var attempt = 0; attempt < maximumSnapshotAttempts; attempt++) + { + if (!TryValidateLease(lease, out slotIndex, out long generation)) + { + return false; + } + + ref ValueSlotMetadataV2 slot = ref _slots.Slot(slotIndex); + long control1 = AtomicControlWord.LoadAcquire(ref slot.Control); + if (_slots.ValidateStructuralControl(control1) != StoreStatus.Success) + { + return false; + } + + ulong directoryBinding = Volatile.Read(ref slot.DirectoryBinding); + int keyLength = Volatile.Read(ref slot.KeyLength); + int observedDescriptorLength = Volatile.Read(ref slot.DescriptorLength); + int observedValueLength = Volatile.Read(ref slot.ValueLength); + int publicationIntent = Volatile.Read(ref slot.PublicationIntent); + long keyOffset = Volatile.Read(ref slot.KeyOffset); + long observedDescriptorOffset = Volatile.Read(ref slot.DescriptorOffset); + long observedPayloadOffset = Volatile.Read(ref slot.PayloadOffset); + Reach(LockFreeCheckpointId.ProjectAfterMetadataReadBeforeControlRevalidation); + long control2 = AtomicControlWord.LoadAcquire(ref slot.Control); + if (control2 != control1) + { + if (_slots.ValidateStructuralControl(control2) != StoreStatus.Success) + { + return false; + } + + continue; + } + + ulong rawControl = unchecked((ulong)control1); + int state = (int)(rawControl & 0x7UL); + long observedGeneration = (long)((rawControl >> 3) & 0x1_ffff_ffffUL); + bool lifecycleInvalid = state is not ( + LockFreeSlotTable.PublishedState + or LockFreeSlotTable.RemoveRequestedState) + || observedGeneration != generation; + + long expectedKeyOffset = _layout.KeyStorageOffset + ((long)slotIndex * _layout.KeyStride); + long expectedDescriptorOffset = + _layout.DescriptorStorageOffset + ((long)slotIndex * _layout.DescriptorStride); + long expectedPayloadOffset = + _layout.PayloadStorageOffset + ((long)slotIndex * _layout.PayloadStride); + bool metadataInvalid = directoryBinding != lease.SlotBinding + || keyLength is < 1 || keyLength > _layout.MaxKeyBytes + || observedDescriptorLength < 0 + || observedDescriptorLength > _layout.MaxDescriptorBytes + || observedValueLength < 0 || observedValueLength > _layout.MaxValueBytes + || publicationIntent is not ( + (int)SlotPublicationIntent.ExplicitReservation + or (int)SlotPublicationIntent.AtomicPublication) + || keyOffset != expectedKeyOffset + || observedDescriptorOffset != expectedDescriptorOffset + || observedPayloadOffset != expectedPayloadOffset; + + if (!_storeControl.IsReady + || !_leases.TryGetActiveSlotBinding(lease, out ulong confirmedBinding) + || confirmedBinding != lease.SlotBinding) + { + return false; + } + + long finalControl = AtomicControlWord.LoadAcquire(ref slot.Control); + if (_slots.ValidateStructuralControl(finalControl) != StoreStatus.Success) + { + return false; + } + + if (finalControl != control1) + { + continue; + } + + if (lifecycleInvalid || metadataInvalid) + { + // A stable nonprojectable lifecycle while the exact lease is + // still Active is impossible. A copied-token release race was + // filtered by the active-record revalidation above and expires + // benignly instead of poisoning the store. + _ = CorruptHere(); + return false; + } + + valueLength = observedValueLength; + descriptorLength = observedDescriptorLength; + payloadOffset = expectedPayloadOffset; + descriptorOffset = expectedDescriptorOffset; + return true; + } + + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void Reach(LockFreeCheckpointId checkpoint) => + LockFreeCheckpoint.Reach(ref _checkpoint, checkpoint); + + private static int ControlState(long control) => (int)((ulong)control & 0x7UL); + + private static ulong ControlParticipant(long control) => ((ulong)control >> 36) & 0x0fff_ffffUL; + + private static StoreStatus InitializeMapping( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + ulong pidNamespaceId, + in LockFreeOperationBudget budget) + { + StoreStatus cleared = Clear(region.Pointer, layout.RequiredBytes, budget); + if (cleared != StoreStatus.Success) + { + return cleared; + } + + ref StoreHeaderV2 header = ref Header(region); + Volatile.Write(ref header.Control, LayoutV2Constants.StoreInitializing); + + header.LayoutMajorVersion = LayoutV2Constants.LayoutMajorVersion; + header.LayoutMinorVersion = LayoutV2Constants.LayoutMinorVersion; + header.HeaderLength = layout.HeaderLength; + header.ResourceProtocolVersion = LayoutV2Constants.ResourceProtocolVersion; + header.RequiredFeatures = LayoutV2Constants.RequiredFeatures; + header.OptionalFeatures = LayoutV2Constants.OptionalFeatures; + header.TotalBytes = layout.TotalBytes; + header.StoreId = CreateStoreId(); + header.Sequence = 0; + header.SlotCount = layout.SlotCount; + header.LeaseRecordCount = layout.LeaseRecordCount; + header.ParticipantRecordCount = layout.ParticipantRecordCount; + header.MaxKeyBytes = layout.MaxKeyBytes; + header.MaxDescriptorBytes = layout.MaxDescriptorBytes; + header.MaxValueBytes = layout.MaxValueBytes; + header.ParticipantIndexBits = layout.ParticipantIndexBits; + header.ParticipantGenerationBits = layout.ParticipantGenerationBits; + header.ParticipantOffset = layout.ParticipantOffset; + header.ParticipantLength = layout.ParticipantLength; + header.ParticipantStride = layout.ParticipantStride; + header.PrimaryLaneCount = layout.PrimaryLaneCount; + header.PrimaryBucketCount = layout.PrimaryBucketCount; + header.PrimaryBucketStride = layout.PrimaryBucketStride; + header.PrimaryDirectoryOffset = layout.PrimaryDirectoryOffset; + header.PrimaryDirectoryLength = layout.PrimaryDirectoryLength; + header.OverflowDirectoryOffset = layout.OverflowDirectoryOffset; + header.OverflowDirectoryLength = layout.OverflowDirectoryLength; + header.OverflowStride = layout.OverflowStride; + header.LeaseStride = layout.LeaseStride; + header.LeaseRegistryOffset = layout.LeaseRegistryOffset; + header.LeaseRegistryLength = layout.LeaseRegistryLength; + header.SlotMetadataStride = layout.SlotMetadataStride; + header.KeyStride = layout.KeyStride; + header.SlotMetadataOffset = layout.SlotMetadataOffset; + header.SlotMetadataLength = layout.SlotMetadataLength; + header.KeyStorageOffset = layout.KeyStorageOffset; + header.KeyStorageLength = layout.KeyStorageLength; + header.DescriptorStride = layout.DescriptorStride; + header.PayloadStride = layout.PayloadStride; + header.DescriptorStorageOffset = layout.DescriptorStorageOffset; + header.DescriptorStorageLength = layout.DescriptorStorageLength; + header.PayloadStorageOffset = layout.PayloadStorageOffset; + header.PayloadStorageLength = layout.PayloadStorageLength; + header.PidNamespaceId = pidNamespaceId; + header.PidNamespaceMode = OperatingSystem.IsLinux() && pidNamespaceId == 0 + ? LayoutV2Constants.PidNamespaceRecoveryMixed + : LayoutV2Constants.PidNamespaceRecoveryEnabled; + + var participants = new LockFreeParticipantRegistry(region, layout); + StoreStatus participantsInitialized = participants.InitializeRecords(budget); + if (participantsInitialized != StoreStatus.Success) + { + return participantsInitialized; + } + + StoreStatus leasesInitialized = InitializeLeases(region, layout, budget); + if (leasesInitialized != StoreStatus.Success) + { + return leasesInitialized; + } + + StoreStatus slotsInitialized = InitializeSlots(region, layout, budget); + if (slotsInitialized != StoreStatus.Success) + { + return slotsInitialized; + } + + StoreStatus bound = budget.Check(); + if (bound != StoreStatus.Success) + { + return bound; + } + + Volatile.Write(ref header.Control, LayoutV2Constants.StoreReady); + header.Magic = LayoutV2Constants.Magic; + return StoreStatus.Success; + } + + private static ulong CaptureStorePidNamespaceId() + { + if (OperatingSystem.IsWindows()) + { + return 0; + } + + return SharedMemoryStore.Leasing.LeaseOwnerClassifier.TryObserveLinuxPidNamespaceId( + Environment.ProcessId, + out ulong pidNamespaceId) + ? pidNamespaceId + : 0; + } + + private static StoreOpenStatus AdmitPidNamespace( + ref StoreHeaderV2 header, + ulong currentPidNamespaceId) + { + long mode = AtomicControlWord.LoadAcquire(ref header.PidNamespaceMode); + if (mode is not (LayoutV2Constants.PidNamespaceRecoveryEnabled + or LayoutV2Constants.PidNamespaceRecoveryMixed)) + { + return StoreOpenStatus.IncompatibleLayout; + } + + if (OperatingSystem.IsWindows()) + { + return header.PidNamespaceId == 0 + ? StoreOpenStatus.Success + : StoreOpenStatus.IncompatibleLayout; + } + + if (mode == LayoutV2Constants.PidNamespaceRecoveryEnabled + && (header.PidNamespaceId == 0 + || currentPidNamespaceId == 0 + || header.PidNamespaceId != currentPidNamespaceId)) + { + AtomicControlWord.StoreRelease( + ref header.PidNamespaceMode, + LayoutV2Constants.PidNamespaceRecoveryMixed); + } + + return StoreOpenStatus.Success; + } + + private static StoreStatus InitializeLeases( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + in LockFreeOperationBudget budget) + { + long freeControl = unchecked((long)AtomicControlWord.EncodeLease( + LayoutV2Constants.LeaseFree, + generation: 1, + participantToken: 0)); + for (var index = 0; index < layout.LeaseRecordCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref LeaseRecordV2 record = ref *(LeaseRecordV2*)( + region.Pointer + layout.LeaseRegistryOffset + ((long)index * layout.LeaseStride)); + record.SlotBinding = 0; + record.AcquireSequence = 0; + Volatile.Write(ref record.Control, freeControl); + } + + return StoreStatus.Success; + } + + private static StoreStatus InitializeSlots( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + in LockFreeOperationBudget budget) + { + long freeControl = unchecked((long)AtomicControlWord.EncodeSlot( + LayoutV2Constants.SlotFree, + generation: 1, + participantToken: 0)); + for (var index = 0; index < layout.SlotCount; index++) + { + StoreStatus bound = budget.CheckPeriodic(index); + if (bound != StoreStatus.Success) + { + return bound; + } + + ref ValueSlotMetadataV2 slot = ref *(ValueSlotMetadataV2*)( + region.Pointer + layout.SlotMetadataOffset + ((long)index * layout.SlotMetadataStride)); + slot.KeyOffset = layout.KeyStorageOffset + ((long)index * layout.KeyStride); + slot.DescriptorOffset = layout.DescriptorStorageOffset + ((long)index * layout.DescriptorStride); + slot.PayloadOffset = layout.PayloadStorageOffset + ((long)index * layout.PayloadStride); + Volatile.Write(ref slot.Control, freeControl); + } + + return StoreStatus.Success; + } + + private static ref StoreHeaderV2 Header(MemoryMappedStoreRegion region) => + ref *(StoreHeaderV2*)region.Pointer; + + private static ulong CreateStoreId() + { + Span bytes = stackalloc byte[sizeof(ulong)]; + RandomNumberGenerator.Fill(bytes); + ulong value = BitConverter.ToUInt64(bytes); + return value == 0 ? 1UL : value; + } + + private static StoreStatus Clear( + byte* pointer, + long length, + in LockFreeOperationBudget budget) + { + const int ClearChunkBytes = 64 * 1024; + long cleared = 0; + var chunkIndex = 0; + while (cleared < length) + { + StoreStatus bound = budget.CheckPeriodic(chunkIndex++); + if (bound != StoreStatus.Success) + { + return bound; + } + + int chunk = (int)Math.Min(ClearChunkBytes, length - cleared); + new Span(pointer + cleared, chunk).Clear(); + cleared += chunk; + } + + return StoreStatus.Success; + } + + private static StoreOpenStatus ToOpenStatus(StoreStatus status) + { + return status switch + { + StoreStatus.Success => StoreOpenStatus.Success, + StoreStatus.StoreBusy => StoreOpenStatus.StoreBusy, + StoreStatus.OperationCanceled => StoreOpenStatus.OperationCanceled, + StoreStatus.AccessDenied => StoreOpenStatus.AccessDenied, + StoreStatus.UnsupportedPlatform => StoreOpenStatus.UnsupportedPlatform, + _ => StoreOpenStatus.MappingFailed + }; + } +} diff --git a/src/SharedMemoryStore/LockFree/LockFreeTelemetry.cs b/src/SharedMemoryStore/LockFree/LockFreeTelemetry.cs new file mode 100644 index 0000000..92c6c8c --- /dev/null +++ b/src/SharedMemoryStore/LockFree/LockFreeTelemetry.cs @@ -0,0 +1,128 @@ +namespace SharedMemoryStore.LockFree; + +/// +/// Per-handle, process-local observational counters for layout-v2 protocol +/// activity. No protocol decision reads these values, so lost or delayed +/// updates can affect diagnostics only, never correctness or progress. +/// +internal sealed class LockFreeTelemetry +{ + private long _overflowScanCount; + private int _lastObservedOverflowScanLength; + private int _maxObservedOverflowScanLength; + private long _casRetryCount; + private long _helpedTransitionCount; + private long _contentionBudgetExhaustionCount; + private long _invalidTokenCount; + private long _staleTokenCount; + private long _recoveryAttemptCount; + private long _recoveredTransitionCount; + private long _currentOwnerClassificationCount; + private long _liveOwnerClassificationCount; + private long _staleOwnerClassificationCount; + private long _unsupportedOwnerClassificationCount; + private long _inconsistentOwnerClassificationCount; + private long _changingOwnerClassificationCount; + + internal long OverflowScanCount => Volatile.Read(ref _overflowScanCount); + internal int LastObservedOverflowScanLength => Volatile.Read(ref _lastObservedOverflowScanLength); + internal int MaxObservedOverflowScanLength => Volatile.Read(ref _maxObservedOverflowScanLength); + internal long CasRetryCount => Volatile.Read(ref _casRetryCount); + internal long HelpedTransitionCount => Volatile.Read(ref _helpedTransitionCount); + internal long ContentionBudgetExhaustionCount => Volatile.Read(ref _contentionBudgetExhaustionCount); + internal long InvalidTokenCount => Volatile.Read(ref _invalidTokenCount); + internal long StaleTokenCount => Volatile.Read(ref _staleTokenCount); + internal long RecoveryAttemptCount => Volatile.Read(ref _recoveryAttemptCount); + internal long RecoveredTransitionCount => Volatile.Read(ref _recoveredTransitionCount); + internal long CurrentOwnerClassificationCount => Volatile.Read(ref _currentOwnerClassificationCount); + internal long LiveOwnerClassificationCount => Volatile.Read(ref _liveOwnerClassificationCount); + internal long StaleOwnerClassificationCount => Volatile.Read(ref _staleOwnerClassificationCount); + internal long UnsupportedOwnerClassificationCount => Volatile.Read(ref _unsupportedOwnerClassificationCount); + internal long InconsistentOwnerClassificationCount => Volatile.Read(ref _inconsistentOwnerClassificationCount); + internal long ChangingOwnerClassificationCount => Volatile.Read(ref _changingOwnerClassificationCount); + + internal void RecordOverflowScan(int scannedCellCount) + { + if (scannedCellCount <= 0) + { + return; + } + + Interlocked.Increment(ref _overflowScanCount); + Volatile.Write(ref _lastObservedOverflowScanLength, scannedCellCount); + + int observed = Volatile.Read(ref _maxObservedOverflowScanLength); + while (scannedCellCount > observed) + { + int exchanged = Interlocked.CompareExchange( + ref _maxObservedOverflowScanLength, + scannedCellCount, + observed); + if (exchanged == observed) + { + break; + } + + observed = exchanged; + } + } + + internal void RecordCasLoss(int count = 1) => AddPositive(ref _casRetryCount, count); + + internal void RecordHelpedTransition(int count = 1) => + AddPositive(ref _helpedTransitionCount, count); + + internal void RecordContentionBudgetExhaustion() => + Interlocked.Increment(ref _contentionBudgetExhaustionCount); + + internal void RecordInvalidToken(bool stale) + { + if (stale) + { + Interlocked.Increment(ref _staleTokenCount); + } + else + { + Interlocked.Increment(ref _invalidTokenCount); + } + } + + internal void RecordRecoveryAttempt(int count = 1) => + AddPositive(ref _recoveryAttemptCount, count); + + internal void RecordRecoveredTransition(int count = 1) => + AddPositive(ref _recoveredTransitionCount, count); + + internal void RecordOwnerClassification(ParticipantClassificationKind kind) + { + switch (kind) + { + case ParticipantClassificationKind.CurrentProcess: + Interlocked.Increment(ref _currentOwnerClassificationCount); + break; + case ParticipantClassificationKind.Live: + Interlocked.Increment(ref _liveOwnerClassificationCount); + break; + case ParticipantClassificationKind.Stale: + Interlocked.Increment(ref _staleOwnerClassificationCount); + break; + case ParticipantClassificationKind.Unsupported: + Interlocked.Increment(ref _unsupportedOwnerClassificationCount); + break; + case ParticipantClassificationKind.Inconsistent: + Interlocked.Increment(ref _inconsistentOwnerClassificationCount); + break; + case ParticipantClassificationKind.Changing: + Interlocked.Increment(ref _changingOwnerClassificationCount); + break; + } + } + + private static void AddPositive(ref long counter, int count) + { + if (count > 0) + { + Interlocked.Add(ref counter, count); + } + } +} diff --git a/src/SharedMemoryStore/LockFree/ParticipantIncarnation.cs b/src/SharedMemoryStore/LockFree/ParticipantIncarnation.cs new file mode 100644 index 0000000..464d45b --- /dev/null +++ b/src/SharedMemoryStore/LockFree/ParticipantIncarnation.cs @@ -0,0 +1,52 @@ +namespace SharedMemoryStore.LockFree; + +/// +/// Stable, double-read snapshot of one mapped participant-record incarnation. +/// The compact token identifies the record and incarnation; PID/start identity +/// is retained separately so PID reuse can be classified conservatively. +/// +internal readonly record struct ParticipantIncarnation( + int RecordIndex, + int Generation, + ulong Token, + int State, + int ProcessId, + int IdentityKind, + long ProcessStartValue, + long OpenSequence, + ulong PidNamespaceId, + int ReservedValue, + long Control); + +/// Outcome of stabilizing and classifying an exact participant token. +internal enum ParticipantClassificationKind +{ + CurrentProcess, + Live, + Stale, + Unsupported, + Inconsistent, + Changing +} + +/// Classification paired with the exact snapshot that justified it. +internal readonly record struct ParticipantClassification( + ParticipantClassificationKind Kind, + ParticipantIncarnation Incarnation) +{ + internal bool IsRecoverable(bool recoverCurrentProcess) => + Kind == ParticipantClassificationKind.Stale + || (recoverCurrentProcess && Kind == ParticipantClassificationKind.CurrentProcess); +} + +/// Record-local participant transition result used by recovery/help callers. +internal enum ParticipantTransitionResult +{ + Succeeded, + AlreadyCompleted, + ReferencesRemain, + LiveOwner, + Unsupported, + Inconsistent, + Changed +} diff --git a/src/SharedMemoryStore/LockFree/ParticipantToken.cs b/src/SharedMemoryStore/LockFree/ParticipantToken.cs new file mode 100644 index 0000000..796cb35 --- /dev/null +++ b/src/SharedMemoryStore/LockFree/ParticipantToken.cs @@ -0,0 +1,82 @@ +namespace SharedMemoryStore.LockFree; + +internal readonly struct ParticipantToken +{ + private ParticipantToken(ulong value, int recordIndex, int generation) + { + Value = value; + RecordIndex = recordIndex; + Generation = generation; + } + + public ulong Value { get; } + public int RecordIndex { get; } + public int Generation { get; } + + public static ulong Encode(int recordIndex, int generation, int participantCount) + { + ArgumentOutOfRangeException.ThrowIfLessThan(participantCount, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(participantCount, 1_048_575); + ArgumentOutOfRangeException.ThrowIfNegative(recordIndex); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(recordIndex, participantCount); + + var indexBits = RequiredBits(participantCount + 1); + var generationBits = 28 - indexBits; + var maximumGeneration = (1 << generationBits) - 1; + ArgumentOutOfRangeException.ThrowIfLessThan(generation, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(generation, maximumGeneration); + return ((ulong)(uint)generation << indexBits) | (uint)(recordIndex + 1); + } + + public static ParticipantToken Decode(ulong raw, int participantCount) + { + var indexBits = RequiredBits(checked(participantCount + 1)); + var indexMask = (1UL << indexBits) - 1; + var indexPlusOne = raw & indexMask; + var generation = raw >> indexBits; + if (indexPlusOne == 0 || indexPlusOne > (ulong)participantCount || generation == 0) + { + throw new ArgumentOutOfRangeException(nameof(raw)); + } + + return new ParticipantToken(raw, checked((int)indexPlusOne - 1), checked((int)generation)); + } + + /// + /// Validates the complete 28-bit wire token without throwing. Capacity + /// classifiers use this on shared owner fields so an out-of-range encoded + /// record index cannot be mistaken for ordinary occupancy. + /// + public static bool IsStructurallyValid(ulong raw, int participantCount) + { + if (participantCount is < 1 or > 1_048_575 + || raw is 0 or > 0x0fff_ffffUL) + { + return false; + } + + int indexBits = RequiredBits(participantCount + 1); + ulong indexMask = (1UL << indexBits) - 1; + ulong indexPlusOne = raw & indexMask; + ulong generation = raw >> indexBits; + ulong maximumGeneration = (1UL << (28 - indexBits)) - 1; + return indexPlusOne is >= 1 + && indexPlusOne <= (ulong)participantCount + && generation is >= 1 + && generation <= maximumGeneration; + } + + private static int RequiredBits(int distinctValues) + { + var bits = 0; + var value = distinctValues - 1; + do + { + bits++; + value >>= 1; + } + while (value != 0); + + return bits; + } +} diff --git a/src/SharedMemoryStore/LockFree/SpillSummary.cs b/src/SharedMemoryStore/LockFree/SpillSummary.cs new file mode 100644 index 0000000..790729a --- /dev/null +++ b/src/SharedMemoryStore/LockFree/SpillSummary.cs @@ -0,0 +1,88 @@ +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.LockFree; + +/// +/// Versioned negative-cache token for one canonical primary bucket. A present +/// token requires overflow lookup; an empty token preserves the identity of the +/// insertion generation that was proved absent so delayed setters cannot ABA +/// through the initial zero value. +/// +internal readonly struct SpillSummary +{ + internal const int SlotIndexBits = 20; + internal const int SlotGenerationBits = 33; + internal const int PresentBitIndex = SlotIndexBits + SlotGenerationBits; + + private const ulong SlotIndexMask = (1UL << SlotIndexBits) - 1; + private const ulong SlotGenerationMask = (1UL << SlotGenerationBits) - 1; + private const ulong IdentityMask = (1UL << PresentBitIndex) - 1; + private const ulong PresentMask = 1UL << PresentBitIndex; + private const ulong EncodedMask = (1UL << (PresentBitIndex + 1)) - 1; + + private SpillSummary(ulong value, bool isPresent, int slotIndex, long generation) + { + Value = value; + IsPresent = isPresent; + SlotIndex = slotIndex; + Generation = generation; + } + + internal ulong Value { get; } + + internal bool IsPresent { get; } + + internal bool IsInitial => Value == 0; + + internal int SlotIndex { get; } + + internal long Generation { get; } + + internal ulong Binding => IsInitial ? 0 : IndexBinding.Encode(SlotIndex, Generation); + + internal ulong EmptyValue => Value & IdentityMask; + + internal static ulong EncodePresent(ulong binding) => Encode(binding) | PresentMask; + + internal static ulong EncodeEmpty(ulong binding) => Encode(binding); + + internal static SpillSummary Decode(ulong raw) + { + if (raw == 0) + { + return default; + } + + if ((raw & ~EncodedMask) != 0) + { + throw new ArgumentOutOfRangeException(nameof(raw)); + } + + ulong indexPlusOne = raw & SlotIndexMask; + ulong generation = (raw >> SlotIndexBits) & SlotGenerationMask; + if (indexPlusOne == 0 + || indexPlusOne > LayoutV2Constants.MaximumSlotCount + || generation == 0) + { + throw new ArgumentOutOfRangeException(nameof(raw)); + } + + return new SpillSummary( + raw, + (raw & PresentMask) != 0, + checked((int)indexPlusOne - 1), + checked((long)generation)); + } + + private static ulong Encode(ulong binding) + { + IndexBinding decoded = IndexBinding.Decode(binding); + if ((uint)decoded.SlotIndex >= LayoutV2Constants.MaximumSlotCount) + { + throw new ArgumentOutOfRangeException(nameof(binding)); + } + + return ((ulong)decoded.Generation << SlotIndexBits) + | checked((uint)(decoded.SlotIndex + 1)); + } +} diff --git a/src/SharedMemoryStore/MemoryStore.cs b/src/SharedMemoryStore/MemoryStore.cs index ff01c10..a529db8 100644 --- a/src/SharedMemoryStore/MemoryStore.cs +++ b/src/SharedMemoryStore/MemoryStore.cs @@ -1,12 +1,17 @@ using System.Buffers; using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Runtime.CompilerServices; using System.Threading; using SharedMemoryStore.Diagnostics; +using SharedMemoryStore.Engines; using SharedMemoryStore.Ingest; using SharedMemoryStore.Interop; using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; using SharedMemoryStore.Leasing; using SharedMemoryStore.Lifecycle; +using SharedMemoryStore.LockFree; using SharedMemoryStore.Options; using SharedMemoryStore.Slots; @@ -31,6 +36,7 @@ public sealed unsafe class MemoryStore : IDisposable private readonly bool _leaseRecoveryEnabled; private readonly ReservationMemoryManager _reservationMemory; private readonly StoreLifecycleGate _lifecycle = new(); + private readonly IStoreEngine? _engine; private long _indexCompactionCount; private bool _disposed; @@ -40,6 +46,9 @@ private MemoryStore( StoreLayout layout, bool leaseRecoveryEnabled) { + _engine = null; + Profile = StoreProfile.Legacy; + ProtocolInfo = new StoreProtocolInfo(StoreProfile.Legacy, 1, 2, 1, 0, 0); _region = region; _synchronization = synchronization; _layout = layout; @@ -54,6 +63,36 @@ private MemoryStore( _reservationMemory = new ReservationMemoryManager(region, layout); } + internal MemoryStore(IStoreEngine engine) + { + ArgumentNullException.ThrowIfNull(engine); + _engine = engine; + Profile = engine.Profile; + ProtocolInfo = engine.ProtocolInfo; + _region = null!; + _synchronization = null!; + _layout = default; + _index = null!; + _slots = null!; + _writer = null!; + _reader = null!; + _reclaimer = null!; + _leases = null!; + _diagnostics = null!; + _reservationMemory = null!; + } + + /// + /// Gets the explicitly selected layout and concurrency profile used by this handle. + /// + public StoreProfile Profile { get; } + + /// + /// Gets the immutable persisted layout and resource-protocol identity, independently of the + /// package version. + /// + public StoreProtocolInfo ProtocolInfo { get; } + /// /// Creates or opens a named store using the supplied options and the default bounded wait policy. /// @@ -63,7 +102,8 @@ public static StoreOpenStatus TryCreateOrOpen(in SharedMemoryStoreOptions option } /// - /// Creates or opens a named store using the supplied options and caller-selected wait policy. + /// Creates or opens a named store using the supplied options and caller-selected cold-path + /// lifecycle/participant wait policy. /// public static StoreOpenStatus TryCreateOrOpen( in SharedMemoryStoreOptions options, @@ -88,55 +128,141 @@ public static StoreOpenStatus TryCreateOrOpen( return validation; } + if (options.Profile == StoreProfile.LockFree + && !LayoutV2Constants.IsSupportedArchitecture(RuntimeInformation.ProcessArchitecture)) + { + return StoreOpenStatus.UnsupportedPlatform; + } + var waitStartTimestamp = Stopwatch.GetTimestamp(); - var mappingStatus = SharedStorePlatform.TryOpen(options, waitOptions, out var region, out var synchronization); - if (mappingStatus != StoreOpenStatus.Success || region is null || synchronization is null) + StoreOpenStatus mappingStatus = SharedStorePlatform.TryBeginOpen( + options, + waitOptions, + waitStartTimestamp, + out SharedStoreOpenScope? openScope); + if (mappingStatus != StoreOpenStatus.Success || openScope is null) { return mappingStatus; } - MemoryStore candidate; + if (options.Profile == StoreProfile.LockFree) + { + StoreOpenStatus lockFreeStatus; + IStoreEngine? engine = null; + try + { + using (openScope) + { + lockFreeStatus = StoreEngineFactory.TryCreateLockFreeUnderColdGate( + options, + waitOptions, + waitStartTimestamp, + openScope.Region, + openScope.Synchronization, + openScope.Disposition, + out engine); + if (lockFreeStatus == StoreOpenStatus.Success && engine is not null) + { + openScope.TransferResourceOwnership(); + } + } + } + catch (UnauthorizedAccessException) + { + engine?.Dispose(); + lockFreeStatus = StoreOpenStatus.AccessDenied; + } + catch (Exception) + { + engine?.Dispose(); + lockFreeStatus = StoreOpenStatus.MappingFailed; + } + + if (lockFreeStatus != StoreOpenStatus.Success || engine is null) + { + store = null; + return lockFreeStatus; + } + + try + { + // Facade construction occurs only after the platform gates are + // released. Its failure cleanup may dispose the Linux region + // and enter .lifecycle through exact-owner release. + store = StoreEngineFactory.WrapOwnedEngine(engine); + return StoreOpenStatus.Success; + } + catch (UnauthorizedAccessException) + { + store = null; + return StoreOpenStatus.AccessDenied; + } + catch (Exception) + { + store = null; + return StoreOpenStatus.MappingFailed; + } + } + + StoreStatus legacyRemainingStatus = SharedStorePlatform.TryGetRemainingWaitOptions( + waitOptions, + waitStartTimestamp, + out _); + if (legacyRemainingStatus != StoreStatus.Success) + { + openScope.Dispose(); + return ToOpenStatus(legacyRemainingStatus); + } + + MemoryStore? candidate = null; + StoreOpenStatus initializeStatus; try { - candidate = new MemoryStore(region, synchronization, layout, options.EnableLeaseRecovery); + using (openScope) + { + candidate = new MemoryStore( + openScope.Region, + openScope.Synchronization, + layout, + options.EnableLeaseRecovery); + openScope.TransferResourceOwnership(); + initializeStatus = candidate.InitializeOrValidate( + options, + openScope.Disposition); + } } catch (UnauthorizedAccessException) { - region.Dispose(); - synchronization.Dispose(); + candidate?.DisposeUninitialized(); return StoreOpenStatus.AccessDenied; } catch (Exception) { - region.Dispose(); - synchronization.Dispose(); + candidate?.DisposeUninitialized(); return StoreOpenStatus.MappingFailed; } - StoreOpenStatus initializeStatus; - if (!candidate.TryEnterStoreLock(waitOptions.RemainingSince(waitStartTimestamp), out var lockStatus)) + if (initializeStatus != StoreOpenStatus.Success) { candidate.DisposeUninitialized(); - return ToOpenStatus(lockStatus); + return initializeStatus; } try { - initializeStatus = candidate.InitializeOrValidate(options); + store = StoreEngineFactory.WrapLegacy(candidate); + return StoreOpenStatus.Success; } - finally + catch (UnauthorizedAccessException) { - candidate.ExitStoreLock(); + store = null; + return StoreOpenStatus.AccessDenied; } - - if (initializeStatus != StoreOpenStatus.Success) + catch (Exception) { - candidate.Dispose(); - return initializeStatus; + store = null; + return StoreOpenStatus.MappingFailed; } - - store = candidate; - return StoreOpenStatus.Success; } /// @@ -148,7 +274,7 @@ public StoreStatus TryPublish(ReadOnlySpan key, ReadOnlySpan value, } /// - /// Publishes immutable payload bytes using the supplied wait policy for shared synchronization. + /// Publishes immutable payload bytes using the supplied profile-specific bounded wait policy. /// public StoreStatus TryPublish( ReadOnlySpan key, @@ -156,6 +282,23 @@ public StoreStatus TryPublish( ReadOnlySpan descriptor, StoreWaitOptions waitOptions) { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.TryPublish(key, value, descriptor, remainingWait); + } + } + if (!TryEnterOperation(waitOptions, out var operation, out var enterStatus)) { return Record(enterStatus); @@ -231,11 +374,14 @@ public StoreStatus TryPublish( /// /// /// The reservation remains invisible to readers until commit succeeds. + /// A reservation is a single-producer lifecycle; concurrent access through copied reservation + /// structs is unsupported. /// Callers write through immediate views or advanced /// direct-I/O views and must advance exactly /// bytes before commit. Disposing an active reservation aborts it, /// and descriptor bytes are immutable after reservation creation. /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] public StoreStatus TryReserve( ReadOnlySpan key, int payloadLength, @@ -246,7 +392,7 @@ public StoreStatus TryReserve( } /// - /// Reserves store-owned payload storage using the supplied wait policy for shared synchronization. + /// Reserves store-owned payload storage using the supplied profile-specific bounded wait policy. /// public StoreStatus TryReserve( ReadOnlySpan key, @@ -255,6 +401,31 @@ public StoreStatus TryReserve( StoreWaitOptions waitOptions, out ValueReservation reservation) { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + reservation = default; + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + var status = _engine.TryReserve( + key, + payloadLength, + descriptor, + remainingWait, + out var handle); + reservation = status == StoreStatus.Success ? new ValueReservation(this, handle) : default; + return status; + } + } + reservation = default; if (!TryEnterOperation(waitOptions, out var operation, out var enterStatus)) @@ -335,7 +506,7 @@ public StoreStatus TryPublishSegments( } /// - /// Publishes a segmented payload after acquiring shared synchronization with the supplied wait policy. + /// Publishes a segmented payload using the supplied profile-specific bounded wait policy. /// public StoreStatus TryPublishSegments( ReadOnlySpan key, @@ -344,6 +515,29 @@ public StoreStatus TryPublishSegments( StoreWaitOptions waitOptions, out long copiedBytes) { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + copiedBytes = 0; + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.TryPublishSegments( + key, + payload, + descriptor, + remainingWait, + out copiedBytes); + } + } + copiedBytes = 0; if (payload.Length > int.MaxValue) { @@ -436,10 +630,30 @@ public StoreStatus TryAcquire(ReadOnlySpan key, out ValueLease lease) } /// - /// Acquires a read lease using the supplied wait policy for shared synchronization. + /// Acquires a shared read lease using the supplied profile-specific bounded wait policy. /// public StoreStatus TryAcquire(ReadOnlySpan key, StoreWaitOptions waitOptions, out ValueLease lease) { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + lease = default; + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + var status = _engine.TryAcquire(key, remainingWait, out var handle); + lease = status == StoreStatus.Success ? new ValueLease(this, handle) : default; + return status; + } + } + lease = default; if (!TryEnterOperation(waitOptions, out var operation, out var enterStatus)) @@ -501,7 +715,8 @@ public StoreStatus TryAcquire(ReadOnlySpan key, StoreWaitOptions waitOptio } /// - /// Removes the value identified by the supplied key and reclaims its slot when no active lease protects it. + /// Logically removes the value identified by the supplied key and cooperatively reclaims its + /// storage after every protecting lease is released or safely recovered. /// public StoreStatus TryRemove(ReadOnlySpan key) { @@ -509,10 +724,31 @@ public StoreStatus TryRemove(ReadOnlySpan key) } /// - /// Removes the value identified by the supplied key using the supplied wait policy for shared synchronization. + /// Logically removes the value using the supplied profile-specific bounded wait policy. After + /// the removal ordering point the key is logically absent. The RemovePending + /// () result reports either a protecting lease or + /// incomplete bounded post-removal work; physical + /// reclamation may complete cooperatively after this call returns. /// public StoreStatus TryRemove(ReadOnlySpan key, StoreWaitOptions waitOptions) { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.TryRemove(key, remainingWait); + } + } + if (!TryEnterOperation(waitOptions, out var operation, out var enterStatus)) { return Record(enterStatus); @@ -558,19 +794,50 @@ public StoreStatus TryRemove(ReadOnlySpan key, StoreWaitOptions waitOption /// /// Explicitly recovers stale active lease records according to the supplied owner policy. /// + /// + /// When is true, the caller + /// must first quiesce lease acquisition, projection, borrowed-span use, and release across + /// every current-process handle attached to this mapping, and keep them quiescent until this + /// call returns. False remains safe during normal concurrent lease activity. + /// public StoreStatus TryRecoverLeases(in LeaseRecoveryOptions options, out LeaseRecoveryReport report) { return TryRecoverLeases(options, StoreWaitOptions.Default, out report); } /// - /// Explicitly recovers stale active lease records using the supplied wait policy for shared synchronization. + /// Explicitly recovers stale active lease records using the supplied profile-specific bounded wait policy. /// + /// + /// When is true, the caller + /// must first quiesce lease acquisition, projection, borrowed-span use, and release across + /// every current-process handle attached to this mapping, and keep them quiescent until this + /// call returns. The library deliberately adds no hot-path gate for this administrative + /// test/controlled-shutdown override. + /// public StoreStatus TryRecoverLeases( in LeaseRecoveryOptions options, StoreWaitOptions waitOptions, out LeaseRecoveryReport report) { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + report = default; + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.TryRecoverLeases(options, remainingWait, out report); + } + } + if (!TryEnterOperation(waitOptions, out var operation, out var enterStatus)) { report = default; @@ -622,13 +889,31 @@ public StoreStatus TryRecoverReservations(in ReservationRecoveryOptions options, } /// - /// Explicitly recovers stale pending reservations using the supplied wait policy for shared synchronization. + /// Explicitly recovers stale pending reservations using the supplied profile-specific bounded wait policy. /// public StoreStatus TryRecoverReservations( in ReservationRecoveryOptions options, StoreWaitOptions waitOptions, out ReservationRecoveryReport report) { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + report = default; + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.TryRecoverReservations(options, remainingWait, out report); + } + } + if (!TryEnterOperation(waitOptions, out var operation, out var enterStatus)) { report = default; @@ -689,10 +974,31 @@ public StoreStatus TryGetDiagnostics(out DiagnosticsSnapshot snapshot) } /// - /// Attempts to create a caller-formatted diagnostic snapshot using the supplied wait policy. + /// Attempts to create a bounded, moment-in-time diagnostic snapshot using the supplied + /// profile-specific wait policy without imposing a global data-path pause. /// public StoreStatus TryGetDiagnostics(StoreWaitOptions waitOptions, out DiagnosticsSnapshot snapshot) { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + snapshot = engineEnterStatus == StoreStatus.StoreDisposed + ? CreateDisposedSnapshot() + : default; + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.TryGetDiagnostics(remainingWait, out snapshot); + } + } + if (!TryEnterOperation(waitOptions, out var operation, out var enterStatus)) { snapshot = enterStatus == StoreStatus.StoreDisposed ? CreateDisposedSnapshot() : default; @@ -725,10 +1031,17 @@ public StoreStatus TryGetDiagnostics(StoreWaitOptions waitOptions, out Diagnosti } /// - /// Releases this store handle and invalidates future operations, lease spans, and reservation spans. + /// Releases this local store handle and invalidates future operations, lease spans, and + /// reservation spans without disposing other handles attached to the mapping. /// public void Dispose() { + if (!_lifecycle.IsDisposingOrDisposed + && _engine is ILockFreeCheckpointEmitter checkpointEmitter) + { + checkpointEmitter.ReachCheckpoint(LockFreeCheckpointId.DisposalBeforeLocalGateClose); + } + if (!_lifecycle.TryBeginDispose()) { return; @@ -736,30 +1049,446 @@ public void Dispose() try { - var lockTaken = TryEnterStoreLock(StoreWaitOptions.Default, out _); - if (lockTaken) + if (_engine is not null) { - try - { - DisposeCore(); - } - finally - { - ExitStoreLock(); - } - } - else - { - DisposeCore(); + _engine.Dispose(); + return; } + + // TryBeginDispose has closed local entry and drained every active + // operation, so teardown does not need the ordinary cross-process + // operation lock. DisposeCore closes that descriptor before the + // mapping's exact-owner cleanup can enter Linux .lifecycle. + DisposeCore(); } finally { _lifecycle.CompleteDispose(); - _synchronization.Dispose(); } } + internal ReservationHandle CreateLegacyReservationHandle( + int slotIndex, + SlotLifecycleId lifecycleId, + int payloadLength) + { + return new ReservationHandle( + unchecked((ulong)Header.StoreId), + unchecked((ulong)lifecycleId.ReuseEpoch), + EncodeLegacySlotBinding(slotIndex, lifecycleId.Generation), + payloadLength); + } + + internal LeaseHandle CreateLegacyLeaseHandle( + int slotIndex, + SlotLifecycleId lifecycleId, + int leaseRecordId) + { + return new LeaseHandle( + unchecked((ulong)Header.StoreId), + unchecked((ulong)lifecycleId.ReuseEpoch), + EncodeLegacySlotBinding(slotIndex, lifecycleId.Generation), + checked((uint)(leaseRecordId + 1))); + } + + internal bool IsLeaseActive(in LeaseHandle handle) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return false; + } + + using (engineOperation) + { + return _engine.IsLeaseActive(handle); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return false; + } + + return TryDecodeLegacyLeaseHandle(handle, out var slotIndex, out var lifecycleId, out var leaseRecordId) + && IsLeaseActive(slotIndex, lifecycleId, leaseRecordId); + } + + internal int GetValueLength(in LeaseHandle handle) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return 0; + } + + using (engineOperation) + { + return _engine.GetValueLength(handle); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return 0; + } + + return TryDecodeLegacyLeaseHandle(handle, out var slotIndex, out var lifecycleId, out _) + ? GetValueLength(slotIndex, lifecycleId) + : 0; + } + + internal int GetDescriptorLength(in LeaseHandle handle) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return 0; + } + + using (engineOperation) + { + return _engine.GetDescriptorLength(handle); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return 0; + } + + return TryDecodeLegacyLeaseHandle(handle, out var slotIndex, out var lifecycleId, out _) + ? GetDescriptorLength(slotIndex, lifecycleId) + : 0; + } + + internal ReadOnlySpan GetValueSpan(LeaseHandle handle) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return ReadOnlySpan.Empty; + } + + using (engineOperation) + { + return _engine.GetValueSpan(handle); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return ReadOnlySpan.Empty; + } + + return TryDecodeLegacyLeaseHandle(handle, out var slotIndex, out var lifecycleId, out _) + ? GetValueSpan(slotIndex, lifecycleId) + : ReadOnlySpan.Empty; + } + + internal ReadOnlySpan GetDescriptorSpan(LeaseHandle handle) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return ReadOnlySpan.Empty; + } + + using (engineOperation) + { + return _engine.GetDescriptorSpan(handle); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return ReadOnlySpan.Empty; + } + + return TryDecodeLegacyLeaseHandle(handle, out var slotIndex, out var lifecycleId, out _) + ? GetDescriptorSpan(slotIndex, lifecycleId) + : ReadOnlySpan.Empty; + } + + internal StoreStatus ReleaseLease(in LeaseHandle handle, StoreWaitOptions waitOptions) + { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.ReleaseLease(handle, remainingWait); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return StoreStatus.StoreDisposed; + } + + return TryDecodeLegacyLeaseHandle(handle, out var slotIndex, out var lifecycleId, out var leaseRecordId) + ? ReleaseLease(slotIndex, lifecycleId, leaseRecordId, waitOptions) + : StoreStatus.InvalidLease; + } + + internal bool IsReservationPending(in ReservationHandle handle) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return false; + } + + using (engineOperation) + { + return _engine.IsReservationPending(handle); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return false; + } + + return TryDecodeLegacyReservationHandle(handle, out var slotIndex, out var lifecycleId) + && IsReservationPending(slotIndex, lifecycleId); + } + + internal int GetReservationBytesWritten(in ReservationHandle handle) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return 0; + } + + using (engineOperation) + { + return _engine.GetReservationBytesWritten(handle); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return 0; + } + + return TryDecodeLegacyReservationHandle(handle, out var slotIndex, out var lifecycleId) + ? GetReservationBytesWritten(slotIndex, lifecycleId) + : 0; + } + + internal Span GetReservationSpan(ReservationHandle handle, int sizeHint) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return Span.Empty; + } + + using (engineOperation) + { + return _engine.GetReservationSpan(handle, sizeHint); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return Span.Empty; + } + + return TryDecodeLegacyReservationHandle(handle, out var slotIndex, out var lifecycleId) + ? GetReservationSpan(slotIndex, lifecycleId, sizeHint) + : Span.Empty; + } + + internal Memory GetReservationMemory(ReservationHandle handle, int sizeHint) + { + if (_engine is not null) + { + if (!_lifecycle.TryEnter(out var engineOperation)) + { + return Memory.Empty; + } + + using (engineOperation) + { + return _engine.DangerousGetReservationMemory(handle, sizeHint); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return Memory.Empty; + } + + return TryDecodeLegacyReservationHandle(handle, out var slotIndex, out var lifecycleId) + ? GetReservationMemory(slotIndex, lifecycleId, sizeHint) + : Memory.Empty; + } + + internal StoreStatus AdvanceReservation( + in ReservationHandle handle, + int byteCount, + StoreWaitOptions waitOptions) + { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.AdvanceReservation(handle, byteCount, remainingWait); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return StoreStatus.StoreDisposed; + } + + return TryDecodeLegacyReservationHandle(handle, out var slotIndex, out var lifecycleId) + ? AdvanceReservation(slotIndex, lifecycleId, byteCount, waitOptions) + : StoreStatus.InvalidReservation; + } + + internal StoreStatus CommitReservation(in ReservationHandle handle, StoreWaitOptions waitOptions) + { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.CommitReservation(handle, remainingWait); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return StoreStatus.StoreDisposed; + } + + return TryDecodeLegacyReservationHandle(handle, out var slotIndex, out var lifecycleId) + ? CommitReservation(slotIndex, lifecycleId, waitOptions) + : StoreStatus.InvalidReservation; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal StoreStatus AbortReservation( + in ReservationHandle handle, + bool countAbort, + StoreWaitOptions waitOptions) + { + if (_engine is not null) + { + if (!TryEnterEngineOperation( + waitOptions, + out var engineOperation, + out StoreWaitOptions remainingWait, + out StoreStatus engineEnterStatus)) + { + return _engine.RecordFacadeStatus(engineEnterStatus); + } + + using (engineOperation) + { + return _engine.AbortReservation(handle, remainingWait); + } + } + + if (_lifecycle.IsDisposingOrDisposed) + { + return StoreStatus.StoreDisposed; + } + + return TryDecodeLegacyReservationHandle(handle, out var slotIndex, out var lifecycleId) + ? AbortReservation(slotIndex, lifecycleId, countAbort, waitOptions) + : StoreStatus.InvalidReservation; + } + + internal static int DecodeLegacySlotIndex(ulong slotBinding) => unchecked((int)(uint)slotBinding) - 1; + + internal static int DecodeLegacyLeaseRecordId(in LeaseHandle handle) => unchecked((int)handle.LeaseToken) - 1; + + internal static SlotLifecycleId DecodeLegacyLifecycle(in LeaseHandle handle) => + new(checked((int)(handle.SlotBinding >> 32)), unchecked((long)handle.ParticipantToken)); + + internal static SlotLifecycleId DecodeLegacyLifecycle(in ReservationHandle handle) => + new(checked((int)(handle.SlotBinding >> 32)), unchecked((long)handle.ParticipantToken)); + + private static ulong EncodeLegacySlotBinding(int slotIndex, int generation) => + ((ulong)checked((uint)generation) << 32) | checked((uint)(slotIndex + 1)); + + private bool TryDecodeLegacyReservationHandle( + in ReservationHandle handle, + out int slotIndex, + out SlotLifecycleId lifecycleId) + { + slotIndex = -1; + lifecycleId = default; + if (handle.StoreId != unchecked((ulong)Header.StoreId)) + { + return false; + } + + slotIndex = DecodeLegacySlotIndex(handle.SlotBinding); + lifecycleId = DecodeLegacyLifecycle(handle); + return slotIndex >= 0 && slotIndex < _layout.SlotCount && lifecycleId.IsValid; + } + + private bool TryDecodeLegacyLeaseHandle( + in LeaseHandle handle, + out int slotIndex, + out SlotLifecycleId lifecycleId, + out int leaseRecordId) + { + slotIndex = -1; + lifecycleId = default; + leaseRecordId = -1; + if (handle.StoreId != unchecked((ulong)Header.StoreId)) + { + return false; + } + + slotIndex = DecodeLegacySlotIndex(handle.SlotBinding); + lifecycleId = DecodeLegacyLifecycle(handle); + leaseRecordId = DecodeLegacyLeaseRecordId(handle); + return slotIndex >= 0 && slotIndex < _layout.SlotCount + && leaseRecordId >= 0 && leaseRecordId < _layout.LeaseRecordCount + && lifecycleId.IsValid; + } + internal bool IsLeaseActive(int slotIndex, SlotLifecycleId lifecycleId, int leaseRecordId) { if (!TryEnterOperation(StoreWaitOptions.Default, out var operation, out _)) @@ -895,26 +1624,28 @@ internal StoreStatus ReleaseLease( } } - internal StoreLayout Layout => _layout; + private MemoryStore LegacyCore => _engine is Engines.LegacyV12.LegacyV12StoreEngine legacy ? legacy.Core : this; + + internal StoreLayout Layout => LegacyCore._layout; - internal ref StoreHeader Header => ref *(StoreHeader*)_region.Pointer; + internal ref StoreHeader Header => ref *(StoreHeader*)LegacyCore._region.Pointer; - internal ref SharedSlotMetadata GetSlotForTesting(int slotIndex) => ref _slots.GetSlot(slotIndex); + internal ref SharedSlotMetadata GetSlotForTesting(int slotIndex) => ref LegacyCore._slots.GetSlot(slotIndex); - internal ref SharedLeaseRecord GetLeaseRecordForTesting(int leaseRecordId) => ref _leases.GetRecord(leaseRecordId); + internal ref SharedLeaseRecord GetLeaseRecordForTesting(int leaseRecordId) => ref LegacyCore._leases.GetRecord(leaseRecordId); - internal void SetSlotSearchCursorForTesting(int nextSearch) => _slots.SetNextSearchForTesting(nextSearch); + internal void SetSlotSearchCursorForTesting(int nextSearch) => LegacyCore._slots.SetNextSearchForTesting(nextSearch); - internal void SetLeaseSearchCursorForTesting(int nextSearch) => _leases.SetNextSearchForTesting(nextSearch); + internal void SetLeaseSearchCursorForTesting(int nextSearch) => LegacyCore._leases.SetNextSearchForTesting(nextSearch); internal void SetSlotLifecycleForTesting(int slotIndex, SlotLifecycleId lifecycleId) { - ref var slot = ref _slots.GetSlot(slotIndex); + ref var slot = ref LegacyCore._slots.GetSlot(slotIndex); slot.Generation = lifecycleId.Generation; slot.ReuseEpoch = lifecycleId.ReuseEpoch; } - internal IndexStateCounts CountIndexStatesForTesting() => _index.CountStates(); + internal IndexStateCounts CountIndexStatesForTesting() => LegacyCore._index.CountStates(); internal bool IsReservationPending(int slotIndex, SlotLifecycleId lifecycleId) { @@ -1157,22 +1888,58 @@ internal StoreStatus AbortReservation( } } - private StoreOpenStatus InitializeOrValidate(SharedMemoryStoreOptions options) + private StoreOpenStatus InitializeOrValidate( + SharedMemoryStoreOptions options, + RegionOpenDisposition disposition) { + // Existing regions are mapped at their actual backing capacity so an + // opposite-profile header can be rejected without first projecting the + // caller's requested (possibly much larger) view. Prove the complete + // fixed v1 header is mapped before creating a ref to it. + if (_region.Capacity < Marshal.SizeOf()) + { + return StoreOpenStatus.IncompatibleLayout; + } + ref var header = ref Header; - if (options.OpenMode == OpenMode.CreateNew || header.Magic == 0) + if (disposition == RegionOpenDisposition.CreatedNew) { if (options.OpenMode == OpenMode.OpenExisting) { return StoreOpenStatus.IncompatibleLayout; } + // Initialization clears RequiredBytes and publishes TotalBytes in + // the header. Both lengths must describe storage actually mapped by + // this handle before either write is attempted. + if (_region.Capacity < _layout.RequiredBytes + || _region.Capacity < _layout.TotalBytes) + { + return StoreOpenStatus.IncompatibleLayout; + } + InitializeHeader(); _slots.Initialize(); _leases.Initialize(); return StoreOpenStatus.Success; } + if (options.OpenMode == OpenMode.CreateNew) + { + return StoreOpenStatus.AlreadyExists; + } + + if (header.Magic == 0) + { + // An existing unpublished mapping may belong to an older creator + // that exposed the region before entering the ordinary gate. This + // opener has no proof of initialization ownership and must not + // mutate it. + return options.OpenMode == OpenMode.CreateOrOpen + ? StoreOpenStatus.StoreBusy + : StoreOpenStatus.IncompatibleLayout; + } + if (header.Magic != LayoutConstants.Magic || header.LayoutMajorVersion != LayoutConstants.LayoutMajorVersion || !_layout.MatchesHeader(header) @@ -1181,6 +1948,17 @@ private StoreOpenStatus InitializeOrValidate(SharedMemoryStoreOptions options) return StoreOpenStatus.IncompatibleLayout; } + // Header and section validation proves the requested v1 topology, but + // it does not prove that a live backing file was not truncated after the + // header was committed. Never accept the layout unless both its declared + // extent and its computed required extent fit in the actual view. + if (header.TotalBytes < _layout.RequiredBytes + || _region.Capacity < header.TotalBytes + || _region.Capacity < _layout.RequiredBytes) + { + return StoreOpenStatus.IncompatibleLayout; + } + return Volatile.Read(ref header.StoreState) == LayoutConstants.StoreUnsupported ? StoreOpenStatus.UnsupportedPlatform : StoreOpenStatus.Success; @@ -1268,6 +2046,72 @@ private StoreStatus ValidateReservationInput(ReadOnlySpan key, int payload : StoreStatus.Success; } + private bool TryEnterEngineOperation( + StoreWaitOptions waitOptions, + out StoreLifecycleGate.Operation operation, + out StoreWaitOptions remainingWait, + out StoreStatus status) + { + long started = Stopwatch.GetTimestamp(); + status = _lifecycle.TryEnter(waitOptions, started, out operation); + if (status != StoreStatus.Success) + { + remainingWait = default; + return false; + } + + if (TryGetRemainingWaitOptions( + waitOptions, + started, + out remainingWait, + out status)) + { + return true; + } + + operation.Dispose(); + operation = default; + return false; + } + + private static bool TryGetRemainingWaitOptions( + StoreWaitOptions waitOptions, + long started, + out StoreWaitOptions remainingWait, + out StoreStatus status) + { + remainingWait = default; + if (!waitOptions.IsValid) + { + status = StoreStatus.UnknownFailure; + return false; + } + + if (waitOptions.CancellationToken.IsCancellationRequested) + { + status = StoreStatus.OperationCanceled; + return false; + } + + if (waitOptions.IsInfinite || waitOptions.Timeout == TimeSpan.Zero) + { + remainingWait = waitOptions; + status = StoreStatus.Success; + return true; + } + + TimeSpan remaining = waitOptions.Timeout - Stopwatch.GetElapsedTime(started); + if (remaining <= TimeSpan.Zero) + { + status = StoreStatus.StoreBusy; + return false; + } + + remainingWait = new StoreWaitOptions(remaining, waitOptions.CancellationToken); + status = StoreStatus.Success; + return true; + } + private bool TryEnterOperation( StoreWaitOptions waitOptions, out StoreLifecycleGate.Operation operation, @@ -1359,9 +2203,14 @@ private static StoreOpenStatus ToOpenStatus(StoreStatus status) private void DisposeUninitialized() { - DisposeCore(); - _synchronization.Dispose(); - _lifecycle.CompleteDispose(); + try + { + DisposeCore(); + } + finally + { + _lifecycle.CompleteDispose(); + } } private void DisposeCore() @@ -1373,11 +2222,26 @@ private void DisposeCore() _disposed = true; _reservationMemory.Dispose(); - _region.Dispose(); + try + { + // Region disposal may acquire Linux .lifecycle and commit final- + // owner cleanup. Retire the ordinary lock descriptor first so no + // reopening participant can inherit an obsolete inode generation. + _synchronization.Dispose(); + } + finally + { + _region.Dispose(); + } } - private DiagnosticsSnapshot CreateDisposedSnapshot() + internal DiagnosticsSnapshot CreateDisposedSnapshot() { + if (_engine is not null) + { + return _engine.CreateDisposedDiagnosticsSnapshot(); + } + return _diagnostics.CreateSnapshot( _layout.TotalBytes, _layout.SlotCount, @@ -1417,6 +2281,17 @@ private StoreStatus Record(StoreStatus status) return status; } + /// + /// Records a status rejected by the outer profile-neutral facade. This is + /// deliberately diagnostics-only so it remains safe after failed lifecycle + /// entry and while another thread may be disposing mapped resources. + /// + internal StoreStatus RecordFacadeStatus(StoreStatus status) + { + _diagnostics.Record(status); + return status; + } + private void MaybeCompactIndex() { var state = _index.CountStates(); diff --git a/src/SharedMemoryStore/Options/SharedMemoryStoreOptionsValidator.cs b/src/SharedMemoryStore/Options/SharedMemoryStoreOptionsValidator.cs index 78cce0d..482e36e 100644 --- a/src/SharedMemoryStore/Options/SharedMemoryStoreOptionsValidator.cs +++ b/src/SharedMemoryStore/Options/SharedMemoryStoreOptionsValidator.cs @@ -1,4 +1,5 @@ using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; namespace SharedMemoryStore.Options; @@ -41,10 +42,30 @@ public static StoreOptionsValidationResult ValidateDetailed(SharedMemoryStoreOpt failures.Add(new StoreOptionsValidationFailure(nameof(options.OpenMode), "OpenMode must be a defined value.")); } + if (!Enum.IsDefined(options.Profile)) + { + failures.Add(new StoreOptionsValidationFailure(nameof(options.Profile), "Profile must be a defined value.")); + } + + if (options.Profile == StoreProfile.LockFree + && (options.ParticipantRecordCount < 1 || options.ParticipantRecordCount > 1_048_575)) + { + failures.Add(new StoreOptionsValidationFailure( + nameof(options.ParticipantRecordCount), + "ParticipantRecordCount must be between 1 and 1,048,575 for the lock-free profile.")); + } + if (options.SlotCount <= 0) { failures.Add(new StoreOptionsValidationFailure(nameof(options.SlotCount), "SlotCount must be greater than zero.")); } + else if (options.Profile == StoreProfile.LockFree + && options.SlotCount > LayoutV2Constants.MaximumSlotCount) + { + failures.Add(new StoreOptionsValidationFailure( + nameof(options.SlotCount), + "SlotCount must be between 1 and 1,048,575 for the lock-free profile.")); + } if (options.MaxKeyBytes <= 0) { @@ -76,6 +97,37 @@ public static StoreOptionsValidationResult ValidateDetailed(SharedMemoryStoreOpt return Invalid(failures); } + if (options.Profile == StoreProfile.LockFree) + { + StoreLayoutV2 layoutV2; + try + { + layoutV2 = StoreLayoutV2.FromOptions(options); + } + catch (OverflowException) + { + return Invalid(new StoreOptionsValidationFailure(nameof(options.TotalBytes), "Layout size calculation overflowed.")); + } + catch (ArgumentOutOfRangeException exception) + { + return Invalid(new StoreOptionsValidationFailure(exception.ParamName ?? nameof(options), exception.Message)); + } + + if (!layoutV2.FitsWithinTotalBytes()) + { + return new StoreOptionsValidationResult( + StoreOpenStatus.InsufficientCapacity, + new[] + { + new StoreOptionsValidationFailure( + nameof(options.TotalBytes), + $"TotalBytes must be at least {layoutV2.RequiredBytes} for the configured capacities.") + }); + } + + return new StoreOptionsValidationResult(StoreOpenStatus.Success, Array.Empty()); + } + try { layout = StoreLayout.FromOptions(options); diff --git a/src/SharedMemoryStore/Properties/AssemblyInfo.cs b/src/SharedMemoryStore/Properties/AssemblyInfo.cs index f99dbf2..7458769 100644 --- a/src/SharedMemoryStore/Properties/AssemblyInfo.cs +++ b/src/SharedMemoryStore/Properties/AssemblyInfo.cs @@ -3,4 +3,7 @@ [assembly: InternalsVisibleTo("SharedMemoryStore.UnitTests")] [assembly: InternalsVisibleTo("SharedMemoryStore.ContractTests")] [assembly: InternalsVisibleTo("SharedMemoryStore.IntegrationTests")] +[assembly: InternalsVisibleTo("SharedMemoryStore.LinearizabilityTests")] [assembly: InternalsVisibleTo("SharedMemoryStore.Benchmarks")] +[assembly: InternalsVisibleTo("SharedMemoryStore.LockFreeAgent")] +[assembly: InternalsVisibleTo("SharedMemoryStore.SyncProbe")] diff --git a/src/SharedMemoryStore/SharedMemoryStore.csproj b/src/SharedMemoryStore/SharedMemoryStore.csproj index 2c43658..74103ca 100644 --- a/src/SharedMemoryStore/SharedMemoryStore.csproj +++ b/src/SharedMemoryStore/SharedMemoryStore.csproj @@ -9,7 +9,7 @@ true true SharedMemoryStore - 1.0.2 + 2.0.0 SharedMemoryStore contributors SharedMemoryStore A bounded named shared-memory key-value store for opaque binary values. @@ -19,7 +19,7 @@ git https://github.com/rantri/SharedMemoryStore README.md - NuGet SharedMemoryStore 1.0.2 preserves Linux, Windows, and same-host Docker support, corrects Linux layout-mismatch reporting, and rejects unsupported managed processes early while preserving the public API, status values, BCL-only runtime surface, layout 1.2, and resource naming 1. Repository source adds independently versioned native C++ and Python 0.1.0 sibling distributions using C ABI 1.0, validated with .NET on Windows and Linux; v1.0.2 neither includes them in NuGet nor publishes them to PyPI or a native package registry. + NuGet SharedMemoryStore 2.0.0 adds an explicitly selected C# layout 2.0 lock-free key-value profile and resource protocol 2, including zero-copy reservation publication, shared read leases, generation-fenced removal/reuse, participant records, explicit recovery, and additive diagnostics. The legacy profile remains the default and preserves mapped layout 1.2, resource naming 1, existing status numbers, and Linux, Windows, and same-host Docker support. Layout 1.2 and 2.0 mappings are distinct: upgrade and rollback require drain, close, recreate, and application-owned republish; there is no in-place conversion or mixed-layout writer mode. Independently versioned C++ and Python 0.1.0 distributions remain layout 1.2-only and fail closed on layout 2.0. true snupkg SharedMemoryStore diff --git a/src/SharedMemoryStore/SharedMemoryStoreOptions.cs b/src/SharedMemoryStore/SharedMemoryStoreOptions.cs index c7c9d8b..d104a18 100644 --- a/src/SharedMemoryStore/SharedMemoryStoreOptions.cs +++ b/src/SharedMemoryStore/SharedMemoryStoreOptions.cs @@ -1,4 +1,5 @@ using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; using SharedMemoryStore.Options; namespace SharedMemoryStore; @@ -18,11 +19,35 @@ public enum OpenMode CreateOrOpen = 2 } +/// +/// Selects the shared-memory layout and concurrency protocol used by a store. +/// +public enum StoreProfile +{ + /// + /// Uses the compatible layout-v1.2 implementation and its legacy synchronization protocol. + /// This remains the default profile. + /// + Legacy = 0, + + /// + /// Uses the layout-v2 lock-free implementation. Progress is system-wide lock-free, not + /// wait-free for every caller under sustained contention. + /// + LockFree = 1 +} + /// /// Configuration used when creating or opening a bounded named shared-memory store. /// public sealed class SharedMemoryStoreOptions { + /// + /// Gets the explicitly requested shared-memory layout and concurrency profile. The default + /// is ; opening never auto-selects an existing opposite profile. + /// + public StoreProfile Profile { get; init; } = StoreProfile.Legacy; + /// Gets the OS-visible name of the mapped store. public string Name { get; init; } = string.Empty; @@ -47,6 +72,12 @@ public sealed class SharedMemoryStoreOptions /// Gets the maximum number of simultaneously active lease records. public int LeaseRecordCount { get; init; } + /// + /// Gets the layout-v2 participant-record capacity. Each open store handle consumes one record; + /// the default is 64. Legacy stores ignore this value. + /// + public int ParticipantRecordCount { get; init; } = 64; + /// Gets a value indicating whether explicit stale lease recovery is enabled. public bool EnableLeaseRecovery { get; init; } @@ -68,6 +99,46 @@ public static long CalculateRequiredBytes( leaseRecordCount); } + /// + /// Calculates the minimum mapped-region length for an explicitly selected store profile. + /// + /// + /// The lock-free profile validates its layout-v2 slot and participant limits. The existing + /// profile-less overload retains layout-v1.2 sizing. + /// + public static long CalculateRequiredBytes( + StoreProfile profile, + int slotCount, + int maxValueBytes, + int maxDescriptorBytes, + int maxKeyBytes, + int leaseRecordCount, + int participantRecordCount = 64) + { + if (!Enum.IsDefined(profile)) + { + throw new ArgumentOutOfRangeException(nameof(profile)); + } + + if (profile == StoreProfile.Legacy) + { + return CalculateRequiredBytes( + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount); + } + + return StoreLayoutV2.CalculateRequiredBytes( + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount); + } + /// /// Creates valid ordinary store options and derives the required mapped-region size. /// @@ -83,6 +154,7 @@ public static SharedMemoryStoreOptions Create( { return new SharedMemoryStoreOptions { + Profile = StoreProfile.Legacy, Name = name, OpenMode = openMode, SlotCount = slotCount, @@ -100,6 +172,47 @@ public static SharedMemoryStoreOptions Create( }; } + /// + /// Creates layout-v2 lock-free store options and derives the required mapped-region size. + /// + /// + /// Selection is explicit and never converts an existing layout-v1.2 mapping. One participant + /// record is consumed by each successfully opened handle. + /// + public static SharedMemoryStoreOptions CreateLockFree( + string name, + int slotCount, + int maxValueBytes, + int maxDescriptorBytes, + int maxKeyBytes, + int leaseRecordCount, + int participantRecordCount = 64, + OpenMode openMode = OpenMode.CreateOrOpen, + bool enableLeaseRecovery = false) + { + return new SharedMemoryStoreOptions + { + Profile = StoreProfile.LockFree, + Name = name, + OpenMode = openMode, + SlotCount = slotCount, + MaxValueBytes = maxValueBytes, + MaxDescriptorBytes = maxDescriptorBytes, + MaxKeyBytes = maxKeyBytes, + LeaseRecordCount = leaseRecordCount, + ParticipantRecordCount = participantRecordCount, + EnableLeaseRecovery = enableLeaseRecovery, + TotalBytes = CalculateRequiredBytes( + StoreProfile.LockFree, + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount) + }; + } + /// /// Validates this option instance and returns actionable public validation details. /// @@ -120,7 +233,17 @@ public static StoreOptionsValidationResult Validate(SharedMemoryStoreOptions? op /// /// Options for explicit owner-controlled stale lease recovery. /// -/// When true, active lease records owned by the current process may be recovered for tests and controlled shutdown. +/// +/// When true, active lease records owned by the current process may be recovered +/// for tests and controlled shutdown only after the caller has quiesced all +/// current-process lease acquisition, projection, borrowed-span use, and release +/// across every handle attached to the mapping. That process-wide quiescence must +/// remain in force until recovery returns. Concurrent use of this override with +/// current-process lease activity is unsupported and is not guarded by a +/// hot-path gate. False remains safe during normal concurrent lease activity. +/// A live record still in its Claiming initialization phase remains protected +/// until participant closing or recovery proves its claimant is quiescent. +/// public readonly record struct LeaseRecoveryOptions(bool RecoverCurrentProcessLeases); /// diff --git a/src/SharedMemoryStore/StoreProtocolInfo.cs b/src/SharedMemoryStore/StoreProtocolInfo.cs new file mode 100644 index 0000000..a5ddf6d --- /dev/null +++ b/src/SharedMemoryStore/StoreProtocolInfo.cs @@ -0,0 +1,19 @@ +namespace SharedMemoryStore; + +/// +/// Describes the persisted layout and shared-resource protocol used by a store handle. These +/// protocol versions are independent of the NuGet package version and must be checked separately. +/// +/// The selected store profile. +/// The persisted layout major version. +/// The persisted layout minor version. +/// The version of the named-resource protocol. +/// Feature bits every compatible opener must understand. +/// Feature bits a compatible opener may safely ignore. +public readonly record struct StoreProtocolInfo( + StoreProfile Profile, + int LayoutMajorVersion, + int LayoutMinorVersion, + int ResourceProtocolVersion, + ulong RequiredFeatures, + ulong OptionalFeatures); diff --git a/src/SharedMemoryStore/StoreStatus.cs b/src/SharedMemoryStore/StoreStatus.cs index 6e9b9f7..f86b797 100644 --- a/src/SharedMemoryStore/StoreStatus.cs +++ b/src/SharedMemoryStore/StoreStatus.cs @@ -32,11 +32,16 @@ public enum StoreOpenStatus /// The runtime failed to create or open the memory mapping. MappingFailed = 8, - /// The store could not be opened because shared synchronization was busy for the selected wait policy. + /// + /// The selected open bound expired during cold lifecycle coordination or participant claim. + /// StoreBusy = 9, - /// The open or create operation was canceled before shared synchronization was acquired. - OperationCanceled = 10 + /// The open or create operation was canceled before a handle became active. + OperationCanceled = 10, + + /// No reusable layout-v2 participant record is currently available. + ParticipantTableFull = 11 } /// @@ -74,7 +79,10 @@ public enum StoreStatus /// The lease has already been released. LeaseAlreadyReleased = 9, - /// Removal was requested while active leases still protect the slot. + /// + /// The key is logically absent, but active leases protect its generation or bounded + /// post-removal classification and physical reclamation work remains incomplete. + /// RemovePending = 10, /// The current platform does not support the requested operation. @@ -107,9 +115,12 @@ public enum StoreStatus /// The supplied key is empty or otherwise invalid. InvalidKey = 20, - /// Shared synchronization was busy for the selected wait policy. + /// + /// The selected wait bound expired: legacy shared synchronization was busy, or a lock-free + /// operation exhausted its bounded local retry, revalidation, helping, or backoff budget. + /// StoreBusy = 21, - /// The operation was canceled before shared synchronization was acquired. + /// The operation was canceled before its documented ordering point. OperationCanceled = 22 } diff --git a/src/SharedMemoryStore/StoreWaitOptions.cs b/src/SharedMemoryStore/StoreWaitOptions.cs index 6701be5..c6d2e39 100644 --- a/src/SharedMemoryStore/StoreWaitOptions.cs +++ b/src/SharedMemoryStore/StoreWaitOptions.cs @@ -4,22 +4,27 @@ namespace SharedMemoryStore; /// -/// Selects how long public store operations may wait for shared synchronization. +/// Bounds work performed by public store operations. Legacy stores apply the bound to shared +/// synchronization; lock-free stores apply it to local retry, revalidation, helping, and backoff. /// /// -/// Maximum time to wait for shared synchronization, for no wait, -/// or for an explicit unbounded wait. +/// Maximum operation wait/work bound, for the minimum safe attempt, +/// or for an explicit unbounded local retry policy. +/// +/// +/// Optional token that cancels the operation when observed before its ordering point. /// -/// Optional token that cancels the wait before synchronization is acquired. public readonly record struct StoreWaitOptions(TimeSpan Timeout, CancellationToken CancellationToken = default) { /// Gets the production default wait policy: one second and no cancellation token. public static StoreWaitOptions Default { get; } = new(TimeSpan.FromSeconds(1)); - /// Gets a policy that returns immediately when shared synchronization is busy. + /// Gets a policy that performs only the minimum safe attempt without optional waiting. public static StoreWaitOptions NoWait { get; } = new(TimeSpan.Zero); - /// Gets a policy that waits indefinitely for callers that intentionally want legacy blocking behavior. + /// + /// Gets an unbounded policy for intentional legacy blocking or lock-free local retry. + /// public static StoreWaitOptions Infinite { get; } = new(System.Threading.Timeout.InfiniteTimeSpan); /// Gets a value indicating whether this policy waits indefinitely. diff --git a/src/SharedMemoryStore/ValueLease.cs b/src/SharedMemoryStore/ValueLease.cs index db1bb8f..427126b 100644 --- a/src/SharedMemoryStore/ValueLease.cs +++ b/src/SharedMemoryStore/ValueLease.cs @@ -1,69 +1,70 @@ +using SharedMemoryStore.Engines; using SharedMemoryStore.Layout; namespace SharedMemoryStore; /// -/// Struct token that protects one published value generation and projects read-only spans over shared memory. +/// Shared read-lease token that protects one published value generation and projects read-only +/// spans over shared memory. Several leases may protect the same generation. Borrowed spans remain +/// valid only until this exact lease is released or recovered, or until the local store is disposed. /// public readonly struct ValueLease : IDisposable { private readonly MemoryStore? _store; - private readonly int _slotIndex; - private readonly SlotLifecycleId _lifecycleId; - private readonly int _leaseRecordId; + private readonly LeaseHandle _handle; - internal ValueLease(MemoryStore store, int slotIndex, SlotLifecycleId lifecycleId, int leaseRecordId) + internal ValueLease(MemoryStore store, in LeaseHandle handle) { _store = store; - _slotIndex = slotIndex; - _lifecycleId = lifecycleId; - _leaseRecordId = leaseRecordId; + _handle = handle; + } + + internal ValueLease(MemoryStore store, int slotIndex, SlotLifecycleId lifecycleId, int leaseRecordId) + : this(store, store.CreateLegacyLeaseHandle(slotIndex, lifecycleId, leaseRecordId)) + { } /// Gets a value indicating whether this token still references an active lease record. - public bool IsValid => IsActive; + public bool IsValid => _store?.IsLeaseActive(_handle) == true; /// Gets the value span length for the protected slot generation. - public int ValueLength => IsActive ? _store!.GetValueLength(_slotIndex, _lifecycleId) : 0; + public int ValueLength => _store?.GetValueLength(_handle) ?? 0; /// Gets the descriptor span length for the protected slot generation. - public int DescriptorLength => IsActive ? _store!.GetDescriptorLength(_slotIndex, _lifecycleId) : 0; - - /// Gets a read-only span over the protected value bytes. - public ReadOnlySpan ValueSpan => IsActive ? _store!.GetValueSpan(_slotIndex, _lifecycleId) : ReadOnlySpan.Empty; - - /// Gets a read-only span over the protected descriptor bytes. - public ReadOnlySpan DescriptorSpan => IsActive ? _store!.GetDescriptorSpan(_slotIndex, _lifecycleId) : ReadOnlySpan.Empty; + public int DescriptorLength => _store?.GetDescriptorLength(_handle) ?? 0; /// - /// Releases the lease exactly once and returns the deterministic release status. + /// Gets a borrowed read-only span over the protected value bytes, valid until lease release, + /// recovery, or local store disposal. /// - public StoreStatus Release() - { - return _store?.ReleaseLease(_slotIndex, _lifecycleId, _leaseRecordId) ?? StoreStatus.InvalidLease; - } + public ReadOnlySpan ValueSpan => + _store is null ? ReadOnlySpan.Empty : _store.GetValueSpan(_handle); /// - /// Releases the lease exactly once using the supplied wait policy. + /// Gets a borrowed read-only span over the protected descriptor bytes, valid until lease + /// release, recovery, or local store disposal. /// - public StoreStatus Release(StoreWaitOptions waitOptions) - { - return _store?.ReleaseLease(_slotIndex, _lifecycleId, _leaseRecordId, waitOptions) ?? StoreStatus.InvalidLease; - } + public ReadOnlySpan DescriptorSpan => + _store is null ? ReadOnlySpan.Empty : _store.GetDescriptorSpan(_handle); + + /// Releases the lease exactly once and returns the deterministic release status. + public StoreStatus Release() => + _store?.ReleaseLease(_handle, StoreWaitOptions.Default) ?? StoreStatus.InvalidLease; /// - /// Releases the lease on a best-effort basis when it is still active. + /// Releases the lease exactly once using the supplied profile-specific bounded wait policy. /// - public void Dispose() - { - _ = Release(); - } + public StoreStatus Release(StoreWaitOptions waitOptions) => + _store?.ReleaseLease(_handle, waitOptions) ?? StoreStatus.InvalidLease; + + /// Releases the lease on a best-effort basis when it is still active. + public void Dispose() => _ = Release(); - internal int LeaseRecordIdForTesting => _leaseRecordId; + internal int LeaseRecordIdForTesting => MemoryStore.DecodeLegacyLeaseRecordId(_handle); - internal int SlotIndexForTesting => _slotIndex; + internal int SlotIndexForTesting => MemoryStore.DecodeLegacySlotIndex(_handle.SlotBinding); - internal SlotLifecycleId LifecycleIdForTesting => _lifecycleId; + internal SlotLifecycleId LifecycleIdForTesting => MemoryStore.DecodeLegacyLifecycle(_handle); - private bool IsActive => _store?.IsLeaseActive(_slotIndex, _lifecycleId, _leaseRecordId) == true; + internal LeaseHandle HandleForEngine => _handle; } diff --git a/src/cpp/src/internal.hpp b/src/cpp/src/internal.hpp index 5528515..c9916b8 100644 --- a/src/cpp/src/internal.hpp +++ b/src/cpp/src/internal.hpp @@ -25,6 +25,7 @@ static_assert( "SharedMemoryStore mapped layout requires a little-endian target."); constexpr std::int32_t magic = 0x31534D53; +constexpr std::int32_t lock_free_magic = 0x32534D53; constexpr std::int32_t alignment = 8; constexpr std::int32_t store_initializing = 0; diff --git a/src/cpp/src/platform_linux.cpp b/src/cpp/src/platform_linux.cpp index b0eb02c..adcb977 100644 --- a/src/cpp/src/platform_linux.cpp +++ b/src/cpp/src/platform_linux.cpp @@ -16,6 +16,10 @@ #include #include +#if !defined(F_OFD_SETLK) +#define F_OFD_SETLK 37 +#endif + namespace sms::detail { namespace { @@ -24,15 +28,21 @@ using clock_type = std::chrono::steady_clock; class FileState { public: explicit FileState(std::string path) : path_(std::move(path)) { - fd_ = ::open(path_.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (fd_ >= 0) ::fchmod(fd_, 0600); + const auto descriptor = ::open(path_.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + fd_.store(descriptor, std::memory_order_release); + if (descriptor >= 0) ::fchmod(descriptor, 0600); + } + ~FileState() { retire(); } + int fd() const noexcept { return fd_.load(std::memory_order_acquire); } + bool usable() const noexcept { return fd() >= 0; } + void retire() noexcept { + const auto descriptor = fd_.exchange(-1, std::memory_order_acq_rel); + if (descriptor >= 0) ::close(descriptor); } - ~FileState() { if (fd_ >= 0) ::close(fd_); } - int fd() const noexcept { return fd_; } std::timed_mutex mutex; private: std::string path_; - int fd_{-1}; + std::atomic fd_{-1}; }; std::mutex file_states_gate; @@ -43,7 +53,7 @@ std::shared_ptr get_file_state(const std::string& raw_path) { auto path = std::filesystem::absolute(raw_path, error).lexically_normal().string(); if (error) path = raw_path; std::lock_guard guard(file_states_gate); - if (auto found = file_states[path].lock()) return found; + if (auto found = file_states[path].lock(); found && found->usable()) return found; auto created = std::make_shared(path); file_states[path] = created; return created; @@ -54,7 +64,7 @@ class LinuxFileLock final : public SharedLock { explicit LinuxFileLock(std::shared_ptr state) : state_(std::move(state)) {} ~LinuxFileLock() override { release(); } - bool usable() const noexcept { return state_ && state_->fd() >= 0; } + bool usable() const noexcept { return state_ && state_->usable(); } sms_status acquire(const Wait& wait) noexcept override { if (!usable()) return errno == EACCES ? SMS_STATUS_ACCESS_DENIED : SMS_STATUS_UNKNOWN_FAILURE; @@ -69,6 +79,10 @@ class LinuxFileLock final : public SharedLock { local_held_ = state_->mutex.try_lock_for(std::chrono::milliseconds(wait.milliseconds)); } if (!local_held_) return SMS_STATUS_STORE_BUSY; + if (!usable()) { + release(); + return SMS_STATUS_UNKNOWN_FAILURE; + } for (;;) { struct flock request{}; @@ -76,7 +90,7 @@ class LinuxFileLock final : public SharedLock { request.l_whence = SEEK_SET; request.l_start = 0; request.l_len = 1; - if (::fcntl(state_->fd(), F_SETLK, &request) == 0) { + if (::fcntl(state_->fd(), F_OFD_SETLK, &request) == 0) { held_ = true; return SMS_STATUS_SUCCESS; } @@ -84,7 +98,9 @@ class LinuxFileLock final : public SharedLock { if (error != EACCES && error != EAGAIN) { release(); if (error == EACCES || error == EPERM) return SMS_STATUS_ACCESS_DENIED; - if (error == ENOSYS || error == ENOTSUP) return SMS_STATUS_UNSUPPORTED_PLATFORM; + if (error == EINVAL || error == ENOSYS || error == ENOTSUP || error == EOPNOTSUPP) { + return SMS_STATUS_UNSUPPORTED_PLATFORM; + } return SMS_STATUS_UNKNOWN_FAILURE; } if (!wait.infinite()) { @@ -108,12 +124,21 @@ class LinuxFileLock final : public SharedLock { request.l_whence = SEEK_SET; request.l_start = 0; request.l_len = 1; - ::fcntl(state_->fd(), F_SETLK, &request); + int result{}; + do { + result = ::fcntl(state_->fd(), F_OFD_SETLK, &request); + } while (result != 0 && errno == EINTR); + if (result != 0) { + // Closing a descriptor releases every OFD lock attached to it. + // Retire before the local gate is reopened so local work never + // proceeds while foreign participants remain excluded. + state_->retire(); + } held_ = false; } if (local_held_) { - state_->mutex.unlock(); local_held_ = false; + state_->mutex.unlock(); } } @@ -235,7 +260,8 @@ bool write_owners(const std::string& path, const std::vector& owner void delete_stale(const ResourceName& resource) noexcept { ::unlink(resource.linux_region_path.c_str()); - ::unlink(resource.linux_lock_path.c_str()); + // Keep the operation-lock inode as a permanent rendezvous, matching the + // lifecycle inode. It has no generation state and prevents pathname split. ::unlink(resource.linux_owners_path.c_str()); ::unlink((resource.linux_owners_path + ".tmp").c_str()); } diff --git a/src/cpp/src/platform_windows.cpp b/src/cpp/src/platform_windows.cpp index f258a43..a380eb2 100644 --- a/src/cpp/src/platform_windows.cpp +++ b/src/cpp/src/platform_windows.cpp @@ -110,8 +110,13 @@ PlatformOpenResult platform_open(const ResourceName& resource, const Options& op } } + // Map the existing section's actual extent. Requesting the caller-computed + // layout-v1.2 length here can fail before Store::initialize_or_validate can + // inspect an SMS2 header when the v1 request is larger than the v2 region. + // A zero byte count is the Windows API's header-first/full-section form; + // the store validates identity and dimensions before projecting layout data. auto* data = static_cast(MapViewOfFile( - mapping, FILE_MAP_ALL_ACCESS, 0, 0, static_cast(options.total_bytes))); + mapping, FILE_MAP_ALL_ACCESS, 0, 0, 0)); if (!data) { const auto error = GetLastError(); CloseHandle(mapping); diff --git a/src/cpp/src/store.cpp b/src/cpp/src/store.cpp index 41f9de3..d2f6dd1 100644 --- a/src/cpp/src/store.cpp +++ b/src/cpp/src/store.cpp @@ -123,6 +123,10 @@ sms_open_status Store::initialize_or_validate(const Options& options) noexcept { initialize_header(); return SMS_OPEN_SUCCESS; } + // Native ABI 1.0 is intentionally layout-v1.2-only. Recognize SMS2 solely + // to reject it before any v1 directory, slot, lease, descriptor, or payload + // address is calculated. + if (h.Magic == lock_free_magic) return SMS_OPEN_INCOMPATIBLE_LAYOUT; if (h.Magic != magic || h.LayoutMajorVersion != SMS_LAYOUT_MAJOR_VERSION || !layout_.matches(h) || !layout_.bounds_valid(h)) { return SMS_OPEN_INCOMPATIBLE_LAYOUT; @@ -901,9 +905,11 @@ sms_status Store::get_layout(const Wait& wait, Layout& result) noexcept { void Store::close() noexcept { if (closed_.exchange(true, std::memory_order_acq_rel)) return; gate_.lock(); + // Linux region close can enter lifecycle cleanup. Retire the ordinary + // lock descriptor before that cleanup can observe a final owner. + lock_.reset(); if (region_) region_->close(); region_.reset(); - lock_.reset(); gate_.unlock(); } diff --git a/tests/SharedMemoryStore.ContractTests/DiagnosticsContractTests.cs b/tests/SharedMemoryStore.ContractTests/DiagnosticsContractTests.cs index 627253d..af44dcc 100644 --- a/tests/SharedMemoryStore.ContractTests/DiagnosticsContractTests.cs +++ b/tests/SharedMemoryStore.ContractTests/DiagnosticsContractTests.cs @@ -24,4 +24,117 @@ public void DiagnosticSnapshotDistinguishesIndexStatesAndProbeCounters() Assert.True(diagnostics.MaxObservedProbeLength >= diagnostics.LastObservedProbeLength); ReliabilityAssertions.AssertIndexHealthAddsUp(diagnostics); } + + [Fact] + public void DisposedLegacyDiagnosticsRetainTheHistoricalSnapshotContract() + { + MemoryStore store = ContractStoreFactory.Create( + ContractStoreFactory.Options(slotCount: 4, maxKeyBytes: 8, leaseRecordCount: 4)); + try + { + for (byte value = 1; value <= 4; value++) + { + Assert.Equal(StoreStatus.Success, store.TryPublish([value], [value])); + } + + for (byte value = 1; value <= 3; value++) + { + Assert.Equal(StoreStatus.Success, store.TryRemove([value])); + } + + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([0x7f], out _)); + DiagnosticsSnapshot live = store.GetDiagnostics(); + Assert.True(live.IndexCompactionCount > 0); + + store.Dispose(); + + StoreStatus status = store.TryGetDiagnostics(out DiagnosticsSnapshot disposed); + + Assert.Equal(StoreStatus.StoreDisposed, status); + Assert.Equal(StoreProfile.Legacy, disposed.Profile); + Assert.Equal(live.ProtocolInfo, disposed.ProtocolInfo); + Assert.Equal(live.TotalBytes, disposed.TotalBytes); + Assert.Equal(live.SlotCount, disposed.SlotCount); + Assert.Equal(live.IndexEntryCount, disposed.IndexEntryCount); + Assert.Equal(live.IndexCompactionCount, disposed.IndexCompactionCount); + Assert.Equal(live.GetFailureCount(StoreStatus.NotFound), disposed.GetFailureCount(StoreStatus.NotFound)); + Assert.Equal(live.GetFailureCount(StoreStatus.StoreDisposed), disposed.GetFailureCount(StoreStatus.StoreDisposed)); + Assert.Equal(live.LastFailureStatus, disposed.LastFailureStatus); + Assert.Equal(0, disposed.FreeSlotCount); + Assert.Equal(0, disposed.PublishedSlotCount); + Assert.Equal(0, disposed.OccupiedIndexEntryCount); + Assert.Equal(0, disposed.UsableIndexCapacity); + + DiagnosticsSnapshot formatted = store.GetDiagnostics(); + + Assert.Equal(disposed.TotalBytes, formatted.TotalBytes); + Assert.Equal(disposed.SlotCount, formatted.SlotCount); + Assert.Equal(disposed.IndexEntryCount, formatted.IndexEntryCount); + Assert.Equal(disposed.IndexCompactionCount, formatted.IndexCompactionCount); + Assert.Equal( + disposed.GetFailureCount(StoreStatus.StoreDisposed) + 1, + formatted.GetFailureCount(StoreStatus.StoreDisposed)); + Assert.Equal(StoreStatus.StoreDisposed, formatted.LastFailureStatus); + } + finally + { + store.Dispose(); + } + } + + [Fact] + public void DisposedLockFreeDiagnosticsUseCachedMetricsAndLiveLocalCounters() + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-disposed-diagnostics-{Guid.NewGuid():N}", + slotCount: 3, + maxValueBytes: 8, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: 4, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + MemoryStore store = ContractStoreFactory.Create(options); + try + { + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + Assert.Equal(StoreStatus.DuplicateKey, store.TryPublish([1], [2])); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([2], out _)); + DiagnosticsSnapshot live = store.GetDiagnostics(); + + store.Dispose(); + + StoreStatus status = store.TryGetDiagnostics(out DiagnosticsSnapshot disposed); + + Assert.Equal(StoreStatus.StoreDisposed, status); + Assert.Equal(StoreProfile.LockFree, disposed.Profile); + Assert.Equal(live.ProtocolInfo, disposed.ProtocolInfo); + Assert.Equal(live.TotalBytes, disposed.TotalBytes); + Assert.Equal(live.SlotCount, disposed.SlotCount); + Assert.Equal(live.ParticipantRecordCount, disposed.ParticipantRecordCount); + Assert.Equal(live.IndexEntryCount, disposed.IndexEntryCount); + Assert.Equal(live.PublishedSlotCount, disposed.PublishedSlotCount); + Assert.Equal(live.PrimaryDirectoryOccupancy, disposed.PrimaryDirectoryOccupancy); + Assert.Equal(live.GetFailureCount(StoreStatus.DuplicateKey), disposed.GetFailureCount(StoreStatus.DuplicateKey)); + Assert.Equal(live.GetFailureCount(StoreStatus.NotFound), disposed.GetFailureCount(StoreStatus.NotFound)); + Assert.Equal(live.GetFailureCount(StoreStatus.StoreDisposed), disposed.GetFailureCount(StoreStatus.StoreDisposed)); + Assert.Equal(live.LastFailureStatus, disposed.LastFailureStatus); + + DiagnosticsSnapshot formatted = store.GetDiagnostics(); + + Assert.Equal(disposed.ProtocolInfo, formatted.ProtocolInfo); + Assert.Equal(disposed.TotalBytes, formatted.TotalBytes); + Assert.Equal(disposed.SlotCount, formatted.SlotCount); + Assert.Equal(disposed.IndexEntryCount, formatted.IndexEntryCount); + Assert.Equal( + disposed.GetFailureCount(StoreStatus.StoreDisposed) + 1, + formatted.GetFailureCount(StoreStatus.StoreDisposed)); + Assert.Equal(StoreStatus.StoreDisposed, formatted.LastFailureStatus); + } + finally + { + store.Dispose(); + } + } } diff --git a/tests/SharedMemoryStore.ContractTests/LockFreeDiagnosticsContractTests.cs b/tests/SharedMemoryStore.ContractTests/LockFreeDiagnosticsContractTests.cs new file mode 100644 index 0000000..28845c6 --- /dev/null +++ b/tests/SharedMemoryStore.ContractTests/LockFreeDiagnosticsContractTests.cs @@ -0,0 +1,347 @@ +using System.Reflection; +using SharedMemoryStore.Diagnostics; +using SharedMemoryStore.Interop; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.ContractTests; + +public sealed class LockFreeDiagnosticsContractTests +{ + [Fact] + public void SnapshotSurfaceAddsV2PressureAndRecoverySignalsWithoutRemovingLegacyMembers() + { + Type snapshot = typeof(DiagnosticsSnapshot); + string[] legacyProperties = + [ + nameof(DiagnosticsSnapshot.TotalBytes), + nameof(DiagnosticsSnapshot.SlotCount), + nameof(DiagnosticsSnapshot.FreeSlotCount), + nameof(DiagnosticsSnapshot.PublishedSlotCount), + nameof(DiagnosticsSnapshot.PendingRemovalCount), + nameof(DiagnosticsSnapshot.ActiveLeaseCount), + nameof(DiagnosticsSnapshot.ActiveReservationCount), + nameof(DiagnosticsSnapshot.IndexEntryCount), + nameof(DiagnosticsSnapshot.TombstoneIndexEntryCount), + nameof(DiagnosticsSnapshot.LastFailureStatus) + ]; + string[] v2Properties = + [ + "Profile", + "ProtocolInfo", + "InitializingSlotCount", + "ReservedSlotCount", + "ReclaimingSlotCount", + "RetiredSlotCount", + "ClaimingLeaseCount", + "RecoveringLeaseCount", + "FreeLeaseCount", + "RetiredLeaseCount", + "ParticipantRecordCount", + "FreeParticipantCount", + "RegisteringParticipantCount", + "ActiveParticipantCount", + "ClosingParticipantCount", + "RecoveringParticipantCount", + "ReclaimingParticipantCount", + "RetiredParticipantCount", + "IsParticipantTableExhausted", + "PrimaryDirectoryOccupancy", + "SpilledBucketCount", + "OverflowDirectoryOccupancy", + "OverflowScanCount", + "MaxObservedOverflowScanLength", + "CasRetryCount", + "HelpedTransitionCount", + "ContentionBudgetExhaustionCount", + "InvalidTokenCount", + "StaleTokenCount", + "RecoveryAttemptCount", + "RecoveredTransitionCount", + "CurrentOwnerClassificationCount", + "LiveOwnerClassificationCount", + "StaleOwnerClassificationCount", + "UnsupportedOwnerClassificationCount", + "InconsistentOwnerClassificationCount", + "ChangingOwnerClassificationCount" + ]; + + Assert.All(legacyProperties, name => AssertReadableProperty(snapshot, name)); + Assert.All(v2Properties, name => AssertReadableProperty(snapshot, name)); + Assert.Equal(typeof(StoreProfile), snapshot.GetProperty("Profile")!.PropertyType); + Assert.Equal(typeof(StoreProtocolInfo), snapshot.GetProperty("ProtocolInfo")!.PropertyType); + Assert.Equal(typeof(bool), snapshot.GetProperty("IsParticipantTableExhausted")!.PropertyType); + } + + [Fact] + public void BoundedScannerReportsSlotLeaseParticipantAndSpillOccupancy() + { + const int slotCount = 17; + const int participantCount = 2; + string name = $"sms-v2-diagnostics-scan-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions create = Options( + name, + OpenMode.CreateNew, + slotCount, + participantCount); + using MemoryStore first = Open(create); + StoreLayoutV2 layout = StoreLayoutV2.FromOptions(create); + IReadOnlyList collidingKeys = FindCollidingKeys( + layout.PrimaryBucketCount - 1, + slotCount); + + foreach (byte[] key in collidingKeys) + { + Assert.Equal(StoreStatus.Success, first.TryPublish(key, [1])); + } + + Assert.Equal(StoreStatus.Success, first.TryAcquire(collidingKeys[0], out ValueLease lease)); + using MemoryStore second = Open(Options( + name, + OpenMode.OpenExisting, + slotCount, + participantCount)); + LockFreeDiagnostics diagnostics = CreateDiagnostics(first, create); + + DiagnosticsSnapshot snapshot = diagnostics.CreateSnapshot(); + + Assert.Equal(StoreProfile.LockFree, snapshot.Profile); + Assert.Equal(first.ProtocolInfo, snapshot.ProtocolInfo); + Assert.Equal(slotCount, snapshot.SlotCount); + Assert.Equal(slotCount, snapshot.PublishedSlotCount); + Assert.Equal(1, snapshot.ActiveLeaseCount); + Assert.Equal(2, snapshot.ActiveParticipantCount); + Assert.Equal(0, snapshot.FreeParticipantCount); + Assert.True(snapshot.IsParticipantTableExhausted); + Assert.Equal(16, snapshot.PrimaryDirectoryOccupancy); + Assert.True(snapshot.SpilledBucketCount >= 1); + Assert.Equal(1, snapshot.OverflowDirectoryOccupancy); + Assert.Equal( + snapshot.SlotCount, + snapshot.FreeSlotCount + + snapshot.InitializingSlotCount + + snapshot.ReservedSlotCount + + snapshot.PublishedSlotCount + + snapshot.PendingRemovalCount + + snapshot.ReclaimingSlotCount + + snapshot.RetiredSlotCount); + Assert.Equal( + snapshot.ParticipantRecordCount, + snapshot.FreeParticipantCount + + snapshot.RegisteringParticipantCount + + snapshot.ActiveParticipantCount + + snapshot.ClosingParticipantCount + + snapshot.RecoveringParticipantCount + + snapshot.ReclaimingParticipantCount + + snapshot.RetiredParticipantCount); + + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public void LocalCountersExposeRetryHelpTokenAndRecoveryPressureWithoutMutatingOwnership() + { + SharedMemoryStoreOptions options = Options( + $"sms-v2-diagnostics-counters-{Guid.NewGuid():N}", + OpenMode.CreateNew, + slotCount: 3, + participantCount: 2); + using MemoryStore store = Open(options); + LockFreeDiagnostics diagnostics = CreateDiagnostics(store, options); + + Assert.Equal(StoreStatus.StoreBusy, diagnostics.RecordStatus(StoreStatus.StoreBusy)); + Assert.Equal(StoreStatus.StoreFull, diagnostics.RecordStatus(StoreStatus.StoreFull)); + diagnostics.RecordOverflowScan(scannedCellCount: 7); + diagnostics.RecordOverflowScan(scannedCellCount: 3); + diagnostics.RecordCasRetry(4); + diagnostics.RecordHelpedTransition(2); + diagnostics.RecordInvalidToken(stale: false); + diagnostics.RecordInvalidToken(stale: true); + diagnostics.RecordRecoveryAttempt(3); + diagnostics.RecordRecoveredTransition(2); + diagnostics.RecordOwnerClassification(ParticipantClassificationKind.CurrentProcess); + diagnostics.RecordOwnerClassification(ParticipantClassificationKind.Live); + diagnostics.RecordOwnerClassification(ParticipantClassificationKind.Stale); + diagnostics.RecordOwnerClassification(ParticipantClassificationKind.Unsupported); + diagnostics.RecordOwnerClassification(ParticipantClassificationKind.Inconsistent); + diagnostics.RecordOwnerClassification(ParticipantClassificationKind.Changing); + + DiagnosticsSnapshot snapshot = diagnostics.CreateSnapshot(); + + Assert.Equal(2, snapshot.OverflowScanCount); + Assert.Equal(7, snapshot.MaxObservedOverflowScanLength); + Assert.Equal(4, snapshot.CasRetryCount); + Assert.Equal(2, snapshot.HelpedTransitionCount); + Assert.Equal(1, snapshot.ContentionBudgetExhaustionCount); + Assert.Equal(1, snapshot.InvalidTokenCount); + Assert.Equal(1, snapshot.StaleTokenCount); + Assert.Equal(3, snapshot.RecoveryAttemptCount); + Assert.Equal(2, snapshot.RecoveredTransitionCount); + Assert.Equal(1, snapshot.CurrentOwnerClassificationCount); + Assert.Equal(1, snapshot.LiveOwnerClassificationCount); + Assert.Equal(1, snapshot.StaleOwnerClassificationCount); + Assert.Equal(1, snapshot.UnsupportedOwnerClassificationCount); + Assert.Equal(1, snapshot.InconsistentOwnerClassificationCount); + Assert.Equal(1, snapshot.ChangingOwnerClassificationCount); + Assert.Equal(1, snapshot.CapacityPressureCount); + Assert.Equal(1, snapshot.GetFailureCount(StoreStatus.StoreBusy)); + Assert.Equal(1, snapshot.GetFailureCount(StoreStatus.StoreFull)); + } + + [Fact] + public async Task ConcurrentOverflowScanMaximumIsMonotonic() + { + for (var round = 0; round < 256; round++) + { + var telemetry = new LockFreeTelemetry(); + using var start = new Barrier(participantCount: 3); + Task maximum = Task.Run(() => + { + start.SignalAndWait(); + telemetry.RecordOverflowScan(4_096); + }); + Task contender = Task.Run(() => + { + start.SignalAndWait(); + telemetry.RecordOverflowScan(4_095); + }); + + start.SignalAndWait(); + await Task.WhenAll(maximum, contender); + Assert.Equal(4_096, telemetry.MaxObservedOverflowScanLength); + } + } + + [Fact] + public void LegacySnapshotRetainsExistingValuesAndMarksV2OnlySignalsNotApplicable() + { + using MemoryStore store = ContractStoreFactory.Create( + ContractStoreFactory.Options(slotCount: 2)); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + + DiagnosticsSnapshot snapshot = store.GetDiagnostics(); + + Assert.Equal(StoreProfile.Legacy, snapshot.Profile); + Assert.Equal(new StoreProtocolInfo(StoreProfile.Legacy, 1, 2, 1, 0, 0), snapshot.ProtocolInfo); + Assert.Equal(1, snapshot.PublishedSlotCount); + Assert.Equal(0, snapshot.PrimaryDirectoryOccupancy); + Assert.Equal(0, snapshot.SpilledBucketCount); + Assert.Equal(0, snapshot.OverflowDirectoryOccupancy); + Assert.Equal(0, snapshot.ParticipantRecordCount); + Assert.False(snapshot.IsParticipantTableExhausted); + Assert.Equal(0, snapshot.CasRetryCount); + Assert.Equal(0, snapshot.HelpedTransitionCount); + } + + [Fact] + public void PublicLockFreeDiagnosticsExposeTheOpenedProfileAndCurrentOccupancy() + { + SharedMemoryStoreOptions options = Options( + $"sms-v2-diagnostics-public-{Guid.NewGuid():N}", + OpenMode.CreateNew, + slotCount: 3, + participantCount: 2); + using MemoryStore store = Open(options); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + + StoreStatus status = store.TryGetDiagnostics(out DiagnosticsSnapshot snapshot); + + Assert.Equal(StoreStatus.Success, status); + Assert.Equal(StoreProfile.LockFree, snapshot.Profile); + Assert.Equal(store.ProtocolInfo, snapshot.ProtocolInfo); + Assert.Equal(1, snapshot.PublishedSlotCount); + Assert.Equal(1, snapshot.ActiveParticipantCount); + Assert.Equal(1, snapshot.FreeParticipantCount); + } + + private static void AssertReadableProperty(Type type, string name) + { + PropertyInfo? property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.Public); + Assert.True(property is not null, $"Required additive diagnostics property '{name}' is missing."); + Assert.NotNull(property!.GetMethod); + Assert.True(property.GetMethod!.IsPublic); + } + + private static LockFreeDiagnostics CreateDiagnostics( + MemoryStore store, + SharedMemoryStoreOptions options) + { + object? engineValue = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store); + object engine = Assert.IsAssignableFrom(engineValue); + var region = Assert.IsType( + engine.GetType() + .GetField("_region", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + return new LockFreeDiagnostics( + region, + StoreLayoutV2.FromOptions(options), + store.ProtocolInfo); + } + + private static MemoryStore Open(SharedMemoryStoreOptions options) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options( + string name, + OpenMode mode, + int slotCount, + int participantCount) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 8, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: Math.Max(4, slotCount), + participantRecordCount: participantCount, + openMode: mode, + enableLeaseRecovery: true); + + private static IReadOnlyList FindCollidingKeys(int bucketMask, int required) + { + var groups = new Dictionary<(int First, int Second), List>(); + Span key = stackalloc byte[4]; + for (var value = 0; value < 1_000_000; value++) + { + BitConverter.TryWriteBytes(key, value); + ulong hash = StoreKey.Hash(key); + int first = (int)(Mix(hash) & (uint)bucketMask); + int second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & (uint)bucketMask); + if (second == first) + { + second = (first + 1) & bucketMask; + } + + var pair = (first, second); + if (!groups.TryGetValue(pair, out List? keys)) + { + keys = []; + groups.Add(pair, keys); + } + + keys.Add(key.ToArray()); + if (keys.Count == required) + { + return keys; + } + } + + throw new InvalidOperationException("Unable to generate the required colliding key set."); + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } +} diff --git a/tests/SharedMemoryStore.ContractTests/LockFreeLayoutContractTests.cs b/tests/SharedMemoryStore.ContractTests/LockFreeLayoutContractTests.cs new file mode 100644 index 0000000..c7e1c90 --- /dev/null +++ b/tests/SharedMemoryStore.ContractTests/LockFreeLayoutContractTests.cs @@ -0,0 +1,1336 @@ +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.InteropServices; +using System.Text.Json; + +namespace SharedMemoryStore.ContractTests; + +/// +/// Freezes the language-neutral layout-2.0 wire contract without taking a +/// compile-time dependency on the implementation types. The reflection is +/// intentional: this test is introduced before the v2 layout and must compile +/// while failing specifically because that layout is not implemented yet. +/// +public sealed class LockFreeLayoutContractTests +{ + private const long MaximumDirectoryTargetIndex = (1L << 22) - 1; + private const long MaximumSlotGeneration = (1L << 33) - 1; + private const int MaximumLockFreeSlotCount = 1_048_575; + private static readonly Assembly StoreAssembly = typeof(MemoryStore).Assembly; + + [Fact] + public void PublishedV2ManifestMatchesTheExecutableRecordContract() + { + var root = new DirectoryInfo(AppContext.BaseDirectory); + while (root is not null && !File.Exists(Path.Combine(root.FullName, "SharedMemoryStore.slnx"))) + { + root = root.Parent; + } + + Assert.NotNull(root); + var path = Path.Combine(root!.FullName, "protocol", "fixtures", "v2.0", "manifest.json"); + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var protocol = document.RootElement.GetProperty("protocol"); + var records = document.RootElement.GetProperty("records"); + var directoryLocation = document.RootElement.GetProperty("directory_location"); + var directoryOperation = document.RootElement.GetProperty("directory_operation"); + var publicationIntent = document.RootElement.GetProperty("publication_intent"); + var pidNamespaceIdentity = document.RootElement.GetProperty("pid_namespace_identity"); + var sizing = document.RootElement.GetProperty("sizing"); + + Assert.Equal(2, protocol.GetProperty("layout_major").GetInt32()); + Assert.Equal(0, protocol.GetProperty("layout_minor").GetInt32()); + Assert.Equal(2, protocol.GetProperty("resource_protocol").GetInt32()); + Assert.Equal(7UL, protocol.GetProperty("required_features").GetUInt64()); + Assert.Equal( + 1UL, + protocol.GetProperty("required_feature_bits") + .GetProperty("versioned_empty_spill_summary") + .GetUInt64()); + Assert.Equal( + 2UL, + protocol.GetProperty("required_feature_bits") + .GetProperty("publication_intent") + .GetUInt64()); + Assert.Equal( + 4UL, + protocol.GetProperty("required_feature_bits") + .GetProperty("pid_namespace_identity") + .GetUInt64()); + Assert.Equal("32534d53", protocol.GetProperty("magic_integer_hex").GetString()); + Assert.Equal(512, records.GetProperty("store_header").GetProperty("size").GetInt32()); + Assert.Equal( + 264, + records.GetProperty("store_header") + .GetProperty("fields") + .GetProperty("pid_namespace_id") + .GetInt32()); + Assert.Equal( + 272, + records.GetProperty("store_header") + .GetProperty("fields") + .GetProperty("pid_namespace_mode") + .GetInt32()); + Assert.Equal(64, records.GetProperty("participant").GetProperty("size").GetInt32()); + Assert.Equal( + 32, + records.GetProperty("participant") + .GetProperty("fields") + .GetProperty("pid_namespace_id") + .GetInt32()); + Assert.Equal(128, records.GetProperty("primary_directory_bucket").GetProperty("size").GetInt32()); + Assert.Equal(64, records.GetProperty("lease").GetProperty("size").GetInt32()); + Assert.Equal(128, records.GetProperty("value_slot").GetProperty("size").GetInt32()); + Assert.Equal( + 52, + records.GetProperty("value_slot") + .GetProperty("fields") + .GetProperty("publication_intent") + .GetInt32()); + + Assert.Equal(52, publicationIntent.GetProperty("field_offset").GetInt32()); + Assert.Equal(0, publicationIntent.GetProperty("values").GetProperty("none").GetInt32()); + Assert.Equal( + 1, + publicationIntent.GetProperty("values").GetProperty("explicit_reservation").GetInt32()); + Assert.Equal( + 2, + publicationIntent.GetProperty("values").GetProperty("atomic_publication").GetInt32()); + Assert.Equal(264, pidNamespaceIdentity.GetProperty("header_id_offset").GetInt32()); + Assert.Equal(272, pidNamespaceIdentity.GetProperty("header_mode_offset").GetInt32()); + Assert.Equal(32, pidNamespaceIdentity.GetProperty("participant_id_offset").GetInt32()); + Assert.Equal( + 1, + pidNamespaceIdentity.GetProperty("modes").GetProperty("recovery_enabled").GetInt32()); + Assert.Equal( + 2, + pidNamespaceIdentity.GetProperty("modes").GetProperty("mixed_or_unproven").GetInt32()); + + AssertJsonBitRange(directoryLocation, "kind_bits", 0, 1); + AssertJsonBitRange(directoryLocation, "index_bits", 2, 23); + AssertJsonBitRange(directoryLocation, "slot_generation_bits", 24, 56); + AssertJsonBitRange(directoryLocation, "reserved_bits", 57, 63); + + AssertJsonBitRange(directoryOperation, "intent_bits", 0, 1); + AssertJsonBitRange(directoryOperation, "phase_bits", 2, 4); + AssertJsonBitRange(directoryOperation, "target_kind_bits", 5, 6); + AssertJsonBitRange(directoryOperation, "target_index_bits", 7, 28); + AssertJsonBitRange(directoryOperation, "slot_generation_bits", 29, 61); + AssertJsonBitRange(directoryOperation, "reserved_bits", 62, 63); + + Assert.Equal(1, sizing.GetProperty("slot_count_min").GetInt32()); + Assert.Equal(MaximumLockFreeSlotCount, sizing.GetProperty("slot_count_max").GetInt32()); + } + + [Theory] + [InlineData(64, 7, 21, 0x7f, 0x1f_ffff)] + [InlineData(1, 1, 27, 0x1, 0x7ff_ffff)] + [InlineData(1_048_575, 20, 8, 0x0f_ffff, 0xff)] + public void ParticipantTokenSplitAndMasksAreDerivedFromConfiguredCapacity( + int participantCount, + int expectedIndexBits, + int expectedGenerationBits, + int expectedIndexMask, + int expectedGenerationMask) + { + object layout = CreateLayout(participantCount: participantCount); + + Assert.Equal(expectedIndexBits, GetInt64(layout, "ParticipantIndexBits", "ParticipantTokenIndexBits")); + Assert.Equal(expectedGenerationBits, GetInt64(layout, "ParticipantGenerationBits", "ParticipantTokenGenerationBits")); + Assert.Equal(expectedIndexMask, GetInt64(layout, "ParticipantIndexMask", "ParticipantTokenIndexMask")); + Assert.Equal(expectedGenerationMask, GetInt64(layout, "ParticipantGenerationMask", "ParticipantTokenGenerationMask")); + } + + [Fact] + public void DefaultParticipantTokenEncodesIndexPlusOneAndRetiresAtConfiguredTerminalGeneration() + { + object layout = CreateLayout(participantCount: 64); + Type tokenType = RequireType("SharedMemoryStore.LockFree.ParticipantToken"); + const int recordIndex = 63; + const int terminalGeneration = 0x1f_ffff; + + ulong encoded = Encode( + tokenType, + new Dictionary + { + ["recordIndex"] = recordIndex, + ["index"] = recordIndex, + ["generation"] = terminalGeneration, + ["incarnation"] = terminalGeneration, + ["participantCount"] = 64, + ["indexBits"] = 7 + }); + + Assert.Equal(((ulong)terminalGeneration << 7) | 64UL, encoded); + Assert.Equal(terminalGeneration, GetInt64(layout, "ParticipantGenerationMask", "ParticipantTokenGenerationMask")); + + Exception error = Assert.ThrowsAny(() => Encode( + tokenType, + new Dictionary + { + ["recordIndex"] = recordIndex, + ["index"] = recordIndex, + ["generation"] = terminalGeneration + 1, + ["incarnation"] = terminalGeneration + 1, + ["participantCount"] = 64, + ["indexBits"] = 7 + })); + + Assert.IsAssignableFrom(Unwrap(error)); + } + + [Theory] + [InlineData(0, 1, 0x0000_0000_8000_0001UL)] + [InlineData(5, 9, 0x0000_0004_8000_0006UL)] + [InlineData(2_147_483_646, 8_589_934_591, ulong.MaxValue)] + public void IndexBindingCodecUsesThirtyOneIndexBitsAndThirtyThreeGenerationBits( + int slotIndex, + long generation, + ulong expected) + { + Type bindingType = RequireType("SharedMemoryStore.LockFree.IndexBinding"); + + ulong encoded = Encode( + bindingType, + new Dictionary + { + ["slotIndex"] = slotIndex, + ["index"] = slotIndex, + ["generation"] = generation + }); + + Assert.Equal(expected, encoded); + AssertDecoded(bindingType, encoded, ("slotIndex", slotIndex), ("generation", generation)); + } + + [Theory] + [InlineData(0, 1, 0x0020_0000_0010_0001UL, 0x0000_0000_0010_0001UL)] + [InlineData(5, 9, 0x0020_0000_0090_0006UL, 0x0000_0000_0090_0006UL)] + [InlineData(1_048_574, 8_589_934_591, 0x003f_ffff_ffff_ffffUL, 0x001f_ffff_ffff_ffffUL)] + public void SpillSummaryCodecCarriesTwentyBitIndexThirtyThreeBitGenerationAndVersionedEmpty( + int slotIndex, + long generation, + ulong expectedPresent, + ulong expectedEmpty) + { + Type bindingType = RequireType("SharedMemoryStore.LockFree.IndexBinding"); + Type summaryType = RequireType("SharedMemoryStore.LockFree.SpillSummary"); + ulong binding = Encode( + bindingType, + new Dictionary + { + ["slotIndex"] = slotIndex, + ["index"] = slotIndex, + ["generation"] = generation + }); + var values = new Dictionary { ["binding"] = binding }; + ulong present = EncodeNamed(summaryType, "Present", values); + ulong empty = EncodeNamed(summaryType, "Empty", values); + object presentDecoded = Decode(summaryType, present); + object emptyDecoded = Decode(summaryType, empty); + + Assert.Equal(expectedPresent, present); + Assert.Equal(expectedEmpty, empty); + Assert.Equal(1, GetInt64(presentDecoded, "IsPresent")); + Assert.Equal(0, GetInt64(emptyDecoded, "IsPresent")); + Assert.Equal(binding, GetUInt64(presentDecoded, "Binding")); + Assert.Equal(binding, GetUInt64(emptyDecoded, "Binding")); + Assert.Equal(empty, GetUInt64(presentDecoded, "EmptyValue")); + } + + [Fact] + public void SpillSummaryRejectsMalformedAndReservedTokensAndNeverReturnsToInitialZero() + { + Type bindingType = RequireType("SharedMemoryStore.LockFree.IndexBinding"); + Type summaryType = RequireType("SharedMemoryStore.LockFree.SpillSummary"); + ulong firstBinding = Encode( + bindingType, + new Dictionary { ["slotIndex"] = 0, ["index"] = 0, ["generation"] = 1L }); + ulong laterBinding = Encode( + bindingType, + new Dictionary { ["slotIndex"] = 0, ["index"] = 0, ["generation"] = 2L }); + ulong presentFirst = EncodeNamed( + summaryType, + "Present", + new Dictionary { ["binding"] = firstBinding }); + ulong emptyFirst = GetUInt64(Decode(summaryType, presentFirst), "EmptyValue"); + ulong presentLater = EncodeNamed( + summaryType, + "Present", + new Dictionary { ["binding"] = laterBinding }); + + Assert.Equal(0UL, GetUInt64(Decode(summaryType, 0), "Value")); + Assert.NotEqual(0UL, emptyFirst); + Assert.Equal(3, new HashSet { presentFirst, emptyFirst, presentLater }.Count); + AssertDecodeRejected(summaryType, 1UL << 53); + AssertDecodeRejected(summaryType, (1UL << 53) | 1UL); + AssertDecodeRejected(summaryType, presentFirst | (1UL << 54)); + + ulong outOfRangeBinding = Encode( + bindingType, + new Dictionary + { + ["slotIndex"] = 1_048_575, + ["index"] = 1_048_575, + ["generation"] = 1L + }); + Exception encodeError = Assert.ThrowsAny(() => EncodeNamed( + summaryType, + "Present", + new Dictionary { ["binding"] = outOfRangeBinding })); + Assert.IsAssignableFrom(Unwrap(encodeError)); + } + + [Fact] + public void RequiredFeatureMaskFencesEveryOlderV2DraftBothWays() + { + const ulong zeroFeatureDraft = 0; + const ulong spillOnlyDraft = 1; + Type constantsType = RequireType("SharedMemoryStore.LayoutV2.LayoutV2Constants"); + ulong newRequiredFeatures = GetStaticUInt64(constantsType, "RequiredFeatures"); + ulong spillFeature = GetStaticUInt64( + constantsType, + "SpillSummaryVersionedEmptyRequiredFeature"); + ulong publicationIntentFeature = GetStaticUInt64( + constantsType, + "PublicationIntentRequiredFeature"); + ulong pidNamespaceFeature = GetStaticUInt64( + constantsType, + "PidNamespaceIdentityRequiredFeature"); + MethodInfo matches = constantsType.GetMethod( + "MatchesRequiredFeatures", + BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + ?? throw new Xunit.Sdk.XunitException( + "LayoutV2Constants.MatchesRequiredFeatures is absent."); + + Assert.Equal(1UL, spillFeature); + Assert.Equal(2UL, publicationIntentFeature); + Assert.Equal(4UL, pidNamespaceFeature); + Assert.Equal( + spillFeature | publicationIntentFeature | pidNamespaceFeature, + newRequiredFeatures); + Assert.Equal(7UL, newRequiredFeatures); + Assert.True((bool)matches.Invoke(null, [newRequiredFeatures])!); + Assert.False((bool)matches.Invoke(null, [zeroFeatureDraft])!); + Assert.False((bool)matches.Invoke(null, [spillOnlyDraft])!); + Assert.False((bool)matches.Invoke(null, [publicationIntentFeature])!); + Assert.False((bool)matches.Invoke(null, [spillFeature | publicationIntentFeature])!); + Assert.NotEqual(0UL, newRequiredFeatures & ~spillOnlyDraft); + } + + [Fact] + public void PublicationIntentEnumAssignmentsAreWireStable() + { + Type intentType = RequireType("SharedMemoryStore.LayoutV2.SlotPublicationIntent"); + + Assert.True(intentType.IsEnum); + Assert.Equal(0, Convert.ToInt32(Enum.Parse(intentType, "None"))); + Assert.Equal(1, Convert.ToInt32(Enum.Parse(intentType, "ExplicitReservation"))); + Assert.Equal(2, Convert.ToInt32(Enum.Parse(intentType, "AtomicPublication"))); + } + + [Theory] + [InlineData(1, 0, 1, 0x0000_0000_0100_0001UL)] + [InlineData(2, 17, 9, 0x0000_0000_0900_0046UL)] + [InlineData(2, MaximumDirectoryTargetIndex, MaximumSlotGeneration, 0x01ff_ffff_ffff_fffeUL)] + public void DirectoryLocationCodecUsesKindTargetIndexAndExactSlotGeneration( + int kind, + long cellIndex, + long generation, + ulong expected) + { + Type locationType = RequireType("SharedMemoryStore.LockFree.DirectoryLocation"); + + ulong encoded = Encode( + locationType, + new Dictionary + { + ["kind"] = kind, + ["targetKind"] = kind, + ["index"] = cellIndex, + ["cellIndex"] = cellIndex, + ["targetIndex"] = cellIndex, + ["generation"] = generation, + ["slotGeneration"] = generation + }); + + Assert.Equal(expected, encoded); + AssertDecoded( + locationType, + encoded, + ("kind", kind), + ("index", cellIndex), + ("generation", generation)); + } + + [Theory] + [InlineData(1, 1, 0, 0, 1, 0x0000_0000_2000_0005UL)] + [InlineData(1, 4, 0, 0, 9, 0x0000_0001_2000_0011UL)] + [InlineData(2, 3, 1, 4, 5, 0x0000_0000_a000_022eUL)] + [InlineData(2, 5, 2, MaximumDirectoryTargetIndex, MaximumSlotGeneration, 0x3fff_ffff_ffff_ffd6UL)] + public void DirectoryOperationCodecUsesIntentPhaseTargetAndExactSlotGeneration( + int intent, + int phase, + int targetKind, + long targetIndex, + long generation, + ulong expected) + { + Type operationType = RequireType("SharedMemoryStore.LockFree.DirectoryOperation"); + + ulong encoded = Encode( + operationType, + new Dictionary + { + ["intent"] = intent, + ["phase"] = phase, + ["kind"] = targetKind, + ["targetKind"] = targetKind, + ["index"] = targetIndex, + ["cellIndex"] = targetIndex, + ["targetIndex"] = targetIndex, + ["generation"] = generation, + ["slotGeneration"] = generation + }); + + Assert.Equal(expected, encoded); + AssertDecoded( + operationType, + encoded, + ("intent", intent), + ("phase", phase), + ("kind", targetKind), + ("index", targetIndex), + ("generation", generation)); + } + + [Fact] + public void DirectoryLocationRejectsZeroOrOverflowGenerationOutOfRangeTargetAndEveryReservedBit() + { + Type locationType = RequireType("SharedMemoryStore.LockFree.DirectoryLocation"); + + AssertEncodeRejected( + locationType, + new Dictionary + { + ["kind"] = 1, + ["index"] = 0L, + ["generation"] = 0L + }); + AssertEncodeRejected( + locationType, + new Dictionary + { + ["kind"] = 1, + ["index"] = 0L, + ["generation"] = MaximumSlotGeneration + 1 + }); + AssertEncodeRejected( + locationType, + new Dictionary + { + ["kind"] = 1, + ["index"] = MaximumDirectoryTargetIndex + 1, + ["generation"] = 1L + }); + + AssertDecodeRejected(locationType, 1UL); // Primary/index zero, but generation zero. + const ulong valid = 0x0000_0000_0100_0001UL; + for (var bit = 57; bit <= 63; bit++) + { + AssertDecodeRejected(locationType, valid | (1UL << bit)); + } + } + + [Fact] + public void DirectoryOperationRejectsZeroOrOverflowGenerationOutOfRangeTargetAndEveryReservedBit() + { + Type operationType = RequireType("SharedMemoryStore.LockFree.DirectoryOperation"); + + AssertEncodeRejected( + operationType, + new Dictionary + { + ["intent"] = 1, + ["phase"] = 1, + ["targetKind"] = 0, + ["targetIndex"] = 0L, + ["generation"] = 0L + }); + AssertEncodeRejected( + operationType, + new Dictionary + { + ["intent"] = 1, + ["phase"] = 1, + ["targetKind"] = 0, + ["targetIndex"] = 0L, + ["generation"] = MaximumSlotGeneration + 1 + }); + AssertEncodeRejected( + operationType, + new Dictionary + { + ["intent"] = 2, + ["phase"] = 3, + ["targetKind"] = 1, + ["targetIndex"] = MaximumDirectoryTargetIndex + 1, + ["generation"] = 1L + }); + + AssertDecodeRejected(operationType, 5UL); // Insert/Prepared, but generation zero. + const ulong valid = 0x0000_0000_2000_0005UL; + for (var bit = 62; bit <= 63; bit++) + { + AssertDecodeRejected(operationType, valid | (1UL << bit)); + } + } + + [Fact] + public void ParticipantSlotAndLeaseControlWordsUseTheDocumentedBitPartitions() + { + Type controlType = RequireType("SharedMemoryStore.LockFree.AtomicControlWord"); + const int participantState = 2; + const int participantIncarnation = 5; + const int pid = 1_234; + ulong participant = EncodeNamed( + controlType, + "Participant", + new Dictionary + { + ["state"] = participantState, + ["incarnation"] = participantIncarnation, + ["generation"] = participantIncarnation, + ["pid"] = pid, + ["processId"] = pid + }); + + Assert.Equal( + (ulong)participantState | ((ulong)participantIncarnation << 3) | ((ulong)pid << 31), + participant); + Assert.Equal(0UL, participant >> 63); + + const int lifecycleState = 2; + const long lifecycleGeneration = 9; + const int participantToken = 0x0123_4567; + ulong expectedOwned = (ulong)lifecycleState + | ((ulong)lifecycleGeneration << 3) + | ((ulong)participantToken << 36); + var ownedParts = new Dictionary + { + ["state"] = lifecycleState, + ["generation"] = lifecycleGeneration, + ["incarnation"] = lifecycleGeneration, + ["participantToken"] = participantToken, + ["token"] = participantToken + }; + + Assert.Equal(expectedOwned, EncodeNamed(controlType, "Slot", ownedParts)); + Assert.Equal(expectedOwned, EncodeNamed(controlType, "Lease", ownedParts)); + } + + [Fact] + public void SharedRecordStridesAndFieldOffsetsAreExactAndAtomicWordsAreAligned() + { + AssertRecord( + "SharedMemoryStore.LayoutV2.StoreHeaderV2", + 512, + ("PidNamespaceId", 264), + ("PidNamespaceMode", 272)); + + AssertRecord( + "SharedMemoryStore.LayoutV2.ParticipantRecordV2", + 64, + ("Control", 0), + ("IdentityKind", 8), + ("Reserved", 12), + ("ProcessStartValue", 16), + ("OpenSequence", 24), + ("PidNamespaceId", 32)); + + AssertRecord( + "SharedMemoryStore.LayoutV2.PrimaryDirectoryBucketV2", + 128, + ("SpillSummary", 0), + ("Mutation", 8), + ("Lanes", 16)); + + AssertRecord( + "SharedMemoryStore.LayoutV2.LeaseRecordV2", + 64, + ("Control", 0), + ("SlotBinding", 8), + ("AcquireSequence", 16)); + + AssertRecord( + "SharedMemoryStore.LayoutV2.ValueSlotMetadataV2", + 128, + ("Control", 0), + ("DirectoryBinding", 8), + ("DirectoryLocation", 16), + ("DirectoryOperation", 24), + ("KeyHash", 32), + ("KeyLength", 40), + ("DescriptorLength", 44), + ("ValueLength", 48), + ("PublicationIntent", 52), + ("BytesAdvanced", 56), + ("CommitSequence", 64), + ("KeyOffset", 72), + ("DescriptorOffset", 80), + ("PayloadOffset", 88)); + } + + [Fact] + public void LayoutSectionsFollowTheCanonicalOrderSizesStridesAndAlignment() + { + const int slots = 3; + const int leases = 5; + const int participants = 64; + object layout = CreateLayout( + slotCount: slots, + leaseRecordCount: leases, + participantCount: participants, + maxKeyBytes: 7, + maxDescriptorBytes: 9, + maxValueBytes: 17); + + long headerLength = GetInt64(layout, "HeaderLength"); + long participantOffset = GetInt64(layout, "ParticipantOffset", "ParticipantRecordsOffset"); + long participantLength = GetInt64(layout, "ParticipantLength", "ParticipantRecordsLength"); + long primaryOffset = GetInt64(layout, "PrimaryDirectoryOffset", "PrimaryBucketOffset"); + long primaryLength = GetInt64(layout, "PrimaryDirectoryLength", "PrimaryBucketLength"); + long overflowOffset = GetInt64(layout, "OverflowDirectoryOffset", "OverflowOffset"); + long overflowLength = GetInt64(layout, "OverflowDirectoryLength", "OverflowLength"); + long leaseOffset = GetInt64(layout, "LeaseRegistryOffset", "LeaseOffset"); + long leaseLength = GetInt64(layout, "LeaseRegistryLength", "LeaseLength"); + long slotOffset = GetInt64(layout, "SlotMetadataOffset", "ValueSlotMetadataOffset"); + long slotLength = GetInt64(layout, "SlotMetadataLength", "ValueSlotMetadataLength"); + long keyOffset = GetInt64(layout, "KeyStorageOffset", "KeyOffset"); + long keyLength = GetInt64(layout, "KeyStorageLength", "KeyLength"); + long descriptorOffset = GetInt64(layout, "DescriptorStorageOffset", "DescriptorOffset"); + long descriptorLength = GetInt64(layout, "DescriptorStorageLength", "DescriptorLength"); + long payloadOffset = GetInt64(layout, "PayloadStorageOffset", "PayloadOffset"); + long payloadLength = GetInt64(layout, "PayloadStorageLength", "PayloadLength"); + + Assert.Equal(0, headerLength % 64); + Assert.Equal(headerLength, participantOffset); + Assert.Equal(participants * 64, participantLength); + Assert.Equal(64, GetInt64(layout, "ParticipantStride", "ParticipantRecordStride")); + + Assert.Equal(Align64(participantOffset + participantLength), primaryOffset); + Assert.Equal(128, GetInt64(layout, "PrimaryBucketStride", "PrimaryDirectoryBucketStride")); + Assert.Equal(GetInt64(layout, "BucketCount", "PrimaryBucketCount") * 128, primaryLength); + + Assert.Equal(Align8(primaryOffset + primaryLength), overflowOffset); + Assert.Equal(slots * 8, overflowLength); + Assert.Equal(8, GetInt64(layout, "OverflowStride", "OverflowBindingStride")); + + Assert.Equal(Align64(overflowOffset + overflowLength), leaseOffset); + Assert.Equal(leases * 64, leaseLength); + Assert.Equal(64, GetInt64(layout, "LeaseStride", "LeaseRecordStride")); + + Assert.Equal(Align64(leaseOffset + leaseLength), slotOffset); + Assert.Equal(slots * 128, slotLength); + Assert.Equal(128, GetInt64(layout, "SlotMetadataStride", "ValueSlotMetadataStride")); + + Assert.Equal(Align8(slotOffset + slotLength), keyOffset); + Assert.Equal(slots * 8, keyLength); + Assert.Equal(8, GetInt64(layout, "KeyStride")); + + Assert.Equal(Align8(keyOffset + keyLength), descriptorOffset); + Assert.Equal(slots * 16, descriptorLength); + Assert.Equal(16, GetInt64(layout, "DescriptorStride")); + + Assert.Equal(Align8(descriptorOffset + descriptorLength), payloadOffset); + Assert.Equal(slots * 24, payloadLength); + Assert.Equal(24, GetInt64(layout, "PayloadStride")); + Assert.Equal(Align8(payloadOffset + payloadLength), GetInt64(layout, "RequiredBytes")); + + foreach (long offset in new[] + { + participantOffset, primaryOffset, overflowOffset, leaseOffset, + slotOffset, keyOffset, descriptorOffset, payloadOffset + }) + { + Assert.Equal(0, offset % 8); + } + + foreach (long offset in new[] { participantOffset, primaryOffset, leaseOffset, slotOffset }) + { + Assert.Equal(0, offset % 64); + } + } + + [Fact] + public void LockFreeSlotCountAcceptsTheContractMaximumAndRejectsTheNextValue() + { + object maximum = CreateLayout( + slotCount: MaximumLockFreeSlotCount, + leaseRecordCount: 1, + participantCount: 1, + maxKeyBytes: 1, + maxDescriptorBytes: 0, + maxValueBytes: 1); + + Assert.Equal(MaximumLockFreeSlotCount, GetInt64(maximum, "SlotCount")); + Assert.Equal(1 << 22, GetInt64(maximum, "PrimaryLaneCount")); + + Exception error = Assert.ThrowsAny(() => CreateLayout( + slotCount: MaximumLockFreeSlotCount + 1, + leaseRecordCount: 1, + participantCount: 1, + maxKeyBytes: 1, + maxDescriptorBytes: 0, + maxValueBytes: 1)); + + Assert.IsType(Unwrap(error)); + } + + [Fact] + public void LayoutArithmeticThrowsInsteadOfWrapping() + { + Exception error = Assert.ThrowsAny(() => CreateLayout( + slotCount: 1, + leaseRecordCount: 1, + participantCount: 64, + maxKeyBytes: 1, + maxDescriptorBytes: 0, + maxValueBytes: int.MaxValue)); + + Assert.IsType(Unwrap(error)); + } + + [Fact] + public void LayoutV2ExplicitlyGatesSupportToX64() + { + Type constantsType = RequireType("SharedMemoryStore.LayoutV2.LayoutV2Constants"); + MethodInfo gate = constantsType + .GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .SingleOrDefault(method => + method.ReturnType == typeof(bool) + && method.GetParameters() is [{ ParameterType: var parameterType }] + && parameterType == typeof(Architecture) + && method.Name.Contains("Support", StringComparison.OrdinalIgnoreCase)) + ?? throw new Xunit.Sdk.XunitException( + $"{constantsType.FullName} must expose the testable x64 architecture gate."); + + Assert.True((bool)gate.Invoke(null, [Architecture.X64])!); + Assert.False((bool)gate.Invoke(null, [Architecture.X86])!); + Assert.False((bool)gate.Invoke(null, [Architecture.Arm64])!); + } + + [Fact] + public void AtomicControlWordRmwWrappersCallSequentiallyConsistentInterlockedPrimitives() + { + Type atomicType = RequireType("SharedMemoryStore.LockFree.AtomicControlWord"); + MethodInfo[] methods = atomicType.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + + Assert.Contains(methods, method => CallsInterlocked(method, nameof(Interlocked.CompareExchange))); + Assert.Contains(methods, method => CallsInterlocked(method, nameof(Interlocked.Exchange))); + } + + [Fact] + public void PublicProfileAwareSizingAndCreateHelperUseTheCanonicalV2Layout() + { + Type profileType = RequireType("SharedMemoryStore.StoreProfile"); + object lockFree = Enum.Parse(profileType, "LockFree"); + MethodInfo calculate = typeof(SharedMemoryStoreOptions) + .GetMethods(BindingFlags.Static | BindingFlags.Public) + .SingleOrDefault(method => + method.Name == nameof(SharedMemoryStoreOptions.CalculateRequiredBytes) + && method.GetParameters().Length == 7 + && method.GetParameters()[0].ParameterType == profileType) + ?? throw new Xunit.Sdk.XunitException("The profile-aware CalculateRequiredBytes overload is absent."); + + object?[] sizingArguments = [lockFree, 3, 17, 9, 7, 5, 64]; + long publicBytes = Convert.ToInt64(calculate.Invoke(null, sizingArguments)); + long internalBytes = GetInt64(CreateLayout( + slotCount: 3, + leaseRecordCount: 5, + participantCount: 64, + maxKeyBytes: 7, + maxDescriptorBytes: 9, + maxValueBytes: 17), "RequiredBytes"); + + MethodInfo create = typeof(SharedMemoryStoreOptions) + .GetMethods(BindingFlags.Static | BindingFlags.Public) + .SingleOrDefault(method => method.Name == "CreateLockFree") + ?? throw new Xunit.Sdk.XunitException("SharedMemoryStoreOptions.CreateLockFree is absent."); + object options = create.Invoke( + null, + BindArguments( + create.GetParameters(), + new Dictionary + { + ["name"] = $"sms-layout-contract-{Guid.NewGuid():N}", + ["slotCount"] = 3, + ["maxValueBytes"] = 17, + ["maxDescriptorBytes"] = 9, + ["maxKeyBytes"] = 7, + ["leaseRecordCount"] = 5, + ["participantRecordCount"] = 64, + ["participantCount"] = 64 + }))!; + + Assert.Equal(internalBytes, publicBytes); + Assert.Equal(publicBytes, GetInt64(options, "TotalBytes")); + } + + private static object CreateLayout( + int slotCount = 3, + int leaseRecordCount = 5, + int participantCount = 64, + int maxKeyBytes = 7, + int maxDescriptorBytes = 9, + int maxValueBytes = 17) + { + Type layoutType = RequireType("SharedMemoryStore.LayoutV2.StoreLayoutV2"); + var values = new Dictionary + { + ["totalBytes"] = 0L, + ["slotCount"] = slotCount, + ["leaseRecordCount"] = leaseRecordCount, + ["participantRecordCount"] = participantCount, + ["participantCount"] = participantCount, + ["maxKeyBytes"] = maxKeyBytes, + ["maxDescriptorBytes"] = maxDescriptorBytes, + ["maxValueBytes"] = maxValueBytes + }; + + foreach (ConstructorInfo constructor in layoutType + .GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .OrderByDescending(candidate => candidate.GetParameters().Length)) + { + if (TryBindArguments(constructor.GetParameters(), values, out object?[]? arguments)) + { + return constructor.Invoke(arguments); + } + } + + foreach (MethodInfo factory in layoutType + .GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .Where(method => method.ReturnType == layoutType) + .OrderByDescending(candidate => candidate.GetParameters().Length)) + { + if (TryBindArguments(factory.GetParameters(), values, out object?[]? arguments)) + { + return factory.Invoke(null, arguments)!; + } + } + + throw new Xunit.Sdk.XunitException( + $"{layoutType.FullName} needs a dimension-based constructor or factory for contract validation."); + } + + private static Type RequireType(string fullName) + { + return StoreAssembly.GetType(fullName, throwOnError: false) + ?? throw new Xunit.Sdk.XunitException($"Required layout-2.0 type {fullName} is absent."); + } + + private static ulong Encode(Type codecType, IReadOnlyDictionary values) + { + IEnumerable factories = codecType + .GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .Where(method => + method.Name.Contains("Encode", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("Pack", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("Create", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("FromParts", StringComparison.OrdinalIgnoreCase)) + .OrderByDescending(method => method.GetParameters().Length); + + foreach (MethodInfo factory in factories) + { + if (!TryBindArguments(factory.GetParameters(), values, out object?[]? arguments)) + { + continue; + } + + object? result = factory.Invoke(null, arguments); + return ToRaw(result, codecType); + } + + foreach (ConstructorInfo constructor in codecType + .GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .OrderByDescending(candidate => candidate.GetParameters().Length)) + { + if (TryBindArguments(constructor.GetParameters(), values, out object?[]? arguments)) + { + return ToRaw(constructor.Invoke(arguments), codecType); + } + } + + throw new Xunit.Sdk.XunitException( + $"{codecType.FullName} needs an encode/create factory or constructor for its documented parts."); + } + + private static void AssertEncodeRejected( + Type codecType, + IReadOnlyDictionary values) + { + Exception error = Assert.ThrowsAny(() => Encode(codecType, values)); + Assert.IsAssignableFrom(Unwrap(error)); + } + + private static void AssertDecodeRejected(Type codecType, ulong raw) + { + Exception error = Assert.ThrowsAny(() => Decode(codecType, raw)); + Assert.IsAssignableFrom(Unwrap(error)); + } + + private static object Decode(Type codecType, ulong raw) + { + foreach (MethodInfo method in codecType + .GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .Where(method => + method.Name.Contains("Decode", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("FromRaw", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("FromValue", StringComparison.OrdinalIgnoreCase))) + { + var values = new Dictionary + { + ["raw"] = raw, + ["value"] = raw, + ["word"] = raw, + ["encoded"] = raw + }; + if (!TryBindArguments(method.GetParameters(), values, out object?[]? arguments)) + { + continue; + } + + object? decoded = method.Invoke(null, arguments); + if (decoded is not null && decoded is not bool) + { + return decoded; + } + } + + throw new Xunit.Sdk.XunitException($"{codecType.FullName} needs a scalar decode method."); + } + + private static ulong EncodeNamed( + Type codecType, + string family, + IReadOnlyDictionary values) + { + foreach (MethodInfo method in codecType + .GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .Where(method => method.Name.Contains(family, StringComparison.OrdinalIgnoreCase)) + .Where(method => + method.Name.Contains("Encode", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("Pack", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("Create", StringComparison.OrdinalIgnoreCase))) + { + if (TryBindArguments(method.GetParameters(), values, out object?[]? arguments)) + { + return ToRaw(method.Invoke(null, arguments), codecType); + } + } + + throw new Xunit.Sdk.XunitException( + $"{codecType.FullName} needs a {family} encode/pack/create method with the documented fields."); + } + + private static ulong ToRaw(object? value, Type codecType) + { + Assert.NotNull(value); + Type valueType = value.GetType(); + if (IsIntegral(valueType)) + { + return Convert.ToUInt64(value); + } + + return GetUInt64(value, "Value", "RawValue", "EncodedValue", "PackedValue", "Word", "Control"); + } + + private static void AssertDecoded(Type codecType, ulong raw, params (string Name, object Expected)[] expectedParts) + { + object? decoded = null; + foreach (MethodInfo method in codecType + .GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .Where(method => + method.Name.Contains("Decode", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("FromRaw", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("FromValue", StringComparison.OrdinalIgnoreCase))) + { + ParameterInfo[] parameters = method.GetParameters(); + var values = new Dictionary + { + ["raw"] = raw, + ["value"] = raw, + ["word"] = raw, + ["encoded"] = raw + }; + + if (!TryBindArguments(parameters, values, out object?[]? arguments)) + { + continue; + } + + object? result = method.Invoke(null, arguments); + if (result is bool success) + { + Assert.True(success); + foreach ((string name, object expected) in expectedParts) + { + int index = Array.FindIndex(parameters, parameter => + parameter.IsOut && SemanticNameMatches(parameter.Name, name)); + Assert.True(index >= 0, $"{method.Name} does not decode {name}."); + Assert.Equal(Convert.ToUInt64(expected), Convert.ToUInt64(arguments![index])); + } + + return; + } + + decoded = result; + if (decoded is not null) + { + break; + } + } + + Assert.NotNull(decoded); + foreach ((string name, object expected) in expectedParts) + { + Assert.Equal(Convert.ToUInt64(expected), checked((ulong)GetInt64(decoded, name))); + } + } + + private static void AssertRecord(string fullName, int expectedSize, params (string Field, int Offset)[] fields) + { + Type recordType = RequireType(fullName); + Assert.True(recordType.IsValueType, $"{fullName} must be a fixed-width value type."); + Assert.Equal(expectedSize, Marshal.SizeOf(recordType)); + + foreach ((string expectedName, int expectedOffset) in fields) + { + FieldInfo field = FindField(recordType, expectedName, expectedOffset); + int actualOffset = checked((int)Marshal.OffsetOf(recordType, field.Name)); + Assert.Equal(expectedOffset, actualOffset); + if (field.FieldType == typeof(long) || field.FieldType == typeof(ulong)) + { + Assert.Equal(0, actualOffset % 8); + } + } + } + + private static void AssertJsonBitRange( + JsonElement parent, + string propertyName, + int expectedFirst, + int expectedLast) + { + int[] actual = parent.GetProperty(propertyName) + .EnumerateArray() + .Select(static item => item.GetInt32()) + .ToArray(); + Assert.Equal(new[] { expectedFirst, expectedLast }, actual); + } + + private static FieldInfo FindField(Type type, string expectedName, int expectedOffset) + { + string normalized = Normalize(expectedName); + string singular = normalized.TrimEnd('s'); + return type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .Where(field => + { + string actual = Normalize(field.Name); + return actual == normalized + || actual.Contains(normalized, StringComparison.Ordinal) + || actual.StartsWith(singular, StringComparison.Ordinal); + }) + .SingleOrDefault(field => checked((int)Marshal.OffsetOf(type, field.Name)) == expectedOffset) + ?? throw new Xunit.Sdk.XunitException($"{type.FullName} is missing field {expectedName}."); + } + + private static long GetInt64(object instance, params string[] candidateNames) + { + Type type = instance.GetType(); + BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; + foreach (string name in candidateNames) + { + string normalized = Normalize(name); + PropertyInfo? property = type.GetProperties(flags) + .SingleOrDefault(candidate => Normalize(candidate.Name) == normalized); + if (property is not null) + { + return Convert.ToInt64(property.GetValue(instance)); + } + + FieldInfo? field = type.GetFields(flags) + .SingleOrDefault(candidate => Normalize(candidate.Name) == normalized); + if (field is not null) + { + return Convert.ToInt64(field.GetValue(instance)); + } + } + + throw new Xunit.Sdk.XunitException( + $"{type.FullName} is missing integral member {string.Join("/", candidateNames)}."); + } + + private static ulong GetUInt64(object instance, params string[] candidateNames) + { + Type type = instance.GetType(); + BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; + foreach (string name in candidateNames) + { + string normalized = Normalize(name); + PropertyInfo? property = type.GetProperties(flags) + .SingleOrDefault(candidate => Normalize(candidate.Name) == normalized); + if (property is not null) + { + return Convert.ToUInt64(property.GetValue(instance)); + } + + FieldInfo? field = type.GetFields(flags) + .SingleOrDefault(candidate => Normalize(candidate.Name) == normalized); + if (field is not null) + { + return Convert.ToUInt64(field.GetValue(instance)); + } + } + + throw new Xunit.Sdk.XunitException( + $"{type.FullName} is missing integral member {string.Join("/", candidateNames)}."); + } + + private static ulong GetStaticUInt64(Type type, string name) + { + BindingFlags flags = BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic; + FieldInfo? field = type.GetField(name, flags); + if (field is not null) + { + return Convert.ToUInt64(field.GetValue(null)); + } + + PropertyInfo? property = type.GetProperty(name, flags); + if (property is not null) + { + return Convert.ToUInt64(property.GetValue(null)); + } + + throw new Xunit.Sdk.XunitException($"{type.FullName} is missing static integral member {name}."); + } + + private static object?[] BindArguments(ParameterInfo[] parameters, IReadOnlyDictionary values) + { + Assert.True(TryBindArguments(parameters, values, out object?[]? arguments)); + return arguments!; + } + + private static bool TryBindArguments( + ParameterInfo[] parameters, + IReadOnlyDictionary values, + out object?[]? arguments) + { + arguments = new object?[parameters.Length]; + for (var index = 0; index < parameters.Length; index++) + { + ParameterInfo parameter = parameters[index]; + Type targetType = parameter.ParameterType.IsByRef + ? parameter.ParameterType.GetElementType()! + : parameter.ParameterType; + + if (parameter.IsOut) + { + arguments[index] = targetType.IsValueType ? Activator.CreateInstance(targetType) : null; + continue; + } + + object? supplied = FindValue(parameter.Name, values); + if (supplied is null) + { + if (parameter.HasDefaultValue) + { + arguments[index] = parameter.DefaultValue; + continue; + } + + arguments = null; + return false; + } + + try + { + arguments[index] = ConvertArgument(supplied, targetType); + } + catch (Exception) + { + arguments = null; + return false; + } + } + + return true; + } + + private static object? FindValue(string? parameterName, IReadOnlyDictionary values) + { + foreach ((string name, object value) in values) + { + if (SemanticNameMatches(parameterName, name)) + { + return value; + } + } + + return null; + } + + private static bool SemanticNameMatches(string? left, string right) + { + string normalizedLeft = Normalize(left ?? string.Empty); + string normalizedRight = Normalize(right); + if (normalizedLeft == normalizedRight) + { + return true; + } + + return (normalizedLeft, normalizedRight) switch + { + ("participantrecordcount", "participantcount") or ("participantcount", "participantrecordcount") => true, + ("recordindex", "index") or ("index", "recordindex") => true, + ("cellindex", "index") or ("index", "cellindex") => true, + ("targetindex", "index") or ("index", "targetindex") => true, + ("targetindex", "cellindex") or ("cellindex", "targetindex") => true, + ("targetkind", "kind") or ("kind", "targetkind") => true, + ("generation", "incarnation") or ("incarnation", "generation") => true, + ("rawvalue", "raw") or ("raw", "rawvalue") => true, + ("encodedvalue", "encoded") or ("encoded", "encodedvalue") => true, + _ => false + }; + } + + private static object ConvertArgument(object value, Type targetType) + { + if (targetType.IsInstanceOfType(value)) + { + return value; + } + + if (targetType.IsEnum) + { + return Enum.ToObject(targetType, value); + } + + return Convert.ChangeType(value, targetType); + } + + private static bool IsIntegral(Type type) + { + return type == typeof(byte) + || type == typeof(sbyte) + || type == typeof(short) + || type == typeof(ushort) + || type == typeof(int) + || type == typeof(uint) + || type == typeof(long) + || type == typeof(ulong); + } + + private static Exception Unwrap(Exception exception) + { + while (exception is TargetInvocationException { InnerException: not null } target) + { + exception = target.InnerException!; + } + + return exception; + } + + private static bool CallsInterlocked(MethodInfo method, string calledMethodName) + { + MethodBody? body = method.GetMethodBody(); + byte[]? il = body?.GetILAsByteArray(); + if (il is null) + { + return false; + } + + for (var position = 0; position < il.Length;) + { + OpCode opcode; + byte first = il[position++]; + if (first == 0xfe) + { + opcode = MultiByteOpCodes[il[position++]]; + } + else + { + opcode = SingleByteOpCodes[first]; + } + + int operandSize = GetOperandSize(opcode.OperandType, il, position); + if (opcode.OperandType == OperandType.InlineMethod) + { + int token = BitConverter.ToInt32(il, position); + try + { + MethodBase? called = method.Module.ResolveMethod( + token, + method.DeclaringType?.GetGenericArguments(), + method.GetGenericArguments()); + if (called?.DeclaringType == typeof(Interlocked) && called.Name == calledMethodName) + { + return true; + } + } + catch (ArgumentException) + { + // Invalid tokens cannot represent the required call. + } + } + + position += operandSize; + } + + return false; + } + + private static int GetOperandSize(OperandType operandType, byte[] il, int operandPosition) + { + return operandType switch + { + OperandType.InlineNone => 0, + OperandType.ShortInlineBrTarget or OperandType.ShortInlineI or OperandType.ShortInlineVar => 1, + OperandType.InlineVar => 2, + OperandType.InlineI or OperandType.InlineBrTarget or OperandType.InlineField + or OperandType.InlineMethod or OperandType.InlineSig or OperandType.InlineString + or OperandType.InlineTok or OperandType.InlineType or OperandType.ShortInlineR => 4, + OperandType.InlineI8 or OperandType.InlineR => 8, + OperandType.InlineSwitch => 4 + (BitConverter.ToInt32(il, operandPosition) * 4), + _ => throw new InvalidOperationException($"Unknown IL operand type {operandType}.") + }; + } + + private static readonly OpCode[] SingleByteOpCodes = BuildOpCodeTable(multiByte: false); + private static readonly OpCode[] MultiByteOpCodes = BuildOpCodeTable(multiByte: true); + + private static OpCode[] BuildOpCodeTable(bool multiByte) + { + var table = new OpCode[256]; + foreach (FieldInfo field in typeof(OpCodes).GetFields(BindingFlags.Public | BindingFlags.Static)) + { + var opcode = (OpCode)field.GetValue(null)!; + ushort value = unchecked((ushort)opcode.Value); + if ((!multiByte && value < 0x100) || (multiByte && (value & 0xff00) == 0xfe00)) + { + table[value & 0xff] = opcode; + } + } + + return table; + } + + private static long Align8(long value) => checked(value + 7) & ~7L; + + private static long Align64(long value) => checked(value + 63) & ~63L; + + private static string Normalize(string value) + { + return new string(value.Where(char.IsLetterOrDigit).Select(char.ToLowerInvariant).ToArray()); + } +} diff --git a/tests/SharedMemoryStore.ContractTests/LockFreeLeaseContractTests.cs b/tests/SharedMemoryStore.ContractTests/LockFreeLeaseContractTests.cs new file mode 100644 index 0000000..fc5cdb1 --- /dev/null +++ b/tests/SharedMemoryStore.ContractTests/LockFreeLeaseContractTests.cs @@ -0,0 +1,285 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using SharedMemoryStore.Interop; + +namespace SharedMemoryStore.ContractTests; + +public sealed class LockFreeLeaseContractTests +{ + [Fact] + public void SharedLeasesProjectExactImmutableDescriptorAndPayloadBytes() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(leaseRecordCount: 4); + byte[] payload = Enumerable.Range(0, 64).Select(static value => (byte)value).ToArray(); + byte[] descriptor = [0xa1, 0xb2, 0xc3]; + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x11], payload, descriptor)); + + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x11], out var first)); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x11], out var second)); + Assert.True(first.IsValid); + Assert.True(second.IsValid); + Assert.Equal(payload.Length, first.ValueLength); + Assert.Equal(descriptor.Length, first.DescriptorLength); + Assert.True(first.ValueSpan.SequenceEqual(payload)); + Assert.True(first.DescriptorSpan.SequenceEqual(descriptor)); + Assert.True(second.ValueSpan.SequenceEqual(payload)); + Assert.True(second.DescriptorSpan.SequenceEqual(descriptor)); + + var copiedFirst = first; + Assert.Equal(StoreStatus.Success, first.Release()); + Assert.False(first.IsValid); + Assert.True(first.ValueSpan.IsEmpty); + Assert.True(first.DescriptorSpan.IsEmpty); + Assert.Contains( + copiedFirst.Release(), + new[] { StoreStatus.LeaseAlreadyReleased, StoreStatus.InvalidLease }); + + // Releasing one shared lease cannot invalidate another lease over the + // same immutable generation. + Assert.True(second.IsValid); + Assert.True(second.ValueSpan.SequenceEqual(payload)); + Assert.Equal(StoreStatus.Success, second.Release()); + } + + [Fact] + public void NormalRecoveryPreservesLiveCurrentProcessLeasesAcrossHandles() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(leaseRecordCount: 4); + using MemoryStore attached = OpenExisting(owned.Name, leaseRecordCount: 4); + byte[] payload = [0x61, 0x62, 0x63]; + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x61], payload)); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x61], out var first)); + Assert.Equal(StoreStatus.Success, attached.TryAcquire([0x61], out var second)); + + // False is the normal concurrently safe policy. It must preserve live + // current-process records regardless of which local handle created them. + Assert.True(first.ValueSpan.SequenceEqual(payload)); + Assert.True(second.ValueSpan.SequenceEqual(payload)); + Assert.Equal( + StoreStatus.Success, + owned.Store.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: false), + out LeaseRecoveryReport report)); + Assert.Equal(0, report.RecoveredLeaseCount); + Assert.Equal(2, report.ActiveLeaseCount); + Assert.True(first.IsValid); + Assert.True(second.IsValid); + Assert.Equal(StoreStatus.Success, first.Release()); + Assert.Equal(StoreStatus.Success, second.Release()); + } + + [Fact] + public void MissingAndExhaustedLeaseOutcomesReturnInvalidTokensAndReleasedCapacityIsReusable() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(leaseRecordCount: 2); + Assert.Equal(StoreProfile.LockFree, owned.Store.Profile); + Assert.Equal(2, owned.Store.ProtocolInfo.LayoutMajorVersion); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x21], [1, 2, 3])); + + Assert.Equal(StoreStatus.NotFound, owned.Store.TryAcquire([0xff], out var missing)); + Assert.False(missing.IsValid); + Assert.Equal(StoreStatus.InvalidLease, missing.Release()); + + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x21], out var first)); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x21], out var second)); + Assert.Equal(StoreStatus.LeaseTableFull, owned.Store.TryAcquire([0x21], out var exhausted)); + Assert.False(exhausted.IsValid); + + Assert.Equal(StoreStatus.Success, first.Release()); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x21], out var replacement)); + Assert.True(replacement.IsValid); + Assert.Equal(StoreStatus.Success, second.Release()); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public void PreCanceledAcquireLeavesNoLeaseClaimAndLaterAcquireSucceeds() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x31], [3, 1])); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.Equal( + StoreStatus.OperationCanceled, + owned.Store.TryAcquire( + [0x31], + new StoreWaitOptions(TimeSpan.FromSeconds(1), cancellation.Token), + out var canceled)); + Assert.False(canceled.IsValid); + Assert.Equal(StoreStatus.InvalidLease, canceled.Release()); + + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x31], out var lease)); + Assert.True(lease.ValueSpan.SequenceEqual(new byte[] { 3, 1 })); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public void DisposingOneHandleInvalidatesItsBorrowedLeaseWithoutExposingMappedMemory() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var owned = CreateStore(leaseRecordCount: 2); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x41], [4, 1])); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x41], out var lease)); + Assert.True(lease.IsValid); + + owned.Dispose(); + + Assert.False(lease.IsValid); + Assert.Equal(0, lease.ValueLength); + Assert.Equal(0, lease.DescriptorLength); + Assert.True(lease.ValueSpan.IsEmpty); + Assert.True(lease.DescriptorSpan.IsEmpty); + Assert.Contains(lease.Release(), new[] { StoreStatus.StoreDisposed, StoreStatus.InvalidLease }); + } + + [Fact] + public void AcquireProjectionAndReleaseNeverEnterTheNamedOperationSynchronizer() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(leaseRecordCount: 2); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x51], [5, 1], [9])); + using var held = new HeldOperationSynchronizer(owned.Name); + var stopwatch = Stopwatch.StartNew(); + + Assert.Equal( + StoreStatus.Success, + owned.Store.TryAcquire([0x51], StoreWaitOptions.NoWait, out var lease)); + Assert.True(lease.ValueSpan.SequenceEqual(new byte[] { 5, 1 })); + Assert.True(lease.DescriptorSpan.SequenceEqual(new byte[] { 9 })); + Assert.Equal(StoreStatus.Success, lease.Release(StoreWaitOptions.NoWait)); + + stopwatch.Stop(); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(1)); + } + + private static OwnedStore CreateStore(int leaseRecordCount) + { + string name = $"sms-v2-lease-contract-{Guid.NewGuid():N}"; + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 4, + maxValueBytes: 128, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 4, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return new OwnedStore(name, Assert.IsType(store)); + } + + private static MemoryStore OpenExisting(string name, int leaseRecordCount) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 4, + maxValueBytes: 128, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 4, + openMode: OpenMode.OpenExisting, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed class OwnedStore(string name, MemoryStore store) : IDisposable + { + public string Name { get; } = name; + + public MemoryStore Store { get; } = store; + + public void Dispose() => Store.Dispose(); + } + + private sealed class HeldOperationSynchronizer : IDisposable + { + private readonly ManualResetEventSlim _ready = new(); + private readonly ManualResetEventSlim _release = new(); + private readonly Thread _thread; + private Exception? _failure; + + public HeldOperationSynchronizer(string storeName) + { + _thread = new Thread(() => Hold(storeName)) { IsBackground = true }; + _thread.Start(); + Assert.True(_ready.Wait(TimeSpan.FromSeconds(5))); + if (_failure is not null) + { + throw new InvalidOperationException("Unable to hold the operation synchronizer.", _failure); + } + } + + public void Dispose() + { + _release.Set(); + Assert.True(_thread.Join(TimeSpan.FromSeconds(5))); + _ready.Dispose(); + _release.Dispose(); + if (_failure is not null) + { + throw new InvalidOperationException("The synchronization holder failed.", _failure); + } + } + + private void Hold(string storeName) + { + try + { + using var synchronization = SharedStorePlatform.CreateSynchronization( + PlatformResourceName.Create(storeName)); + StoreStatus status = synchronization.TryEnter(StoreWaitOptions.Infinite); + if (status != StoreStatus.Success) + { + throw new InvalidOperationException($"Unable to enter operation synchronizer: {status}."); + } + + _ready.Set(); + _release.Wait(); + synchronization.Exit(); + } + catch (Exception error) + { + _failure = error; + _ready.Set(); + } + } + } +} diff --git a/tests/SharedMemoryStore.ContractTests/LockFreePackageContractTests.cs b/tests/SharedMemoryStore.ContractTests/LockFreePackageContractTests.cs new file mode 100644 index 0000000..cd0c902 --- /dev/null +++ b/tests/SharedMemoryStore.ContractTests/LockFreePackageContractTests.cs @@ -0,0 +1,188 @@ +using System.Reflection; +using System.Xml.Linq; + +namespace SharedMemoryStore.ContractTests; + +public sealed class LockFreePackageContractTests +{ + private static readonly Assembly StoreAssembly = typeof(MemoryStore).Assembly; + + [Fact] + public void PackageMetadataSeparatesNuGetLayoutAndResourceProtocolVersions() + { + string project = File.ReadAllText(Path.Combine( + FindRepositoryRoot(), + "src", + "SharedMemoryStore", + "SharedMemoryStore.csproj")); + var document = XDocument.Parse(project); + XElement propertyGroup = Assert.Single(document.Root!.Elements("PropertyGroup")); + + Assert.Equal("2.0.0", propertyGroup.Element("Version")?.Value); + Assert.Equal(new Version(2, 0, 0, 0), StoreAssembly.GetName().Version); + + string releaseNotes = propertyGroup.Element("PackageReleaseNotes")?.Value ?? string.Empty; + Assert.Contains("layout 2.0", releaseNotes, StringComparison.OrdinalIgnoreCase); + Assert.Contains("resource protocol 2", releaseNotes, StringComparison.OrdinalIgnoreCase); + Assert.Contains("legacy", releaseNotes, StringComparison.OrdinalIgnoreCase); + Assert.Contains("default", releaseNotes, StringComparison.OrdinalIgnoreCase); + Assert.Contains("lock-free", releaseNotes, StringComparison.OrdinalIgnoreCase); + Assert.Contains("C++", releaseNotes, StringComparison.Ordinal); + Assert.Contains("Python", releaseNotes, StringComparison.Ordinal); + Assert.Contains("layout 1.2", releaseNotes, StringComparison.OrdinalIgnoreCase); + Assert.Contains("no in-place", releaseNotes, StringComparison.OrdinalIgnoreCase); + } + + internal static void AssertEveryAdditiveLockFreePublicSymbolHasPackagedXmlDocumentation() + { + IReadOnlyDictionary members = LoadDocumentationMembers(); + string[] expectedMembers = + [ + "T:SharedMemoryStore.StoreProfile", + "F:SharedMemoryStore.StoreProfile.Legacy", + "F:SharedMemoryStore.StoreProfile.LockFree", + "P:SharedMemoryStore.SharedMemoryStoreOptions.Profile", + "P:SharedMemoryStore.SharedMemoryStoreOptions.ParticipantRecordCount", + "M:SharedMemoryStore.SharedMemoryStoreOptions.CalculateRequiredBytes(SharedMemoryStore.StoreProfile,System.Int32,System.Int32,System.Int32,System.Int32,System.Int32,System.Int32)", + "M:SharedMemoryStore.SharedMemoryStoreOptions.CreateLockFree(System.String,System.Int32,System.Int32,System.Int32,System.Int32,System.Int32,System.Int32,SharedMemoryStore.OpenMode,System.Boolean)", + "T:SharedMemoryStore.StoreProtocolInfo", + "P:SharedMemoryStore.StoreProtocolInfo.Profile", + "P:SharedMemoryStore.StoreProtocolInfo.LayoutMajorVersion", + "P:SharedMemoryStore.StoreProtocolInfo.LayoutMinorVersion", + "P:SharedMemoryStore.StoreProtocolInfo.ResourceProtocolVersion", + "P:SharedMemoryStore.StoreProtocolInfo.RequiredFeatures", + "P:SharedMemoryStore.StoreProtocolInfo.OptionalFeatures", + "P:SharedMemoryStore.MemoryStore.Profile", + "P:SharedMemoryStore.MemoryStore.ProtocolInfo", + "F:SharedMemoryStore.StoreOpenStatus.ParticipantTableFull", + "P:SharedMemoryStore.DiagnosticsSnapshot.Profile", + "P:SharedMemoryStore.DiagnosticsSnapshot.ProtocolInfo", + "P:SharedMemoryStore.DiagnosticsSnapshot.InitializingSlotCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ReservedSlotCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ReclaimingSlotCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.RetiredSlotCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ClaimingLeaseCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.RecoveringLeaseCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.FreeLeaseCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.RetiredLeaseCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ParticipantRecordCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.FreeParticipantCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.RegisteringParticipantCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ActiveParticipantCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ClosingParticipantCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.RecoveringParticipantCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ReclaimingParticipantCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.RetiredParticipantCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.IsParticipantTableExhausted", + "P:SharedMemoryStore.DiagnosticsSnapshot.PrimaryDirectoryOccupancy", + "P:SharedMemoryStore.DiagnosticsSnapshot.SpilledBucketCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.OverflowDirectoryOccupancy", + "P:SharedMemoryStore.DiagnosticsSnapshot.OverflowScanCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.MaxObservedOverflowScanLength", + "P:SharedMemoryStore.DiagnosticsSnapshot.CasRetryCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.HelpedTransitionCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ContentionBudgetExhaustionCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.InvalidTokenCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.StaleTokenCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.RecoveryAttemptCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.RecoveredTransitionCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.CurrentOwnerClassificationCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.LiveOwnerClassificationCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.StaleOwnerClassificationCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.UnsupportedOwnerClassificationCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.InconsistentOwnerClassificationCount", + "P:SharedMemoryStore.DiagnosticsSnapshot.ChangingOwnerClassificationCount" + ]; + + var missing = expectedMembers.Where(member => !members.ContainsKey(member)).ToArray(); + Assert.True(missing.Length == 0, "Missing XML documentation members: " + string.Join(", ", missing)); + Assert.All(expectedMembers, member => Assert.False(string.IsNullOrWhiteSpace(members[member]))); + } + + [Fact] + public void ChangedConcurrencyAndLifetimeContractsAreExplicitInPackagedXmlDocumentation() + { + IReadOnlyDictionary members = LoadDocumentationMembers(); + + AssertContainsAll( + members["F:SharedMemoryStore.StoreProfile.LockFree"], + "lock-free", + "wait-free"); + AssertContainsAll( + members["P:SharedMemoryStore.SharedMemoryStoreOptions.ParticipantRecordCount"], + "layout-v2", + "handle", + "64"); + AssertContainsAll( + members["T:SharedMemoryStore.StoreProtocolInfo"], + "independent", + "package"); + AssertContainsAll( + members["T:SharedMemoryStore.StoreWaitOptions"], + "legacy", + "lock-free", + "local"); + AssertContainsAll( + members["F:SharedMemoryStore.StoreStatus.RemovePending"], + "logically absent", + "bounded", + "physical reclamation"); + AssertContainsAll( + members["F:SharedMemoryStore.StoreStatus.StoreBusy"], + "legacy", + "lock-free", + "local retry"); + AssertContainsAll( + members["F:SharedMemoryStore.StoreStatus.OperationCanceled"], + "ordering point"); + AssertContainsAll( + members["M:SharedMemoryStore.MemoryStore.TryRemove(System.ReadOnlySpan{System.Byte},SharedMemoryStore.StoreWaitOptions)"], + "logically absent", + "RemovePending", + "physical"); + AssertContainsAll( + members["T:SharedMemoryStore.ValueReservation"], + "single-producer", + "copied", + "concurrent"); + AssertContainsAll( + members["P:SharedMemoryStore.ValueLease.ValueSpan"], + "release", + "store disposal"); + } + + private static IReadOnlyDictionary LoadDocumentationMembers() + { + string xmlPath = Path.ChangeExtension(StoreAssembly.Location, ".xml"); + Assert.True(File.Exists(xmlPath), $"Expected generated package documentation at '{xmlPath}'."); + + return XDocument.Load(xmlPath) + .Descendants("member") + .Where(member => member.Attribute("name") is not null) + .ToDictionary( + member => member.Attribute("name")!.Value, + member => Normalize(member.Value), + StringComparer.Ordinal); + } + + private static void AssertContainsAll(string documentation, params string[] expectedFragments) + { + Assert.All( + expectedFragments, + fragment => Assert.Contains(fragment, documentation, StringComparison.OrdinalIgnoreCase)); + } + + private static string Normalize(string value) => + string.Join(' ', value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + + private static string FindRepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + return directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + } +} diff --git a/tests/SharedMemoryStore.ContractTests/LockFreeProfileApiContractTests.cs b/tests/SharedMemoryStore.ContractTests/LockFreeProfileApiContractTests.cs new file mode 100644 index 0000000..519da21 --- /dev/null +++ b/tests/SharedMemoryStore.ContractTests/LockFreeProfileApiContractTests.cs @@ -0,0 +1,350 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.ContractTests; + +public sealed class LockFreeProfileApiContractTests +{ + private static readonly Assembly StoreAssembly = typeof(Store).Assembly; + + [Fact] + public void EveryAdditiveLockFreePublicSymbolHasPackagedXmlDocumentation() + { + LockFreePackageContractTests.AssertEveryAdditiveLockFreePublicSymbolHasPackagedXmlDocumentation(); + } + + [Fact] + public void StoreProfileHasOnlyTheStableProfileAssignments() + { + var profileType = RequirePublicType("SharedMemoryStore.StoreProfile"); + + Assert.True(profileType.IsEnum); + Assert.Equal(new[] { "Legacy", "LockFree" }, Enum.GetNames(profileType)); + Assert.Equal(0, Convert.ToInt32(Enum.Parse(profileType, "Legacy"))); + Assert.Equal(1, Convert.ToInt32(Enum.Parse(profileType, "LockFree"))); + } + + [Fact] + public void OptionsExposeInitOnlyProfileAndParticipantCapacityWithLegacyDefaults() + { + var profileType = RequirePublicType("SharedMemoryStore.StoreProfile"); + var optionsType = typeof(SharedMemoryStoreOptions); + var profile = RequireProperty(optionsType, "Profile", profileType); + var participantCount = RequireProperty(optionsType, "ParticipantRecordCount", typeof(int)); + + AssertInitOnly(profile); + AssertInitOnly(participantCount); + + var options = new SharedMemoryStoreOptions(); + Assert.Equal("Legacy", profile.GetValue(options)?.ToString()); + Assert.Equal(64, participantCount.GetValue(options)); + } + + [Fact] + public void LegacySizingAndCreateSignaturesRemainUnchanged() + { + var optionsType = typeof(SharedMemoryStoreOptions); + + var calculate = RequireMethod( + optionsType, + nameof(SharedMemoryStoreOptions.CalculateRequiredBytes), + BindingFlags.Public | BindingFlags.Static, + typeof(int), typeof(int), typeof(int), typeof(int), typeof(int)); + Assert.Equal(typeof(long), calculate.ReturnType); + AssertParameterNames( + calculate, + "slotCount", "maxValueBytes", "maxDescriptorBytes", "maxKeyBytes", "leaseRecordCount"); + Assert.All(calculate.GetParameters(), parameter => Assert.False(parameter.IsOptional)); + + var create = RequireMethod( + optionsType, + nameof(SharedMemoryStoreOptions.Create), + BindingFlags.Public | BindingFlags.Static, + typeof(string), typeof(int), typeof(int), typeof(int), typeof(int), typeof(int), + typeof(OpenMode), typeof(bool)); + Assert.Equal(optionsType, create.ReturnType); + AssertParameterNames( + create, + "name", "slotCount", "maxValueBytes", "maxDescriptorBytes", "maxKeyBytes", + "leaseRecordCount", "openMode", "enableLeaseRecovery"); + AssertOptionalDefault(create.GetParameters()[6], OpenMode.CreateOrOpen); + AssertOptionalDefault(create.GetParameters()[7], false); + } + + [Fact] + public void ExistingCreateAlwaysSelectsLegacyProfile() + { + var profileType = RequirePublicType("SharedMemoryStore.StoreProfile"); + var profile = RequireProperty(typeof(SharedMemoryStoreOptions), "Profile", profileType); + + var options = SharedMemoryStoreOptions.Create( + "contract-create-legacy", + slotCount: 2, + maxValueBytes: 32, + maxDescriptorBytes: 8, + maxKeyBytes: 16, + leaseRecordCount: 2); + + Assert.Equal("Legacy", profile.GetValue(options)?.ToString()); + } + + [Fact] + public void ProfileAwareSizingIsAdditiveAndParticipantCapacityIsOptional() + { + var profileType = RequirePublicType("SharedMemoryStore.StoreProfile"); + var calculate = RequireMethod( + typeof(SharedMemoryStoreOptions), + nameof(SharedMemoryStoreOptions.CalculateRequiredBytes), + BindingFlags.Public | BindingFlags.Static, + profileType, typeof(int), typeof(int), typeof(int), typeof(int), typeof(int), typeof(int)); + + Assert.Equal(typeof(long), calculate.ReturnType); + AssertParameterNames( + calculate, + "profile", "slotCount", "maxValueBytes", "maxDescriptorBytes", "maxKeyBytes", + "leaseRecordCount", "participantRecordCount"); + AssertOptionalDefault(calculate.GetParameters()[6], 64); + } + + [Fact] + public void CreateLockFreeHasTheSpecifiedAdditiveSignatureAndDefaults() + { + var create = RequireMethod( + typeof(SharedMemoryStoreOptions), + "CreateLockFree", + BindingFlags.Public | BindingFlags.Static, + typeof(string), typeof(int), typeof(int), typeof(int), typeof(int), typeof(int), + typeof(int), typeof(OpenMode), typeof(bool)); + + Assert.Equal(typeof(SharedMemoryStoreOptions), create.ReturnType); + AssertParameterNames( + create, + "name", "slotCount", "maxValueBytes", "maxDescriptorBytes", "maxKeyBytes", + "leaseRecordCount", "participantRecordCount", "openMode", "enableLeaseRecovery"); + AssertOptionalDefault(create.GetParameters()[6], 64); + AssertOptionalDefault(create.GetParameters()[7], OpenMode.CreateOrOpen); + AssertOptionalDefault(create.GetParameters()[8], false); + } + + [Theory] + [InlineData(0)] + [InlineData(1_048_576)] + public void LockFreeParticipantCapacityOutsideTheContractRangeIsInvalid(int participantRecordCount) + { + var profileType = RequirePublicType("SharedMemoryStore.StoreProfile"); + var calculate = RequireMethod( + typeof(SharedMemoryStoreOptions), + nameof(SharedMemoryStoreOptions.CalculateRequiredBytes), + BindingFlags.Public | BindingFlags.Static, + profileType, typeof(int), typeof(int), typeof(int), typeof(int), typeof(int), typeof(int)); + var lockFree = Enum.Parse(profileType, "LockFree"); + var requiredBytes = (long)calculate.Invoke( + null, + new[] { lockFree, 2, 32, 8, 16, 2, 64 })!; + + var options = new SharedMemoryStoreOptions + { + Name = $"contract-invalid-participants-{participantRecordCount}", + OpenMode = OpenMode.CreateNew, + SlotCount = 2, + MaxValueBytes = 32, + MaxDescriptorBytes = 8, + MaxKeyBytes = 16, + LeaseRecordCount = 2, + TotalBytes = requiredBytes + }; + RequireProperty(typeof(SharedMemoryStoreOptions), "Profile", profileType).SetValue(options, lockFree); + RequireProperty(typeof(SharedMemoryStoreOptions), "ParticipantRecordCount", typeof(int)) + .SetValue(options, participantRecordCount); + + Assert.Equal(StoreOpenStatus.InvalidOptions, options.Validate().Status); + } + + [Fact] + public void LockFreeSlotCapacityMaximumIsAcceptedAndTheNextValueIsRejectedWithoutChangingLegacySizing() + { + const int maximumLockFreeSlotCount = 1_048_575; + const int firstRejectedLockFreeSlotCount = maximumLockFreeSlotCount + 1; + + long maximumBytes = SharedMemoryStoreOptions.CalculateRequiredBytes( + StoreProfile.LockFree, + maximumLockFreeSlotCount, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: 1, + leaseRecordCount: 1, + participantRecordCount: 1); + + Assert.True(maximumBytes > 0); + Assert.Throws(() => + SharedMemoryStoreOptions.CalculateRequiredBytes( + StoreProfile.LockFree, + firstRejectedLockFreeSlotCount, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: 1, + leaseRecordCount: 1, + participantRecordCount: 1)); + + long legacyBytes = SharedMemoryStoreOptions.CalculateRequiredBytes( + StoreProfile.Legacy, + firstRejectedLockFreeSlotCount, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: 1, + leaseRecordCount: 1, + participantRecordCount: 1); + Assert.True(legacyBytes > 0); + } + + [Fact] + public void StoreProtocolInfoIsTheSpecifiedReadonlyRecordValue() + { + var profileType = RequirePublicType("SharedMemoryStore.StoreProfile"); + var protocolType = RequirePublicType("SharedMemoryStore.StoreProtocolInfo"); + + Assert.True(protocolType.IsValueType); + Assert.True(protocolType.IsSealed); + Assert.NotNull(protocolType.GetCustomAttribute()); + + var constructor = protocolType.GetConstructor(new[] + { + profileType, typeof(int), typeof(int), typeof(int), typeof(ulong), typeof(ulong) + }); + Assert.NotNull(constructor); + AssertParameterNames( + constructor!, + "Profile", "LayoutMajorVersion", "LayoutMinorVersion", "ResourceProtocolVersion", + "RequiredFeatures", "OptionalFeatures"); + + RequireProperty(protocolType, "Profile", profileType); + RequireProperty(protocolType, "LayoutMajorVersion", typeof(int)); + RequireProperty(protocolType, "LayoutMinorVersion", typeof(int)); + RequireProperty(protocolType, "ResourceProtocolVersion", typeof(int)); + RequireProperty(protocolType, "RequiredFeatures", typeof(ulong)); + RequireProperty(protocolType, "OptionalFeatures", typeof(ulong)); + } + + [Fact] + public void MemoryStoreExposesImmutableProfileAndProtocolIdentity() + { + var profileType = RequirePublicType("SharedMemoryStore.StoreProfile"); + var protocolType = RequirePublicType("SharedMemoryStore.StoreProtocolInfo"); + var profile = RequireProperty(typeof(Store), "Profile", profileType); + var protocol = RequireProperty(typeof(Store), "ProtocolInfo", protocolType); + + Assert.Null(profile.SetMethod); + Assert.Null(protocol.SetMethod); + } + + [Fact] + public void StoreOpenStatusAppendsParticipantTableFullWithoutRenumberingLegacyValues() + { + var expected = new Dictionary + { + ["Success"] = 0, + ["AlreadyExists"] = 1, + ["NotFound"] = 2, + ["InvalidOptions"] = 3, + ["IncompatibleLayout"] = 4, + ["UnsupportedPlatform"] = 5, + ["InsufficientCapacity"] = 6, + ["AccessDenied"] = 7, + ["MappingFailed"] = 8, + ["StoreBusy"] = 9, + ["OperationCanceled"] = 10, + ["ParticipantTableFull"] = 11 + }; + + Assert.Equal(expected.Keys, Enum.GetNames()); + Assert.All(expected, pair => + Assert.Equal(pair.Value, Convert.ToInt32(Enum.Parse(pair.Key)))); + } + + [Fact] + public void StoreStatusNumericAssignmentsRemainUnchanged() + { + var expected = new Dictionary + { + ["Success"] = 0, + ["DuplicateKey"] = 1, + ["NotFound"] = 2, + ["KeyTooLarge"] = 3, + ["ValueTooLarge"] = 4, + ["DescriptorTooLarge"] = 5, + ["StoreFull"] = 6, + ["LeaseTableFull"] = 7, + ["InvalidLease"] = 8, + ["LeaseAlreadyReleased"] = 9, + ["RemovePending"] = 10, + ["UnsupportedPlatform"] = 11, + ["StoreDisposed"] = 12, + ["CorruptStore"] = 13, + ["AccessDenied"] = 14, + ["UnknownFailure"] = 15, + ["InvalidReservation"] = 16, + ["ReservationIncomplete"] = 17, + ["ReservationAlreadyCompleted"] = 18, + ["ReservationWriteOutOfRange"] = 19, + ["InvalidKey"] = 20, + ["StoreBusy"] = 21, + ["OperationCanceled"] = 22 + }; + + Assert.Equal(expected.Keys, Enum.GetNames()); + Assert.All(expected, pair => + Assert.Equal(pair.Value, Convert.ToInt32(Enum.Parse(pair.Key)))); + } + + private static Type RequirePublicType(string fullName) + { + var type = StoreAssembly.GetType(fullName, throwOnError: false, ignoreCase: false); + Assert.True(type is not null, $"Required public type '{fullName}' is missing."); + Assert.True(type!.IsPublic, $"Required type '{fullName}' must be public."); + return type; + } + + private static PropertyInfo RequireProperty(Type declaringType, string name, Type propertyType) + { + var property = declaringType.GetProperty(name, BindingFlags.Public | BindingFlags.Instance); + Assert.True(property is not null, $"Required public property '{declaringType.FullName}.{name}' is missing."); + Assert.Equal(propertyType, property!.PropertyType); + Assert.NotNull(property.GetMethod); + Assert.True(property.GetMethod!.IsPublic); + return property; + } + + private static MethodInfo RequireMethod( + Type declaringType, + string name, + BindingFlags bindingFlags, + params Type[] parameterTypes) + { + var method = declaringType.GetMethod(name, bindingFlags, binder: null, parameterTypes, modifiers: null); + Assert.True( + method is not null, + $"Required method '{declaringType.FullName}.{name}({string.Join(", ", parameterTypes.Select(type => type.Name))})' is missing."); + return method!; + } + + private static void AssertInitOnly(PropertyInfo property) + { + Assert.NotNull(property.SetMethod); + Assert.True(property.SetMethod!.IsPublic); + Assert.Contains( + typeof(IsExternalInit), + property.SetMethod.ReturnParameter.GetRequiredCustomModifiers()); + } + + private static void AssertParameterNames(MethodBase method, params string[] expectedNames) + { + Assert.Equal(expectedNames, method.GetParameters().Select(parameter => parameter.Name)); + } + + private static void AssertOptionalDefault(ParameterInfo parameter, object expectedDefault) + { + Assert.True(parameter.IsOptional, $"Parameter '{parameter.Name}' must be optional."); + Assert.Equal(expectedDefault, parameter.DefaultValue); + } +} diff --git a/tests/SharedMemoryStore.ContractTests/LockFreePublishContractTests.cs b/tests/SharedMemoryStore.ContractTests/LockFreePublishContractTests.cs new file mode 100644 index 0000000..6ec0b46 --- /dev/null +++ b/tests/SharedMemoryStore.ContractTests/LockFreePublishContractTests.cs @@ -0,0 +1,327 @@ +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using SharedMemoryStore.Interop; + +namespace SharedMemoryStore.ContractTests; + +public sealed class LockFreePublishContractTests +{ + [Fact] + public void SimpleAndReservationPublicationAreInvisibleUntilCommitAndThenExact() + { + using var store = CreateLockFreeStore(slotCount: 4); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1, 2, 3], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var simple)); + Assert.Equal(new byte[] { 1, 2, 3 }, simple.ValueSpan.ToArray()); + Assert.Equal(new byte[] { 7 }, simple.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, simple.Release()); + + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 3, [8], out var reservation)); + reservation.GetSpan(3).Fill(9); + Assert.Equal(StoreStatus.Success, reservation.Advance(2)); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([2], out _)); + Assert.Equal(StoreStatus.ReservationIncomplete, reservation.Commit()); + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + Assert.Equal(StoreStatus.Success, reservation.Commit()); + + Assert.Equal(StoreStatus.Success, store.TryAcquire([2], out var reserved)); + Assert.Equal(new byte[] { 9, 9, 9 }, reserved.ValueSpan.ToArray()); + Assert.Equal(new byte[] { 8 }, reserved.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, reserved.Release()); + } + + [Fact] + public void SegmentedPublicationCopiesTheLogicalSequenceAndDescriptorExactly() + { + using var store = CreateLockFreeStore(); + var sequence = SequenceFactory.Create([1, 2], [3], [4, 5]); + + Assert.Equal(StoreStatus.Success, store.TryPublishSegments([1], sequence, [6], out var copied)); + Assert.Equal(5, copied); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, lease.ValueSpan.ToArray()); + Assert.Equal(new byte[] { 6 }, lease.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public void EmptyOversizedAndZeroLengthBoundariesReturnStableStatuses() + { + using var store = CreateLockFreeStore( + slotCount: 6, + maxValueBytes: 3, + maxDescriptorBytes: 1, + maxKeyBytes: 2); + + Assert.Equal(StoreStatus.InvalidKey, store.TryPublish([], [1])); + Assert.Equal(StoreStatus.InvalidKey, store.TryReserve([], 1, default, out _)); + Assert.Equal(StoreStatus.KeyTooLarge, store.TryPublish([1, 2, 3], [1])); + Assert.Equal(StoreStatus.ValueTooLarge, store.TryPublish([1], [1, 2, 3, 4])); + Assert.Equal(StoreStatus.ValueTooLarge, store.TryReserve([1], -1, default, out _)); + Assert.Equal(StoreStatus.ValueTooLarge, store.TryReserve([1], 4, default, out _)); + Assert.Equal(StoreStatus.DescriptorTooLarge, store.TryPublish([1], [1], [1, 2])); + Assert.Equal(StoreStatus.DescriptorTooLarge, store.TryReserve([1], 1, [1, 2], out _)); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var emptySimple)); + Assert.Equal(0, emptySimple.ValueLength); + Assert.True(emptySimple.ValueSpan.IsEmpty); + Assert.Equal(StoreStatus.Success, emptySimple.Release()); + + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 0, default, out var emptyReservation)); + Assert.Equal(0, emptyReservation.PayloadLength); + Assert.Equal(StoreStatus.Success, emptyReservation.Commit()); + Assert.Equal(StoreStatus.Success, store.TryAcquire([2], out var emptyReserved)); + Assert.True(emptyReserved.ValueSpan.IsEmpty); + Assert.Equal(StoreStatus.Success, emptyReserved.Release()); + } + + [Fact] + public void DuplicateAndCapacityStatusesDoNotCreateSecondCurrentValue() + { + using var store = CreateLockFreeStore(slotCount: 1); + + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + Assert.Equal(StoreStatus.DuplicateKey, store.TryPublish([1], [2])); + Assert.Equal(StoreStatus.DuplicateKey, store.TryReserve([1], 1, default, out _)); + Assert.Equal(StoreStatus.StoreFull, store.TryPublish([2], [2])); + Assert.Equal(StoreStatus.Success, reservation.Abort()); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [3])); + Assert.Equal(StoreStatus.DuplicateKey, store.TryPublish([1], [4])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(3, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public void PreCanceledPublicationLeavesNoKeySlotOrCopiedBytes() + { + using var store = CreateLockFreeStore(slotCount: 3); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var wait = new StoreWaitOptions(TimeSpan.FromSeconds(1), cancellation.Token); + var sequence = new ReadOnlySequence(new byte[] { 2 }); + + Assert.Equal(StoreStatus.OperationCanceled, store.TryPublish([1], [1], default, wait)); + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryPublishSegments([2], sequence, default, wait, out var copied)); + Assert.Equal(0, copied); + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryReserve([3], 1, default, wait, out var canceledReservation)); + Assert.False(canceledReservation.IsValid); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + Assert.Equal(StoreStatus.Success, store.TryPublishSegments([2], sequence, default, out copied)); + Assert.Equal(1, copied); + Assert.Equal(StoreStatus.Success, store.TryReserve([3], 1, default, out var reservation)); + Assert.Equal(StoreStatus.Success, reservation.Abort()); + } + + [Fact] + public void PublishReserveAdvanceCommitAndSegmentsNeverEnterOperationSynchronizer() + { + using var store = CreateLockFreeStore(slotCount: 4); + using var held = new HeldOperationSynchronizer(storeName: StoreName(store)); + var sequence = new ReadOnlySequence(new byte[] { 2 }); + var stopwatch = Stopwatch.StartNew(); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1], default, StoreWaitOptions.NoWait)); + Assert.Equal( + StoreStatus.Success, + store.TryPublishSegments([2], sequence, default, StoreWaitOptions.NoWait, out var copied)); + Assert.Equal(1, copied); + Assert.Equal( + StoreStatus.Success, + store.TryReserve([3], 1, default, StoreWaitOptions.NoWait, out var reservation)); + reservation.GetSpan()[0] = 3; + Assert.Equal(StoreStatus.Success, reservation.Advance(1, StoreWaitOptions.NoWait)); + Assert.Equal(StoreStatus.Success, reservation.Commit(StoreWaitOptions.NoWait)); + + stopwatch.Stop(); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(1)); + } + + [Fact] + public void WarmedSimpleSegmentedAndReservationPublicationAllocateZeroBytes() + { + const int WarmupIterations = 16; + const int MeasuredIterations = 64; + using var store = CreateLockFreeStore(slotCount: 3 * (WarmupIterations + MeasuredIterations)); + var simpleValue = new byte[] { 1 }; + var segmentValue = new byte[] { 2 }; + var segmented = new ReadOnlySequence(segmentValue); + + PublishBatch(store, 0, WarmupIterations, simpleValue, segmented); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + var before = GC.GetAllocatedBytesForCurrentThread(); + PublishBatch(store, WarmupIterations, MeasuredIterations, simpleValue, segmented); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(0, allocated); + } + + private static void PublishBatch( + MemoryStore store, + int start, + int count, + byte[] simpleValue, + ReadOnlySequence segmented) + { + Span key = stackalloc byte[5]; + for (var offset = 0; offset < count; offset++) + { + var id = start + offset; + BitConverter.TryWriteBytes(key[1..], id); + + key[0] = 1; + RequireSuccess(store.TryPublish(key, simpleValue)); + + key[0] = 2; + RequireSuccess(store.TryPublishSegments(key, segmented, default, out var copied)); + if (copied != 1) + { + throw new InvalidOperationException("Segmented publish copied an unexpected length."); + } + + key[0] = 3; + RequireSuccess(store.TryReserve(key, 1, default, out var reservation)); + reservation.GetSpan()[0] = 3; + RequireSuccess(reservation.Advance(1)); + RequireSuccess(reservation.Commit()); + } + } + + private static void RequireSuccess(StoreStatus status) + { + if (status != StoreStatus.Success) + { + throw new InvalidOperationException($"Expected Success, received {status}."); + } + } + + private static MemoryStore CreateLockFreeStore( + int slotCount = 8, + int maxValueBytes = 16, + int maxDescriptorBytes = 4, + int maxKeyBytes = 8) + { + var name = $"sms-v2-publish-contract-{Guid.NewGuid():N}"; + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount: Math.Max(8, slotCount), + participantRecordCount: 4, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + var status = MemoryStore.TryCreateOrOpen(options, out var store); + + Assert.Equal(StoreOpenStatus.Success, status); + var result = Assert.IsType(store); + Assert.Equal(StoreProfile.LockFree, result.Profile); + StoreNames.Add(result, new StoreNameHolder(name)); + return result; + } + + private static readonly ConditionalWeakTable StoreNames = new(); + + private static string StoreName(MemoryStore store) => StoreNames.GetValue( + store, + static _ => throw new InvalidOperationException("The lock-free store name was not registered.")).Name; + + private sealed record StoreNameHolder(string Name); + + private sealed class HeldOperationSynchronizer : IDisposable + { + private readonly ManualResetEventSlim _ready = new(); + private readonly ManualResetEventSlim _release = new(); + private readonly Thread _thread; + private Exception? _failure; + + public HeldOperationSynchronizer(string storeName) + { + _thread = new Thread(() => Hold(storeName)) { IsBackground = true }; + _thread.Start(); + Assert.True(_ready.Wait(TimeSpan.FromSeconds(5)), "The operation synchronizer holder did not start."); + if (_failure is not null) + { + throw new InvalidOperationException("The operation synchronizer could not be held.", _failure); + } + } + + public void Dispose() + { + _release.Set(); + Assert.True(_thread.Join(TimeSpan.FromSeconds(5)), "The operation synchronizer holder did not stop."); + _ready.Dispose(); + _release.Dispose(); + if (_failure is not null) + { + throw new InvalidOperationException("The operation synchronizer holder failed.", _failure); + } + } + + private void Hold(string storeName) + { + try + { + using var synchronization = SharedStorePlatform.CreateSynchronization( + PlatformResourceName.Create(storeName)); + var status = synchronization.TryEnter(StoreWaitOptions.Infinite); + if (status != StoreStatus.Success) + { + throw new InvalidOperationException($"Unable to enter operation synchronizer: {status}."); + } + + _ready.Set(); + _release.Wait(); + synchronization.Exit(); + } + catch (Exception error) + { + _failure = error; + _ready.Set(); + } + } + } + + private static class SequenceFactory + { + public static ReadOnlySequence Create(params byte[][] segments) + { + BufferSegment? first = null; + BufferSegment? last = null; + foreach (var segment in segments) + { + last = last is null ? first = new BufferSegment(segment) : last.Append(segment); + } + + return new ReadOnlySequence(first!, 0, last!, last!.Memory.Length); + } + } + + private sealed class BufferSegment : ReadOnlySequenceSegment + { + public BufferSegment(byte[] memory) + { + Memory = memory; + } + + public BufferSegment Append(byte[] memory) + { + var segment = new BufferSegment(memory) { RunningIndex = RunningIndex + Memory.Length }; + Next = segment; + return segment; + } + } +} diff --git a/tests/SharedMemoryStore.ContractTests/LockFreeRemoveContractTests.cs b/tests/SharedMemoryStore.ContractTests/LockFreeRemoveContractTests.cs new file mode 100644 index 0000000..71ca741 --- /dev/null +++ b/tests/SharedMemoryStore.ContractTests/LockFreeRemoveContractTests.cs @@ -0,0 +1,226 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using SharedMemoryStore.Interop; + +namespace SharedMemoryStore.ContractTests; + +public sealed class LockFreeRemoveContractTests +{ + [Fact] + public void InfiniteRemoveWithoutLeasesReturnsSuccessAndMakesCapacityReusable() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(slotCount: 1, leaseRecordCount: 4); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x11], [1, 2, 3])); + + Assert.Equal( + StoreStatus.Success, + owned.Store.TryRemove([0x11], StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.NotFound, owned.Store.TryAcquire([0x11], out var absent)); + Assert.False(absent.IsValid); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x11], [4, 5, 6])); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x11], out var replacement)); + Assert.True(replacement.ValueSpan.SequenceEqual(new byte[] { 4, 5, 6 })); + Assert.Equal(StoreStatus.Success, replacement.Release()); + Assert.Equal(StoreStatus.DuplicateKey, owned.Store.TryPublish([0x11], [7])); + } + + [Fact] + public void InfiniteRemoveWithActiveLeaseReturnsPendingAndPreservesBytesUntilReusable() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(slotCount: 1, leaseRecordCount: 4); + byte[] original = [2, 4, 6, 8]; + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x21], original, [9])); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x21], out var lease)); + + Assert.Equal( + StoreStatus.RemovePending, + owned.Store.TryRemove([0x21], StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.NotFound, owned.Store.TryAcquire([0x21], out var rejected)); + Assert.False(rejected.IsValid); + Assert.Equal(StoreStatus.DuplicateKey, owned.Store.TryPublish([0x21], [1])); + Assert.True(lease.IsValid); + Assert.True(lease.ValueSpan.SequenceEqual(original)); + Assert.True(lease.DescriptorSpan.SequenceEqual(new byte[] { 9 })); + + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x21], [1, 3, 5])); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x21], out var replacement)); + Assert.True(replacement.ValueSpan.SequenceEqual(new byte[] { 1, 3, 5 })); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public void NoWaitAfterLogicalRemovalReturnsConservativePendingAndCooperativeWorkRestoresCapacity() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(slotCount: 1, leaseRecordCount: 8_192); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x31], [3, 1])); + + Assert.Equal(StoreStatus.RemovePending, owned.Store.TryRemove([0x31], StoreWaitOptions.NoWait)); + Assert.Equal(StoreStatus.NotFound, owned.Store.TryAcquire([0x31], out _)); + + // The key is already logically absent. A later allocator/helper must be + // able to finish physical unlink/reclaim without a global maintenance owner. + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x31], [3, 2])); + } + + [Fact] + public void PreCanceledRemoveDoesNotCrossLogicalOrderingPointOrLoseTheValue() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(slotCount: 2, leaseRecordCount: 4); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x41], [4, 1])); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.Equal( + StoreStatus.OperationCanceled, + owned.Store.TryRemove( + [0x41], + new StoreWaitOptions(TimeSpan.FromSeconds(1), cancellation.Token))); + Assert.Equal(StoreStatus.Success, owned.Store.TryAcquire([0x41], out var preserved)); + Assert.True(preserved.ValueSpan.SequenceEqual(new byte[] { 4, 1 })); + Assert.Equal(StoreStatus.Success, preserved.Release()); + } + + [Fact] + public void RemoveValidatesKeysAndReturnsStableMissingStatus() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(slotCount: 2, leaseRecordCount: 4, maxKeyBytes: 2); + Assert.Equal(StoreStatus.InvalidKey, owned.Store.TryRemove([])); + Assert.Equal(StoreStatus.KeyTooLarge, owned.Store.TryRemove([1, 2, 3])); + Assert.Equal(StoreStatus.NotFound, owned.Store.TryRemove([1])); + } + + [Fact] + public void RemoveNeverEntersTheNamedOperationSynchronizer() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var owned = CreateStore(slotCount: 2, leaseRecordCount: 4); + Assert.Equal(StoreStatus.Success, owned.Store.TryPublish([0x51], [5, 1])); + using var held = new HeldOperationSynchronizer(owned.Name); + var stopwatch = Stopwatch.StartNew(); + + StoreStatus status = owned.Store.TryRemove([0x51], StoreWaitOptions.NoWait); + + stopwatch.Stop(); + Assert.Contains(status, new[] { StoreStatus.Success, StoreStatus.RemovePending }); + Assert.Equal(StoreStatus.NotFound, owned.Store.TryAcquire([0x51], StoreWaitOptions.NoWait, out _)); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(1)); + } + + private static OwnedStore CreateStore( + int slotCount, + int leaseRecordCount, + int maxKeyBytes = 8) + { + string name = $"sms-v2-remove-contract-{Guid.NewGuid():N}"; + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 64, + maxDescriptorBytes: 8, + maxKeyBytes, + leaseRecordCount, + participantRecordCount: 4, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus openStatus = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, openStatus); + return new OwnedStore(name, Assert.IsType(store)); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed class OwnedStore(string name, MemoryStore store) : IDisposable + { + public string Name { get; } = name; + + public MemoryStore Store { get; } = store; + + public void Dispose() => Store.Dispose(); + } + + private sealed class HeldOperationSynchronizer : IDisposable + { + private readonly ManualResetEventSlim _ready = new(); + private readonly ManualResetEventSlim _release = new(); + private readonly Thread _thread; + private Exception? _failure; + + public HeldOperationSynchronizer(string storeName) + { + _thread = new Thread(() => Hold(storeName)) { IsBackground = true }; + _thread.Start(); + Assert.True(_ready.Wait(TimeSpan.FromSeconds(5))); + if (_failure is not null) + { + throw new InvalidOperationException("Unable to hold the operation synchronizer.", _failure); + } + } + + public void Dispose() + { + _release.Set(); + Assert.True(_thread.Join(TimeSpan.FromSeconds(5))); + _ready.Dispose(); + _release.Dispose(); + if (_failure is not null) + { + throw new InvalidOperationException("The synchronization holder failed.", _failure); + } + } + + private void Hold(string storeName) + { + try + { + using var synchronization = SharedStorePlatform.CreateSynchronization( + PlatformResourceName.Create(storeName)); + StoreStatus status = synchronization.TryEnter(StoreWaitOptions.Infinite); + if (status != StoreStatus.Success) + { + throw new InvalidOperationException($"Unable to enter operation synchronizer: {status}."); + } + + _ready.Set(); + _release.Wait(); + synchronization.Exit(); + } + catch (Exception error) + { + _failure = error; + _ready.Set(); + } + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/AssemblyInfo.cs b/tests/SharedMemoryStore.IntegrationTests/AssemblyInfo.cs new file mode 100644 index 0000000..539e914 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/AssemblyInfo.cs @@ -0,0 +1,6 @@ +// Integration classes frequently create independent 8-12 process workloads. +// Running those classes concurrently measures host oversubscription and process +// startup scheduling rather than the bounded cross-process protocol exercised +// inside each test. Keep class execution serial while preserving every test's +// real multi-process concurrency. +[assembly: Xunit.CollectionBehavior(DisableTestParallelization = true)] diff --git a/tests/SharedMemoryStore.IntegrationTests/LinuxFileLockIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LinuxFileLockIntegrationTests.cs new file mode 100644 index 0000000..c2eceae --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LinuxFileLockIntegrationTests.cs @@ -0,0 +1,498 @@ +using System.Diagnostics; +using System.Reflection; +using System.Runtime.Loader; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using Microsoft.Win32.SafeHandles; +using SharedMemoryStore.Interop; + +namespace SharedMemoryStore.IntegrationTests; + +[SupportedOSPlatform("linux")] +public sealed class LinuxFileLockIntegrationTests +{ + private static readonly TimeSpan AgentTimeout = TimeSpan.FromSeconds(10); + + [Fact] + [Trait("Category", "Integration")] + public async Task SharedWrapperHandoffsRemainExclusiveAndReusable() + { + if (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + PlatformResourceName resource = PlatformResourceName.Create( + $"sms-linux-file-lock-handoff-{Guid.NewGuid():N}"); + string path = resource.LinuxLifecycleLockPath; + LinuxFileLock? shared = null; + try + { + Assert.Equal(StoreStatus.Success, LinuxFileLock.TryOpen(path, out shared)); + Assert.NotNull(shared); + LinuxFileLock fileLock = shared; + var start = new ManualResetEventSlim(false); + int active = 0; + int maximumActive = 0; + int successfulHandoffs = 0; + Task[] workers = Enumerable.Range(0, 4) + .Select(_ => Task.Run(() => + { + start.Wait(); + for (var iteration = 0; iteration < 1_000; iteration++) + { + Assert.Equal( + StoreStatus.Success, + fileLock.TryAcquire(new StoreWaitOptions(TimeSpan.FromSeconds(5)))); + int entered = Interlocked.Increment(ref active); + UpdateMaximum(ref maximumActive, entered); + Thread.Yield(); + Interlocked.Decrement(ref active); + fileLock.Release(); + Interlocked.Increment(ref successfulHandoffs); + } + })) + .ToArray(); + + start.Set(); + await Task.WhenAll(workers).WaitAsync(TimeSpan.FromSeconds(30)); + Assert.Equal(4_000, Volatile.Read(ref successfulHandoffs)); + Assert.Equal(1, Volatile.Read(ref maximumActive)); + + Assert.Equal(StoreStatus.Success, fileLock.TryAcquire(StoreWaitOptions.NoWait)); + fileLock.Release(); + } + finally + { + shared?.Dispose(); + File.Delete(path); + } + } + + [Fact] + [Trait("Category", "Integration")] + public void OpenFileDescriptionLocksContendAcrossAssemblyLoadContexts() + { + if (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + PlatformResourceName resource = PlatformResourceName.Create( + $"sms-linux-file-lock-alc-{Guid.NewGuid():N}"); + string path = resource.LinuxSynchronizationPath; + var firstContext = new AssemblyLoadContext("sms-lock-first", isCollectible: true); + var secondContext = new AssemblyLoadContext("sms-lock-second", isCollectible: true); + IDisposable? first = null; + IDisposable? second = null; + try + { + (string firstStatus, IDisposable? firstLock) = + TryAcquireInContext(firstContext, path, "Infinite"); + first = firstLock; + Assert.Equal(nameof(StoreStatus.Success), firstStatus); + Assert.NotNull(first); + + (string secondStatus, IDisposable? secondLock) = + TryAcquireInContext(secondContext, path, "NoWait"); + second = secondLock; + Assert.Equal(nameof(StoreStatus.StoreBusy), secondStatus); + Assert.Null(second); + Assert.Equal("RESULT StoreBusy", RunForeignProbe(path)); + + first.Dispose(); + first = null; + + (secondStatus, second) = TryAcquireInContext(secondContext, path, "NoWait"); + Assert.Equal(nameof(StoreStatus.Success), secondStatus); + Assert.NotNull(second); + second.Dispose(); + second = null; + + Assert.Equal("RESULT Success", RunForeignProbe(path)); + } + finally + { + second?.Dispose(); + first?.Dispose(); + firstContext.Unload(); + secondContext.Unload(); + File.Delete(path); + } + } + + [Fact] + [Trait("Category", "Integration")] + public void ManagedAndNativeOfdDescriptorsExcludeEachOtherInsideOneProcess() + { + if (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + PlatformResourceName resource = PlatformResourceName.Create( + $"sms-linux-file-lock-native-{Guid.NewGuid():N}"); + string path = resource.LinuxSynchronizationPath; + NativeOfdLock? nativeHolder = null; + LinuxFileLock? managedHolder = null; + try + { + Assert.True(NativeOfdLock.TryAcquire(path, out nativeHolder)); + Assert.NotNull(nativeHolder); + Assert.Equal( + StoreStatus.StoreBusy, + LinuxFileLock.TryAcquire(path, StoreWaitOptions.NoWait, out managedHolder)); + Assert.Null(managedHolder); + Assert.Equal("RESULT StoreBusy", RunForeignProbe(path)); + + nativeHolder.Dispose(); + nativeHolder = null; + + Assert.Equal( + StoreStatus.Success, + LinuxFileLock.TryAcquire(path, StoreWaitOptions.NoWait, out managedHolder)); + Assert.NotNull(managedHolder); + Assert.False(NativeOfdLock.TryAcquire(path, out nativeHolder)); + Assert.Null(nativeHolder); + Assert.Equal("RESULT StoreBusy", RunForeignProbe(path)); + + managedHolder.Dispose(); + managedHolder = null; + Assert.Equal("RESULT Success", RunForeignProbe(path)); + } + finally + { + nativeHolder?.Dispose(); + managedHolder?.Dispose(); + File.Delete(path); + } + } + + [Fact] + [Trait("Category", "Integration")] + public async Task ConcurrentFinalCloseAndReopenKeepOnePersistentLockRendezvous() + { + if (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + string name = $"sms-linux-lock-rendezvous-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.Create( + name, + slotCount: 4, + maxValueBytes: 64, + maxDescriptorBytes: 16, + maxKeyBytes: 16, + leaseRecordCount: 8); + PlatformResourceName resource = PlatformResourceName.Create(name); + MemoryStore? current = null; + try + { + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(options, out current)); + Assert.NotNull(current); + + for (var iteration = 0; iteration < 3; iteration++) + { + MemoryStore closing = current; + current = null; + using var start = new ManualResetEventSlim(false); + MemoryStore? reopened = null; + StoreOpenStatus reopenStatus = StoreOpenStatus.MappingFailed; + Task close = Task.Run(() => + { + start.Wait(); + closing.Dispose(); + }); + Task reopen = Task.Run(() => + { + start.Wait(); + reopenStatus = MemoryStore.TryCreateOrOpen(options, out reopened); + }); + + start.Set(); + await Task.WhenAll(close, reopen).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(StoreOpenStatus.Success, reopenStatus); + current = Assert.IsType(reopened); + Assert.True(File.Exists(resource.LinuxSynchronizationPath)); + byte[] key = [(byte)(iteration + 1)]; + + Assert.Equal( + StoreStatus.Success, + LinuxFileLock.TryAcquire( + resource.LinuxSynchronizationPath, + StoreWaitOptions.NoWait, + out LinuxFileLock? holder)); + using (Assert.IsType(holder)) + { + Assert.Equal("RESULT StoreBusy", RunForeignProbe(resource.LinuxSynchronizationPath)); + Assert.Equal( + StoreStatus.StoreBusy, + current.TryPublish(key, [42], default, StoreWaitOptions.NoWait)); + } + + Assert.Equal( + StoreStatus.Success, + current.TryPublish(key, [42], default, StoreWaitOptions.NoWait)); + Assert.Equal( + StoreStatus.Success, + current.TryRemove(key, StoreWaitOptions.NoWait)); + } + + current.Dispose(); + current = null; + Assert.True(File.Exists(resource.LinuxSynchronizationPath)); + Assert.Equal("RESULT Success", RunForeignProbe(resource.LinuxSynchronizationPath)); + } + finally + { + current?.Dispose(); + File.Delete(resource.LinuxSynchronizationPath); + } + } + + [Theory] + [InlineData("operation")] + [InlineData("lifecycle")] + [Trait("Category", "Integration")] + public void DisposingLocalContenderDoesNotReleaseForeignProcessExclusion(string lockKind) + { + if (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + PlatformResourceName resource = PlatformResourceName.Create( + $"sms-linux-file-lock-{lockKind}-{Guid.NewGuid():N}"); + string path = lockKind switch + { + "operation" => resource.LinuxSynchronizationPath, + "lifecycle" => resource.LinuxLifecycleLockPath, + _ => throw new ArgumentOutOfRangeException(nameof(lockKind)) + }; + + LinuxFileLock? holder = null; + LinuxFileLock? contender = null; + LinuxFileLock? sameThreadContender = null; + try + { + Assert.Equal( + StoreStatus.Success, + LinuxFileLock.TryAcquire(path, StoreWaitOptions.Infinite, out holder)); + Assert.NotNull(holder); + + Assert.Equal( + StoreStatus.StoreBusy, + LinuxFileLock.TryAcquire(path, StoreWaitOptions.NoWait, out sameThreadContender)); + Assert.Null(sameThreadContender); + + StoreStatus contenderStatus = StoreStatus.UnknownFailure; + Exception? contenderFailure = null; + var contenderThread = new Thread(() => + { + try + { + contenderStatus = LinuxFileLock.TryAcquire( + path, + new StoreWaitOptions(TimeSpan.FromMilliseconds(100)), + out contender); + } + catch (Exception exception) + { + contenderFailure = exception; + } + }) + { + IsBackground = true, + Name = "SharedMemoryStore local file-lock contender" + }; + + contenderThread.Start(); + Assert.True(contenderThread.Join(TimeSpan.FromSeconds(5)), "The local contender did not finish."); + Assert.Null(contenderFailure); + Assert.Equal(StoreStatus.StoreBusy, contenderStatus); + Assert.Null(contender); + + Assert.Equal("RESULT StoreBusy", RunForeignProbe(path)); + + holder.Dispose(); + holder = null; + + Assert.Equal("RESULT Success", RunForeignProbe(path)); + } + finally + { + sameThreadContender?.Dispose(); + contender?.Dispose(); + holder?.Dispose(); + File.Delete(path); + } + } + + private static string RunForeignProbe(string path) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(LocateAgentAssembly()); + startInfo.ArgumentList.Add("linux-file-lock-probe"); + startInfo.ArgumentList.Add(path); + + using Process process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start the Linux file-lock probe."); + if (!process.WaitForExit((int)AgentTimeout.TotalMilliseconds)) + { + process.Kill(entireProcessTree: true); + throw new TimeoutException("The Linux file-lock probe did not finish."); + } + + string output = process.StandardOutput.ReadToEnd().Trim(); + string error = process.StandardError.ReadToEnd().Trim(); + Assert.True( + process.ExitCode == 0, + $"Linux file-lock probe exited {process.ExitCode}. stdout={output} stderr={error}"); + return output; + } + + private static void UpdateMaximum(ref int target, int candidate) + { + int observed; + while (candidate > (observed = Volatile.Read(ref target)) + && Interlocked.CompareExchange(ref target, candidate, observed) != observed) + { + } + } + + private static (string Status, IDisposable? FileLock) TryAcquireInContext( + AssemblyLoadContext context, + string path, + string waitProperty) + { + Assembly assembly = context.LoadFromAssemblyPath(typeof(MemoryStore).Assembly.Location); + Type lockType = assembly.GetType( + "SharedMemoryStore.Interop.LinuxFileLock", + throwOnError: true)!; + Type waitType = assembly.GetType( + "SharedMemoryStore.StoreWaitOptions", + throwOnError: true)!; + object wait = waitType.GetProperty( + waitProperty, + BindingFlags.Public | BindingFlags.Static)!.GetValue(null)!; + MethodInfo method = lockType.GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(candidate => + candidate.Name == "TryAcquire" + && candidate.GetParameters().Length == 3); + object?[] arguments = [path, wait, null]; + object status = method.Invoke(null, arguments)!; + return (status.ToString()!, arguments[2] as IDisposable); + } + + private static string LocateAgentAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + string root = directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + return File.Exists(path) + ? path + : throw new FileNotFoundException("Lock-free agent was not built.", path); + } + + private sealed class NativeOfdLock : IDisposable + { + private const int OpenFileDescriptionSetLock = 37; + private readonly FileStream _stream; + private bool _disposed; + + private NativeOfdLock(FileStream stream) + { + _stream = stream; + } + + internal static bool TryAcquire(string path, out NativeOfdLock? holder) + { + holder = null; + LinuxSharedMemoryDirectory.EnsureExists(Path.GetDirectoryName(path) ?? "."); + var stream = new FileStream(path, new FileStreamOptions + { + Mode = FileMode.OpenOrCreate, + Access = FileAccess.ReadWrite, + Share = FileShare.ReadWrite | FileShare.Delete, + UnixCreateMode = LinuxSharedMemoryDirectory.PrivateFileMode + }); + var request = NativeFlock.Create(type: 1); + if (Fcntl(stream.SafeFileHandle, OpenFileDescriptionSetLock, ref request) != 0) + { + stream.Dispose(); + return false; + } + + holder = new NativeOfdLock(stream); + return true; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + var request = NativeFlock.Create(type: 2); + _ = Fcntl(_stream.SafeFileHandle, OpenFileDescriptionSetLock, ref request); + _stream.Dispose(); + } + + [DllImport("libc", EntryPoint = "fcntl", SetLastError = true)] + private static extern int Fcntl( + SafeFileHandle fileDescriptor, + int command, + ref NativeFlock request); + + [StructLayout(LayoutKind.Explicit, Size = 32)] + private struct NativeFlock + { + [FieldOffset(0)] + internal short Type; + + [FieldOffset(2)] + internal short Whence; + + [FieldOffset(8)] + internal long Start; + + [FieldOffset(16)] + internal long Length; + + [FieldOffset(24)] + internal int ProcessId; + + internal static NativeFlock Create(short type) => new() + { + Type = type, + Whence = 0, + Start = 0, + Length = 1, + ProcessId = 0 + }; + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LinuxOwnerAnchorIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LinuxOwnerAnchorIntegrationTests.cs new file mode 100644 index 0000000..164a903 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LinuxOwnerAnchorIntegrationTests.cs @@ -0,0 +1,387 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using SharedMemoryStore.Interop; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.IntegrationTests; + +[SupportedOSPlatform("linux")] +public sealed class LinuxOwnerAnchorIntegrationTests +{ + private static readonly TimeSpan AgentTimeout = TimeSpan.FromSeconds(20); + + [Fact] + [Trait("Category", "Integration")] + public void ColdOpenReclaimsUnlockedOrphanWithoutDisturbingLiveOrUncertainAnchors() + { + if (!IsQualifiedLinuxHost()) + { + return; + } + + string name = "sms-owner-anchor-sweep-" + Guid.NewGuid().ToString("N"); + PlatformResourceName resource = PlatformResourceName.Create(name); + SharedMemoryStoreOptions createOptions = Options(name, OpenMode.CreateNew); + SharedMemoryStoreOptions openOptions = Options(name, OpenMode.OpenExisting); + Store? sibling = null; + Store? opener = null; + LinuxOwnerAnchor? lockedOrphan = null; + string? ambiguousPath = null; + string? ambiguousTarget = null; + string? malformedPath = null; + string? fifoPath = null; + try + { + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(createOptions, out sibling)); + Assert.NotNull(sibling); + string siblingOwner = Assert.Single(ReadOwnerLines(resource.LinuxOwnersPath)); + Guid siblingToken = ParseOwnerToken(siblingOwner); + string siblingAnchorPath = LinuxOwnerAnchor.GetPath(resource.LinuxOwnersPath, siblingToken); + + Guid referencedStaleToken = Guid.NewGuid(); + string referencedStalePath = CreateUnlockedAnchor( + resource.LinuxOwnersPath, + referencedStaleToken); + string referencedStaleOwner = string.Join( + ':', + int.MaxValue.ToString(CultureInfo.InvariantCulture), + "proc-stale-tail", + referencedStaleToken.ToString("N")); + File.AppendAllText( + resource.LinuxOwnersPath, + referencedStaleOwner + Environment.NewLine); + File.SetUnixFileMode( + resource.LinuxOwnersPath, + LinuxSharedMemoryDirectory.PrivateFileMode); + + Guid staleToken = Guid.NewGuid(); + string stalePath = CreateUnlockedAnchor(resource.LinuxOwnersPath, staleToken); + Guid lockedToken = Guid.NewGuid(); + lockedOrphan = LinuxOwnerAnchor.Create(resource.LinuxOwnersPath, lockedToken); + string lockedPath = lockedOrphan.AnchorPath; + Guid ambiguousToken = Guid.NewGuid(); + ambiguousPath = LinuxOwnerAnchor.GetPath(resource.LinuxOwnersPath, ambiguousToken); + ambiguousTarget = resource.LinuxOwnersPath + ".anchor-test-target"; + File.WriteAllBytes(ambiguousTarget, []); + File.CreateSymbolicLink(ambiguousPath, ambiguousTarget); + malformedPath = resource.LinuxOwnersPath + ".anchor.not-a-valid-owner-token"; + File.WriteAllBytes(malformedPath, []); + fifoPath = CreateFifoAnchor(resource.LinuxOwnersPath, Guid.NewGuid()); + + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(openOptions, out opener)); + Assert.NotNull(opener); + + Assert.False(File.Exists(stalePath)); + // Once the first authoritative live witness is found, an attach + // preserves every committed tail record. Consequently the sweep + // may remove the truly unreferenced orphan above, but must defer + // this referenced stale anchor to a full close/no-live scan. + Assert.True(File.Exists(referencedStalePath)); + Assert.Contains( + referencedStaleOwner, + ReadOwnerLines(resource.LinuxOwnersPath), + StringComparer.Ordinal); + Assert.True(File.Exists(siblingAnchorPath)); + Assert.Contains(siblingOwner, ReadOwnerLines(resource.LinuxOwnersPath), StringComparer.Ordinal); + Assert.True(File.Exists(lockedPath)); + Assert.True(File.Exists(ambiguousPath)); + Assert.True(File.Exists(ambiguousTarget)); + Assert.True(File.Exists(malformedPath)); + Assert.True(File.Exists(fifoPath)); + Assert.Equal( + LinuxOwnerAnchorState.Locked, + LinuxOwnerAnchor.Probe( + resource.LinuxOwnersPath, + lockedToken, + honorLocalRegistry: false)); + Assert.Equal( + LinuxOwnerAnchorState.Ambiguous, + LinuxOwnerAnchor.Probe( + resource.LinuxOwnersPath, + ambiguousToken, + honorLocalRegistry: false)); + + Assert.Equal(StoreStatus.Success, sibling!.TryPublish([0x55], [0x7A])); + Assert.Equal(StoreStatus.Success, sibling.TryAcquire([0x55], out ValueLease lease)); + Assert.Equal(0x7A, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, sibling.TryRemove([0x55])); + } + finally + { + opener?.Dispose(); + sibling?.Dispose(); + lockedOrphan?.Dispose(); + DeleteIfExists(ambiguousPath); + DeleteIfExists(malformedPath); + DeleteIfExists(fifoPath); + DeleteIfExists(ambiguousTarget); + } + } + + [Fact] + [Trait("Category", "Integration")] + public async Task ForeignPidViewCannotDeleteLiveMappingAndCrashUnlocksAnchorForRecreation() + { + if (!IsQualifiedLinuxHost()) + { + return; + } + + string name = "sms-owner-anchor-pidns-" + Guid.NewGuid().ToString("N"); + PlatformResourceName resource = PlatformResourceName.Create(name); + SharedMemoryStoreOptions createOptions = Options(name, OpenMode.CreateNew); + SharedMemoryStoreOptions openOptions = Options(name, OpenMode.OpenExisting); + Store? original = null; + Store? survivor = null; + Store? recreated = null; + using Process child = StartOwner(name, keyValue: 77); + try + { + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(createOptions, out original)); + Assert.NotNull(original); + + // The helper starts only after the mapping exists. + child.Start(); + int childPid = await ReadReadyProcessIdAsync(child); + string[] owners = ReadOwnerLines(resource.LinuxOwnersPath); + Assert.Equal(2, owners.Length); + string childOwner = Assert.Single(owners, owner => ParseOwnerProcessId(owner) == childPid); + string[] childParts = childOwner.Split(':', 3); + Assert.Equal(3, childParts.Length); + Assert.True(Guid.TryParseExact(childParts[2], "N", out Guid childToken)); + string anchorPath = LinuxOwnerAnchor.GetPath(resource.LinuxOwnersPath, childToken); + Assert.Equal( + LinuxOwnerAnchorState.Locked, + LinuxOwnerAnchor.Probe( + resource.LinuxOwnersPath, + childToken, + honorLocalRegistry: false)); + + string foreignView = string.Join(':', int.MaxValue, childParts[1], childParts[2]); + RewriteOwner(resource, childOwner, foreignView); + + original.Dispose(); + original = null; + Assert.True(File.Exists(resource.LinuxRegionPath)); + Assert.True(File.Exists(anchorPath)); + + // PID/start-token probing alone sees no such process. The locked + // anchor is authoritative and preserves the live child's mapping. + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(openOptions, out survivor)); + Assert.NotNull(survivor); + Assert.Equal(StoreStatus.Success, survivor.TryAcquire(BitConverter.GetBytes(77), out ValueLease lease)); + Assert.Equal(77, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + survivor.Dispose(); + survivor = null; + + Kill(child); + Assert.Equal( + LinuxOwnerAnchorState.Unlocked, + LinuxOwnerAnchor.Probe( + resource.LinuxOwnersPath, + childToken, + honorLocalRegistry: false)); + + // flock is released by process teardown. The next cold lifecycle + // operation drops the stale line and anchor before recreating. + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(createOptions, out recreated)); + Assert.NotNull(recreated); + Assert.False(File.Exists(anchorPath)); + Assert.Single(ReadOwnerLines(resource.LinuxOwnersPath)); + } + finally + { + Kill(child); + recreated?.Dispose(); + survivor?.Dispose(); + original?.Dispose(); + } + } + + private static Process StartOwner(string name, int keyValue) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(LocateOwnerToolAssembly()); + foreach (string argument in new[] + { + "live", + name, + "4", + "64", + "8", + "8", + "8", + keyValue.ToString(CultureInfo.InvariantCulture) + }) + { + startInfo.ArgumentList.Add(argument); + } + + return new Process { StartInfo = startInfo }; + } + + private static async Task ReadReadyProcessIdAsync(Process process) + { + string? line = await process.StandardOutput.ReadLineAsync().WaitAsync(AgentTimeout); + if (line is null || !line.StartsWith("READY ", StringComparison.Ordinal)) + { + string error = await process.StandardError.ReadToEndAsync(); + throw new Xunit.Sdk.XunitException( + "Owner helper did not become ready. stdout=" + line + " stderr=" + error); + } + + return int.Parse(line.AsSpan(6), NumberStyles.None, CultureInfo.InvariantCulture); + } + + private static void RewriteOwner( + PlatformResourceName resource, + string expectedOwner, + string replacementOwner) + { + Assert.Equal( + StoreStatus.Success, + LinuxFileLock.TryAcquire( + resource.LinuxLifecycleLockPath, + StoreWaitOptions.Infinite, + out LinuxFileLock? lifecycleLock)); + using (Assert.IsType(lifecycleLock)) + { + string[] owners = ReadOwnerLines(resource.LinuxOwnersPath); + Assert.Contains(expectedOwner, owners, StringComparer.Ordinal); + string temporaryPath = resource.LinuxOwnersPath + ".test.tmp"; + try + { + File.WriteAllLines( + temporaryPath, + owners.Select(owner => string.Equals(owner, expectedOwner, StringComparison.Ordinal) + ? replacementOwner + : owner)); + File.SetUnixFileMode(temporaryPath, LinuxSharedMemoryDirectory.PrivateFileMode); + File.Move(temporaryPath, resource.LinuxOwnersPath, overwrite: true); + } + finally + { + File.Delete(temporaryPath); + } + } + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode openMode) => new() + { + Name = name, + OpenMode = openMode, + SlotCount = 4, + MaxValueBytes = 64, + MaxDescriptorBytes = 8, + MaxKeyBytes = 8, + LeaseRecordCount = 8, + EnableLeaseRecovery = true, + TotalBytes = SharedMemoryStoreOptions.CalculateRequiredBytes(4, 64, 8, 8, 8) + }; + + private static string[] ReadOwnerLines(string path) + { + Assert.True(File.Exists(path)); + return File.ReadAllLines(path) + .Select(static line => line.Trim()) + .Where(static line => line.Length != 0) + .ToArray(); + } + + private static int ParseOwnerProcessId(string owner) => + int.Parse( + owner.AsSpan(0, owner.IndexOf(':')), + NumberStyles.None, + CultureInfo.InvariantCulture); + + private static Guid ParseOwnerToken(string owner) + { + string[] parts = owner.Split(':', 3); + Assert.Equal(3, parts.Length); + Assert.True(Guid.TryParseExact(parts[2], "N", out Guid token)); + return token; + } + + private static string CreateUnlockedAnchor(string ownersPath, Guid token) + { + string path = LinuxOwnerAnchor.GetPath(ownersPath, token); + File.WriteAllBytes(path, []); + File.SetUnixFileMode(path, LinuxSharedMemoryDirectory.PrivateFileMode); + return path; + } + + private static string CreateFifoAnchor(string ownersPath, Guid token) + { + string path = LinuxOwnerAnchor.GetPath(ownersPath, token); + int result = MkFifo(path, 0x180); // 0600 + Assert.True(result == 0, $"mkfifo failed with errno {Marshal.GetLastPInvokeError()}."); + return path; + } + + private static void DeleteIfExists(string? path) + { + if (path is not null) + { + File.Delete(path); + } + } + + [DllImport("libc", EntryPoint = "mkfifo", SetLastError = true)] + private static extern int MkFifo( + [MarshalAs(UnmanagedType.LPUTF8Str)] string path, + uint mode); + + private static string LocateOwnerToolAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + string root = directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LeaseOwnerTool", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LeaseOwnerTool.dll"); + return File.Exists(path) + ? path + : throw new FileNotFoundException("Lease owner helper was not built.", path); + } + + private static bool IsQualifiedLinuxHost() => + OperatingSystem.IsLinux() + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private static void Kill(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(5_000); + } + } + catch + { + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LinuxOwnerReleaseMarkerIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LinuxOwnerReleaseMarkerIntegrationTests.cs new file mode 100644 index 0000000..2d5ec9a --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LinuxOwnerReleaseMarkerIntegrationTests.cs @@ -0,0 +1,510 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using SharedMemoryStore.IntegrationTests.TestSupport; +using SharedMemoryStore.Interop; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.IntegrationTests; + +[SupportedOSPlatform("linux")] +public sealed class LinuxOwnerReleaseMarkerIntegrationTests +{ + private static readonly TimeSpan DisposeCompletionLimit = TimeSpan.FromSeconds(2); + + [Fact] + [Trait("Category", "Integration")] + public async Task HeldLifecycleBoundsDisposeKeepsSiblingUsableAndNextOpenRemovesOnlyExactGhost() + { + if (!IsQualifiedLinuxHost()) + { + return; + } + + var name = $"sms-owner-release-exact-{Guid.NewGuid():N}"; + var resource = PlatformResourceName.Create(name); + var createOptions = CreateOptions(name, OpenMode.CreateNew, participantRecordCount: 3); + var openOptions = CreateOptions(name, OpenMode.OpenExisting, participantRecordCount: 3); + Store? first = null; + Store? sibling = null; + Store? next = null; + LinuxFileLockHolder? heldLifecycle = null; + + try + { + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(createOptions, out first)); + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(openOptions, out sibling)); + Assert.NotNull(first); + Assert.NotNull(sibling); + + var ownersBefore = ReadOwnerLines(resource.LinuxOwnersPath); + Assert.Equal(2, ownersBefore.Length); + heldLifecycle = LinuxFileLockHolder.Acquire(resource.LinuxLifecycleLockPath); + Assert.Equal( + StoreStatus.Success, + LinuxFileLock.TryAcquire( + resource.LinuxSynchronizationPath, + StoreWaitOptions.NoWait, + out var ordinaryOperationLock)); + Assert.NotNull(ordinaryOperationLock); + ordinaryOperationLock!.Dispose(); + + var stopwatch = Stopwatch.StartNew(); + var disposeTask = Task.Run(first!.Dispose); + await disposeTask.WaitAsync(DisposeCompletionLimit); + stopwatch.Stop(); + first = null; + + Assert.True(stopwatch.Elapsed <= DisposeCompletionLimit); + var markers = ReadFinalizedMarkers(resource); + var marker = Assert.Single(markers); + Assert.Equal(LinuxSharedMemoryDirectory.PrivateFileMode, File.GetUnixFileMode(marker.Path)); + Assert.Contains(marker.Owner, ownersBefore, StringComparer.Ordinal); + Guid releasedOwnerToken = ParseOwnerToken(marker.Owner); + Assert.Equal( + LinuxOwnerAnchorState.Missing, + LinuxOwnerAnchor.Probe(resource.LinuxOwnersPath, releasedOwnerToken)); + Assert.False(File.Exists(LinuxOwnerAnchor.GetPath(resource.LinuxOwnersPath, releasedOwnerToken))); + string siblingOwner = Assert.Single( + ownersBefore, + owner => !string.Equals(owner, marker.Owner, StringComparison.Ordinal)); + Assert.Equal( + LinuxOwnerAnchorState.Locked, + LinuxOwnerAnchor.Probe(resource.LinuxOwnersPath, ParseOwnerToken(siblingOwner))); + Assert.Equal( + ownersBefore.OrderBy(static owner => owner, StringComparer.Ordinal), + ReadOwnerLines(resource.LinuxOwnersPath).OrderBy(static owner => owner, StringComparer.Ordinal)); + + Assert.Equal(StoreStatus.Success, sibling!.TryPublish([1], [42])); + Assert.Equal(StoreStatus.Success, sibling.TryAcquire([1], out var lease)); + Assert.Equal(42, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + + heldLifecycle.Dispose(); + heldLifecycle = null; + + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(openOptions, out next)); + Assert.NotNull(next); + var ownersAfter = ReadOwnerLines(resource.LinuxOwnersPath); + Assert.Equal(2, ownersAfter.Length); + Assert.DoesNotContain(marker.Owner, ownersAfter, StringComparer.Ordinal); + Assert.Contains(ownersBefore.Single(owner => !string.Equals(owner, marker.Owner, StringComparison.Ordinal)), ownersAfter, StringComparer.Ordinal); + Assert.Empty(GetFinalizedMarkerPaths(resource)); + Assert.Equal(StoreStatus.Success, sibling.TryRemove([1])); + } + finally + { + heldLifecycle?.Dispose(); + next?.Dispose(); + sibling?.Dispose(); + first?.Dispose(); + } + } + + [Fact] + [Trait("Category", "Integration")] + public async Task FinalOwnerMarkerAllowsCreateNewWhileReleasingProcessRemainsAlive() + { + if (!IsQualifiedLinuxHost()) + { + return; + } + + var name = $"sms-owner-release-recreate-{Guid.NewGuid():N}"; + var resource = PlatformResourceName.Create(name); + var options = CreateOptions(name, OpenMode.CreateNew, participantRecordCount: 1); + Store? first = null; + Store? recreated = null; + LinuxFileLockHolder? heldLifecycle = null; + + try + { + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(options, out first)); + Assert.NotNull(first); + var originalOwner = Assert.Single(ReadOwnerLines(resource.LinuxOwnersPath)); + heldLifecycle = LinuxFileLockHolder.Acquire(resource.LinuxLifecycleLockPath); + + var disposeTask = Task.Run(first!.Dispose); + await disposeTask.WaitAsync(DisposeCompletionLimit); + first = null; + var marker = Assert.Single(ReadFinalizedMarkers(resource)); + Assert.Equal(originalOwner, marker.Owner); + Assert.Equal(Environment.ProcessId, ParseOwnerProcessId(marker.Owner)); + Assert.Equal( + LinuxOwnerAnchorState.Missing, + LinuxOwnerAnchor.Probe(resource.LinuxOwnersPath, ParseOwnerToken(marker.Owner))); + + heldLifecycle!.Dispose(); + heldLifecycle = null; + + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(options, out recreated)); + Assert.NotNull(recreated); + var recreatedOwner = Assert.Single(ReadOwnerLines(resource.LinuxOwnersPath)); + Assert.NotEqual(originalOwner, recreatedOwner); + Assert.Empty(GetFinalizedMarkerPaths(resource)); + } + finally + { + heldLifecycle?.Dispose(); + recreated?.Dispose(); + first?.Dispose(); + } + } + + [Fact] + [Trait("Category", "Integration")] + public async Task ConcurrentBlockedReleasesPublishDistinctPrivateMarkersWithoutLoss() + { + if (!IsQualifiedLinuxHost()) + { + return; + } + + const int ownerCount = 8; + var name = $"sms-owner-release-concurrent-{Guid.NewGuid():N}"; + var resource = PlatformResourceName.Create(name); + var createOptions = CreateOptions(name, OpenMode.CreateNew, participantRecordCount: ownerCount + 1); + var openOptions = CreateOptions(name, OpenMode.OpenExisting, participantRecordCount: ownerCount + 1); + var recreateOptions = CreateOptions(name, OpenMode.CreateNew, participantRecordCount: ownerCount + 1); + var stores = new List(ownerCount); + Store? recreated = null; + LinuxFileLockHolder? heldLifecycle = null; + + try + { + for (var index = 0; index < ownerCount; index++) + { + var status = Store.TryCreateOrOpen(index == 0 ? createOptions : openOptions, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + stores.Add(Assert.IsType(store)); + } + + var ownersBefore = ReadOwnerLines(resource.LinuxOwnersPath); + Assert.Equal(ownerCount, ownersBefore.Length); + heldLifecycle = LinuxFileLockHolder.Acquire(resource.LinuxLifecycleLockPath); + + using var start = new ManualResetEventSlim(); + var releaseTasks = stores + .Select(store => Task.Run(() => + { + start.Wait(); + store.Dispose(); + })) + .ToArray(); + start.Set(); + await Task.WhenAll(releaseTasks).WaitAsync(DisposeCompletionLimit); + stores.Clear(); + + var markers = ReadFinalizedMarkers(resource); + Assert.Equal(ownerCount, markers.Length); + Assert.Equal(ownerCount, markers.Select(static marker => marker.Path).Distinct(StringComparer.Ordinal).Count()); + Assert.Equal(ownerCount, markers.Select(static marker => marker.Owner).Distinct(StringComparer.Ordinal).Count()); + Assert.Equal( + ownersBefore.OrderBy(static owner => owner, StringComparer.Ordinal), + markers.Select(static marker => marker.Owner).OrderBy(static owner => owner, StringComparer.Ordinal)); + Assert.All( + markers, + marker => Assert.Equal( + LinuxSharedMemoryDirectory.PrivateFileMode, + File.GetUnixFileMode(marker.Path))); + Assert.All( + markers, + marker => Assert.Equal( + LinuxOwnerAnchorState.Missing, + LinuxOwnerAnchor.Probe(resource.LinuxOwnersPath, ParseOwnerToken(marker.Owner)))); + + heldLifecycle!.Dispose(); + heldLifecycle = null; + + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(recreateOptions, out recreated)); + Assert.NotNull(recreated); + Assert.Empty(GetFinalizedMarkerPaths(resource)); + Assert.Single(ReadOwnerLines(resource.LinuxOwnersPath)); + } + finally + { + heldLifecycle?.Dispose(); + recreated?.Dispose(); + foreach (var store in stores) + { + store.Dispose(); + } + } + } + + [Fact] + [Trait("Category", "Integration")] + public void MalformedFinalizedMarkerFailsOpenClosedAndIsPreserved() + { + if (!IsQualifiedLinuxHost()) + { + return; + } + + var name = $"sms-owner-release-malformed-{Guid.NewGuid():N}"; + var resource = PlatformResourceName.Create(name); + var createOptions = CreateOptions(name, OpenMode.CreateNew, participantRecordCount: 2); + var openOptions = CreateOptions(name, OpenMode.OpenExisting, participantRecordCount: 2); + var markerPath = resource.LinuxOwnersPath + + ".released." + + Guid.NewGuid().ToString("N") + + ".ready"; + + using var store = IntegrationStoreFactory.Create(createOptions); + File.WriteAllText(markerPath, "not-an-owner-record"); + File.SetUnixFileMode(markerPath, LinuxSharedMemoryDirectory.PrivateFileMode); + try + { + var status = Store.TryCreateOrOpen(openOptions, out var rejected); + + rejected?.Dispose(); + Assert.Equal(StoreOpenStatus.MappingFailed, status); + Assert.Null(rejected); + Assert.True(File.Exists(markerPath)); + Assert.Equal(StoreStatus.Success, store.TryPublish([7], [9])); + Assert.Equal(StoreStatus.Success, store.TryRemove([7])); + } + finally + { + File.Delete(markerPath); + } + } + + [Fact] + [Trait("Category", "Integration")] + public async Task OpenBlockedBeforeMappingDoesNotPublishOwnerOrReleaseMarker() + { + if (!IsQualifiedLinuxHost()) + { + return; + } + + var name = $"sms-owner-release-open-failure-{Guid.NewGuid():N}"; + var resource = PlatformResourceName.Create(name); + var createOptions = CreateOptions(name, OpenMode.CreateNew, participantRecordCount: 3); + var openOptions = CreateOptions(name, OpenMode.OpenExisting, participantRecordCount: 3); + using var anchor = IntegrationStoreFactory.Create(createOptions); + var anchorOwner = Assert.Single(ReadOwnerLines(resource.LinuxOwnersPath)); + LinuxFileLockHolder? operationLock = null; + Store? next = null; + + try + { + operationLock = LinuxFileLockHolder.Acquire(resource.LinuxSynchronizationPath); + var failedOpen = Task.Run(() => + { + var status = Store.TryCreateOrOpen( + openOptions, + new StoreWaitOptions(TimeSpan.FromMilliseconds(600)), + out var rejected); + rejected?.Dispose(); + return status; + }); + + await Task.Delay(TimeSpan.FromMilliseconds(100)); + Assert.False(failedOpen.IsCompleted); + Assert.Equal([anchorOwner], ReadOwnerLines(resource.LinuxOwnersPath)); + Assert.Empty(ReadFinalizedMarkers(resource)); + Assert.Equal(StoreOpenStatus.StoreBusy, await failedOpen.WaitAsync(TimeSpan.FromSeconds(3))); + + Assert.Equal([anchorOwner], ReadOwnerLines(resource.LinuxOwnersPath)); + Assert.Empty(ReadFinalizedMarkers(resource)); + Assert.Equal( + LinuxOwnerAnchorState.Locked, + LinuxOwnerAnchor.Probe(resource.LinuxOwnersPath, ParseOwnerToken(anchorOwner))); + Assert.Equal(StoreStatus.Success, anchor.TryPublish([3], [5])); + + operationLock.Dispose(); + operationLock = null; + + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(openOptions, out next)); + Assert.NotNull(next); + var ownersAfter = ReadOwnerLines(resource.LinuxOwnersPath); + Assert.Contains(anchorOwner, ownersAfter, StringComparer.Ordinal); + Assert.Empty(GetFinalizedMarkerPaths(resource)); + Assert.Equal(StoreStatus.Success, anchor.TryRemove([3])); + } + finally + { + operationLock?.Dispose(); + next?.Dispose(); + } + } + + private static bool IsQualifiedLinuxHost() + { + return OperatingSystem.IsLinux() + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + } + + private static SharedMemoryStoreOptions CreateOptions( + string name, + OpenMode openMode, + int participantRecordCount) + { + return SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 4, + maxValueBytes: 64, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount: 8, + participantRecordCount, + openMode); + } + + private static string[] ReadOwnerLines(string path) + { + Assert.True(File.Exists(path)); + return File.ReadAllLines(path) + .Select(static line => line.Trim()) + .Where(static line => line.Length != 0) + .ToArray(); + } + + private static ReleaseMarker[] ReadFinalizedMarkers(PlatformResourceName resource) + { + return GetFinalizedMarkerPaths(resource) + .Select(path => new ReleaseMarker(path, File.ReadAllText(path).Trim())) + .ToArray(); + } + + private static string[] GetFinalizedMarkerPaths(PlatformResourceName resource) + { + var directory = Path.GetDirectoryName(resource.LinuxOwnersPath)!; + var pattern = Path.GetFileName(resource.LinuxOwnersPath) + ".released.*.ready"; + return Directory.Exists(directory) + ? Directory.GetFiles(directory, pattern, SearchOption.TopDirectoryOnly) + : []; + } + + private static int ParseOwnerProcessId(string owner) + { + return int.Parse( + owner.AsSpan(0, owner.IndexOf(':')), + System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture); + } + + private static Guid ParseOwnerToken(string owner) + { + string[] parts = owner.Split(':', 3); + Assert.Equal(3, parts.Length); + Assert.True(Guid.TryParseExact(parts[2], "N", out Guid token)); + return token; + } + + private static async Task WaitForOwnerCountAsync(string ownersPath, int expectedCount) + { + var deadline = Stopwatch.StartNew(); + while (deadline.Elapsed < TimeSpan.FromSeconds(2)) + { + if (File.Exists(ownersPath) + && File.ReadAllLines(ownersPath).Count(static line => line.Trim().Length != 0) == expectedCount) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(10)); + } + + throw new TimeoutException($"The owner sidecar did not reach {expectedCount} records."); + } + + private sealed record ReleaseMarker(string Path, string Owner); + + private sealed class LinuxFileLockHolder : IDisposable + { + private readonly string _path; + private readonly ManualResetEventSlim _ready = new(); + private readonly ManualResetEventSlim _release = new(); + private readonly ManualResetEventSlim _finished = new(); + private readonly Thread _ownerThread; + private Exception? _failure; + private bool _disposed; + + private LinuxFileLockHolder(string path) + { + _path = path; + _ownerThread = new Thread(HoldLock) + { + IsBackground = true, + Name = "SharedMemoryStore file-lock test holder" + }; + _ownerThread.Start(); + if (!_ready.Wait(TimeSpan.FromSeconds(5))) + { + throw new TimeoutException("The file-lock holder did not start."); + } + + if (_failure is not null) + { + throw new InvalidOperationException("The file-lock holder failed to acquire the lock.", _failure); + } + } + + public static LinuxFileLockHolder Acquire(string path) => new(path); + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _release.Set(); + if (!_finished.Wait(TimeSpan.FromSeconds(5))) + { + throw new TimeoutException("The file-lock holder did not release the lock."); + } + + _release.Dispose(); + _ready.Dispose(); + _finished.Dispose(); + if (_failure is not null) + { + throw new InvalidOperationException("The file-lock holder failed.", _failure); + } + } + + private void HoldLock() + { + LinuxFileLock? heldLock = null; + try + { + var status = LinuxFileLock.TryAcquire( + _path, + StoreWaitOptions.Infinite, + out heldLock); + if (status != StoreStatus.Success || heldLock is null) + { + throw new InvalidOperationException($"File-lock acquisition returned {status}."); + } + + _ready.Set(); + _release.Wait(); + } + catch (Exception exception) + { + _failure = exception; + _ready.Set(); + } + finally + { + try + { + heldLock?.Dispose(); + } + catch (Exception exception) + { + _failure ??= exception; + } + finally + { + _finished.Set(); + } + } + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeBroadcastLeaseIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeBroadcastLeaseIntegrationTests.cs new file mode 100644 index 0000000..be111bd --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeBroadcastLeaseIntegrationTests.cs @@ -0,0 +1,440 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using SharedMemoryStore.Interop; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeBroadcastLeaseIntegrationTests +{ + private const int ReaderCount = 12; + private const int SlotCount = 1; + private const int MaxValueBytes = 4_096; + private const int MaxDescriptorBytes = 16; + private const int MaxKeyBytes = 16; + private const int LeaseRecordCount = 16; + private const int ParticipantRecordCount = 16; + private static readonly TimeSpan AgentTimeout = TimeSpan.FromSeconds(45); + + [Fact] + [Trait("Category", "Integration")] + public void TwelveProcessesHoldThroughLogicalRemoveThenFinalReleaseReclaimsExactlyOneSlot() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-broadcast-leases-{Guid.NewGuid():N}"; + byte[] key = [0x42, 0x52, 0x4f, 0x41, 0x44, 0x43, 0x41, 0x53, 0x54]; + byte[] value = CreatePayload(); + byte[] descriptor = [0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0]; + using var store = CreateStore(name); + Assert.Equal(StoreStatus.Success, store.TryPublish(key, value, descriptor)); + + string directory = Path.Combine(Path.GetTempPath(), $"sms-v2-broadcast-gate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string releasePath = Path.Combine(directory, "release.all"); + string[] readyPaths = Enumerable.Range(0, ReaderCount) + .Select(index => Path.Combine(directory, $"reader-{index}.ready")) + .ToArray(); + Process[] readers = readyPaths + .Select(readyPath => StartAgent(HoldCommand(name, key, value, descriptor, readyPath, releasePath))) + .ToArray(); + + try + { + WaitForReadyFiles(readyPaths, readers, AgentTimeout); + + // Every ready file is written only after its process has activated + // and validated a lease. At this point twelve independent processes + // simultaneously protect the same immutable generation. + Assert.All(readers, static process => Assert.False(process.HasExited)); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out var controllerLease)); + Assert.Equal(value.Length, controllerLease.ValueLength); + Assert.Equal(descriptor.Length, controllerLease.DescriptorLength); + Assert.Equal(Checksum(value, descriptor), Checksum(controllerLease.ValueSpan, controllerLease.DescriptorSpan)); + Assert.Equal(StoreStatus.Success, controllerLease.Release()); + + Assert.Equal(StoreStatus.RemovePending, store.TryRemove(key)); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(key, out var rejected)); + Assert.False(rejected.IsValid); + Assert.Equal(StoreStatus.DuplicateKey, store.TryPublish(key, [0xff])); + Assert.Equal(StoreStatus.StoreFull, store.TryPublish([0x7f], [0x7f])); + + File.WriteAllText(releasePath, "release"); + AgentResult[] results = WaitForAgents(readers, AgentTimeout); + Assert.All(results, static result => + { + Assert.True( + result.ExitCode == 0, + "Agent exit code: " + + result.ExitCode.ToString(CultureInfo.InvariantCulture) + + Environment.NewLine + + "stdout: " + + result.StandardOutput + + Environment.NewLine + + "stderr: " + + result.StandardError); + Assert.Contains("OK lease-hold released=1", result.StandardOutput); + }); + + // The last exact lease release cooperatively performs one safe + // physical reclaim. With one configured slot, successful + // republish proves that no second slot or global maintenance owner + // masked a leak. + byte[] replacement = value.ToArray(); + for (var index = 0; index < replacement.Length; index++) + { + replacement[index] ^= 0xa5; + } + + Assert.Equal(StoreStatus.Success, store.TryPublish(key, replacement, descriptor)); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out var replacementLease)); + Assert.Equal( + Checksum(replacement, descriptor), + Checksum(replacementLease.ValueSpan, replacementLease.DescriptorSpan)); + Assert.Equal(StoreStatus.Success, replacementLease.Release()); + } + finally + { + try + { + File.WriteAllText(releasePath, "release"); + } + catch + { + // Best effort before process termination. + } + + foreach (Process reader in readers) + { + Kill(reader); + reader.Dispose(); + } + + try + { + Directory.Delete(directory, recursive: true); + } + catch + { + // Unique temporary artifacts can be reclaimed after a racing process exit. + } + } + } + + [Fact] + [Trait("Category", "Integration")] + public void LinuxDefaultBoundedColdOpenConvergesAcrossThreeTwelveProcessWaves() + { + if (!OperatingSystem.IsLinux() + || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + string name = $"sms-v2-linux-open-contention-{Guid.NewGuid():N}"; + byte[] key = [0x4f, 0x50, 0x45, 0x4e]; + byte[] value = [0x62, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64]; + byte[] descriptor = [0x01]; + using var store = CreateStore(name); + Assert.Equal(StoreStatus.Success, store.TryPublish(key, value, descriptor)); + + // Every agent uses MemoryStore.TryCreateOrOpen(options), so each open + // retains the public one-second default bound. Repeated waves also force + // normal owner removal and any bounded-release marker reconciliation + // before the next twelve-way attach burst. + for (var wave = 0; wave < 3; wave++) + { + RunDefaultBoundedOpenWave(name, key, value, descriptor, wave); + MemoryStore verifier = OpenStore(name); + verifier.Dispose(); + Assert.Equal( + 1, + File.ReadAllLines(PlatformResourceName.Create(name).LinuxOwnersPath) + .Count(static line => line.Trim().Length != 0)); + } + } + + private static void RunDefaultBoundedOpenWave( + string name, + byte[] key, + byte[] value, + byte[] descriptor, + int wave) + { + string directory = Path.Combine( + Path.GetTempPath(), + $"sms-v2-linux-open-wave-{wave}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string releasePath = Path.Combine(directory, "release.all"); + string[] readyPaths = Enumerable.Range(0, ReaderCount) + .Select(index => Path.Combine(directory, $"reader-{index}.ready")) + .ToArray(); + Process[] readers = []; + try + { + readers = readyPaths + .Select(readyPath => StartAgent( + HoldCommand(name, key, value, descriptor, readyPath, releasePath))) + .ToArray(); + WaitForReadyFiles(readyPaths, readers, AgentTimeout); + Assert.All(readers, static process => Assert.False(process.HasExited)); + Assert.Equal( + ReaderCount + 1, + File.ReadAllLines(PlatformResourceName.Create(name).LinuxOwnersPath) + .Count(static line => line.Trim().Length != 0)); + + File.WriteAllText(releasePath, "release"); + AgentResult[] results = WaitForAgents(readers, AgentTimeout); + Assert.All(results, static result => + { + Assert.True( + result.ExitCode == 0, + "Default-bounded opener failed. Exit=" + + result.ExitCode.ToString(CultureInfo.InvariantCulture) + + Environment.NewLine + + "stdout: " + + result.StandardOutput + + Environment.NewLine + + "stderr: " + + result.StandardError); + Assert.Contains("OK lease-hold released=1", result.StandardOutput); + }); + } + finally + { + try + { + File.WriteAllText(releasePath, "release"); + } + catch + { + // Best effort before process termination. + } + + foreach (Process reader in readers) + { + Kill(reader); + reader.Dispose(); + } + + try + { + Directory.Delete(directory, recursive: true); + } + catch + { + // Unique temporary artifacts can be reclaimed after a racing process exit. + } + } + } + + private static MemoryStore CreateStore(string name) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + SlotCount, + MaxValueBytes, + MaxDescriptorBytes, + MaxKeyBytes, + LeaseRecordCount, + ParticipantRecordCount, + OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore OpenStore(string name) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + SlotCount, + MaxValueBytes, + MaxDescriptorBytes, + MaxKeyBytes, + LeaseRecordCount, + ParticipantRecordCount, + OpenMode.OpenExisting, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static string[] HoldCommand( + string name, + byte[] key, + byte[] value, + byte[] descriptor, + string readyPath, + string releasePath) => + [ + "lease-hold", + name, + SlotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + ParticipantRecordCount.ToString(CultureInfo.InvariantCulture), + Convert.ToHexString(key), + Convert.ToHexString(value), + Convert.ToHexString(descriptor), + readyPath, + releasePath + ]; + + private static void WaitForReadyFiles( + IReadOnlyList readyPaths, + IReadOnlyList processes, + TimeSpan timeout) + { + long deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (readyPaths.Any(static path => !File.Exists(path))) + { + Process? failed = processes.FirstOrDefault(static process => process.HasExited); + if (failed is not null) + { + throw new Xunit.Sdk.XunitException( + "Reader exited before the lease barrier. Exit=" + + failed.ExitCode.ToString(CultureInfo.InvariantCulture) + + Environment.NewLine + + failed.StandardOutput.ReadToEnd() + + Environment.NewLine + + failed.StandardError.ReadToEnd()); + } + + if (Stopwatch.GetTimestamp() >= deadline) + { + throw new TimeoutException("Twelve readers did not acquire their leases before the barrier timeout."); + } + + spin.SpinOnce(); + } + } + + private static AgentResult[] WaitForAgents(IReadOnlyList processes, TimeSpan timeout) + { + long deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + var results = new AgentResult[processes.Count]; + for (var index = 0; index < processes.Count; index++) + { + long remaining = deadline - Stopwatch.GetTimestamp(); + int milliseconds = remaining <= 0 + ? 0 + : (int)Math.Min(int.MaxValue, Math.Ceiling(remaining * 1000d / Stopwatch.Frequency)); + Process process = processes[index]; + if (!process.WaitForExit(milliseconds)) + { + throw new TimeoutException("Broadcast reader agents exceeded their shared timeout."); + } + + results[index] = new AgentResult( + process.ExitCode, + process.StandardOutput.ReadToEnd(), + process.StandardError.ReadToEnd()); + } + + return results; + } + + private static Process StartAgent(string[] command) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(LocateAgentAssembly()); + foreach (string argument in command) + { + startInfo.ArgumentList.Add(argument); + } + + return Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start the lock-free lease agent."); + } + + private static string LocateAgentAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + string root = directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + if (!File.Exists(path)) + { + throw new FileNotFoundException("Lock-free lease agent was not built.", path); + } + + return path; + } + + private static byte[] CreatePayload() + { + var payload = new byte[MaxValueBytes]; + for (var index = 0; index < payload.Length; index++) + { + payload[index] = (byte)((index * 31) ^ (index >> 3)); + } + + return payload; + } + + private static ulong Checksum(ReadOnlySpan value, ReadOnlySpan descriptor) + { + ulong checksum = 14_695_981_039_346_656_037UL; + foreach (byte item in value) + { + checksum = unchecked((checksum ^ item) * 1_099_511_628_211UL); + } + + foreach (byte item in descriptor) + { + checksum = unchecked((checksum ^ item) * 1_099_511_628_211UL); + } + + return checksum; + } + + private static void Kill(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(5_000); + } + } + catch + { + // Cleanup is best effort for unique test resources. + } + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private readonly record struct AgentResult(int ExitCode, string StandardOutput, string StandardError); +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeChurnIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeChurnIntegrationTests.cs new file mode 100644 index 0000000..b2db764 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeChurnIntegrationTests.cs @@ -0,0 +1,562 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text.Json; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeChurnIntegrationTests +{ + private const int SlotCount = 64; + private const int MaxValueBytes = 16; + private const int MaxDescriptorBytes = 0; + private const int MaxKeyBytes = 8; + private const int LeaseRecordCount = 128; + private const int ParticipantRecordCount = 8; + private const int WorkerCount = 2; + private const int DefaultIterationsPerWorker = 1_024; + private const int FixedKeySlotCount = 32; + private const int FixedKeyParticipantRecordCount = 16; + private const int FixedKeyWorkerCount = 8; + private const int FixedKeyTrialCount = 3; + private const int FixedKeyIterationsPerWorker = 100_000; + private static readonly int IterationsPerWorker = GetIterationsPerWorker(); + private static readonly TimeSpan WorkloadTimeout = GetWorkloadTimeout(); + private static readonly TimeSpan FixedKeyWorkloadTimeout = TimeSpan.FromMinutes(2); + + [Fact] + [Trait("Category", "Integration")] + public void BenchmarkFixedKeysSurviveRepeatedEightProcessPublishRemoveChurn() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = Enumerable.Range(1, FixedKeyWorkerCount) + .Select(static value => BitConverter.GetBytes((long)value)) + .ToArray(); + // SyncProbe worker ids 2, 4, and 7 use catalog keys 3, 5, and 8. + // Those exact benchmark keys serialize through canonical bucket 11. + int sharedBucket = CanonicalBucket(keys[2], FixedKeySlotCount); + Assert.Equal(11, sharedBucket); + Assert.Equal(sharedBucket, CanonicalBucket(keys[4], FixedKeySlotCount)); + Assert.Equal(sharedBucket, CanonicalBucket(keys[7], FixedKeySlotCount)); + + for (var trial = 0; trial < FixedKeyTrialCount; trial++) + { + RunFixedKeyTrial(trial, keys); + } + } + + [Fact] + [Trait("Category", "Integration")] + public void CollisionHeavyMultiProcessRemoveReuseRestoresCapacityAndKeepsLateLatencyBounded() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-collision-churn-{Guid.NewGuid():N}"; + byte[][] collisionKeys = GenerateCanonicalBucketCollisions(SlotCount, targetBucket: 0); + Assert.Equal(SlotCount, collisionKeys.Length); + Assert.All(collisionKeys, key => Assert.Equal(0, CanonicalBucket(key))); + + using var store = CreateStore(name); + string directory = Path.Combine(Path.GetTempPath(), $"sms-v2-churn-gate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string goPath = Path.Combine(directory, "go"); + string[] readyPaths = Enumerable.Range(0, WorkerCount) + .Select(index => Path.Combine(directory, $"worker-{index}.ready")) + .ToArray(); + Process[] workers = Enumerable.Range(0, WorkerCount) + .Select(worker => StartAgent(ChurnCommand( + name, + worker, + collisionKeys.Where((_, index) => index % WorkerCount == worker).ToArray(), + readyPaths[worker], + goPath))) + .ToArray(); + + try + { + WaitForReadyFiles(readyPaths, workers, WorkloadTimeout); + File.WriteAllText(goPath, "go"); + ChurnResult[] results = WaitForWorkers(workers, WorkloadTimeout) + .Select(ParseResult) + .ToArray(); + + Assert.True(results.Length == WorkerCount); + Assert.All(results, result => + { + Assert.Equal(IterationsPerWorker, result.Iterations); + Assert.Equal(SlotCount / WorkerCount, result.CollisionKeyCount); + Assert.InRange(result.RemovePendingCount, 0, result.Iterations); + AssertLatencyStable(result.EarlyPublishP99Ticks, result.LatePublishP99Ticks, "publish", result.WorkerId); + AssertLatencyStable(result.EarlyMissingP99Ticks, result.LateMissingP99Ticks, "missing", result.WorkerId); + }); + + // Every churn lifecycle ended absent. Filling all configured slots + // with keys sharing one canonical bucket proves exact reclamation, + // overflow-directory reuse, and absence of owner-controlled leaks. + for (var index = 0; index < collisionKeys.Length; index++) + { + Assert.Equal(StoreStatus.Success, store.TryPublish(collisionKeys[index], [(byte)index])); + } + + Assert.Equal(StoreStatus.StoreFull, store.TryPublish([0xff], [0xff])); + for (var index = 0; index < collisionKeys.Length; index++) + { + Assert.Equal(StoreStatus.Success, store.TryAcquire(collisionKeys[index], out var lease)); + Assert.Equal((byte)index, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, store.TryRemove(collisionKeys[index])); + } + + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out var final)); + Assert.Equal(SlotCount, final.FreeSlotCount); + Assert.Equal(0, final.PublishedSlotCount); + Assert.Equal(0, final.PendingRemovalCount); + Assert.Equal(0, final.ActiveLeaseCount); + Assert.Equal(0, final.ActiveReservationCount); + Assert.Equal(0, final.InitializingSlotCount); + Assert.Equal(0, final.ReservedSlotCount); + Assert.Equal(0, final.ReclaimingSlotCount); + Assert.Equal(0, final.PrimaryDirectoryOccupancy); + Assert.Equal(0, final.SpilledBucketCount); + Assert.Equal(0, final.OverflowDirectoryOccupancy); + } + finally + { + try + { + File.WriteAllText(goPath, "go"); + } + catch + { + // Best effort before worker termination. + } + + foreach (Process worker in workers) + { + Kill(worker); + worker.Dispose(); + } + + try + { + Directory.Delete(directory, recursive: true); + } + catch + { + // Unique temporary artifacts can be reclaimed after process exit. + } + } + } + + private static void RunFixedKeyTrial(int trial, byte[][] keys) + { + string name = $"sms-v2-fixed-key-churn-{trial}-{Guid.NewGuid():N}"; + using var store = CreateStore( + name, + FixedKeySlotCount, + FixedKeyParticipantRecordCount); + string directory = Path.Combine( + Path.GetTempPath(), + $"sms-v2-fixed-key-gate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string goPath = Path.Combine(directory, "go"); + string[] readyPaths = Enumerable.Range(0, FixedKeyWorkerCount) + .Select(index => Path.Combine(directory, $"worker-{index}.ready")) + .ToArray(); + Process[] workers = Enumerable.Range(0, FixedKeyWorkerCount) + .Select(worker => StartAgent(ChurnCommand( + name, + worker, + [keys[worker]], + readyPaths[worker], + goPath, + FixedKeySlotCount, + FixedKeyParticipantRecordCount, + FixedKeyIterationsPerWorker))) + .ToArray(); + + try + { + WaitForReadyFiles(readyPaths, workers, FixedKeyWorkloadTimeout); + File.WriteAllText(goPath, "go"); + ChurnResult[] results = WaitForWorkers(workers, FixedKeyWorkloadTimeout) + .Select(ParseResult) + .ToArray(); + Assert.Equal(FixedKeyWorkerCount, results.Length); + Assert.All(results, result => + { + Assert.Equal(FixedKeyIterationsPerWorker, result.Iterations); + Assert.Equal(1, result.CollisionKeyCount); + }); + + // End-of-wave capacity recovery proves no failed delayed helper + // left a target cell, location, slot, or terminal corruption latch + // behind after the process-level collision schedule. + byte[][] capacityKeys = Enumerable.Range(0, FixedKeySlotCount) + .Select(index => BitConverter.GetBytes(0x1000_0000L + index)) + .ToArray(); + for (var index = 0; index < capacityKeys.Length; index++) + { + Assert.Equal(StoreStatus.Success, store.TryPublish(capacityKeys[index], [(byte)index])); + } + + Assert.Equal(StoreStatus.StoreFull, store.TryPublish([0xff], [0xff])); + foreach (byte[] key in capacityKeys) + { + Assert.Equal(StoreStatus.Success, store.TryRemove(key)); + } + + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out var final)); + Assert.Equal(FixedKeySlotCount, final.FreeSlotCount); + Assert.Equal(0, final.PublishedSlotCount); + Assert.Equal(0, final.PendingRemovalCount); + Assert.Equal(0, final.ActiveLeaseCount); + Assert.Equal(0, final.ActiveReservationCount); + Assert.Equal(0, final.InitializingSlotCount); + Assert.Equal(0, final.ReservedSlotCount); + Assert.Equal(0, final.ReclaimingSlotCount); + Assert.Equal(0, final.PrimaryDirectoryOccupancy); + Assert.Equal(0, final.SpilledBucketCount); + Assert.Equal(0, final.OverflowDirectoryOccupancy); + } + finally + { + try + { + File.WriteAllText(goPath, "go"); + } + catch + { + // Best effort before worker termination. + } + + foreach (Process worker in workers) + { + Kill(worker); + worker.Dispose(); + } + + try + { + Directory.Delete(directory, recursive: true); + } + catch + { + // Unique temporary artifacts can be reclaimed after process exit. + } + } + } + + private static MemoryStore CreateStore(string name) + => CreateStore(name, SlotCount, ParticipantRecordCount); + + private static MemoryStore CreateStore( + string name, + int slotCount, + int participantRecordCount) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + MaxValueBytes, + MaxDescriptorBytes, + MaxKeyBytes, + LeaseRecordCount, + participantRecordCount, + OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static string[] ChurnCommand( + string name, + int workerId, + byte[][] keys, + string readyPath, + string goPath) => + ChurnCommand( + name, + workerId, + keys, + readyPath, + goPath, + SlotCount, + ParticipantRecordCount, + IterationsPerWorker); + + private static string[] ChurnCommand( + string name, + int workerId, + byte[][] keys, + string readyPath, + string goPath, + int slotCount, + int participantRecordCount, + int iterations) => + [ + "churn-worker", + name, + slotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + participantRecordCount.ToString(CultureInfo.InvariantCulture), + workerId.ToString(CultureInfo.InvariantCulture), + iterations.ToString(CultureInfo.InvariantCulture), + string.Join(';', keys.Select(Convert.ToHexString)), + readyPath, + goPath + ]; + + private static void WaitForReadyFiles( + IReadOnlyList readyPaths, + IReadOnlyList processes, + TimeSpan timeout) + { + long deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (readyPaths.Any(static path => !File.Exists(path))) + { + Process? failed = processes.FirstOrDefault(static process => process.HasExited); + if (failed is not null) + { + throw Failure(failed, "Churn worker exited before the start barrier."); + } + + if (Stopwatch.GetTimestamp() >= deadline) + { + throw new TimeoutException("Churn workers did not reach the start barrier."); + } + + spin.SpinOnce(); + } + } + + private static AgentOutput[] WaitForWorkers(IReadOnlyList processes, TimeSpan timeout) + { + long deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + var results = new AgentOutput[processes.Count]; + for (var index = 0; index < processes.Count; index++) + { + long remaining = deadline - Stopwatch.GetTimestamp(); + int milliseconds = remaining <= 0 + ? 0 + : (int)Math.Min(int.MaxValue, Math.Ceiling(remaining * 1000d / Stopwatch.Frequency)); + Process process = processes[index]; + if (!process.WaitForExit(milliseconds)) + { + throw new TimeoutException("Collision churn workers exceeded their shared bounded timeout."); + } + + string stdout = process.StandardOutput.ReadToEnd(); + string stderr = process.StandardError.ReadToEnd(); + if (process.ExitCode != 0) + { + throw new Xunit.Sdk.XunitException( + $"Churn worker exit={process.ExitCode}{Environment.NewLine}stdout: {stdout}{Environment.NewLine}stderr: {stderr}"); + } + + results[index] = new AgentOutput(stdout, stderr); + } + + return results; + } + + private static ChurnResult ParseResult(AgentOutput output) + { + string line = output.StandardOutput + .Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) + .Single(value => value.StartsWith("RESULT ", StringComparison.Ordinal)); + ChurnResult? result = JsonSerializer.Deserialize(line["RESULT ".Length..]); + return result ?? throw new Xunit.Sdk.XunitException("Churn worker returned an empty result."); + } + + private static void AssertLatencyStable(long early, long late, string operation, int worker) + { + Assert.True(early > 0 && late > 0); + long timerSlack = Math.Max(1, Stopwatch.Frequency / 1_000); // one millisecond for timer quantization/scheduling + long allowed = Math.Max(checked(early * 2), checked(early + timerSlack)); + Assert.True( + late <= allowed, + $"Worker {worker} {operation} p99 regressed: early={early} ticks, late={late} ticks, allowed={allowed} ticks."); + } + + private static byte[][] GenerateCanonicalBucketCollisions(int count, int targetBucket) + { + var keys = new List(count); + for (long candidate = 0; keys.Count < count && candidate < 10_000_000; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + if (CanonicalBucket(key) == targetBucket) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static int CanonicalBucket(ReadOnlySpan key) => + CanonicalBucket(key, SlotCount); + + private static int CanonicalBucket(ReadOnlySpan key, int slotCount) + { + int primaryLanes = NextPowerOfTwo(Math.Max(32, slotCount * 4)); + int bucketMask = (primaryLanes / 8) - 1; + return (int)(Mix(Hash(key)) & (uint)bucketMask); + } + + private static ulong Hash(ReadOnlySpan key) + { + ulong hash = 14_695_981_039_346_656_037UL; + foreach (byte value in key) + { + hash ^= value; + hash *= 1_099_511_628_211UL; + } + + return hash; + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static int NextPowerOfTwo(int value) + { + var result = 1; + while (result < value) + { + result <<= 1; + } + + return result; + } + + private static Process StartAgent(string[] command) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(LocateAgentAssembly()); + foreach (string argument in command) + { + startInfo.ArgumentList.Add(argument); + } + + return Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start collision churn worker."); + } + + private static string LocateAgentAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + string root = directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + return File.Exists(path) ? path : throw new FileNotFoundException("Lock-free churn agent was not built.", path); + } + + private static Xunit.Sdk.XunitException Failure(Process process, string message) => + new( + message + + Environment.NewLine + + process.StandardOutput.ReadToEnd() + + Environment.NewLine + + process.StandardError.ReadToEnd()); + + private static void Kill(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(5_000); + } + } + catch + { + // Cleanup is best effort for unique test resources. + } + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private static int GetIterationsPerWorker() + { + string? configured = Environment.GetEnvironmentVariable("SMS_LOCK_FREE_CHURN_CYCLES"); + if (string.IsNullOrWhiteSpace(configured)) + { + return DefaultIterationsPerWorker; + } + + if (!long.TryParse(configured, NumberStyles.None, CultureInfo.InvariantCulture, out long totalCycles) + || totalCycles < WorkerCount * 512L + || totalCycles > WorkerCount * 100_000_000L) + { + throw new InvalidOperationException( + "SMS_LOCK_FREE_CHURN_CYCLES must be an integer between " + + (WorkerCount * 512L).ToString(CultureInfo.InvariantCulture) + + " and " + + (WorkerCount * 100_000_000L).ToString(CultureInfo.InvariantCulture) + + "."); + } + + return checked((int)((totalCycles + WorkerCount - 1) / WorkerCount)); + } + + private static TimeSpan GetWorkloadTimeout() + { + // Retain the short-test floor while allowing the configured qualification + // tiers to execute millions of full publish/acquire/release/remove cycles. + double scaledSeconds = 60d + (IterationsPerWorker / 20_000d); + return TimeSpan.FromSeconds(Math.Min(7_200d, scaledSeconds)); + } + + private readonly record struct AgentOutput(string StandardOutput, string StandardError); + + private sealed record ChurnResult( + int WorkerId, + int Iterations, + int CollisionKeyCount, + int RemovePendingCount, + long EarlyPublishP99Ticks, + long LatePublishP99Ticks, + long EarlyMissingP99Ticks, + long LateMissingP99Ticks); +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeCrashRecoveryIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeCrashRecoveryIntegrationTests.cs new file mode 100644 index 0000000..8ffa07a --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeCrashRecoveryIntegrationTests.cs @@ -0,0 +1,763 @@ +using System.Diagnostics; +using System.Globalization; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text.Json; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.IntegrationTests; + +[Collection("LockFree crash recovery")] +public sealed class LockFreeCrashRecoveryIntegrationTests +{ + private const int SlotCount = 20; + private const int LeaseRecordCount = 4; + private const int ParticipantRecordCount = 8; + private const int MaxValueBytes = 16; + private const int MaxDescriptorBytes = 4; + private const int MaxKeyBytes = 8; + private static readonly TimeSpan AgentTimeout = TimeSpan.FromSeconds(20); + + public static TheoryData CanonicalCheckpoints + { + get + { + int configuredCases = GetConfiguredRecoveryCases(); + var entries = LockFreeCheckpointCatalog.Entries; + var data = new TheoryData(); + for (var caseIndex = 0; caseIndex < configuredCases; caseIndex++) + { + data.Add(caseIndex, (int)entries[caseIndex % entries.Count].Id); + } + + return data; + } + } + + [Theory] + [MemberData(nameof(CanonicalCheckpoints))] + [Trait("Category", "Integration")] + [Trait("Category", "CrashRecovery")] + public async Task EveryCanonicalCheckpointCanBeKilledRecoveredAndFilledToCapacity( + int caseIndex, + int checkpointValue) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var checkpoint = (LockFreeCheckpointId)checkpointValue; + LockFreeCheckpointEntry catalogEntry = LockFreeCheckpointCatalog.Get(checkpoint); + string name = $"sms-v2-crash-{caseIndex:D5}-{checkpointValue:D2}-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions options = Options(name, OpenMode.CreateNew); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(options, out MemoryStore? opened)); + using var store = Assert.IsType(opened); + + byte[] tokenKey = Key(0xA0, checkpointValue); + byte[] existingKey = Key(0xB0, checkpointValue); + byte[] operationKey = checkpoint + == LockFreeCheckpointId.DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation + ? GenerateBucketPairMate(tokenKey, SlotCount) + : Key(0xC0, checkpointValue); + byte[] recoveryKey = Key(0xD0, checkpointValue); + byte[] unrelatedKey = checkpoint + == LockFreeCheckpointId.DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation + ? GenerateKeyOutsideBuckets(tokenKey, SlotCount) + : Key(0xE0, checkpointValue); + byte[] value = [0x31, unchecked((byte)checkpointValue), 0xC7]; + byte[] descriptor = [0xD5, unchecked((byte)(checkpointValue ^ 0x5A))]; + + Assert.Equal(StoreStatus.Success, store.TryPublish(tokenKey, [0x71, 0x72], [0x73])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(tokenKey, out ValueLease liveLease)); + ValueLease staleLiveCopy = liveLease; + DirectoryReferenceRepair? invalidReferenceRepair = checkpoint + == LockFreeCheckpointId.DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation + ? DirectoryReferenceRepair.Capture(store, tokenKey) + : null; + + if (invalidReferenceRepair is not null) + { + Assert.Equal(StoreStatus.Success, store.TryPublish(unrelatedKey, [0xE1, 0xE2])); + } + + if (NeedsExistingValue(checkpoint)) + { + Assert.Equal(StoreStatus.Success, store.TryPublish(existingKey, value, descriptor)); + } + + using Process agent = StartAgent( + name, + checkpoint, + tokenKey, + existingKey, + operationKey, + recoveryKey, + value, + descriptor); + try + { + CheckpointSignal signal = await ReadCheckpointAsync(agent, checkpoint); + Assert.Equal(checkpointValue, signal.Id); + Assert.Equal(checkpoint.ToString(), signal.Name); + Assert.Equal(catalogEntry.Family.ToString(), signal.Family); + Assert.Equal(catalogEntry.Position.ToString(), signal.Position); + Assert.Equal(catalogEntry.Crash.ToString(), signal.Crash); + Assert.True(signal.ProcessId > 0); + Assert.False(agent.HasExited, "The child must remain paused in the production checkpoint callback."); + + invalidReferenceRepair?.AssertInvalidReferenceIsPresent(); + if (IsStoreFullProofCheckpoint(checkpoint)) + { + DiagnosticsSnapshot saturated = store.GetDiagnostics(); + Assert.Equal(0, saturated.FreeSlotCount); + Assert.Equal(SlotCount, saturated.PublishedSlotCount); + Assert.Equal(0, saturated.ActiveReservationCount); + Assert.Equal(0, saturated.InitializingSlotCount); + Assert.Equal(0, saturated.ReservedSlotCount); + Assert.Equal(0, saturated.ReclaimingSlotCount); + + RemoveStoreFullFillers(store, checkpoint); + DiagnosticsSnapshot paused = store.GetDiagnostics(); + Assert.Equal(SlotCount - 1, paused.FreeSlotCount); + Assert.Equal(0, paused.ActiveReservationCount); + Assert.Equal(0, paused.InitializingSlotCount); + Assert.Equal(0, paused.ReservedSlotCount); + Assert.Equal(0, paused.ReclaimingSlotCount); + } + + // A stopped participant cannot own unrelated progress. + if (invalidReferenceRepair is not null) + { + Assert.Equal(StoreStatus.Success, store.TryAcquire(unrelatedKey, out ValueLease unrelated)); + Assert.True(unrelated.ValueSpan.SequenceEqual([(byte)0xE1, (byte)0xE2])); + Assert.Equal(StoreStatus.Success, unrelated.Release()); + } + else + { + Assert.Equal(StoreStatus.Success, store.TryPublish(unrelatedKey, [0xE1, 0xE2])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(unrelatedKey, out ValueLease unrelated)); + Assert.True(unrelated.ValueSpan.SequenceEqual([(byte)0xE1, (byte)0xE2])); + Assert.Equal(StoreStatus.Success, unrelated.Release()); + Assert.Equal(StoreStatus.Success, store.TryRemove(unrelatedKey)); + } + + Kill(agent); + Assert.True(agent.HasExited, "The deterministic checkpoint participant did not terminate."); + invalidReferenceRepair?.RestoreExactReference(); + + RecoveryEvidence recovery = RecoverStoppedOwners(store); + Assert.Equal(0, recovery.FailedLeaseRecoveries); + Assert.Equal(0, recovery.FailedReservationRecoveries); + Assert.True(recovery.ActiveLeases >= 1, "The controller's live lease must remain classified as active."); + Assert.True(liveLease.IsValid); + Assert.True(liveLease.ValueSpan.SequenceEqual([(byte)0x71, (byte)0x72])); + AssertFullParticipantCapacity(name); + + if (IsSpillSummaryCheckpoint(checkpoint)) + { + DiagnosticsSnapshot beforeMissingLookup = store.GetDiagnostics(); + Assert.Equal(0, beforeMissingLookup.SpilledBucketCount); + Assert.Equal(0, beforeMissingLookup.OverflowDirectoryOccupancy); + + byte[] missingSpillKey = GenerateBucketPairCollisions(count: 18, slotCount: SlotCount)[17]; + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(missingSpillKey, out _)); + + DiagnosticsSnapshot afterMissingLookup = store.GetDiagnostics(); + Assert.Equal(beforeMissingLookup.OverflowScanCount, afterMissingLookup.OverflowScanCount); + } + + if (signal.LeaseToken != 0) + { + var killedToken = new ValueLease( + store, + new LeaseHandle( + signal.StoreId, + signal.ParticipantToken, + signal.SlotBinding, + signal.LeaseToken)); + Assert.False(killedToken.IsValid); + Assert.NotEqual(StoreStatus.Success, killedToken.Release()); + } + + Assert.Equal(StoreStatus.Success, liveLease.Release()); + Assert.False(staleLiveCopy.IsValid); + + // Reuse the same lease table after recovery and prove a copied old + // controller token cannot release a later incarnation either. + Assert.Equal(StoreStatus.Success, store.TryAcquire(tokenKey, out ValueLease reusedLease)); + Assert.NotEqual(StoreStatus.Success, staleLiveCopy.Release()); + Assert.True(reusedLease.IsValid); + Assert.Equal(StoreStatus.Success, reusedLease.Release()); + + RemoveIfPresent(store, tokenKey); + RemoveIfPresent(store, existingKey); + RemoveIfPresent(store, operationKey); + RemoveIfPresent(store, recoveryKey); + RemoveIfPresent(store, unrelatedKey); + if (IsSpillSummaryCheckpoint(checkpoint)) + { + foreach (byte[] spillKey in GenerateBucketPairCollisions(count: 17, slotCount: SlotCount)) + { + RemoveIfPresent(store, spillKey); + } + } + + AssertFullSlotAndLeaseCapacity(store, checkpointValue); + } + finally + { + Kill(agent); + invalidReferenceRepair?.RestoreExactReference(); + if (IsStoreFullProofCheckpoint(checkpoint)) + { + RemoveStoreFullFillers(store, checkpoint); + } + } + } + + private static RecoveryEvidence RecoverStoppedOwners(MemoryStore store) + { + var recoveredLeases = 0; + var recoveredReservations = 0; + var activeLeases = 0; + var failedLeases = 0; + var failedReservations = 0; + + for (var attempt = 0; attempt < 8; attempt++) + { + StoreStatus leaseStatus = store.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: false), + out LeaseRecoveryReport leases); + StoreStatus reservationStatus = store.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: false), + out ReservationRecoveryReport reservations); + + Assert.True( + leaseStatus is StoreStatus.Success or StoreStatus.StoreBusy, + "Lease recovery returned " + leaseStatus); + Assert.True( + reservationStatus is StoreStatus.Success or StoreStatus.StoreBusy, + "Reservation recovery returned " + reservationStatus); + + recoveredLeases += leases.RecoveredLeaseCount; + recoveredReservations += reservations.RecoveredReservationCount; + activeLeases = Math.Max(activeLeases, leases.ActiveLeaseCount); + failedLeases += leases.FailedRecoveryCount; + failedReservations += reservations.FailedRecoveryCount; + if (leaseStatus == StoreStatus.Success && reservationStatus == StoreStatus.Success) + { + return new RecoveryEvidence( + recoveredLeases, + recoveredReservations, + activeLeases, + failedLeases, + failedReservations); + } + } + + throw new Xunit.Sdk.XunitException("Recovery did not converge within eight bounded passes."); + } + + private static void AssertFullSlotAndLeaseCapacity(MemoryStore store, int checkpointValue) + { + var fillKeys = new byte[SlotCount][]; + for (var index = 0; index < SlotCount; index++) + { + fillKeys[index] = [0xF1, unchecked((byte)checkpointValue), unchecked((byte)index)]; + Assert.Equal( + StoreStatus.Success, + RetryStoreBusy(() => store.TryPublish(fillKeys[index], [unchecked((byte)(0x40 + index))]))); + } + + byte[] overflowKey = [0xF2, unchecked((byte)checkpointValue), 0xFF]; + Assert.Equal(StoreStatus.StoreFull, store.TryPublish(overflowKey, [0xFF])); + + var leases = new ValueLease[LeaseRecordCount]; + for (var index = 0; index < leases.Length; index++) + { + Assert.Equal(StoreStatus.Success, store.TryAcquire(fillKeys[0], out leases[index])); + Assert.True(leases[index].IsValid); + } + + Assert.Equal(StoreStatus.LeaseTableFull, store.TryAcquire(fillKeys[0], out _)); + foreach (ValueLease lease in leases) + { + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + foreach (byte[] key in fillKeys) + { + Assert.Equal(StoreStatus.Success, RetryStoreBusy(() => store.TryRemove(key))); + } + } + + private static void AssertFullParticipantCapacity(string name) + { + var handles = new List(ParticipantRecordCount - 1); + try + { + for (var index = 1; index < ParticipantRecordCount; index++) + { + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen(Options(name, OpenMode.OpenExisting), out MemoryStore? opened)); + handles.Add(Assert.IsType(opened)); + } + + Assert.Equal( + StoreOpenStatus.ParticipantTableFull, + MemoryStore.TryCreateOrOpen(Options(name, OpenMode.OpenExisting), out MemoryStore? rejected)); + Assert.Null(rejected); + } + finally + { + foreach (MemoryStore handle in handles) + { + handle.Dispose(); + } + } + } + + private static void RemoveIfPresent(MemoryStore store, byte[] key) + { + for (var attempt = 0; attempt < 16; attempt++) + { + StoreStatus status = store.TryRemove(key); + if (status is StoreStatus.Success or StoreStatus.NotFound) + { + return; + } + + if (status == StoreStatus.RemovePending) + { + _ = store.TryRecoverLeases(new LeaseRecoveryOptions(false), out _); + } + + Assert.True( + status is StoreStatus.RemovePending or StoreStatus.StoreBusy, + "Unexpected cleanup status for " + Convert.ToHexString(key) + ": " + status); + Thread.Yield(); + } + + throw new Xunit.Sdk.XunitException("Key cleanup did not converge: " + Convert.ToHexString(key)); + } + + private static StoreStatus RetryStoreBusy(Func operation) + { + for (var attempt = 0; attempt < 16; attempt++) + { + StoreStatus status = operation(); + if (status != StoreStatus.StoreBusy) + { + return status; + } + + Thread.Yield(); + } + + return StoreStatus.StoreBusy; + } + + private static async Task ReadCheckpointAsync( + Process process, + LockFreeCheckpointId expected) + { + string? line = await process.StandardOutput.ReadLineAsync().WaitAsync(AgentTimeout); + if (line is null || !line.StartsWith("CHECKPOINT ", StringComparison.Ordinal)) + { + string error = await process.StandardError.ReadToEndAsync(); + throw new Xunit.Sdk.XunitException( + "Agent did not reach " + expected + + ". exit=" + (process.HasExited ? process.ExitCode.ToString(CultureInfo.InvariantCulture) : "running") + + Environment.NewLine + "stdout=" + line + + Environment.NewLine + "stderr=" + error); + } + + CheckpointSignal? signal = JsonSerializer.Deserialize(line["CHECKPOINT ".Length..]); + return signal ?? throw new Xunit.Sdk.XunitException("Agent emitted an invalid checkpoint signal: " + line); + } + + private static Process StartAgent( + string name, + LockFreeCheckpointId checkpoint, + byte[] tokenKey, + byte[] existingKey, + byte[] operationKey, + byte[] recoveryKey, + byte[] value, + byte[] descriptor) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(LocateAgentAssembly()); + foreach (string argument in new[] + { + "checkpoint-crash", + name, + SlotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + ParticipantRecordCount.ToString(CultureInfo.InvariantCulture), + ((int)checkpoint).ToString(CultureInfo.InvariantCulture), + Convert.ToHexString(tokenKey), + Convert.ToHexString(existingKey), + Convert.ToHexString(operationKey), + Convert.ToHexString(recoveryKey), + Convert.ToHexString(value), + Convert.ToHexString(descriptor), + "v1" + }) + { + startInfo.ArgumentList.Add(argument); + } + + return Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start the checkpoint crash agent."); + } + + private static string LocateAgentAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + string root = directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + return File.Exists(path) + ? path + : throw new FileNotFoundException("Lock-free checkpoint agent was not built.", path); + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + SlotCount, + MaxValueBytes, + MaxDescriptorBytes, + MaxKeyBytes, + LeaseRecordCount, + ParticipantRecordCount, + openMode, + enableLeaseRecovery: true); + + private static int GetConfiguredRecoveryCases() + { + string? configured = Environment.GetEnvironmentVariable("SMS_LOCK_FREE_RECOVERY_CASES"); + if (string.IsNullOrWhiteSpace(configured)) + { + return LockFreeCheckpointCatalog.Entries.Count; + } + + if (!int.TryParse(configured, NumberStyles.None, CultureInfo.InvariantCulture, out int cases) + || cases is < 1 or > 100_000) + { + throw new InvalidOperationException( + "SMS_LOCK_FREE_RECOVERY_CASES must be an integer between 1 and 100000."); + } + + return cases; + } + + private static bool NeedsExistingValue(LockFreeCheckpointId checkpoint) => checkpoint is + LockFreeCheckpointId.AcquireBeforeLeaseClaimCas + or LockFreeCheckpointId.AcquireAfterLeaseActivationBeforeFinalLookup + or LockFreeCheckpointId.AcquireAfterPublishedRevalidation + or LockFreeCheckpointId.ProjectBeforeHandleValidation + or LockFreeCheckpointId.ProjectAfterMetadataReadBeforeControlRevalidation + or LockFreeCheckpointId.ProjectAfterSpanProjection + or LockFreeCheckpointId.ReleaseBeforeActiveReleaseCas + or LockFreeCheckpointId.ReleaseAfterOwnershipReleaseCas + or LockFreeCheckpointId.ReleaseAfterRecordRecycle + or LockFreeCheckpointId.ReserveAfterExistingLookup + or LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas + or LockFreeCheckpointId.RemoveAfterLeaseClassification + or LockFreeCheckpointId.ReclaimBeforeOwnershipCas + or LockFreeCheckpointId.ReclaimAfterGenerationAdvance + or LockFreeCheckpointId.ReclaimAfterLeaseScanBeforeOwnershipCas + or LockFreeCheckpointId.DirectoryAfterLocationValidation + or LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead + or LockFreeCheckpointId.DirectoryAfterUnlinkDescriptorClearBeforeGenerationAdvance + or LockFreeCheckpointId.ReclaimAfterMetadataValidation; + + private static bool IsSpillSummaryCheckpoint(LockFreeCheckpointId checkpoint) => checkpoint is + LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas + or LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication + or LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan + or LockFreeCheckpointId.DirectoryAfterSpillSummaryClear; + + private static bool IsStoreFullProofCheckpoint(LockFreeCheckpointId checkpoint) => checkpoint is + LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification + or LockFreeCheckpointId.StoreFullAfterExactDoubleCollect; + + private static void RemoveStoreFullFillers( + MemoryStore store, + LockFreeCheckpointId checkpoint) + { + for (var index = 0; index < SlotCount; index++) + { + RemoveIfPresent(store, CreateStoreFullFillerKey(checkpoint, index)); + } + } + + private static byte[] CreateStoreFullFillerKey( + LockFreeCheckpointId checkpoint, + int index) => + BitConverter.GetBytes( + 0x6f00_0000_0000_0000UL + | ((ulong)(byte)checkpoint << 48) + | checked((uint)(index + 1))); + + private static byte[][] GenerateBucketPairCollisions(int count, int slotCount) + { + var keys = new List(count); + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + uint bucketMask = checked((uint)((primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + for (long candidate = 1; keys.Count < count; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + ulong hash = StoreKey.Hash(key); + int first = (int)(Mix(hash) & bucketMask); + int second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & bucketMask); + if (second == first) + { + second = (first + 1) & (int)bucketMask; + } + + if (first == 0 && second == 1) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static byte[] GenerateKeyOutsideBuckets(byte[] excludedKey, int slotCount) + { + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + uint bucketMask = checked((uint)((primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + GetBuckets(StoreKey.Hash(excludedKey), bucketMask, out int excludedFirst, out int excludedSecond); + for (long candidate = 1; ; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + GetBuckets(StoreKey.Hash(key), bucketMask, out int first, out int second); + if (first != excludedFirst + && first != excludedSecond + && second != excludedFirst + && second != excludedSecond) + { + return key; + } + } + } + + private static byte[] GenerateBucketPairMate(byte[] anchorKey, int slotCount) + { + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + uint bucketMask = checked((uint)((primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + GetBuckets(StoreKey.Hash(anchorKey), bucketMask, out int anchorFirst, out int anchorSecond); + for (long candidate = 1; ; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + GetBuckets(StoreKey.Hash(key), bucketMask, out int first, out int second); + if (first == anchorFirst && second == anchorSecond) + { + return key; + } + } + } + + private static void GetBuckets(ulong hash, uint bucketMask, out int first, out int second) + { + first = (int)(Mix(hash) & bucketMask); + second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & bucketMask); + if (second == first) + { + second = (first + 1) & (int)bucketMask; + } + } + + private static int NextPowerOfTwo(int value) + { + var result = 1; + while (result < value) + { + result <<= 1; + } + + return result; + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static byte[] Key(byte prefix, int checkpointValue) => + [prefix, unchecked((byte)checkpointValue)]; + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private static void Kill(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(5_000); + } + } + catch + { + // Unique mappings make best-effort teardown safe after assertion failures. + } + } + + private sealed class DirectoryReferenceRepair + { + private readonly MemoryMappedStoreRegion _region; + private readonly StoreLayoutV2 _layout; + private readonly DirectoryLocation _location; + private readonly ulong _exactBinding; + private readonly ulong _invalidBinding; + private int _repairCompleted; + + private DirectoryReferenceRepair( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + DirectoryLocation location, + ulong exactBinding) + { + _region = region; + _layout = layout; + _location = location; + _exactBinding = exactBinding; + IndexBinding decoded = IndexBinding.Decode(exactBinding); + _invalidBinding = IndexBinding.Encode( + decoded.SlotIndex, + checked(decoded.Generation + 1)); + } + + internal static DirectoryReferenceRepair Capture(MemoryStore store, byte[] key) + { + object engine = ReadPrivate(store, "_engine"); + LockFreeKeyDirectory directory = ReadPrivate(engine, "_directory"); + MemoryMappedStoreRegion region = ReadPrivate(engine, "_region"); + StoreLayoutV2 layout = ReadPrivate(engine, "_layout"); + StoreStatus lookup = directory.TryLookup( + key, + StoreKey.Hash(key), + out ulong binding, + out DirectoryLocation location); + Assert.Equal(StoreStatus.Success, lookup); + return new DirectoryReferenceRepair(region, layout, location, binding); + } + + internal void AssertInvalidReferenceIsPresent() => + Assert.Equal(_invalidBinding, ReadReference()); + + internal void RestoreExactReference() + { + if (Interlocked.Exchange(ref _repairCompleted, 1) != 0) + { + return; + } + + long observed = AtomicControlWord.CompareExchange( + ref Cell(), + unchecked((long)_exactBinding), + unchecked((long)_invalidBinding)); + ulong raw = unchecked((ulong)observed); + Assert.True( + raw == _invalidBinding || raw == _exactBinding, + "The injected directory reference changed before controller repair. observed=" + raw); + } + + private ulong ReadReference() => + unchecked((ulong)AtomicControlWord.LoadAcquire(ref Cell())); + + private unsafe ref long Cell() + { + long offset = _location.Kind switch + { + 1 => PrimaryCellOffset(_layout, checked((int)_location.Index)), + 2 => _layout.OverflowDirectoryOffset + (_location.Index * _layout.OverflowStride), + _ => throw new InvalidOperationException("The captured directory location is invalid.") + }; + return ref *(long*)(_region.Pointer + offset); + } + + private static long PrimaryCellOffset(StoreLayoutV2 layout, int absoluteCellIndex) + { + int bucket = absoluteCellIndex / LayoutV2Constants.PrimaryLanesPerBucket; + int lane = absoluteCellIndex % LayoutV2Constants.PrimaryLanesPerBucket; + return layout.PrimaryDirectoryOffset + + ((long)bucket * layout.PrimaryBucketStride) + + 16 + + (lane * sizeof(long)); + } + + private static T ReadPrivate(object owner, string fieldName) + { + FieldInfo field = owner.GetType().GetField( + fieldName, + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + "Missing field " + owner.GetType().FullName + "." + fieldName + "."); + return Assert.IsAssignableFrom(field.GetValue(owner)); + } + } + + private sealed record CheckpointSignal( + int Id, + string Name, + string Family, + string Position, + string Crash, + int ProcessId, + ulong StoreId, + ulong ParticipantToken, + ulong SlotBinding, + ulong LeaseToken); + + private readonly record struct RecoveryEvidence( + int RecoveredLeases, + int RecoveredReservations, + int ActiveLeases, + int FailedLeaseRecoveries, + int FailedReservationRecoveries); +} + +[CollectionDefinition("LockFree crash recovery", DisableParallelization = true)] +public sealed class LockFreeCrashRecoveryCollection; diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeDiagnosticsIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeDiagnosticsIntegrationTests.cs new file mode 100644 index 0000000..5d17b04 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeDiagnosticsIntegrationTests.cs @@ -0,0 +1,222 @@ +using System.Collections.Concurrent; +using System.Runtime.InteropServices; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeDiagnosticsIntegrationTests +{ + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(15); + + [Fact] + [Trait("Category", "Integration")] + public async Task BoundedSnapshotsRunAlongsideUnrelatedDataProgress() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-live-diagnostics-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions create = Options(name, OpenMode.CreateNew); + SharedMemoryStoreOptions open = Options(name, OpenMode.OpenExisting); + using MemoryStore writer = Open(create); + using MemoryStore observer = Open(open); + var failures = new ConcurrentQueue(); + + Task data = Task.Run(() => + { + for (var iteration = 0; iteration < 2_000; iteration++) + { + byte[] key = BitConverter.GetBytes(iteration); + StoreStatus publish = writer.TryPublish(key, key); + if (publish != StoreStatus.Success) + { + failures.Enqueue($"publish[{iteration}]={publish}"); + return; + } + + StoreStatus acquire = observer.TryAcquire(key, out ValueLease lease); + if (acquire != StoreStatus.Success) + { + failures.Enqueue($"acquire[{iteration}]={acquire}"); + return; + } + + if (!lease.ValueSpan.SequenceEqual(key)) + { + failures.Enqueue($"payload[{iteration}]"); + return; + } + + if (lease.Release() != StoreStatus.Success) + { + failures.Enqueue($"release[{iteration}]"); + return; + } + + StoreStatus remove = writer.TryRemove(key); + if (remove != StoreStatus.Success) + { + failures.Enqueue($"remove[{iteration}]={remove}"); + return; + } + } + }); + + Task scans = Task.Run(() => + { + for (var iteration = 0; iteration < 500; iteration++) + { + Assert.Equal(StoreStatus.Success, observer.TryGetDiagnostics(out DiagnosticsSnapshot snapshot)); + Assert.Equal(StoreProfile.LockFree, snapshot.Profile); + Assert.Equal(observer.ProtocolInfo, snapshot.ProtocolInfo); + Assert.Equal(32, snapshot.SlotCount); + Assert.Equal(4, snapshot.ParticipantRecordCount); + Assert.InRange(snapshot.ActiveParticipantCount, 1, 4); + Assert.Equal( + snapshot.SlotCount, + snapshot.FreeSlotCount + + snapshot.InitializingSlotCount + + snapshot.ReservedSlotCount + + snapshot.PublishedSlotCount + + snapshot.PendingRemovalCount + + snapshot.ReclaimingSlotCount + + snapshot.RetiredSlotCount); + } + }); + + await Task.WhenAll(data, scans).WaitAsync(TestTimeout); + Assert.Empty(failures); + } + + [Fact] + [Trait("Category", "Integration")] + public void PublicOperationsFeedLocalFailureTokenAbortAndRecoveryCounters() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = Open(Options( + $"sms-v2-diagnostics-counters-{Guid.NewGuid():N}", + OpenMode.CreateNew)); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.DuplicateKey, store.TryPublish([1], [8])); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([2], out _)); + + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out ValueLease lease)); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.LeaseAlreadyReleased, lease.Release()); + + Assert.Equal(StoreStatus.Success, store.TryReserve([3], 1, [], out ValueReservation reservation)); + Assert.Equal(StoreStatus.Success, reservation.Abort()); + Assert.Equal(StoreStatus.InvalidReservation, reservation.Abort()); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(true), out LeaseRecoveryReport leaseRecovery)); + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + out ReservationRecoveryReport reservationRecovery)); + + DiagnosticsSnapshot snapshot = store.GetDiagnostics(); + Assert.True(snapshot.GetFailureCount(StoreStatus.DuplicateKey) >= 1); + Assert.True(snapshot.GetFailureCount(StoreStatus.NotFound) >= 1); + Assert.True(snapshot.GetFailureCount(StoreStatus.LeaseAlreadyReleased) >= 1); + Assert.True(snapshot.GetFailureCount(StoreStatus.InvalidReservation) >= 1); + Assert.True(snapshot.StaleTokenCount >= 1); + Assert.True(snapshot.InvalidTokenCount >= 1); + Assert.True(snapshot.AbortedReservationCount >= 1); + Assert.True(snapshot.RecoveryAttemptCount >= leaseRecovery.ScannedRecordCount); + Assert.True(snapshot.RecoveryAttemptCount >= reservationRecovery.ScannedReservationCount); + } + + [Fact] + [Trait("Category", "Integration")] + public async Task ExactContentionHelpingAndOwnerClassificationFeedSharedTelemetrySink() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var paused = new ManualResetEventSlim(initialState: false); + using var resume = new ManualResetEventSlim(initialState: false); + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + if (entry.Id != LockFreeCheckpointId.ReleaseAfterOwnershipReleaseCas) + { + return; + } + + paused.Set(); + resume.Wait(); + }); + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-diagnostics-telemetry-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus open = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + checkpoint, + out MemoryStore? candidate); + Assert.Equal(StoreOpenStatus.Success, open); + using MemoryStore store = Assert.IsType(candidate); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out ValueLease first)); + Task delayedRelease = Task.Run(() => first.Release(StoreWaitOptions.Infinite)); + Assert.True(paused.Wait(TimeSpan.FromSeconds(5))); + + Assert.Equal( + StoreStatus.Success, + store.TryAcquire([1], StoreWaitOptions.Infinite, out ValueLease replacement)); + resume.Set(); + Assert.Equal(StoreStatus.Success, await delayedRelease.WaitAsync(TimeSpan.FromSeconds(5))); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(false), out LeaseRecoveryReport report)); + Assert.Equal(1, report.ActiveLeaseCount); + + DiagnosticsSnapshot snapshot = store.GetDiagnostics(); + Assert.True(snapshot.CasRetryCount > 0); + Assert.True(snapshot.HelpedTransitionCount > 0); + Assert.True(snapshot.CurrentOwnerClassificationCount > 0); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode mode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 32, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 64, + participantRecordCount: 4, + openMode: mode, + enableLeaseRecovery: true); + + private static MemoryStore Open(in SharedMemoryStoreOptions options) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static bool IsSupportedLockFreeHost() => + Environment.Is64BitProcess + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeDisposalIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeDisposalIntegrationTests.cs new file mode 100644 index 0000000..9362016 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeDisposalIntegrationTests.cs @@ -0,0 +1,835 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Runtime.InteropServices; +using SharedMemoryStore.IntegrationTests.TestSupport; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeDisposalIntegrationTests +{ + private const int SlotCount = 8; + private const int SimultaneousRaceRepetitions = 4; + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(10); + + public static TheoryData CheckpointSchedules => new() + { + { DisposalOperation.Publish, (int)LockFreeCheckpointId.PublishAfterCommitPublication }, + { DisposalOperation.Reserve, (int)LockFreeCheckpointId.ReserveAfterReservationPublication }, + { DisposalOperation.ReservationProjection, (int)LockFreeCheckpointId.ProjectAfterSpanProjection }, + { DisposalOperation.Commit, (int)LockFreeCheckpointId.CommitAfterPublicationCas }, + { DisposalOperation.Abort, (int)LockFreeCheckpointId.AbortAfterUnlinkCompletion }, + { DisposalOperation.Acquire, (int)LockFreeCheckpointId.AcquireAfterPublishedRevalidation }, + { DisposalOperation.ValueProjection, (int)LockFreeCheckpointId.ProjectAfterSpanProjection }, + { DisposalOperation.Release, (int)LockFreeCheckpointId.ReleaseAfterRecordRecycle }, + { DisposalOperation.Remove, (int)LockFreeCheckpointId.RemoveAfterLeaseClassification } + }; + + public static TheoryData EveryOperation => new() + { + DisposalOperation.Publish, + DisposalOperation.Reserve, + DisposalOperation.ReservationProjection, + DisposalOperation.Advance, + DisposalOperation.Commit, + DisposalOperation.Abort, + DisposalOperation.Acquire, + DisposalOperation.ValueProjection, + DisposalOperation.DescriptorProjection, + DisposalOperation.Release, + DisposalOperation.Remove, + DisposalOperation.RecoverLeases, + DisposalOperation.RecoverReservations, + DisposalOperation.Diagnostics, + DisposalOperation.RepeatedDispose + }; + + [Theory] + [MemberData(nameof(CheckpointSchedules))] + [Trait("Category", "Integration")] + public async Task InFlightCallKeepsItsLocalMappingAliveWhileSecondHandleProgresses( + DisposalOperation operation, + int checkpointValue) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var checkpoint = (LockFreeCheckpointId)checkpointValue; + using var gate = new CheckpointGate(); + using var context = CreateContext(operation, gate); + gate.Arm(checkpoint); + + Task invocation = Task.Run(() => Invoke(context, operation)); + Assert.True(gate.WaitUntilPaused(TestTimeout), $"{operation} did not reach {checkpoint}."); + + using var disposeStarted = new ManualResetEventSlim(initialState: false); + Task disposal = Task.Run(() => + { + disposeStarted.Set(); + context.First.Dispose(); + }); + + Assert.True(disposeStarted.Wait(TestTimeout)); + AssertSecondHandleProgress(context.Second, context.ProgressKey); + bool disposedBeforeInvocationReturned = await Task.WhenAny( + disposal, + Task.Delay(TimeSpan.FromMilliseconds(250))) == disposal; + + gate.Continue(); + OperationObservation observation = await invocation.WaitAsync(TestTimeout); + await disposal.WaitAsync(TestTimeout); + + Assert.False( + disposedBeforeInvocationReturned, + $"Dispose completed while the {operation} facade call was paused inside its engine."); + AssertDocumentedOutcome(operation, observation); + AssertDisposedTokenSurface(context); + AssertSecondHandleCanReuseCapacity(context); + } + + [Theory] + [MemberData(nameof(EveryOperation))] + [Trait("Category", "Integration")] + public async Task EveryOperationAndTokenCallbackHasOnlyDocumentedDisposalRaceOutcomes( + DisposalOperation operation) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + for (var repetition = 0; repetition < SimultaneousRaceRepetitions; repetition++) + { + using var context = CreateContext(operation); + using var start = new Barrier(participantCount: 3); + var exceptions = new ConcurrentQueue(); + OperationObservation observation = default; + + Task invocation = Task.Run(() => + { + start.SignalAndWait(); + try + { + observation = Invoke(context, operation); + } + catch (Exception exception) + { + exceptions.Enqueue(exception); + } + }); + Task disposal = Task.Run(() => + { + start.SignalAndWait(); + try + { + context.First.Dispose(); + context.First.Dispose(); + } + catch (Exception exception) + { + exceptions.Enqueue(exception); + } + }); + + start.SignalAndWait(); + AssertSecondHandleProgress(context.Second, context.ProgressKey); + await Task.WhenAll(invocation, disposal).WaitAsync(TestTimeout); + + Assert.Empty(exceptions); + AssertDocumentedOutcome(operation, observation); + AssertDisposedTokenSurface(context); + AssertSecondHandleCanReuseCapacity(context); + } + } + + [Fact] + [Trait("Category", "Integration")] + public async Task ConcurrentRepeatedDisposeReturnsEveryOwnedResourceToTheOtherHandle() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var context = CreateContext(DisposalOperation.ReservationProjection); + Task[] disposals = Enumerable.Range(0, 8) + .Select(_ => Task.Run(() => + { + for (var call = 0; call < 32; call++) + { + context.First.Dispose(); + } + })) + .ToArray(); + + await Task.WhenAll(disposals).WaitAsync(TestTimeout); + + AssertDisposedTokenSurface(context); + AssertSecondHandleCanReuseCapacity(context); + } + + [Fact] + [Trait("Category", "Integration")] + public async Task PausedLiveDisposerPublishesRecoverableHandoffWithoutBlockingOtherHandles() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var gate = new CheckpointGate(); + using var context = CreateContext(DisposalOperation.ReservationProjection, gate); + gate.Arm(LockFreeCheckpointId.DisposalAfterParticipantClosingPublication); + + Task disposal = Task.Run(context.First.Dispose); + Assert.True( + gate.WaitUntilPaused(TestTimeout), + "Dispose did not pause after publishing exact participant Closing."); + try + { + Assert.Equal(StoreStatus.Success, context.Second.TryGetDiagnostics(out DiagnosticsSnapshot paused)); + Assert.Equal(1, paused.ClosingParticipantCount); + + Assert.Equal( + StoreStatus.Success, + context.Second.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: false), + StoreWaitOptions.Infinite, + out LeaseRecoveryReport leases)); + Assert.Equal(1, leases.RecoveredLeaseCount); + + Assert.Equal( + StoreStatus.Success, + context.Second.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: false), + StoreWaitOptions.Infinite, + out ReservationRecoveryReport reservations)); + Assert.Equal(1, reservations.RecoveredReservationCount); + + AssertSecondHandleProgress(context.Second, context.ProgressKey); + Assert.Equal(StoreStatus.Success, context.Second.TryGetDiagnostics(out DiagnosticsSnapshot helped)); + Assert.Equal(0, helped.ClosingParticipantCount); + Assert.Equal(0, helped.RecoveringParticipantCount); + } + finally + { + gate.Continue(); + } + + await disposal.WaitAsync(TestTimeout); + LockFreeCheckpointId[] observed = gate.ObservedIds(); + Assert.True( + Array.IndexOf(observed, LockFreeCheckpointId.DisposalAfterParticipantClosingPublication) + < Array.IndexOf(observed, LockFreeCheckpointId.DisposalAfterParticipantRelease)); + AssertSecondHandleCanReuseCapacity(context); + } + + [Fact] + [Trait("Category", "Integration")] + public void DisposeReleasesOwnedLeaseAndReclaimsItsPendingRemovalWithoutAnotherRecoveryCall() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-disposal-pending-remove-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions create = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 8, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + SharedMemoryStoreOptions open = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 8, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.OpenExisting, + enableLeaseRecovery: true); + + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(create, out MemoryStore? owner)); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(open, out MemoryStore? survivor)); + Assert.NotNull(owner); + Assert.NotNull(survivor); + try + { + Assert.Equal(StoreStatus.Success, survivor!.TryPublish(Key(1), [1])); + Assert.Equal(StoreStatus.Success, owner!.TryAcquire(Key(1), out ValueLease lease)); + Assert.Equal(StoreStatus.RemovePending, survivor.TryRemove(Key(1))); + + owner.Dispose(); + + Assert.Equal(StoreStatus.Success, survivor.TryPublish(Key(2), [2])); + Assert.Equal(StoreStatus.Success, survivor.TryRemove(Key(2))); + Assert.Equal(StoreStatus.StoreDisposed, lease.Release()); + } + finally + { + owner?.Dispose(); + survivor?.Dispose(); + } + } + + [Fact] + [Trait("Category", "Integration")] + public async Task HeldColdOpenLockCannotDelayDisposeOrOtherHandleDataProgress() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-disposal-held-cold-lock-{Guid.NewGuid():N}"; + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen(Options(name, OpenMode.CreateNew), out MemoryStore? owner)); + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen(Options(name, OpenMode.OpenExisting), out MemoryStore? survivor)); + Assert.NotNull(owner); + Assert.NotNull(survivor); + + try + { + using var held = new DedicatedColdSynchronizationHolder(name); + var stopwatch = Stopwatch.StartNew(); + Task dispose = Task.Run(owner!.Dispose); + Task progress = Task.Run(() => + { + Assert.Equal(StoreStatus.Success, survivor!.TryPublish(Key(701), [7])); + Assert.Equal(StoreStatus.Success, survivor.TryAcquire(Key(701), out ValueLease lease)); + Assert.Equal(7, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, survivor.TryRemove(Key(701))); + }); + + await Task.WhenAll(dispose, progress).WaitAsync(TimeSpan.FromSeconds(1)); + stopwatch.Stop(); + Assert.True( + stopwatch.Elapsed <= TimeSpan.FromMilliseconds(500), + $"Dispose/data progress waited {stopwatch.Elapsed} on the cold open lock."); + } + finally + { + owner?.Dispose(); + survivor?.Dispose(); + } + } + + [Fact] + [Trait("Category", "Integration")] + public void HeldColdOpenLockIsScopedToItsNamedStore() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string blockedName = $"sms-v2-held-cold-store-a-{Guid.NewGuid():N}"; + string independentName = $"sms-v2-held-cold-store-b-{Guid.NewGuid():N}"; + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen( + Options(blockedName, OpenMode.CreateNew), + out MemoryStore? blockedOwner)); + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen( + Options(independentName, OpenMode.CreateNew), + out MemoryStore? independentOwner)); + Assert.NotNull(blockedOwner); + Assert.NotNull(independentOwner); + + try + { + using var held = new DedicatedColdSynchronizationHolder(blockedName); + Assert.Equal( + StoreOpenStatus.StoreBusy, + MemoryStore.TryCreateOrOpen( + Options(blockedName, OpenMode.OpenExisting), + StoreWaitOptions.NoWait, + out MemoryStore? blockedOpen)); + Assert.Null(blockedOpen); + + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen( + Options(independentName, OpenMode.OpenExisting), + out MemoryStore? independentOpen)); + using (independentOpen) + { + Assert.Equal(StoreStatus.Success, independentOpen!.TryPublish(Key(801), [8])); + Assert.Equal(StoreStatus.Success, independentOpen.TryAcquire(Key(801), out ValueLease lease)); + Assert.Equal(8, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, independentOpen.TryRemove(Key(801))); + } + } + finally + { + blockedOwner?.Dispose(); + independentOwner?.Dispose(); + } + } + + private static DisposalContext CreateContext( + DisposalOperation operation, + CheckpointGate? checkpointGate = null) + { + string name = $"sms-v2-disposal-{operation}-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions create = Options(name, OpenMode.CreateNew); + MemoryStore first; + if (checkpointGate is null) + { + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(create, out var opened)); + first = Assert.IsType(opened); + } + else + { + Assert.Equal( + StoreOpenStatus.Success, + LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + create, + LockFreeCheckpointFactory.CreateInstrumented(checkpointGate.Observe), + out var opened)); + first = Assert.IsType(opened); + } + + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen(Options(name, OpenMode.OpenExisting), out var secondStore)); + MemoryStore second = Assert.IsType(secondStore); + + byte[] publishedKey = Key(1); + byte[] reservationKey = Key(2); + byte[] operationKey = Key(3); + byte[] progressKey = Key(4); + Assert.Equal(StoreStatus.Success, second.TryPublish(publishedKey, [7, 8], [9, 10])); + Assert.Equal(StoreStatus.Success, first.TryAcquire(publishedKey, out ValueLease lease)); + Assert.Equal( + StoreStatus.Success, + first.TryReserve(reservationKey, payloadLength: 2, descriptor: [11], out ValueReservation reservation)); + + if (operation == DisposalOperation.Commit) + { + reservation.GetSpan().Fill(12); + Assert.Equal(StoreStatus.Success, reservation.Advance(2)); + } + + return new DisposalContext( + first, + second, + reservation, + lease, + publishedKey, + reservationKey, + operationKey, + progressKey); + } + + private static OperationObservation Invoke(DisposalContext context, DisposalOperation operation) + { + switch (operation) + { + case DisposalOperation.Publish: + return Status(context.First.TryPublish(context.OperationKey, [21], [22])); + + case DisposalOperation.Reserve: + return Status(context.First.TryReserve( + context.OperationKey, + payloadLength: 1, + descriptor: [22], + out _)); + + case DisposalOperation.ReservationProjection: + { + Span span = context.Reservation.GetSpan(); + return Projection(span.Length); + } + + case DisposalOperation.Advance: + return Status(context.Reservation.Advance(1)); + + case DisposalOperation.Commit: + return Status(context.Reservation.Commit()); + + case DisposalOperation.Abort: + return Status(context.Reservation.Abort()); + + case DisposalOperation.Acquire: + return Status(context.First.TryAcquire(context.PublishedKey, out _)); + + case DisposalOperation.ValueProjection: + { + ReadOnlySpan span = context.Lease.ValueSpan; + return Projection(span.Length); + } + + case DisposalOperation.DescriptorProjection: + { + ReadOnlySpan span = context.Lease.DescriptorSpan; + return Projection(span.Length); + } + + case DisposalOperation.Release: + return Status(context.Lease.Release()); + + case DisposalOperation.Remove: + return Status(context.First.TryRemove(context.PublishedKey)); + + case DisposalOperation.RecoverLeases: + // This theory deliberately races another handle's ordinary work. + // The current-process override requires process-wide quiescence, + // so exercise the concurrency-safe recovery mode here. + return Status(context.First.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: false), + out _)); + + case DisposalOperation.RecoverReservations: + return Status(context.First.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: false), + out _)); + + case DisposalOperation.Diagnostics: + return Status(context.First.TryGetDiagnostics(out _)); + + case DisposalOperation.RepeatedDispose: + context.First.Dispose(); + return Status(StoreStatus.Success); + + default: + throw new ArgumentOutOfRangeException(nameof(operation)); + } + } + + private static void AssertDocumentedOutcome( + DisposalOperation operation, + OperationObservation observation) + { + if (operation is DisposalOperation.ReservationProjection + or DisposalOperation.ValueProjection + or DisposalOperation.DescriptorProjection) + { + // Disposal may invalidate borrowed storage after projection returns, + // so this race may inspect the projection shape but not its bytes. + Assert.Contains(observation.ProjectedLength, new[] { 0, 2 }); + return; + } + + StoreStatus status = Assert.IsType(observation.Status); + Assert.NotEqual(StoreStatus.UnknownFailure, status); + Assert.NotEqual(StoreStatus.CorruptStore, status); + Assert.Contains(status, AllowedStatuses(operation)); + } + + private static IReadOnlyCollection AllowedStatuses(DisposalOperation operation) => + operation switch + { + DisposalOperation.Publish => [StoreStatus.Success, StoreStatus.StoreDisposed], + DisposalOperation.Reserve => [StoreStatus.Success, StoreStatus.StoreDisposed], + DisposalOperation.Advance => + [StoreStatus.Success, StoreStatus.StoreDisposed, StoreStatus.InvalidReservation], + DisposalOperation.Commit => + [StoreStatus.Success, StoreStatus.StoreDisposed, StoreStatus.InvalidReservation, + StoreStatus.ReservationAlreadyCompleted], + DisposalOperation.Abort => + [StoreStatus.Success, StoreStatus.StoreDisposed, StoreStatus.InvalidReservation, + StoreStatus.ReservationAlreadyCompleted], + DisposalOperation.Acquire => + [StoreStatus.Success, StoreStatus.StoreDisposed, StoreStatus.NotFound], + DisposalOperation.Release => + [StoreStatus.Success, StoreStatus.StoreDisposed, StoreStatus.InvalidLease, + StoreStatus.LeaseAlreadyReleased], + DisposalOperation.Remove => + [StoreStatus.Success, StoreStatus.StoreDisposed, StoreStatus.NotFound, + StoreStatus.RemovePending], + DisposalOperation.RecoverLeases or DisposalOperation.RecoverReservations => + [StoreStatus.Success, StoreStatus.StoreDisposed, StoreStatus.StoreBusy, + StoreStatus.UnsupportedPlatform], + DisposalOperation.Diagnostics => + [StoreStatus.Success, StoreStatus.StoreDisposed, StoreStatus.UnsupportedPlatform], + DisposalOperation.RepeatedDispose => [StoreStatus.Success], + _ => throw new ArgumentOutOfRangeException(nameof(operation)) + }; + + private static void AssertSecondHandleProgress(MemoryStore second, byte[] key) + { + Assert.Equal(StoreStatus.Success, second.TryPublish(key, [31, 32], [33])); + Assert.Equal(StoreStatus.Success, second.TryAcquire(key, out ValueLease lease)); + Assert.Equal(new byte[] { 31, 32 }, lease.ValueSpan.ToArray()); + Assert.Equal(new byte[] { 33 }, lease.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, second.TryRemove(key)); + + byte[] reservationKey = Key(5); + Assert.Equal(StoreStatus.Success, second.TryReserve(reservationKey, 1, default, out var reservation)); + reservation.GetSpan()[0] = 34; + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + Assert.Equal(StoreStatus.Success, reservation.Abort()); + } + + private static void AssertDisposedTokenSurface(DisposalContext context) + { + Assert.Equal(StoreStatus.StoreDisposed, context.First.TryPublish(Key(90), [1])); + Assert.False(context.Reservation.IsValid); + Assert.True(context.Reservation.GetSpan().IsEmpty); + Assert.Equal(StoreStatus.StoreDisposed, context.Reservation.Advance(0)); + Assert.Equal(StoreStatus.StoreDisposed, context.Reservation.Commit()); + Assert.Equal(StoreStatus.StoreDisposed, context.Reservation.Abort()); + Assert.False(context.Lease.IsValid); + Assert.True(context.Lease.ValueSpan.IsEmpty); + Assert.True(context.Lease.DescriptorSpan.IsEmpty); + Assert.Equal(StoreStatus.StoreDisposed, context.Lease.Release()); + } + + private static void AssertSecondHandleCanReuseCapacity(DisposalContext context) + { + _ = context.Second.TryRecoverLeases(new LeaseRecoveryOptions(true), out _); + _ = context.Second.TryRecoverReservations(new ReservationRecoveryOptions(true), out _); + RemoveIfPresent(context.Second, context.PublishedKey); + RemoveIfPresent(context.Second, context.ReservationKey); + RemoveIfPresent(context.Second, context.OperationKey); + RemoveIfPresent(context.Second, context.ProgressKey); + RemoveIfPresent(context.Second, Key(5)); + + for (var index = 0; index < SlotCount; index++) + { + Assert.Equal( + StoreStatus.Success, + context.Second.TryPublish(Key(100 + index), [(byte)index])); + } + + Assert.Equal(StoreStatus.StoreFull, context.Second.TryPublish(Key(200), [1])); + for (var index = 0; index < SlotCount; index++) + { + Assert.Equal(StoreStatus.Success, context.Second.TryRemove(Key(100 + index))); + } + } + + private static void RemoveIfPresent(MemoryStore store, byte[] key) + { + StoreStatus status = store.TryRemove(key); + Assert.Contains(status, new[] { StoreStatus.Success, StoreStatus.NotFound }); + } + + private static OperationObservation Status(StoreStatus status) => new(status, 0); + + private static OperationObservation Projection(int length) => new(null, length); + + private static SharedMemoryStoreOptions Options(string name, OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: SlotCount, + maxValueBytes: 32, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount: 16, + participantRecordCount: 4, + openMode: openMode, + enableLeaseRecovery: true); + + private static byte[] Key(int value) => BitConverter.GetBytes(value); + + private static bool IsSupportedLockFreeHost() => + LayoutV2Constants.IsSupportedArchitecture(RuntimeInformation.ProcessArchitecture); + + public enum DisposalOperation + { + Publish, + Reserve, + ReservationProjection, + Advance, + Commit, + Abort, + Acquire, + ValueProjection, + DescriptorProjection, + Release, + Remove, + RecoverLeases, + RecoverReservations, + Diagnostics, + RepeatedDispose + } + + private readonly record struct OperationObservation( + StoreStatus? Status, + int ProjectedLength); + + /// + /// Owns the platform synchronization primitive on one dedicated thread. + /// Windows mutexes must be released by their acquiring thread even when an + /// async xUnit continuation moves the test itself to another thread. + /// + private sealed class DedicatedColdSynchronizationHolder : IDisposable + { + private readonly ManualResetEventSlim _ready = new(initialState: false); + private readonly ManualResetEventSlim _release = new(initialState: false); + private readonly Thread _thread; + private readonly string _name; + private Exception? _failure; + private int _disposed; + + internal DedicatedColdSynchronizationHolder(string name) + { + _name = name; + _thread = new Thread(Hold) + { + IsBackground = true, + Name = "SharedMemoryStore cold-lock test holder" + }; + _thread.Start(); + if (!_ready.Wait(TimeSpan.FromSeconds(5))) + { + throw new TimeoutException("Timed out acquiring the cold store synchronization primitive."); + } + + if (_failure is not null) + { + throw new InvalidOperationException("Unable to hold cold store synchronization.", _failure); + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _release.Set(); + if (!_thread.Join(TimeSpan.FromSeconds(5))) + { + throw new TimeoutException("Cold synchronization holder did not exit."); + } + + _ready.Dispose(); + _release.Dispose(); + } + + private void Hold() + { + try + { + using IDisposable held = PlatformCapabilityProbe.HoldStoreSynchronization(_name); + _ready.Set(); + _release.Wait(); + } + catch (Exception exception) + { + _failure = exception; + _ready.Set(); + } + } + } + + private sealed class DisposalContext : IDisposable + { + internal DisposalContext( + MemoryStore first, + MemoryStore second, + ValueReservation reservation, + ValueLease lease, + byte[] publishedKey, + byte[] reservationKey, + byte[] operationKey, + byte[] progressKey) + { + First = first; + Second = second; + Reservation = reservation; + Lease = lease; + PublishedKey = publishedKey; + ReservationKey = reservationKey; + OperationKey = operationKey; + ProgressKey = progressKey; + } + + internal MemoryStore First { get; } + internal MemoryStore Second { get; } + internal ValueReservation Reservation { get; } + internal ValueLease Lease { get; } + internal byte[] PublishedKey { get; } + internal byte[] ReservationKey { get; } + internal byte[] OperationKey { get; } + internal byte[] ProgressKey { get; } + + public void Dispose() + { + First.Dispose(); + Second.Dispose(); + } + } + + private sealed class CheckpointGate : IDisposable + { + private readonly ManualResetEventSlim _paused = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: false); + private readonly List _observed = []; + private LockFreeCheckpointId? _target; + private int _claimed; + + internal void Arm(LockFreeCheckpointId target) + { + _target = target; + Volatile.Write(ref _claimed, 0); + lock (_observed) + { + _observed.Clear(); + } + _paused.Reset(); + _resume.Reset(); + } + + internal void Observe(LockFreeCheckpointEntry entry) + { + lock (_observed) + { + _observed.Add(entry.Id); + } + + if (_target != entry.Id || Interlocked.CompareExchange(ref _claimed, 1, 0) != 0) + { + return; + } + + _paused.Set(); + if (!_resume.Wait(TestTimeout)) + { + throw new TimeoutException($"Checkpoint {entry.Id} was not resumed."); + } + } + + internal bool WaitUntilPaused(TimeSpan timeout) => _paused.Wait(timeout); + + internal LockFreeCheckpointId[] ObservedIds() + { + // This gate is used by one instrumented engine. Observation order + // is append-only and queried only after its disposal task exits. + lock (_observed) + { + return _observed.ToArray(); + } + } + + internal void Continue() => _resume.Set(); + + public void Dispose() + { + _resume.Set(); + _paused.Dispose(); + _resume.Dispose(); + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeMultiReaderIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeMultiReaderIntegrationTests.cs new file mode 100644 index 0000000..afb4a8d --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeMultiReaderIntegrationTests.cs @@ -0,0 +1,457 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeMultiReaderIntegrationTests +{ + private const int SlotCount = 16; + private const int MaxValueBytes = 128; + private const int MaxDescriptorBytes = 8; + private const int MaxKeyBytes = 8; + private const int LeaseRecordCount = 64; + private const int ParticipantRecordCount = 32; + private static readonly TimeSpan AgentTimeout = TimeSpan.FromSeconds(45); + + [Theory] + [InlineData(1, false)] + [InlineData(6, false)] + [InlineData(12, false)] + [InlineData(1, true)] + [InlineData(6, true)] + [InlineData(12, true)] + [Trait("Category", "Integration")] + public void ReadersVerifySameKeyAndDistributedKeyBytesAcrossProcesses(int readerCount, bool distributedKeys) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-readers-{readerCount}-{distributedKeys}-{Guid.NewGuid():N}"; + using var store = CreateStore(name); + int keyCount = distributedKeys ? readerCount : 1; + for (var index = 0; index < keyCount; index++) + { + Assert.Equal( + StoreStatus.Success, + store.TryPublish(Key(index), Value(index), Descriptor(index))); + } + + using var gate = AgentGate.Create(readerCount); + string[][] commands = Enumerable.Range(0, readerCount) + .Select(index => ReadCommand( + name, + Key(distributedKeys ? index : 0), + Value(distributedKeys ? index : 0), + Descriptor(distributedKeys ? index : 0), + iterations: 64, + gate.GoPath, + gate.ReadyPaths[index])) + .ToArray(); + + AgentResult[] results = RunReaders(commands, gate); + + AssertAgentsSucceeded(results, readerCount); + Assert.All(results, static result => Assert.Contains("OK lease-read", result.StandardOutput)); + } + + [Fact] + [Trait("Category", "Integration")] + public void PausedObserverLeaseDoesNotStopSameKeyOrUnrelatedReaders() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-paused-observer-{Guid.NewGuid():N}"; + using var store = CreateStore(name); + Assert.Equal(StoreStatus.Success, store.TryPublish(Key(1), Value(1), Descriptor(1))); + Assert.Equal(StoreStatus.Success, store.TryPublish(Key(2), Value(2), Descriptor(2))); + + string directory = Path.Combine(Path.GetTempPath(), $"sms-v2-observer-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string observerReady = Path.Combine(directory, "observer.ready"); + string observerRelease = Path.Combine(directory, "observer.release"); + Process? observer = null; + try + { + observer = StartAgent(HoldCommand( + name, + Key(1), + Value(1), + Descriptor(1), + observerReady, + observerRelease)); + WaitForReadyFiles([observerReady], [observer], AgentTimeout); + + const int workerCount = 6; + using var gate = AgentGate.Create(workerCount, directory); + string[][] commands = Enumerable.Range(0, workerCount) + .Select(index => + { + int keyIndex = index % 2 == 0 ? 1 : 2; + return ReadCommand( + name, + Key(keyIndex), + Value(keyIndex), + Descriptor(keyIndex), + iterations: 64, + gate.GoPath, + gate.ReadyPaths[index]); + }) + .ToArray(); + + AgentResult[] workers = RunReaders(commands, gate); + AssertAgentsSucceeded(workers, workerCount); + Assert.False(observer.HasExited, "The observer should remain paused while healthy readers finish."); + + File.WriteAllText(observerRelease, "continue"); + AgentResult observerResult = WaitForAgent(observer, AgentTimeout); + AssertAgentsSucceeded([observerResult], expectedCount: 1); + Assert.Contains("OK lease-hold", observerResult.StandardOutput); + } + finally + { + try + { + File.WriteAllText(observerRelease, "continue"); + } + catch + { + // Best-effort release before terminating the test process. + } + + if (observer is not null) + { + Kill(observer); + observer.Dispose(); + } + + try + { + Directory.Delete(directory, recursive: true); + } + catch + { + // Unique test artifacts can be reclaimed by the OS if cleanup races a process exit. + } + } + } + + private static MemoryStore CreateStore(string name) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + SlotCount, + MaxValueBytes, + MaxDescriptorBytes, + MaxKeyBytes, + LeaseRecordCount, + ParticipantRecordCount, + OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static string[] ReadCommand( + string name, + byte[] key, + byte[] value, + byte[] descriptor, + int iterations, + string goPath, + string readyPath) => + [ + "lease-read", + name, + SlotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + ParticipantRecordCount.ToString(CultureInfo.InvariantCulture), + Convert.ToHexString(key), + Encode(value), + Encode(descriptor), + iterations.ToString(CultureInfo.InvariantCulture), + goPath, + readyPath + ]; + + private static string[] HoldCommand( + string name, + byte[] key, + byte[] value, + byte[] descriptor, + string readyPath, + string releasePath) => + [ + "lease-hold", + name, + SlotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + ParticipantRecordCount.ToString(CultureInfo.InvariantCulture), + Convert.ToHexString(key), + Encode(value), + Encode(descriptor), + readyPath, + releasePath + ]; + + private static AgentResult[] RunReaders(string[][] commands, AgentGate gate) + { + Process[] processes = commands.Select(StartAgent).ToArray(); + try + { + WaitForReadyFiles(gate.ReadyPaths, processes, AgentTimeout); + File.WriteAllText(gate.GoPath, "go"); + return WaitForAgents(processes, AgentTimeout); + } + finally + { + foreach (Process process in processes) + { + Kill(process); + process.Dispose(); + } + } + } + + private static void WaitForReadyFiles( + IReadOnlyList readyPaths, + IReadOnlyList processes, + TimeSpan timeout) + { + long deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (readyPaths.Any(static path => !File.Exists(path))) + { + Process? failed = processes.FirstOrDefault(static process => process.HasExited); + if (failed is not null) + { + var result = new AgentResult( + failed.ExitCode, + failed.StandardOutput.ReadToEnd(), + failed.StandardError.ReadToEnd()); + AssertAgentsSucceeded([result], expectedCount: 1); + } + + if (Stopwatch.GetTimestamp() >= deadline) + { + throw new TimeoutException("Reader agents did not reach their start checkpoint."); + } + + spin.SpinOnce(); + } + } + + private static AgentResult WaitForAgent(Process process, TimeSpan timeout) + { + if (!process.WaitForExit(checked((int)timeout.TotalMilliseconds))) + { + Kill(process); + throw new TimeoutException("Lock-free reader agent exceeded its bounded timeout."); + } + + return new AgentResult( + process.ExitCode, + process.StandardOutput.ReadToEnd(), + process.StandardError.ReadToEnd()); + } + + private static AgentResult[] WaitForAgents(IReadOnlyList processes, TimeSpan timeout) + { + long deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + var results = new AgentResult[processes.Count]; + for (var index = 0; index < processes.Count; index++) + { + long remainingTicks = deadline - Stopwatch.GetTimestamp(); + int remainingMilliseconds = remainingTicks <= 0 + ? 0 + : (int)Math.Min( + int.MaxValue, + Math.Ceiling(remainingTicks * 1000d / Stopwatch.Frequency)); + Process process = processes[index]; + if (!process.WaitForExit(remainingMilliseconds)) + { + Kill(process); + throw new TimeoutException("Lock-free reader agents exceeded their shared bounded timeout."); + } + + results[index] = new AgentResult( + process.ExitCode, + process.StandardOutput.ReadToEnd(), + process.StandardError.ReadToEnd()); + } + + return results; + } + + private static Process StartAgent(string[] command) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(LocateAgentAssembly()); + foreach (string argument in command) + { + startInfo.ArgumentList.Add(argument); + } + + return Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start the lock-free reader agent."); + } + + private static string LocateAgentAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + string root = directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + if (!File.Exists(path)) + { + throw new FileNotFoundException("Lock-free reader agent was not built.", path); + } + + return path; + } + + private static void AssertAgentsSucceeded(AgentResult[] results, int expectedCount) + { + Assert.Equal(expectedCount, results.Length); + Assert.All(results, static result => Assert.True( + result.ExitCode == 0, + "Agent exit code: " + + result.ExitCode.ToString(CultureInfo.InvariantCulture) + + Environment.NewLine + + "stdout: " + + result.StandardOutput + + Environment.NewLine + + "stderr: " + + result.StandardError)); + } + + private static void Kill(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(5_000); + } + } + catch + { + // Cleanup is best effort for unique test resources. + } + } + + private static byte[] Key(int index) => + [(byte)(0x80 | (index & 0x7f)), (byte)(index >> 8), (byte)index]; + + private static byte[] Value(int index) + { + var value = new byte[64]; + for (var offset = 0; offset < value.Length; offset++) + { + value[offset] = (byte)(index * 17 + offset); + } + + return value; + } + + private static byte[] Descriptor(int index) => + [(byte)index, (byte)~index, 0x5a, 0xa5]; + + private static string Encode(byte[] value) => value.Length == 0 ? "-" : Convert.ToHexString(value); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private readonly record struct AgentResult(int ExitCode, string StandardOutput, string StandardError); + + private sealed class AgentGate : IDisposable + { + private AgentGate(string directory, bool ownsDirectory, int participantCount) + { + DirectoryPath = directory; + OwnsDirectory = ownsDirectory; + GoPath = Path.Combine(directory, $"readers-{Guid.NewGuid():N}.go"); + ReadyPaths = Enumerable.Range(0, participantCount) + .Select(index => Path.Combine(directory, $"reader-{Guid.NewGuid():N}-{index}.ready")) + .ToArray(); + } + + public string DirectoryPath { get; } + + public bool OwnsDirectory { get; } + + public string GoPath { get; } + + public string[] ReadyPaths { get; } + + public static AgentGate Create(int participantCount, string? existingDirectory = null) + { + string directory = existingDirectory + ?? Path.Combine(Path.GetTempPath(), $"sms-v2-reader-gate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + return new AgentGate(directory, existingDirectory is null, participantCount); + } + + public void Dispose() + { + foreach (string path in ReadyPaths.Append(GoPath)) + { + try + { + File.Delete(path); + } + catch + { + // Best effort after all reader processes have exited. + } + } + + if (!OwnsDirectory) + { + return; + } + + try + { + Directory.Delete(DirectoryPath, recursive: true); + } + catch + { + // Unique temporary directory may be reclaimed later. + } + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeNoOperationLockIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeNoOperationLockIntegrationTests.cs new file mode 100644 index 0000000..d26c013 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeNoOperationLockIntegrationTests.cs @@ -0,0 +1,410 @@ +using System.Buffers; +using System.Runtime.InteropServices; +using SharedMemoryStore.IntegrationTests.TestSupport; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeNoOperationLockIntegrationTests +{ + private const int SlotCount = 12; + + [Fact] + [Trait("Category", "Integration")] + public void CompleteSteadyStateSurfaceNeverReentersTheColdSynchronization() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + SharedMemoryStoreOptions options = Options( + $"sms-v2-counting-sync-{Guid.NewGuid():N}", + OpenMode.CreateNew); + Assert.Equal( + StoreOpenStatus.Success, + SharedStorePlatform.TryOpenRegion( + options, + StoreWaitOptions.Default, + out MemoryMappedStoreRegion? region)); + Assert.NotNull(region); + + var synchronization = new CountingThrowingSynchronization(); + Assert.Equal( + StoreStatus.Success, + synchronization.TryEnter(StoreWaitOptions.Default)); + IStoreEngine? engine; + StoreOpenStatus open; + using (var openScope = new SharedStoreOpenScope( + region!, + synchronization, + outerLifecycleGate: null, + RegionOpenDisposition.CreatedNew)) + { + open = LockFreeStoreEngine.TryCreateOrOpenUnderColdGate( + options, + StoreWaitOptions.Default, + System.Diagnostics.Stopwatch.GetTimestamp(), + region!, + synchronization, + RegionOpenDisposition.CreatedNew, + out engine); + if (open == StoreOpenStatus.Success && engine is not null) + { + openScope.TransferResourceOwnership(); + } + } + + Assert.Equal(StoreOpenStatus.Success, open); + Assert.NotNull(engine); + var store = new MemoryStore(engine!); + + Assert.Equal(1, synchronization.EnterCount); + Assert.Equal(1, synchronization.ExitCount); + synchronization.ThrowOnEnter = true; + + try + { + ExerciseCompleteSteadyStateSurface(store, new StoreWaitOptions(TimeSpan.FromMilliseconds(250))); + + Assert.Equal(1, synchronization.EnterCount); + Assert.Equal(1, synchronization.ExitCount); + } + finally + { + // Participant unregistration is an exact record-local CAS/help + // lifecycle and must not re-enter the mapping-initialization lock. + synchronization.ThrowOnEnter = false; + store.Dispose(); + } + + Assert.Equal(1, synchronization.EnterCount); + Assert.Equal(1, synchronization.ExitCount); + Assert.Equal(1, synchronization.DisposeCount); + } + + [Fact] + [Trait("Category", "Integration")] + public void FailedColdOpenScopeReleasesGatesBeforeMappedOwnerCleanupExactlyOnce() + { + var events = new List(); + var synchronization = new RecordingSynchronization(events); + var outerLifecycle = new RecordingDisposable(events, "lifecycle-exit"); + Assert.Equal( + StoreStatus.Success, + synchronization.TryEnter(StoreWaitOptions.NoWait)); + + var mapping = System.IO.MemoryMappedFiles.MemoryMappedFile.CreateNew( + mapName: null, + capacity: 4096, + System.IO.MemoryMappedFiles.MemoryMappedFileAccess.ReadWrite); + var accessor = mapping.CreateViewAccessor( + 0, + 4096, + System.IO.MemoryMappedFiles.MemoryMappedFileAccess.ReadWrite); + MemoryMappedStoreRegion region = MemoryMappedStoreRegion.Create( + mapping, + accessor, + () => events.Add("region-owner-cleanup")); + var scope = new SharedStoreOpenScope( + region, + synchronization, + outerLifecycle, + RegionOpenDisposition.CreatedNew); + + scope.Dispose(); + scope.Dispose(); + + Assert.Equal( + ["enter", "exit", "lifecycle-exit", "sync-dispose", "region-owner-cleanup"], + events); + Assert.Equal(1, synchronization.EnterCount); + Assert.Equal(1, synchronization.ExitCount); + Assert.Equal(1, synchronization.DisposeCount); + Assert.Equal(1, outerLifecycle.DisposeCount); + } + + [Fact] + [Trait("Category", "Integration")] + public async Task HeldLegacySynchronizationDoesNotDelayAnySteadyStateOperation() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + SharedMemoryStoreOptions options = Options( + $"sms-v2-held-legacy-sync-{Guid.NewGuid():N}", + OpenMode.CreateNew); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(options, out MemoryStore? candidate)); + using MemoryStore store = Assert.IsType(candidate); + using var blocker = new NamedSynchronizationBlocker(options.Name); + + Task operation = Task.Run(() => ExerciseCompleteSteadyStateSurface(store, StoreWaitOptions.NoWait)); + await operation.WaitAsync(TimeSpan.FromSeconds(2)); + } + + internal static void ExerciseCompleteSteadyStateSurface(MemoryStore store, StoreWaitOptions wait) + { + byte[] simpleKey = Key(1); + byte[] segmentedKey = Key(2); + byte[] committedReservationKey = Key(3); + byte[] abortedReservationKey = Key(4); + byte[] disposedReservationKey = Key(5); + byte[] disposedLeaseKey = Key(6); + + Assert.Equal(StoreStatus.Success, store.TryPublish(simpleKey, [11, 12], [13], wait)); + + var segmentedPayload = new ReadOnlySequence(new byte[] { 21, 22, 23 }); + Assert.Equal( + StoreStatus.Success, + store.TryPublishSegments(segmentedKey, segmentedPayload, [24], wait, out long copiedBytes)); + Assert.Equal(3, copiedBytes); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve(committedReservationKey, 2, [31], wait, out ValueReservation committed)); + Assert.True(committed.IsValid); + Assert.Equal(2, committed.PayloadLength); + Assert.Equal(0, committed.BytesWritten); + Span writable = committed.GetSpan(2); + Assert.Equal(2, writable.Length); + writable[0] = 32; + committed.DangerousGetMemory(1).Span[1] = 33; + Assert.Equal(StoreStatus.Success, committed.Advance(2, wait)); + Assert.Equal(2, committed.BytesWritten); + Assert.Equal(StoreStatus.Success, committed.Commit(wait)); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve(abortedReservationKey, 1, default, wait, out ValueReservation aborted)); + aborted.GetSpan()[0] = 41; + Assert.Equal(StoreStatus.Success, aborted.Abort(wait)); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve(disposedReservationKey, 1, default, wait, out ValueReservation disposedReservation)); + disposedReservation.Dispose(); + Assert.False(disposedReservation.IsValid); + + Assert.Equal(StoreStatus.Success, store.TryAcquire(simpleKey, wait, out ValueLease lease)); + Assert.True(lease.IsValid); + Assert.Equal(2, lease.ValueLength); + Assert.Equal(1, lease.DescriptorLength); + Assert.Equal(new byte[] { 11, 12 }, lease.ValueSpan.ToArray()); + Assert.Equal(new byte[] { 13 }, lease.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, lease.Release(wait)); + + Assert.Equal(StoreStatus.Success, store.TryPublish(disposedLeaseKey, [61], default, wait)); + Assert.Equal(StoreStatus.Success, store.TryAcquire(disposedLeaseKey, wait, out ValueLease disposedLease)); + disposedLease.Dispose(); + Assert.False(disposedLease.IsValid); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(Key(99), wait, out _)); + + AssertRemovalAndReclaim(store, simpleKey, wait); + AssertRemovalAndReclaim(store, segmentedKey, wait); + AssertRemovalAndReclaim(store, committedReservationKey, wait); + AssertRemovalAndReclaim(store, disposedLeaseKey, wait); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(true), wait, out LeaseRecoveryReport leaseReport)); + Assert.Equal(0, leaseReport.RecoveredLeaseCount); + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + wait, + out ReservationRecoveryReport reservationReport)); + Assert.Equal(0, reservationReport.RecoveredReservationCount); + + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(wait, out var diagnostics)); + Assert.Equal(StoreProfile.LockFree, diagnostics.Profile); + Assert.Equal(0, diagnostics.ActiveLeaseCount); + Assert.Equal(0, diagnostics.ActiveReservationCount); + Assert.Equal(0, diagnostics.PublishedSlotCount); + Assert.Equal(0, diagnostics.PendingRemovalCount); + Assert.Equal(SlotCount, diagnostics.FreeSlotCount); + Assert.Equal(StoreProfile.LockFree, store.GetDiagnostics().Profile); + } + + private static void AssertRemovalAndReclaim( + MemoryStore store, + byte[] key, + StoreWaitOptions selectedWait) + { + StoreStatus first = store.TryRemove(key, selectedWait); + Assert.Contains(first, new[] { StoreStatus.Success, StoreStatus.RemovePending }); + if (first == StoreStatus.RemovePending) + { + StoreStatus helped = store.TryRemove(key, new StoreWaitOptions(TimeSpan.FromMilliseconds(250))); + Assert.Contains(helped, new[] { StoreStatus.Success, StoreStatus.NotFound }); + } + + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(key, out _)); + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: SlotCount, + maxValueBytes: 64, + maxDescriptorBytes: 16, + maxKeyBytes: 8, + leaseRecordCount: 16, + participantRecordCount: 4, + openMode, + enableLeaseRecovery: true); + + private static byte[] Key(int value) => BitConverter.GetBytes(value); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && LayoutV2Constants.IsSupportedArchitecture(RuntimeInformation.ProcessArchitecture); + + private sealed class CountingThrowingSynchronization : ISharedStoreSynchronization + { + private int _enterCount; + private int _exitCount; + private int _disposeCount; + + internal bool ThrowOnEnter { get; set; } + + internal int EnterCount => Volatile.Read(ref _enterCount); + + internal int ExitCount => Volatile.Read(ref _exitCount); + + internal int DisposeCount => Volatile.Read(ref _disposeCount); + + public StoreStatus TryEnter(StoreWaitOptions waitOptions) + { + Interlocked.Increment(ref _enterCount); + if (ThrowOnEnter) + { + throw new InvalidOperationException( + "A layout-v2 steady-state operation attempted to enter cold synchronization."); + } + + return StoreStatus.Success; + } + + public void Exit() => Interlocked.Increment(ref _exitCount); + + public void Dispose() => Interlocked.Increment(ref _disposeCount); + } + + private sealed class RecordingSynchronization : ISharedStoreSynchronization + { + private readonly List _events; + private int _enterCount; + private int _exitCount; + private int _disposeCount; + + internal RecordingSynchronization(List events) + { + _events = events; + } + + internal int EnterCount => _enterCount; + + internal int ExitCount => _exitCount; + + internal int DisposeCount => _disposeCount; + + public StoreStatus TryEnter(StoreWaitOptions waitOptions) + { + _enterCount++; + _events.Add("enter"); + return StoreStatus.Success; + } + + public void Exit() + { + _exitCount++; + _events.Add("exit"); + } + + public void Dispose() + { + _disposeCount++; + _events.Add("sync-dispose"); + } + } + + private sealed class RecordingDisposable : IDisposable + { + private readonly List _events; + private readonly string _event; + + internal RecordingDisposable(List events, string @event) + { + _events = events; + _event = @event; + } + + internal int DisposeCount { get; private set; } + + public void Dispose() + { + DisposeCount++; + _events.Add(_event); + } + } + + private sealed class NamedSynchronizationBlocker : IDisposable + { + private readonly string _storeName; + private readonly ManualResetEventSlim _ready = new(initialState: false); + private readonly ManualResetEventSlim _release = new(initialState: false); + private readonly Thread _thread; + private Exception? _failure; + + internal NamedSynchronizationBlocker(string storeName) + { + _storeName = storeName; + _thread = new Thread(Hold) + { + IsBackground = true, + Name = "SharedMemoryStore legacy synchronization blocker" + }; + _thread.Start(); + Assert.True(_ready.Wait(TimeSpan.FromSeconds(5)), "The legacy synchronization was not acquired."); + if (_failure is not null) + { + throw new Xunit.Sdk.XunitException( + $"The legacy synchronization blocker failed: {_failure}"); + } + } + + public void Dispose() + { + _release.Set(); + Assert.True(_thread.Join(TimeSpan.FromSeconds(5)), "The legacy synchronization blocker did not stop."); + _ready.Dispose(); + _release.Dispose(); + if (_failure is not null) + { + throw new Xunit.Sdk.XunitException( + $"The legacy synchronization blocker failed: {_failure}"); + } + } + + private void Hold() + { + try + { + using IDisposable synchronization = PlatformCapabilityProbe.HoldStoreSynchronization(_storeName); + _ready.Set(); + _release.Wait(); + } + catch (Exception exception) + { + _failure = exception; + _ready.Set(); + } + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeOsTraceIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeOsTraceIntegrationTests.cs new file mode 100644 index 0000000..dc90f00 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeOsTraceIntegrationTests.cs @@ -0,0 +1,621 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeOsTraceIntegrationTests +{ + private const int SlotCount = 12; + private const int MaxValueBytes = 64; + private const int MaxDescriptorBytes = 16; + private const int MaxKeyBytes = 8; + private const int LeaseRecordCount = 16; + private const int ParticipantRecordCount = 8; + private const int TraceIterations = 1_024; + private static readonly TimeSpan AgentTimeout = TimeSpan.FromSeconds(75); + + [Fact] + [Trait("Category", "Integration")] + [Trait("Category", "LinuxStrace")] + public void LinuxMarkedSteadyIntervalDoesNotUseStoreOperationLock() + { + if (!IsLinuxX64() || !CommandSucceeds("strace", "--version")) + { + // scripts/validate-lock-free-os.ps1 reports this platform/tool + // boundary as not-qualified. Returning here keeps ordinary local + // test runs portable without misrepresenting qualification evidence. + return; + } + + string name = $"sms-v2-strace-{Guid.NewGuid():N}"; + string markerDirectory = Path.Combine(Path.GetTempPath(), $"sms-v2-strace-{Guid.NewGuid():N}"); + Directory.CreateDirectory(markerDirectory); + string readyPath = Path.Combine(markerDirectory, "ready"); + string goPath = Path.Combine(markerDirectory, "go"); + string donePath = Path.Combine(markerDirectory, "done"); + string tracePrefix = Path.Combine(markerDirectory, "trace"); + string synchronizationPath = PlatformResourceName.Create(name).LinuxSynchronizationPath; + + using MemoryStore store = CreateStore(name); + using Process process = StartTracedAgent( + name, + readyPath, + goPath, + donePath, + tracePrefix); + try + { + Assert.True(WaitForFile(readyPath, AgentTimeout), AgentFailure(process, "The traced agent did not become ready.")); + + // Give the cold open/unlock trace a distinct timestamp from the + // measurement marker even on filesystems with coarse timestamps. + Thread.Sleep(25); + PublishMarker(goPath, "go"); + decimal intervalStart = ToUnixSeconds(File.GetLastWriteTimeUtc(goPath)); + + Assert.True(WaitForFile(donePath, AgentTimeout), AgentFailure(process, "The traced agent did not finish its marked interval.")); + decimal intervalEnd = ToUnixSeconds(File.GetLastWriteTimeUtc(donePath)); + Assert.Equal("ok", File.ReadAllText(donePath)); + Assert.True(process.WaitForExit((int)AgentTimeout.TotalMilliseconds), "The traced agent did not exit after its done marker."); + string output = process.StandardOutput.ReadToEnd(); + string error = process.StandardError.ReadToEnd(); + Assert.True(process.ExitCode == 0, $"The traced agent failed with exit {process.ExitCode}.\nstdout={output}\nstderr={error}"); + + string[] traceFiles = Directory.GetFiles(markerDirectory, "trace*"); + Assert.NotEmpty(traceFiles); + string[] lines = traceFiles.SelectMany(File.ReadLines).ToArray(); + TraceObservation observation = ClassifyTraceLines( + lines, + synchronizationPath, + intervalStart, + intervalEnd); + + Assert.True( + observation.AllTargetLockCalls.Count > 0, + "strace did not observe the expected cold-path operation-lock acquisition; " + + "the no-lock assertion would otherwise be vacuous.\n" + + string.Join(Environment.NewLine, lines)); + Assert.True( + observation.MarkedTargetLockCalls.Count == 0, + "The warmed layout-v2 interval touched the store operation lock.\n" + + string.Join(Environment.NewLine, observation.MarkedTargetLockCalls)); + } + finally + { + Kill(process); + TryDeleteDirectory(markerDirectory); + } + } + + [Fact] + [Trait("Category", "Integration")] + [Trait("Category", "LinuxSignalPause")] + public void LinuxSigStopAtProtocolCheckpointDoesNotBlockUnrelatedProgress() + { + if (!IsLinuxX64() || !CommandSucceeds("kill", "--version")) + { + return; + } + + string name = $"sms-v2-sigstop-{Guid.NewGuid():N}"; + using MemoryStore store = CreateStore(name); + byte[] tokenKey = Key(0x10); + byte[] existingKey = Key(0x20); + byte[] operationKey = Key(0x30); + byte[] unrelatedKey = Key(0x40); + Assert.Equal(StoreStatus.Success, store.TryPublish(tokenKey, [0x11])); + Assert.Equal(StoreStatus.Success, store.TryPublish(existingKey, [0x21])); + + using Process process = StartCheckpointAgent( + "dotnet", + ["exec", LocateAgentAssembly()], + name, + LockFreeCheckpointId.PublishAfterCommitPublication, + tokenKey, + existingKey, + operationKey); + try + { + string checkpoint = ReadCheckpoint(process); + Assert.Contains(nameof(LockFreeCheckpointId.PublishAfterCommitPublication), checkpoint, StringComparison.Ordinal); + + AssertCommand("kill", "-STOP", process.Id.ToString(CultureInfo.InvariantCulture)); + Assert.True(WaitForLinuxStoppedState(process.Id, TimeSpan.FromSeconds(5)), "SIGSTOP did not place the agent in a stopped state."); + + AssertHealthyProgress(store, unrelatedKey); + + AssertCommand("kill", "-CONT", process.Id.ToString(CultureInfo.InvariantCulture)); + process.StandardInput.WriteLine("CONTINUE"); + process.StandardInput.Flush(); + AssertSuccessfulExit(process, "SIGSTOP/SIGCONT checkpoint agent"); + } + finally + { + _ = CommandSucceeds("kill", "-CONT", process.Id.ToString(CultureInfo.InvariantCulture)); + Kill(process); + RemoveIfPresent(store, tokenKey); + RemoveIfPresent(store, existingKey); + RemoveIfPresent(store, operationKey); + } + } + + [Fact] + [Trait("Category", "Integration")] + [Trait("Category", "DockerPause")] + public void LinuxDockerPauseAtProtocolCheckpointDoesNotBlockUnrelatedProgress() + { + if (!string.Equals( + Environment.GetEnvironmentVariable("SMS_RUN_LOCK_FREE_DOCKER_PAUSE_VALIDATION"), + "1", + StringComparison.Ordinal) + || !IsLinuxX64()) + { + return; + } + + Assert.True( + CommandSucceeds("docker", "info", "--format", "{{.ServerVersion}}"), + "Docker was requested but its daemon is unavailable."); + string image = Environment.GetEnvironmentVariable("SMS_LOCK_FREE_DOCKER_IMAGE") + ?? "mcr.microsoft.com/dotnet/runtime:10.0"; + Assert.True( + CommandSucceeds("docker", "image", "inspect", "--format", "{{.Id}}", image), + $"Docker pause validation image is unavailable: {image}"); + + string name = $"sms-v2-docker-pause-{Guid.NewGuid():N}"; + string containerName = $"sms-v2-pause-{Guid.NewGuid():N}"; + using MemoryStore store = CreateStore(name); + byte[] tokenKey = Key(0x50); + byte[] existingKey = Key(0x60); + byte[] operationKey = Key(0x70); + byte[] unrelatedKey = Key(0x80); + Assert.Equal(StoreStatus.Success, store.TryPublish(tokenKey, [0x51])); + Assert.Equal(StoreStatus.Success, store.TryPublish(existingKey, [0x61])); + + string repositoryRoot = FindRepositoryRoot(); + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + string containerAgent = $"/repo/tests/SharedMemoryStore.LockFreeAgent/bin/{configuration}/net10.0/SharedMemoryStore.LockFreeAgent.dll"; + string sharedMemoryDirectory = Path.GetDirectoryName(PlatformResourceName.Create(name).LinuxRegionPath) + ?? throw new DirectoryNotFoundException("Linux shared-memory resource directory was not resolved."); + string[] dockerPrefix = BuildDockerCheckpointPrefix( + containerName, + repositoryRoot, + sharedMemoryDirectory, + image, + containerAgent); + + using Process process = StartCheckpointAgent( + "docker", + dockerPrefix, + name, + LockFreeCheckpointId.PublishAfterCommitPublication, + tokenKey, + existingKey, + operationKey); + try + { + string checkpoint = ReadCheckpoint(process); + Assert.Contains(nameof(LockFreeCheckpointId.PublishAfterCommitPublication), checkpoint, StringComparison.Ordinal); + + AssertCommand("docker", "pause", containerName); + Assert.Equal("true", RunCommand("docker", "inspect", "--format", "{{.State.Paused}}", containerName).Trim()); + AssertHealthyProgress(store, unrelatedKey); + + AssertCommand("docker", "unpause", containerName); + process.StandardInput.WriteLine("CONTINUE"); + process.StandardInput.Flush(); + AssertSuccessfulExit(process, "docker pause/unpause checkpoint agent"); + } + finally + { + _ = CommandSucceeds("docker", "unpause", containerName); + _ = CommandSucceeds("docker", "rm", "--force", containerName); + Kill(process); + RemoveIfPresent(store, tokenKey); + RemoveIfPresent(store, existingKey); + RemoveIfPresent(store, operationKey); + } + } + + [Fact] + [Trait("Category", "TraceSelfTest")] + public void TraceClassifierSeparatesColdAndMarkedStoreLockCalls() + { + const string path = "/dev/shm/SharedMemoryStore/sms-test.lock"; + string[] lines = + [ + "1710000000.100000 fcntl(17, F_OFD_SETLK, {l_type=F_WRLCK}) = 0", + "1710000000.250000 fcntl(18, F_SETLKW, {l_type=F_WRLCK}) = 0", + "1710000000.400000 flock(17, LOCK_UN) = 0" + ]; + + TraceObservation observation = ClassifyTraceLines(lines, path, 1710000000.2m, 1710000000.3m); + + Assert.Equal(2, observation.AllTargetLockCalls.Count); + Assert.Empty(observation.MarkedTargetLockCalls); + } + + [Fact] + [Trait("Category", "TraceSelfTest")] + public void DockerCheckpointPrefixSharesTheHostPidNamespace() + { + string[] prefix = BuildDockerCheckpointPrefix( + "container", + "/repository", + "/dev/shm/SharedMemoryStore", + "runtime-image", + "/repository/agent.dll"); + + Assert.Equal(1, prefix.Count(static argument => string.Equals(argument, "--pid=host", StringComparison.Ordinal))); + Assert.True( + Array.IndexOf(prefix, "--pid=host") < Array.IndexOf(prefix, "runtime-image"), + "The PID namespace option must be a docker-run option, not an agent argument."); + } + + private static string[] BuildDockerCheckpointPrefix( + string containerName, + string repositoryRoot, + string sharedMemoryDirectory, + string image, + string containerAgent) => + [ + "run", "--rm", "--interactive", "--pid=host", "--name", containerName, + "--mount", $"type=bind,source={repositoryRoot},target=/repo,readonly", + "--mount", $"type=bind,source={sharedMemoryDirectory},target={sharedMemoryDirectory}", + image, "dotnet", "exec", containerAgent + ]; + + private static Process StartTracedAgent( + string name, + string readyPath, + string goPath, + string donePath, + string tracePrefix) + { + var start = RedirectedStartInfo("strace"); + foreach (string argument in new[] + { + "-ff", "-ttt", "-yy", "-s", "4096", + "-e", "trace=fcntl,flock", "-o", tracePrefix, + "dotnet", "exec", LocateAgentAssembly(), + "steady-no-lock", name, + SlotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + ParticipantRecordCount.ToString(CultureInfo.InvariantCulture), + TraceIterations.ToString(CultureInfo.InvariantCulture), + readyPath, goPath, donePath + }) + { + start.ArgumentList.Add(argument); + } + + return Process.Start(start) ?? throw new InvalidOperationException("Unable to launch strace."); + } + + private static Process StartCheckpointAgent( + string executable, + IReadOnlyList prefix, + string name, + LockFreeCheckpointId checkpoint, + byte[] tokenKey, + byte[] existingKey, + byte[] operationKey) + { + var start = RedirectedStartInfo(executable); + start.RedirectStandardInput = true; + foreach (string argument in prefix) + { + start.ArgumentList.Add(argument); + } + + foreach (string argument in new[] + { + "checkpoint-crash", name, + SlotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + ParticipantRecordCount.ToString(CultureInfo.InvariantCulture), + ((int)checkpoint).ToString(CultureInfo.InvariantCulture), + Convert.ToHexString(tokenKey), + Convert.ToHexString(existingKey), + Convert.ToHexString(operationKey), + Convert.ToHexString(Key(0x90)), + Convert.ToHexString(new byte[] { 0xA1, 0xA2 }), + Convert.ToHexString(new byte[] { 0xB1 }), + "v1" + }) + { + start.ArgumentList.Add(argument); + } + + return Process.Start(start) ?? throw new InvalidOperationException($"Unable to launch {executable} checkpoint agent."); + } + + private static ProcessStartInfo RedirectedStartInfo(string executable) => new(executable) + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + private static string ReadCheckpoint(Process process) + { + string? line = process.StandardOutput.ReadLineAsync().WaitAsync(AgentTimeout).GetAwaiter().GetResult(); + if (line is null || !line.StartsWith("CHECKPOINT ", StringComparison.Ordinal)) + { + string error = process.StandardError.ReadToEnd(); + throw new Xunit.Sdk.XunitException($"Checkpoint agent did not reach its target. stdout={line}\nstderr={error}"); + } + + return line; + } + + private static void AssertHealthyProgress(MemoryStore store, byte[] key) + { + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0xC1, 0xC2], [0xC3])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + Assert.Equal(new byte[] { 0xC1, 0xC2 }, lease.ValueSpan.ToArray()); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, store.TryRemove(key)); + } + + private static TraceObservation ClassifyTraceLines( + IEnumerable lines, + string targetPath, + decimal intervalStart, + decimal intervalEnd) + { + var all = new List(); + var marked = new List(); + foreach (string line in lines) + { + bool fcntlLock = line.Contains("fcntl(", StringComparison.Ordinal) + && (line.Contains("F_OFD_SETLK", StringComparison.Ordinal) + || line.Contains("F_OFD_SETLKW", StringComparison.Ordinal) + || line.Contains("F_SETLK", StringComparison.Ordinal) + || line.Contains("F_SETLKW", StringComparison.Ordinal)); + bool flock = line.Contains("flock(", StringComparison.Ordinal); + if ((!fcntlLock && !flock) || !line.Contains(targetPath, StringComparison.Ordinal)) + { + continue; + } + + all.Add(line); + if (TryReadTraceTimestamp(line, out decimal timestamp) + && timestamp >= intervalStart + && timestamp <= intervalEnd) + { + marked.Add(line); + } + } + + return new TraceObservation(all, marked); + } + + private static bool TryReadTraceTimestamp(string line, out decimal timestamp) + { + timestamp = 0; + int separator = line.IndexOf(' '); + return separator > 0 + && decimal.TryParse( + line.AsSpan(0, separator), + NumberStyles.AllowDecimalPoint, + CultureInfo.InvariantCulture, + out timestamp); + } + + private static decimal ToUnixSeconds(DateTime utc) => + (utc.ToUniversalTime().Ticks - DateTime.UnixEpoch.Ticks) / (decimal)TimeSpan.TicksPerSecond; + + private static MemoryStore CreateStore(string name) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + SharedMemoryStoreOptions.CreateLockFree( + name, + SlotCount, + MaxValueBytes, + MaxDescriptorBytes, + MaxKeyBytes, + LeaseRecordCount, + ParticipantRecordCount, + OpenMode.CreateNew, + enableLeaseRecovery: true), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static bool WaitForFile(string path, TimeSpan timeout) + { + long started = Stopwatch.GetTimestamp(); + while (!File.Exists(path)) + { + if (Stopwatch.GetElapsedTime(started) >= timeout) + { + return false; + } + + Thread.Sleep(10); + } + + return true; + } + + private static void PublishMarker(string path, string content) + { + // Publish the complete marker atomically. This also fixes the marker's + // mtime before the agent can observe it, so intervalStart cannot move + // past any traced operation begun after the go signal. + string temporaryPath = $"{path}.{Environment.ProcessId.ToString(CultureInfo.InvariantCulture)}.tmp"; + File.WriteAllText(temporaryPath, content); + File.Move(temporaryPath, path); + } + + private static bool WaitForLinuxStoppedState(int processId, TimeSpan timeout) + { + string statusPath = $"/proc/{processId.ToString(CultureInfo.InvariantCulture)}/status"; + long started = Stopwatch.GetTimestamp(); + while (Stopwatch.GetElapsedTime(started) < timeout) + { + if (File.Exists(statusPath) + && File.ReadLines(statusPath).Any(static line => + line.StartsWith("State:", StringComparison.Ordinal) + && line.Contains('T'))) + { + return true; + } + + Thread.Sleep(10); + } + + return false; + } + + private static void AssertSuccessfulExit(Process process, string role) + { + Assert.True(process.WaitForExit((int)AgentTimeout.TotalMilliseconds), $"{role} timed out."); + string output = process.StandardOutput.ReadToEnd(); + string error = process.StandardError.ReadToEnd(); + Assert.True(process.ExitCode == 0, $"{role} failed with exit {process.ExitCode}.\nstdout={output}\nstderr={error}"); + } + + private static string AgentFailure(Process process, string message) + { + if (!process.HasExited) + { + return message; + } + + return $"{message} exit={process.ExitCode}; stdout={process.StandardOutput.ReadToEnd()}; stderr={process.StandardError.ReadToEnd()}"; + } + + private static bool CommandSucceeds(string executable, params string[] arguments) + { + try + { + using Process process = StartCommand(executable, arguments); + Task output = process.StandardOutput.ReadToEndAsync(); + Task error = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(30_000)) + { + process.Kill(entireProcessTree: true); + return false; + } + + _ = output.GetAwaiter().GetResult(); + _ = error.GetAwaiter().GetResult(); + return process.ExitCode == 0; + } + catch + { + return false; + } + } + + private static void AssertCommand(string executable, params string[] arguments) + { + string output = RunCommand(executable, arguments); + _ = output; + } + + private static string RunCommand(string executable, params string[] arguments) + { + using Process process = StartCommand(executable, arguments); + Assert.True(process.WaitForExit(30_000), $"{executable} timed out."); + string output = process.StandardOutput.ReadToEnd(); + string error = process.StandardError.ReadToEnd(); + Assert.True(process.ExitCode == 0, $"{executable} failed with exit {process.ExitCode}.\nstdout={output}\nstderr={error}"); + return output; + } + + private static Process StartCommand(string executable, IEnumerable arguments) + { + ProcessStartInfo start = RedirectedStartInfo(executable); + foreach (string argument in arguments) + { + start.ArgumentList.Add(argument); + } + + return Process.Start(start) ?? throw new InvalidOperationException($"Unable to launch {executable}."); + } + + private static string LocateAgentAssembly() + { + string root = FindRepositoryRoot(); + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + return File.Exists(path) + ? path + : throw new FileNotFoundException("Lock-free OS-trace agent was not built.", path); + } + + private static string FindRepositoryRoot() + { + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + return directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + } + + private static void RemoveIfPresent(MemoryStore store, byte[] key) + { + StoreStatus status = store.TryRemove(key, new StoreWaitOptions(TimeSpan.FromMilliseconds(250))); + Assert.Contains(status, new[] { StoreStatus.Success, StoreStatus.NotFound, StoreStatus.RemovePending }); + } + + private static void Kill(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(5_000); + } + } + catch + { + // Cleanup is best effort for a unique mapped-store name/process. + } + } + + private static void TryDeleteDirectory(string path) + { + try + { + Directory.Delete(path, recursive: true); + } + catch + { + // Trace preservation/cleanup failure must not hide assertion output. + } + } + + private static byte[] Key(byte prefix) => [prefix, 0x01]; + + private static bool IsLinuxX64() => + OperatingSystem.IsLinux() && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed record TraceObservation( + IReadOnlyList AllTargetLockCalls, + IReadOnlyList MarkedTargetLockCalls); +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreePackageIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreePackageIntegrationTests.cs new file mode 100644 index 0000000..916d1ca --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreePackageIntegrationTests.cs @@ -0,0 +1,466 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Runtime.Loader; +using SharedMemoryStore.IntegrationTests.TestSupport; +using SharedMemoryStore.Interop; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreePackageIntegrationTests +{ + private static readonly Lazy ReleasedClient = + new(static () => ReleasedV102Client.Load(), LazyThreadSafetyMode.ExecutionAndPublication); + + public static TheoryData ReleasedOpenMatrix => new() + { + { "smaller", OpenMode.CreateNew }, + { "smaller", OpenMode.OpenExisting }, + { "smaller", OpenMode.CreateOrOpen }, + { "equal", OpenMode.CreateNew }, + { "equal", OpenMode.OpenExisting }, + { "equal", OpenMode.CreateOrOpen }, + { "oversized", OpenMode.CreateNew }, + { "oversized", OpenMode.OpenExisting }, + { "oversized", OpenMode.CreateOrOpen } + }; + + [Theory] + [MemberData(nameof(ReleasedOpenMatrix))] + [Trait("Category", "PackageConsumption")] + public void ReleasedV102PackageFailsClosedOnEveryV2ViewAndOpenMode( + string requestedView, + OpenMode openMode) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-packed-v102-reject-{requestedView}-{openMode}-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions options = LockFreeOptions( + name, + OpenMode.CreateNew, + participantRecordCount: 2, + slotCount: 32, + maxValueBytes: 1024, + maxDescriptorBytes: 64, + maxKeyBytes: 32, + leaseRecordCount: 32); + using MemoryStore store = IntegrationStoreFactory.Create(options); + + long releasedMinimum = ReleasedClient.Value.MinimumRequiredBytes; + Assert.True(options.TotalBytes > releasedMinimum + 8); + long requestedBytes = requestedView switch + { + "smaller" => Math.Max(releasedMinimum, options.TotalBytes - 8), + "equal" => options.TotalBytes, + "oversized" => checked(options.TotalBytes + 4096), + _ => throw new ArgumentOutOfRangeException(nameof(requestedView)) + }; + + string status = ReleasedClient.Value.TryOpen(name, openMode.ToString(), requestedBytes); + + if (openMode == OpenMode.CreateNew) + { + Assert.Equal(nameof(StoreOpenStatus.AlreadyExists), status); + } + else if (requestedView == "oversized" && OperatingSystem.IsWindows()) + { + Assert.Contains( + status, + new[] + { + nameof(StoreOpenStatus.AccessDenied), + nameof(StoreOpenStatus.IncompatibleLayout) + }); + } + else + { + Assert.Equal(nameof(StoreOpenStatus.IncompatibleLayout), status); + } + + Assert.Equal(StoreProfile.LockFree, store.Profile); + Assert.Equal(2, store.ProtocolInfo.LayoutMajorVersion); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [2, 3, 4])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + using (lease) + { + Assert.Equal(new byte[] { 2, 3, 4 }, lease.ValueSpan.ToArray()); + } + } + + [Fact] + [Trait("Category", "PackageConsumption")] + public void ReleasedV102RejectionPreservesLinuxV2LiveOwnerSidecar() + { + if (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + string name = $"sms-packed-v102-linux-owner-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions options = LockFreeOptions(name, OpenMode.CreateNew, participantRecordCount: 2); + PlatformResourceName resourceName = PlatformResourceName.Create(name); + using MemoryStore store = IntegrationStoreFactory.Create(options); + string[] before = ReadNonEmptyLines(resourceName.LinuxOwnersPath); + + string status = ReleasedClient.Value.TryOpen( + name, + nameof(OpenMode.OpenExisting), + options.TotalBytes); + + Assert.Equal(nameof(StoreOpenStatus.IncompatibleLayout), status); + Assert.True(File.Exists(resourceName.LinuxRegionPath)); + Assert.Equal( + before.OrderBy(static line => line, StringComparer.Ordinal), + ReadNonEmptyLines(resourceName.LinuxOwnersPath).OrderBy(static line => line, StringComparer.Ordinal)); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [9])); + } + + [Fact] + [Trait("Category", "PackageConsumption")] + public void ExplicitV2ParticipantCapacityIsConsumedPerHandleAndReusableAfterClose() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-package-participants-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions create = LockFreeOptions(name, OpenMode.CreateNew, participantRecordCount: 2); + SharedMemoryStoreOptions open = LockFreeOptions(name, OpenMode.OpenExisting, participantRecordCount: 2); + + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(create, out var first)); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(open, out var second)); + Assert.Equal(StoreOpenStatus.ParticipantTableFull, MemoryStore.TryCreateOrOpen(open, out var rejected)); + Assert.Null(rejected); + + try + { + second!.Dispose(); + second = null; + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(open, out var replacement)); + using (replacement) + { + Assert.NotNull(replacement); + Assert.Equal(StoreProfile.LockFree, replacement.Profile); + } + } + finally + { + second?.Dispose(); + first?.Dispose(); + } + } + + [Fact] + [Trait("Category", "PackageConsumption")] + public void SameNameUpgradeAndRollbackRequireCloseRecreateAndRepublish() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-package-upgrade-rollback-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions legacyCreate = LegacyOptions(name, OpenMode.CreateNew); + SharedMemoryStoreOptions legacyOpen = LegacyOptions(name, OpenMode.OpenExisting); + SharedMemoryStoreOptions v2Create = LockFreeOptions(name, OpenMode.CreateNew, participantRecordCount: 2); + SharedMemoryStoreOptions v2Open = LockFreeOptions(name, OpenMode.OpenExisting, participantRecordCount: 2); + + using (MemoryStore legacy = IntegrationStoreFactory.Create(legacyCreate)) + { + Assert.Equal(StoreProfile.Legacy, legacy.Profile); + Assert.Equal(StoreStatus.Success, legacy.TryPublish([1], [10])); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, MemoryStore.TryCreateOrOpen(v2Open, out var incompatible)); + Assert.Null(incompatible); + } + + using (MemoryStore v2 = IntegrationStoreFactory.Create(v2Create)) + { + Assert.Equal(StoreProfile.LockFree, v2.Profile); + Assert.Equal(StoreStatus.NotFound, v2.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, v2.TryPublish([1], [20])); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, MemoryStore.TryCreateOrOpen(legacyOpen, out var incompatible)); + Assert.Null(incompatible); + } + + using MemoryStore rolledBack = IntegrationStoreFactory.Create(legacyCreate); + Assert.Equal(StoreProfile.Legacy, rolledBack.Profile); + Assert.Equal(StoreStatus.NotFound, rolledBack.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, rolledBack.TryPublish([1], [30])); + } + + [Fact] + [Trait("Category", "PackageConsumption")] + public void DefaultAndLegacyHelpersNeverAutoSelectAnExistingV2Mapping() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + Assert.Equal(StoreProfile.Legacy, new SharedMemoryStoreOptions().Profile); + Assert.Equal(StoreProfile.Legacy, LegacyOptions("unused", OpenMode.CreateOrOpen).Profile); + Assert.Equal(StoreProfile.LockFree, LockFreeOptions("unused-v2", OpenMode.CreateOrOpen, 1).Profile); + + string name = $"sms-package-default-profile-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions v2 = LockFreeOptions( + name, + OpenMode.CreateNew, + participantRecordCount: 2, + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 4, + leaseRecordCount: 2); + SharedMemoryStoreOptions oversizedDefaultLegacy = LegacyOptions( + name, + OpenMode.OpenExisting, + slotCount: 128, + maxValueBytes: 32 * 1024, + maxDescriptorBytes: 256, + maxKeyBytes: 128, + leaseRecordCount: 256); + using MemoryStore store = IntegrationStoreFactory.Create(v2); + + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(oversizedDefaultLegacy, out var incompatible); + + incompatible?.Dispose(); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, status); + Assert.Null(incompatible); + Assert.Equal(StoreProfile.LockFree, store.Profile); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private static SharedMemoryStoreOptions LockFreeOptions( + string name, + OpenMode openMode, + int participantRecordCount, + int slotCount = 4, + int maxValueBytes = 64, + int maxDescriptorBytes = 8, + int maxKeyBytes = 8, + int leaseRecordCount = 8) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount, + openMode); + + private static SharedMemoryStoreOptions LegacyOptions( + string name, + OpenMode openMode, + int slotCount = 4, + int maxValueBytes = 64, + int maxDescriptorBytes = 8, + int maxKeyBytes = 8, + int leaseRecordCount = 8) => + SharedMemoryStoreOptions.Create( + name, + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + openMode); + + private static string[] ReadNonEmptyLines(string path) => + File.ReadAllLines(path) + .Select(static line => line.Trim()) + .Where(static line => line.Length != 0) + .ToArray(); + + private sealed class ReleasedV102Client + { + private readonly Type _optionsType; + private readonly Type _openModeType; + private readonly MethodInfo _tryCreateOrOpen; + + private ReleasedV102Client(Assembly assembly) + { + _optionsType = RequireType(assembly, "SharedMemoryStore.SharedMemoryStoreOptions"); + _openModeType = RequireType(assembly, "SharedMemoryStore.OpenMode"); + Type storeType = RequireType(assembly, "SharedMemoryStore.MemoryStore"); + _tryCreateOrOpen = storeType + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(method => + { + ParameterInfo[] parameters = method.GetParameters(); + return method.Name == "TryCreateOrOpen" + && parameters.Length == 2 + && parameters[0].ParameterType == _optionsType.MakeByRefType() + && parameters[1].IsOut; + }); + + MethodInfo calculate = _optionsType.GetMethod( + "CalculateRequiredBytes", + BindingFlags.Public | BindingFlags.Static, + binder: null, + [typeof(int), typeof(int), typeof(int), typeof(int), typeof(int)], + modifiers: null) ?? throw new MissingMethodException(_optionsType.FullName, "CalculateRequiredBytes"); + MinimumRequiredBytes = (long)calculate.Invoke(null, [1, 1, 0, 1, 1])!; + } + + public long MinimumRequiredBytes { get; } + + public static ReleasedV102Client Load() + { + string packagePath = ResolvePackagePath(); + string extractionDirectory = Path.Combine( + Path.GetTempPath(), + $"sms-v102-package-{Environment.ProcessId}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(extractionDirectory); + string assemblyPath = Path.Combine(extractionDirectory, "SharedMemoryStore.dll"); + + using (ZipArchive archive = ZipFile.OpenRead(packagePath)) + { + ZipArchiveEntry entry = archive.GetEntry("lib/net10.0/SharedMemoryStore.dll") + ?? throw new InvalidDataException($"Package '{packagePath}' has no net10.0 assembly."); + entry.ExtractToFile(assemblyPath); + } + + var loadContext = new AssemblyLoadContext( + $"SharedMemoryStore-v1.0.2-{Guid.NewGuid():N}", + isCollectible: false); + Assembly assembly = loadContext.LoadFromAssemblyPath(assemblyPath); + Assert.Equal(new Version(1, 0, 2, 0), assembly.GetName().Version); + return new ReleasedV102Client(assembly); + } + + public string TryOpen(string name, string openMode, long totalBytes) + { + object options = Activator.CreateInstance(_optionsType) + ?? throw new InvalidOperationException("Could not create released options instance."); + SetProperty(options, "Name", name); + SetProperty(options, "OpenMode", Enum.Parse(_openModeType, openMode)); + SetProperty(options, "TotalBytes", totalBytes); + SetProperty(options, "SlotCount", 1); + SetProperty(options, "MaxValueBytes", 1); + SetProperty(options, "MaxDescriptorBytes", 0); + SetProperty(options, "MaxKeyBytes", 1); + SetProperty(options, "LeaseRecordCount", 1); + + object?[] arguments = [options, null]; + object result = _tryCreateOrOpen.Invoke(null, arguments) + ?? throw new InvalidOperationException("Released open returned no status."); + if (arguments[1] is IDisposable accidentallyOpened) + { + accidentallyOpened.Dispose(); + } + + return result.ToString() ?? string.Empty; + } + + private static string ResolvePackagePath() + { + var candidates = new List + { + Environment.GetEnvironmentVariable("SMS_V102_PACKAGE_PATH"), + Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + ".nuget", + "packages", + "sharedmemorystore", + "1.0.2", + "sharedmemorystore.1.0.2.nupkg"), + Path.Combine(FindRepositoryRoot(), "artifacts", "package", "SharedMemoryStore.1.0.2.nupkg"), + Path.Combine( + FindRepositoryRoot(), + "artifacts", + "docker-consumer", + "local-packages", + "SharedMemoryStore.1.0.2.nupkg") + }; + + string? existing = candidates.FirstOrDefault(path => !string.IsNullOrWhiteSpace(path) && File.Exists(path)); + if (existing is not null) + { + return Path.GetFullPath(existing); + } + + return RestorePublishedPackage(); + } + + private static string RestorePublishedPackage() + { + string root = Path.Combine( + Path.GetTempPath(), + $"sms-v102-restore-{Environment.ProcessId}-{Guid.NewGuid():N}"); + string packages = Path.Combine(root, "packages"); + Directory.CreateDirectory(root); + string project = Path.Combine(root, "Restore.csproj"); + File.WriteAllText( + project, + """ + + net10.0 + + + """); + + using var process = new Process + { + StartInfo = new ProcessStartInfo( + "dotnet", + $"restore \"{project}\" --packages \"{packages}\"") + { + WorkingDirectory = root, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + } + }; + process.Start(); + Task outputTask = process.StandardOutput.ReadToEndAsync(); + Task errorTask = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(120_000)) + { + process.Kill(entireProcessTree: true); + Assert.Fail("Timed out restoring released package 1.0.2."); + } + + string output = outputTask.GetAwaiter().GetResult(); + string error = errorTask.GetAwaiter().GetResult(); + Assert.True(process.ExitCode == 0, output + Environment.NewLine + error); + + string package = Path.Combine( + packages, + "sharedmemorystore", + "1.0.2", + "sharedmemorystore.1.0.2.nupkg"); + Assert.True(File.Exists(package), $"Restore did not produce '{package}'."); + return package; + } + + private static Type RequireType(Assembly assembly, string fullName) => + assembly.GetType(fullName, throwOnError: true, ignoreCase: false)!; + + private static void SetProperty(object target, string name, object value) + { + PropertyInfo property = target.GetType().GetProperty(name, BindingFlags.Public | BindingFlags.Instance) + ?? throw new MissingMemberException(target.GetType().FullName, name); + property.SetValue(target, value); + } + } + + private static string FindRepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + return directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreePidNamespaceIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreePidNamespaceIntegrationTests.cs new file mode 100644 index 0000000..d5dff69 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreePidNamespaceIntegrationTests.cs @@ -0,0 +1,108 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.IntegrationTests.TestSupport; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreePidNamespaceIntegrationTests +{ + [Fact] + [Trait("Category", "Integration")] + public unsafe void CreatedHeaderCarriesThePlatformPidNamespaceIdentity() + { + if (!IsSupportedHost()) + { + return; + } + + string name = $"sms-v2-pidns-header-{Guid.NewGuid():N}"; + using MemoryStore store = IntegrationStoreFactory.Create( + Options(name, OpenMode.CreateNew)); + MemoryMappedStoreRegion region = ReadRegion(store); + ref StoreHeaderV2 header = ref *(StoreHeaderV2*)region.Pointer; + + if (OperatingSystem.IsLinux()) + { + Assert.NotEqual(0UL, header.PidNamespaceId); + } + else + { + Assert.Equal(0UL, header.PidNamespaceId); + } + Assert.Equal(LayoutV2Constants.PidNamespaceRecoveryEnabled, header.PidNamespaceMode); + } + + [Fact] + [Trait("Category", "Integration")] + public unsafe void LinuxMismatchedHeaderNamespaceDowngradesRecoveryAndKeepsOrdinaryKvAccess() + { + if (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + string name = $"sms-v2-pidns-reject-{Guid.NewGuid():N}"; + using MemoryStore anchor = IntegrationStoreFactory.Create( + Options(name, OpenMode.CreateNew)); + MemoryMappedStoreRegion region = ReadRegion(anchor); + ref StoreHeaderV2 header = ref *(StoreHeaderV2*)region.Pointer; + ulong originalNamespace = header.PidNamespaceId; + Assert.NotEqual(0UL, originalNamespace); + ulong mismatchedNamespace = originalNamespace == ulong.MaxValue + ? originalNamespace - 1 + : originalNamespace + 1; + header.PidNamespaceId = mismatchedNamespace; + try + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting), + out MemoryStore? candidate); + + using MemoryStore opened = Assert.IsType(candidate); + Assert.Equal(StoreOpenStatus.Success, status); + Assert.Equal(mismatchedNamespace, header.PidNamespaceId); + Assert.Equal( + LayoutV2Constants.PidNamespaceRecoveryMixed, + Volatile.Read(ref header.PidNamespaceMode)); + Assert.Equal(StoreStatus.Success, opened.TryPublish([0x31], [0x41, 0x42])); + Assert.Equal(StoreStatus.Success, anchor.TryAcquire([0x31], out ValueLease lease)); + using (lease) + { + Assert.Equal(new byte[] { 0x41, 0x42 }, lease.ValueSpan.ToArray()); + } + Assert.Equal(StoreStatus.Success, anchor.TryRemove([0x31])); + } + finally + { + header.PidNamespaceId = originalNamespace; + } + } + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private static SharedMemoryStoreOptions Options(string name, OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 4, + maxValueBytes: 32, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount: 4, + participantRecordCount: 2, + openMode: openMode, + enableLeaseRecovery: true); + + private static MemoryMappedStoreRegion ReadRegion(MemoryStore store) + { + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + return Assert.IsAssignableFrom(engine.GetType() + .GetField("_region", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeProfileOpenIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeProfileOpenIntegrationTests.cs new file mode 100644 index 0000000..07f747b --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeProfileOpenIntegrationTests.cs @@ -0,0 +1,592 @@ +using System.Globalization; +using System.Runtime.InteropServices; +using SharedMemoryStore.IntegrationTests.TestSupport; +using SharedMemoryStore.Interop; +using SharedMemoryStore.Layout; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeProfileOpenIntegrationTests +{ + [Fact] + [Trait("Category", "Integration")] + public void LinuxLegacyOpenRejectsBackingFileTruncatedBelowTheFixedHeader() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + string name = $"sms-v1-truncated-header-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions create = CreateOptions( + StoreProfile.Legacy, + name, + OpenMode.CreateNew, + participantRecordCount: 2); + SharedMemoryStoreOptions open = CreateOptions( + StoreProfile.Legacy, + name, + OpenMode.OpenExisting, + participantRecordCount: 2); + PlatformResourceName resource = PlatformResourceName.Create(name); + using Store owner = IntegrationStoreFactory.Create(create); + + TruncateLinuxRegion(resource.LinuxRegionPath, Marshal.SizeOf() - 1L); + + StoreOpenStatus status = Store.TryCreateOrOpen(open, out Store? rejected); + + rejected?.Dispose(); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, status); + Assert.Null(rejected); + } + + [Fact] + [Trait("Category", "Integration")] + public void LinuxLegacyOpenRejectsValidHeaderWhosePayloadExtentWasTruncated() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + string name = $"sms-v1-truncated-payload-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions create = CreateOptions( + StoreProfile.Legacy, + name, + OpenMode.CreateNew, + participantRecordCount: 2); + SharedMemoryStoreOptions open = CreateOptions( + StoreProfile.Legacy, + name, + OpenMode.OpenExisting, + participantRecordCount: 2); + PlatformResourceName resource = PlatformResourceName.Create(name); + using Store owner = IntegrationStoreFactory.Create(create); + Assert.True(create.TotalBytes > Marshal.SizeOf()); + + TruncateLinuxRegion(resource.LinuxRegionPath, create.TotalBytes - 1); + + StoreOpenStatus status = Store.TryCreateOrOpen(open, out Store? rejected); + + rejected?.Dispose(); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, status); + Assert.Null(rejected); + } + + [Theory] + [InlineData(OpenMode.OpenExisting)] + [InlineData(OpenMode.CreateOrOpen)] + [Trait("Category", "Integration")] + public void ExistingLegacyHeaderIsRejectedBeforeOversizedLockFreeViewProjection(OpenMode openMode) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var name = $"sms-v1-to-v2-header-first-{Guid.NewGuid():N}"; + var legacyOptions = SharedMemoryStoreOptions.Create( + name, + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 4, + leaseRecordCount: 2, + openMode: OpenMode.CreateNew); + var oversizedLockFreeOptions = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 128, + maxValueBytes: 32 * 1024, + maxDescriptorBytes: 256, + maxKeyBytes: 128, + leaseRecordCount: 256, + participantRecordCount: 8, + openMode: openMode); + + using var legacy = IntegrationStoreFactory.Create(legacyOptions); + Assert.Equal(StoreStatus.Success, legacy.TryPublish([0xA1], [0x5A])); + + var status = Store.TryCreateOrOpen(oversizedLockFreeOptions, out var incompatible); + + incompatible?.Dispose(); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, status); + Assert.Null(incompatible); + Assert.Equal(StoreProfile.Legacy, legacy.Profile); + Assert.Equal(1, legacy.ProtocolInfo.LayoutMajorVersion); + AssertPublishedValue(legacy, [0xA1], [0x5A]); + } + + [Theory] + [InlineData(OpenMode.OpenExisting)] + [InlineData(OpenMode.CreateOrOpen)] + [Trait("Category", "Integration")] + public void ExistingLockFreeHeaderIsRejectedBeforeOversizedLegacyViewProjection(OpenMode openMode) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var name = $"sms-v2-to-v1-header-first-{Guid.NewGuid():N}"; + var lockFreeOptions = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 4, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + var oversizedLegacyOptions = SharedMemoryStoreOptions.Create( + name, + slotCount: 128, + maxValueBytes: 32 * 1024, + maxDescriptorBytes: 256, + maxKeyBytes: 128, + leaseRecordCount: 256, + openMode: openMode); + + using var lockFree = IntegrationStoreFactory.Create(lockFreeOptions); + Assert.Equal(StoreStatus.Success, lockFree.TryPublish([0xA2], [0x5B])); + + var status = Store.TryCreateOrOpen(oversizedLegacyOptions, out var incompatible); + + incompatible?.Dispose(); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, status); + Assert.Null(incompatible); + AssertLockFreeProtocol(lockFree); + AssertPublishedValue(lockFree, [0xA2], [0x5B]); + } + + [Theory] + [InlineData(StoreProfile.Legacy, StoreProfile.LockFree)] + [InlineData(StoreProfile.LockFree, StoreProfile.Legacy)] + [Trait("Category", "Integration")] + public void OppositeProfileCreateNewReportsAlreadyExists( + StoreProfile existingProfile, + StoreProfile requestedProfile) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var name = $"sms-mixed-create-new-{Guid.NewGuid():N}"; + var existingOptions = CreateOptions(existingProfile, name, OpenMode.CreateNew, participantRecordCount: 2); + var requestedOptions = CreateOptions(requestedProfile, name, OpenMode.CreateNew, participantRecordCount: 2); + + using var existing = IntegrationStoreFactory.Create(existingOptions); + Assert.Equal(StoreStatus.Success, existing.TryPublish([0xA3], [0x5C])); + + var status = Store.TryCreateOrOpen(requestedOptions, out var duplicate); + + duplicate?.Dispose(); + Assert.Equal(StoreOpenStatus.AlreadyExists, status); + Assert.Null(duplicate); + AssertPublishedValue(existing, [0xA3], [0x5C]); + } + + [Theory] + [InlineData(StoreProfile.Legacy, StoreProfile.Legacy, OpenMode.CreateNew)] + [InlineData(StoreProfile.Legacy, StoreProfile.Legacy, OpenMode.OpenExisting)] + [InlineData(StoreProfile.Legacy, StoreProfile.Legacy, OpenMode.CreateOrOpen)] + [InlineData(StoreProfile.Legacy, StoreProfile.LockFree, OpenMode.CreateNew)] + [InlineData(StoreProfile.Legacy, StoreProfile.LockFree, OpenMode.OpenExisting)] + [InlineData(StoreProfile.Legacy, StoreProfile.LockFree, OpenMode.CreateOrOpen)] + [InlineData(StoreProfile.LockFree, StoreProfile.Legacy, OpenMode.CreateNew)] + [InlineData(StoreProfile.LockFree, StoreProfile.Legacy, OpenMode.OpenExisting)] + [InlineData(StoreProfile.LockFree, StoreProfile.Legacy, OpenMode.CreateOrOpen)] + [InlineData(StoreProfile.LockFree, StoreProfile.LockFree, OpenMode.CreateNew)] + [InlineData(StoreProfile.LockFree, StoreProfile.LockFree, OpenMode.OpenExisting)] + [InlineData(StoreProfile.LockFree, StoreProfile.LockFree, OpenMode.CreateOrOpen)] + [Trait("Category", "Integration")] + public void ExistingZeroHeaderIsNeverInitializedByAnOpener( + StoreProfile physicalSizingProfile, + StoreProfile requestedProfile, + OpenMode requestedMode) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-zero-header-ownership-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions rawCreate = CreateOptions( + physicalSizingProfile, + name, + OpenMode.CreateNew, + participantRecordCount: 2); + SharedMemoryStoreOptions requested = CreateOptions( + requestedProfile, + name, + requestedMode, + participantRecordCount: 2); + PlatformResourceName resource = PlatformResourceName.Create(name); + + Assert.Equal( + StoreOpenStatus.Success, + SharedStorePlatform.TryOpenRegion( + rawCreate, + StoreWaitOptions.Default, + out MemoryMappedStoreRegion? rawRegion)); + Assert.NotNull(rawRegion); + + try + { + byte[] prefixBefore = ReadPrefix(rawRegion!, 512); + Assert.All(prefixBefore, static value => Assert.Equal(0, value)); + string[] ownersBefore = OperatingSystem.IsLinux() + ? ReadOwnerLines(resource.LinuxOwnersPath) + : []; + + StoreOpenStatus status = Store.TryCreateOrOpen( + requested, + StoreWaitOptions.NoWait, + out Store? rejected); + + rejected?.Dispose(); + StoreOpenStatus expected = requestedMode switch + { + OpenMode.CreateNew => StoreOpenStatus.AlreadyExists, + OpenMode.CreateOrOpen => StoreOpenStatus.StoreBusy, + _ => StoreOpenStatus.IncompatibleLayout + }; + Assert.Equal(expected, status); + Assert.Null(rejected); + Assert.Equal(prefixBefore, ReadPrefix(rawRegion!, prefixBefore.Length)); + if (OperatingSystem.IsLinux()) + { + Assert.Equal( + ownersBefore.OrderBy(static line => line, StringComparer.Ordinal), + ReadOwnerLines(resource.LinuxOwnersPath).OrderBy(static line => line, StringComparer.Ordinal)); + } + } + finally + { + rawRegion!.Dispose(); + } + + SharedMemoryStoreOptions missing = CreateOptions( + requestedProfile, + name, + OpenMode.OpenExisting, + participantRecordCount: 2); + Assert.Equal(StoreOpenStatus.NotFound, Store.TryCreateOrOpen(missing, out Store? absent)); + Assert.Null(absent); + } + + [Theory] + [InlineData(StoreProfile.Legacy, StoreProfile.Legacy, OpenMode.CreateNew)] + [InlineData(StoreProfile.Legacy, StoreProfile.Legacy, OpenMode.OpenExisting)] + [InlineData(StoreProfile.Legacy, StoreProfile.Legacy, OpenMode.CreateOrOpen)] + [InlineData(StoreProfile.Legacy, StoreProfile.LockFree, OpenMode.CreateNew)] + [InlineData(StoreProfile.Legacy, StoreProfile.LockFree, OpenMode.OpenExisting)] + [InlineData(StoreProfile.Legacy, StoreProfile.LockFree, OpenMode.CreateOrOpen)] + [InlineData(StoreProfile.LockFree, StoreProfile.Legacy, OpenMode.CreateNew)] + [InlineData(StoreProfile.LockFree, StoreProfile.Legacy, OpenMode.OpenExisting)] + [InlineData(StoreProfile.LockFree, StoreProfile.Legacy, OpenMode.CreateOrOpen)] + [InlineData(StoreProfile.LockFree, StoreProfile.LockFree, OpenMode.CreateNew)] + [InlineData(StoreProfile.LockFree, StoreProfile.LockFree, OpenMode.OpenExisting)] + [InlineData(StoreProfile.LockFree, StoreProfile.LockFree, OpenMode.CreateOrOpen)] + [Trait("Category", "Integration")] + public void ColdGateIsAcquiredBeforeAnyPhysicalMappingProbe( + StoreProfile creatorProfile, + StoreProfile contenderProfile, + OpenMode contenderMode) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-cold-gate-before-map-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions creator = CreateOptions( + creatorProfile, + name, + OpenMode.CreateNew, + participantRecordCount: 2); + SharedMemoryStoreOptions contender = CreateOptions( + contenderProfile, + name, + contenderMode, + participantRecordCount: 2); + long started = System.Diagnostics.Stopwatch.GetTimestamp(); + Assert.Equal( + StoreOpenStatus.Success, + SharedStorePlatform.TryBeginOpen( + creator, + StoreWaitOptions.Default, + started, + out SharedStoreOpenScope? heldScope)); + Assert.NotNull(heldScope); + + using (heldScope) + { + (StoreOpenStatus Status, Store? Store) result = default; + Exception? contenderFailure = null; + using var contenderFinished = new ManualResetEventSlim(initialState: false); + var contenderThread = new Thread(() => + { + try + { + StoreOpenStatus status = Store.TryCreateOrOpen( + contender, + StoreWaitOptions.NoWait, + out Store? opened); + result = (status, opened); + } + catch (Exception exception) + { + contenderFailure = exception; + } + finally + { + contenderFinished.Set(); + } + }) + { + IsBackground = true, + Name = "SharedMemoryStore cold gate-before-map contender" + }; + contenderThread.Start(); + Assert.True(contenderFinished.Wait(TimeSpan.FromSeconds(5))); + Assert.Null(contenderFailure); + + result.Store?.Dispose(); + Assert.Equal(StoreOpenStatus.StoreBusy, result.Status); + Assert.Null(result.Store); + if (OperatingSystem.IsLinux()) + { + Assert.Single(ReadOwnerLines(PlatformResourceName.Create(name).LinuxOwnersPath)); + } + } + + using Store recreated = IntegrationStoreFactory.Create(creator); + Assert.Equal(creatorProfile, recreated.Profile); + } + + [Fact] + [Trait("Category", "Integration")] + public void ParticipantCapacityIsPerHandleAndAClosedRecordCanBeReused() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var name = $"sms-participant-reuse-{Guid.NewGuid():N}"; + var createOptions = CreateOptions(StoreProfile.LockFree, name, OpenMode.CreateNew, participantRecordCount: 2); + var openOptions = CreateOptions(StoreProfile.LockFree, name, OpenMode.OpenExisting, participantRecordCount: 2); + Store? anchor = null; + Store? second = null; + Store? rejected = null; + Store? replacement = null; + + try + { + var anchorStatus = Store.TryCreateOrOpen(createOptions, out anchor); + var secondStatus = Store.TryCreateOrOpen(openOptions, out second); + var exhaustedStatus = Store.TryCreateOrOpen(openOptions, out rejected); + + second?.Dispose(); + second = null; + var replacementStatus = Store.TryCreateOrOpen(openOptions, out replacement); + + Assert.Equal(StoreOpenStatus.Success, anchorStatus); + Assert.NotNull(anchor); + Assert.Equal(StoreOpenStatus.Success, secondStatus); + Assert.Equal(StoreOpenStatus.ParticipantTableFull, exhaustedStatus); + Assert.Null(rejected); + Assert.Equal(StoreOpenStatus.Success, replacementStatus); + Assert.NotNull(replacement); + AssertLockFreeProtocol(anchor!); + AssertLockFreeProtocol(replacement!); + } + finally + { + replacement?.Dispose(); + rejected?.Dispose(); + second?.Dispose(); + anchor?.Dispose(); + } + } + + [Fact] + [Trait("Category", "Integration")] + public void ClosingFinalParticipantAllowsASecondLockFreeCreateNewLifecycle() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var name = $"sms-participant-final-close-{Guid.NewGuid():N}"; + var options = CreateOptions(StoreProfile.LockFree, name, OpenMode.CreateNew, participantRecordCount: 1); + + using (var first = IntegrationStoreFactory.Create(options)) + { + AssertLockFreeProtocol(first); + } + + using var recreated = IntegrationStoreFactory.Create(options); + AssertLockFreeProtocol(recreated); + } + + [Fact] + [Trait("Category", "Integration")] + public void LinuxLockFreeHandlesKeepV1CompatibleOwnerLinesDuringIncompatibleLegacyOpen() + { + if (!OperatingSystem.IsLinux() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + var name = $"sms-v2-owner-sidecar-{Guid.NewGuid():N}"; + var resourceName = PlatformResourceName.Create(name); + var createOptions = CreateOptions(StoreProfile.LockFree, name, OpenMode.CreateNew, participantRecordCount: 2); + var openOptions = CreateOptions(StoreProfile.LockFree, name, OpenMode.OpenExisting, participantRecordCount: 2); + var legacyOptions = SharedMemoryStoreOptions.Create( + name, + slotCount: 128, + maxValueBytes: 32 * 1024, + maxDescriptorBytes: 256, + maxKeyBytes: 128, + leaseRecordCount: 256, + openMode: OpenMode.OpenExisting); + + using var first = IntegrationStoreFactory.Create(createOptions); + var secondStatus = Store.TryCreateOrOpen(openOptions, out var second); + Assert.Equal(StoreOpenStatus.Success, secondStatus); + Assert.NotNull(second); + + try + { + var ownerLinesBefore = ReadOwnerLines(resourceName.LinuxOwnersPath); + Assert.Equal(2, ownerLinesBefore.Length); + Assert.All(ownerLinesBefore, AssertV1CompatibleCurrentProcessOwnerLine); + + var incompatibleStatus = Store.TryCreateOrOpen(legacyOptions, out var incompatible); + incompatible?.Dispose(); + + Assert.Equal(StoreOpenStatus.IncompatibleLayout, incompatibleStatus); + Assert.Null(incompatible); + Assert.True(File.Exists(resourceName.LinuxRegionPath)); + Assert.Equal( + ownerLinesBefore.OrderBy(static line => line, StringComparer.Ordinal), + ReadOwnerLines(resourceName.LinuxOwnersPath).OrderBy(static line => line, StringComparer.Ordinal)); + + second!.Dispose(); + second = null; + var ownerLinesAfterClose = ReadOwnerLines(resourceName.LinuxOwnersPath); + Assert.Single(ownerLinesAfterClose); + AssertV1CompatibleCurrentProcessOwnerLine(ownerLinesAfterClose[0]); + AssertLockFreeProtocol(first); + } + finally + { + second?.Dispose(); + } + } + + private static bool IsSupportedLockFreeHost() + { + return (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + } + + private static SharedMemoryStoreOptions CreateOptions( + StoreProfile profile, + string name, + OpenMode openMode, + int participantRecordCount) + { + const int slotCount = 4; + const int maxValueBytes = 64; + const int maxDescriptorBytes = 8; + const int maxKeyBytes = 8; + const int leaseRecordCount = 8; + + return profile == StoreProfile.LockFree + ? SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount, + openMode) + : SharedMemoryStoreOptions.Create( + name, + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + openMode); + } + + private static void AssertLockFreeProtocol(Store store) + { + Assert.Equal(StoreProfile.LockFree, store.Profile); + Assert.Equal(StoreProfile.LockFree, store.ProtocolInfo.Profile); + Assert.Equal(2, store.ProtocolInfo.LayoutMajorVersion); + Assert.Equal(0, store.ProtocolInfo.LayoutMinorVersion); + Assert.Equal(2, store.ProtocolInfo.ResourceProtocolVersion); + } + + private static void AssertPublishedValue(Store store, byte[] key, byte[] expectedValue) + { + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + try + { + Assert.Equal(expectedValue, lease.ValueSpan.ToArray()); + } + finally + { + Assert.Equal(StoreStatus.Success, lease.Release()); + } + } + + private static string[] ReadOwnerLines(string path) + { + Assert.True(File.Exists(path)); + return File.ReadAllLines(path) + .Select(static line => line.Trim()) + .Where(static line => line.Length != 0) + .ToArray(); + } + + private static void AssertV1CompatibleCurrentProcessOwnerLine(string ownerLine) + { + var parts = ownerLine.Split(':', 3); + Assert.Equal(3, parts.Length); + Assert.True(int.TryParse(parts[0], NumberStyles.None, CultureInfo.InvariantCulture, out var processId)); + Assert.Equal(Environment.ProcessId, processId); + Assert.True( + parts[1].StartsWith("proc-", StringComparison.Ordinal) + || parts[1].StartsWith("utc-", StringComparison.Ordinal)); + Assert.True(Guid.TryParseExact(parts[2], "N", out _)); + } + + private static void TruncateLinuxRegion(string path, long length) + { + using var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Write, + FileShare.ReadWrite | FileShare.Delete); + stream.SetLength(length); + stream.Flush(flushToDisk: true); + Assert.Equal(length, stream.Length); + } + + private static unsafe byte[] ReadPrefix(MemoryMappedStoreRegion region, int requestedLength) + { + int length = checked((int)Math.Min(region.Capacity, requestedLength)); + var result = new byte[length]; + new ReadOnlySpan(region.Pointer, length).CopyTo(result); + return result; + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreePublishIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreePublishIntegrationTests.cs new file mode 100644 index 0000000..e40b882 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreePublishIntegrationTests.cs @@ -0,0 +1,306 @@ +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreePublishIntegrationTests +{ + [Fact] + [Trait("Category", "Integration")] + public async Task ConcurrentSameKeyPublishersProduceExactlyOneWinnerWithoutLeakingCandidateSlots() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + const int publisherCount = 8; + using var store = CreateStore(slotCount: publisherCount + 1); + using var start = new Barrier(publisherCount + 1); + var statuses = new StoreStatus[publisherCount]; + var publishers = Enumerable.Range(0, publisherCount) + .Select(index => Task.Run(() => + { + start.SignalAndWait(); + statuses[index] = store.TryPublish([0x41], [(byte)index]); + })) + .ToArray(); + + start.SignalAndWait(); + await Task.WhenAll(publishers).WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Single(statuses, static status => status == StoreStatus.Success); + Assert.Equal(publisherCount - 1, statuses.Count(static status => status == StoreStatus.DuplicateKey)); + + // Duplicate candidates must have been returned; all remaining configured + // slots are still available for unrelated keys. + for (var index = 0; index < publisherCount; index++) + { + Assert.Equal(StoreStatus.Success, store.TryPublish(Key(index + 1), [(byte)index])); + } + } + + [Fact] + [Trait("Category", "Integration")] + public async Task ConcurrentUnrelatedKeyPublishersCompleteIndependently() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + const int publisherCount = 12; + using var store = CreateStore(slotCount: publisherCount); + using var start = new Barrier(publisherCount + 1); + var statuses = new StoreStatus[publisherCount]; + var publishers = Enumerable.Range(0, publisherCount) + .Select(index => Task.Run(() => + { + start.SignalAndWait(); + statuses[index] = store.TryPublish(Key(index), [(byte)index]); + })) + .ToArray(); + + start.SignalAndWait(); + await Task.WhenAll(publishers).WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.All(statuses, static status => Assert.Equal(StoreStatus.Success, status)); + } + + [Fact] + [Trait("Category", "Integration")] + public async Task PausedInsertionTransitionDoesNotBlockSameKeyClassificationOrUnrelatedProgress() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var controller = new CheckpointController(LockFreeCheckpointId.DirectoryBeforeDescriptorPublication); + using var store = CreateInstrumentedStore(slotCount: 4, controller); + + var owner = Task.Run(() => store.TryPublish([0x41], [0x11])); + Assert.True( + controller.WaitUntilPaused(TimeSpan.FromSeconds(5)), + "The insertion owner did not reach the configured directory checkpoint."); + + // A same-key contender exercises completion/classification of the + // insert lifecycle while an unrelated publisher proves local progress. + var sameKeyHelper = Task.Run(() => store.TryPublish([0x41], [0x22])); + var unrelated = Task.Run(() => store.TryPublish([0x52], [0x33])); + StoreStatus[] concurrent = await Task.WhenAll(sameKeyHelper, unrelated) + .WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, concurrent[1]); + + controller.Continue(); + StoreStatus ownerStatus = await owner.WaitAsync(TimeSpan.FromSeconds(5)); + StoreStatus[] sameKeyStatuses = [ownerStatus, concurrent[0]]; + Assert.Single(sameKeyStatuses, static status => status == StoreStatus.Success); + Assert.Single(sameKeyStatuses, static status => status == StoreStatus.DuplicateKey); + } + + [Fact] + [Trait("Category", "Integration")] + public async Task CancellationAfterSlotClaimRelinquishesOwnerAndRestoresFullCapacity() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var cancellation = new CancellationTokenSource(); + using var controller = new CheckpointController(LockFreeCheckpointId.DirectoryBeforeDescriptorPublication); + using var store = CreateInstrumentedStore(slotCount: 4, controller); + var wait = new StoreWaitOptions(TimeSpan.FromSeconds(5), cancellation.Token); + + var canceledPublish = Task.Run(() => store.TryPublish([0x61], [0x11], default, wait)); + Assert.True(controller.WaitUntilPaused(TimeSpan.FromSeconds(5))); + cancellation.Cancel(); + controller.Continue(); + + Assert.Equal(StoreStatus.OperationCanceled, await canceledPublish.WaitAsync(TimeSpan.FromSeconds(5))); + AssertFullCapacityCanBePublished(store, firstKey: 0x61, slotCount: 4); + } + + [Fact] + [Trait("Category", "Integration")] + public async Task DeadlineAfterSlotClaimReturnsWithinBoundAndRestoresFullCapacity() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var timeout = TimeSpan.FromMilliseconds(50); + using var controller = new CheckpointController(LockFreeCheckpointId.DirectoryBeforeDescriptorPublication); + using var store = CreateInstrumentedStore(slotCount: 4, controller); + + var expiredPublish = Task.Run(() => + { + var stopwatch = Stopwatch.StartNew(); + StoreStatus status = store.TryPublish( + [0x71], + [0x11], + default, + new StoreWaitOptions(timeout)); + stopwatch.Stop(); + return (Status: status, Elapsed: stopwatch.Elapsed); + }); + Assert.True(controller.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Thread.Sleep(timeout + TimeSpan.FromMilliseconds(50)); + controller.Continue(); + + (StoreStatus status, TimeSpan elapsed) = await expiredPublish.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.StoreBusy, status); + Assert.True( + elapsed <= timeout + TimeSpan.FromMilliseconds(250), + $"Bounded publication took {elapsed} for a {timeout} limit."); + AssertFullCapacityCanBePublished(store, firstKey: 0x71, slotCount: 4); + } + + private static MemoryStore CreateStore(int slotCount) + { + var options = Options($"sms-v2-publish-integration-{Guid.NewGuid():N}", slotCount); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore CreateInstrumentedStore(int slotCount, CheckpointController controller) + { + var options = Options($"sms-v2-instrumented-publish-{Guid.NewGuid():N}", slotCount); + Type? factoryType = typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeInstrumentedStoreFactory", + throwOnError: false, + ignoreCase: false); + Assert.True( + factoryType is not null, + "The lock-free engine must integrate the friend-only checkpoint strategy through LockFreeInstrumentedStoreFactory."); + + MethodInfo? factory = factoryType!.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .SingleOrDefault(method => method.Name == "TryCreateOrOpen" && method.ReturnType == typeof(StoreOpenStatus)); + Assert.NotNull(factory); + + object strategy = LockFreeCheckpointFactory.CreateInstrumented(controller.Observe); + object?[] arguments = BindInstrumentedFactoryArguments(factory!, options, controller.Observe, strategy); + var status = (StoreOpenStatus)factory!.Invoke(null, arguments)!; + MemoryStore? store = arguments.SingleOrDefault(static argument => argument is MemoryStore) as MemoryStore; + + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static object?[] BindInstrumentedFactoryArguments( + MethodInfo factory, + SharedMemoryStoreOptions options, + Action observer, + object strategy) + { + ParameterInfo[] parameters = factory.GetParameters(); + var arguments = new object?[parameters.Length]; + for (var index = 0; index < parameters.Length; index++) + { + ParameterInfo parameter = parameters[index]; + Type parameterType = parameter.ParameterType.IsByRef + ? parameter.ParameterType.GetElementType()! + : parameter.ParameterType; + if (parameter.IsOut && parameterType == typeof(MemoryStore)) + { + arguments[index] = null; + } + else if (parameterType == typeof(SharedMemoryStoreOptions)) + { + arguments[index] = options; + } + else if (parameterType == typeof(StoreWaitOptions)) + { + arguments[index] = StoreWaitOptions.Default; + } + else if (parameterType.IsInstanceOfType(observer)) + { + arguments[index] = observer; + } + else if (parameterType.IsInstanceOfType(strategy)) + { + arguments[index] = strategy; + } + else + { + throw new Xunit.Sdk.XunitException( + $"Unsupported instrumented factory parameter {parameter.Name}: {parameter.ParameterType}."); + } + } + + Assert.Contains(parameters, static parameter => + parameter.IsOut && parameter.ParameterType.GetElementType() == typeof(MemoryStore)); + return arguments; + } + + private static SharedMemoryStoreOptions Options(string name, int slotCount) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: Math.Max(8, slotCount), + participantRecordCount: 4, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + + private static void AssertFullCapacityCanBePublished(MemoryStore store, byte firstKey, int slotCount) + { + for (var index = 0; index < slotCount; index++) + { + Assert.Equal(StoreStatus.Success, store.TryPublish([(byte)(firstKey + index)], [(byte)index])); + } + + Assert.Equal(StoreStatus.StoreFull, store.TryPublish([0xff], [0xff])); + } + + private static byte[] Key(int index) => + [(byte)(index >> 24), (byte)(index >> 16), (byte)(index >> 8), (byte)index]; + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed class CheckpointController : IDisposable + { + private readonly LockFreeCheckpointId _target; + private readonly ManualResetEventSlim _paused = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: false); + private int _targetReached; + + public CheckpointController(LockFreeCheckpointId target) + { + _ = LockFreeCheckpointCatalog.Get(target); + _target = target; + } + + public void Observe(LockFreeCheckpointEntry entry) + { + if (entry.Id != _target || Interlocked.CompareExchange(ref _targetReached, 1, 0) != 0) + { + return; + } + + _paused.Set(); + _resume.Wait(); + } + + public bool WaitUntilPaused(TimeSpan timeout) => _paused.Wait(timeout); + + public void Continue() => _resume.Set(); + + public void Dispose() + { + _resume.Set(); + _paused.Dispose(); + _resume.Dispose(); + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeRawVisibilityIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeRawVisibilityIntegrationTests.cs new file mode 100644 index 0000000..537281f --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeRawVisibilityIntegrationTests.cs @@ -0,0 +1,245 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text.Json; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeRawVisibilityIntegrationTests +{ + private const int SlotCount = 2; + private const int MaxValueBytes = 2_048; + private const int MaxDescriptorBytes = 48; + private const int MaxKeyBytes = 16; + private const int LeaseRecordCount = 32; + private const int ParticipantRecordCount = 16; + private const int Iterations = 256; + private const int KeyCount = 4; + private const int ReaderCount = 3; + private static readonly TimeSpan WorkloadTimeout = TimeSpan.FromSeconds(75); + + [Theory] + [InlineData(0x13579)] + [InlineData(0x24680)] + [Trait("Category", "Integration")] + [Trait("Category", "RawVisibility")] + public void ProductionNoOpFullProtocolPublicationIsVisibleAcrossReuse(int seed) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-raw-visibility-{Guid.NewGuid():N}"; + using MemoryStore store = CreateStore(name); + long startUtcTicks = DateTime.UtcNow.AddSeconds(2).Ticks; + var commands = new List(ReaderCount + 2); + for (var reader = 0; reader < ReaderCount; reader++) + { + commands.Add(Command("raw-visibility-reader", name, seed, startUtcTicks)); + } + + commands.Add(Command("raw-visibility-remover", name, seed, startUtcTicks)); + commands.Add(Command("raw-visibility-publisher", name, seed, startUtcTicks)); + Process[] processes = commands.Select(StartAgent).ToArray(); + try + { + AgentResult[] results = WaitForAgents(processes, WorkloadTimeout); + Assert.All(results, static result => Assert.True( + result.ExitCode == 0, + "Raw visibility agent failed." + + Environment.NewLine + + "exit=" + result.ExitCode.ToString(CultureInfo.InvariantCulture) + + Environment.NewLine + + "stdout: " + result.StandardOutput + + Environment.NewLine + + "stderr: " + result.StandardError)); + + ResultPayload[] payloads = results.Select(ParseResult).ToArray(); + Assert.Equal(ReaderCount, payloads.Count(static result => result.Role == "reader")); + Assert.Single(payloads, static result => result.Role == "remover"); + Assert.Single(payloads, static result => result.Role == "publisher"); + + ResultPayload publisher = payloads.Single(static result => result.Role == "publisher"); + Assert.Equal(Iterations, publisher.Completed); + Assert.Equal(Iterations, publisher.Observations); + Assert.True(publisher.MinimumGeneration > 0); + AssertAggressiveReuse(publisher); + + ResultPayload remover = payloads.Single(static result => result.Role == "remover"); + Assert.Equal(Iterations, remover.Completed); + Assert.Equal(Iterations, remover.Observations); + Assert.True(remover.MinimumGeneration > 0); + AssertAggressiveReuse(remover); + + foreach (ResultPayload reader in payloads.Where(static result => result.Role == "reader")) + { + Assert.Equal(1, reader.Completed); + Assert.True(reader.Observations > 0, "Every independent reader must validate at least one data generation."); + Assert.True(reader.MinimumGeneration > 0); + Assert.True(reader.MaximumGeneration >= reader.MinimumGeneration); + } + } + finally + { + KillAll(processes); + foreach (Process process in processes) + { + process.Dispose(); + } + } + } + + private static MemoryStore CreateStore(string name) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + name, + SlotCount, + MaxValueBytes, + MaxDescriptorBytes, + MaxKeyBytes, + LeaseRecordCount, + ParticipantRecordCount, + OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus open = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, open); + return Assert.IsType(store); + } + + private static void AssertAggressiveReuse(ResultPayload result) + { + long requiredAdvance = (Iterations / SlotCount) - 1; + Assert.True( + result.MaximumGeneration - result.MinimumGeneration >= requiredAdvance, + $"{result.Role} observed insufficient physical reuse: min={result.MinimumGeneration}, " + + $"max={result.MaximumGeneration}, required advance={requiredAdvance}."); + } + + private static string[] Command(string command, string name, int seed, long startUtcTicks) => + [ + command, + name, + SlotCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + ParticipantRecordCount.ToString(CultureInfo.InvariantCulture), + Iterations.ToString(CultureInfo.InvariantCulture), + KeyCount.ToString(CultureInfo.InvariantCulture), + MaxValueBytes.ToString(CultureInfo.InvariantCulture), + seed.ToString(CultureInfo.InvariantCulture), + startUtcTicks.ToString(CultureInfo.InvariantCulture) + ]; + + private static AgentResult[] WaitForAgents(Process[] processes, TimeSpan timeout) + { + long deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + var results = new AgentResult[processes.Length]; + for (var index = 0; index < processes.Length; index++) + { + long remaining = deadline - Stopwatch.GetTimestamp(); + int milliseconds = remaining <= 0 + ? 0 + : (int)Math.Min(int.MaxValue, Math.Ceiling(remaining * 1_000d / Stopwatch.Frequency)); + Process process = processes[index]; + if (!process.WaitForExit(milliseconds)) + { + throw new TimeoutException("Raw visibility agents exceeded their shared bounded timeout."); + } + + results[index] = new AgentResult( + process.ExitCode, + process.StandardOutput.ReadToEnd(), + process.StandardError.ReadToEnd()); + } + + return results; + } + + private static ResultPayload ParseResult(AgentResult result) + { + string line = result.StandardOutput + .Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) + .Single(static value => value.StartsWith("RESULT ", StringComparison.Ordinal)); + return JsonSerializer.Deserialize(line["RESULT ".Length..]) + ?? throw new Xunit.Sdk.XunitException("Raw visibility agent returned an empty result."); + } + + private static Process StartAgent(string[] command) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(LocateAgentAssembly()); + foreach (string argument in command) + { + startInfo.ArgumentList.Add(argument); + } + + return Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start a raw visibility agent."); + } + + private static string LocateAgentAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + string root = directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + return File.Exists(path) + ? path + : throw new FileNotFoundException("Lock-free raw visibility agent was not built.", path); + } + + private static void KillAll(IEnumerable processes) + { + foreach (Process process in processes) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(5_000); + } + } + catch + { + // Cleanup is best effort for a unique mapped-store name. + } + } + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private readonly record struct AgentResult(int ExitCode, string StandardOutput, string StandardError); + + private sealed record ResultPayload( + string Role, + int Completed, + long Observations, + ulong Checksum, + long MinimumGeneration, + long MaximumGeneration); +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeSampleValidationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeSampleValidationTests.cs new file mode 100644 index 0000000..9e7b054 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeSampleValidationTests.cs @@ -0,0 +1,89 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeSampleValidationTests +{ + [Theory] + [InlineData(6)] + [InlineData(12)] + [Trait("Category", "Integration")] + public void BrokerKeySampleKeepsDispatchOutsideStoreAndValidatesKvLifecycles(int workerCount) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + int frameCount = workerCount * 2; + using Process process = StartSample(workerCount, frameCount); + Assert.True(process.WaitForExit(30_000), "Broker-key sample exceeded its bounded timeout."); + string stdout = process.StandardOutput.ReadToEnd(); + string stderr = process.StandardError.ReadToEnd(); + Assert.True( + process.ExitCode == 0, + $"Sample exit={process.ExitCode}{Environment.NewLine}{stdout}{Environment.NewLine}{stderr}"); + + string result = stdout.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) + .Single(line => line.StartsWith("RESULT ", StringComparison.Ordinal)); + IReadOnlyDictionary fields = result["RESULT ".Length..] + .Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Select(token => token.Split('=', 2)) + .ToDictionary(parts => parts[0], parts => parts[1], StringComparer.Ordinal); + + Assert.Equal(workerCount.ToString(), fields["workers"]); + Assert.Equal(frameCount.ToString(), fields["frames"]); + Assert.Equal(frameCount.ToString(), fields["processed"]); + Assert.Equal(fields["workerChecksum"], fields["observerChecksum"]); + Assert.Equal(nameof(StoreStatus.RemovePending), fields["pendingRemove"]); + Assert.Equal(nameof(StoreStatus.NotFound), fields["missing"]); + Assert.Equal(nameof(StoreStatus.Success), fields["diagnostics"]); + Assert.Equal(nameof(StoreProfile.LockFree), fields["profile"]); + Assert.Equal("2.0", fields["layout"]); + Assert.Equal("0", fields["recoveredLeases"]); + Assert.Equal("0", fields["recoveredReservations"]); + } + + private static Process StartSample(int workerCount, int frameCount) + { + var start = new ProcessStartInfo("dotnet") + { + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + start.ArgumentList.Add("exec"); + start.ArgumentList.Add(LocateSampleAssembly()); + start.ArgumentList.Add("--workers"); + start.ArgumentList.Add(workerCount.ToString(System.Globalization.CultureInfo.InvariantCulture)); + start.ArgumentList.Add("--frames"); + start.ArgumentList.Add(frameCount.ToString(System.Globalization.CultureInfo.InvariantCulture)); + return Process.Start(start) ?? throw new InvalidOperationException("Could not start broker-key sample."); + } + + private static string LocateSampleAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? root = new(AppContext.BaseDirectory); + while (root is not null && !File.Exists(Path.Combine(root.FullName, "SharedMemoryStore.slnx"))) + { + root = root.Parent; + } + + string path = Path.Combine( + root?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."), + "samples", + "LockFreeBrokerKeys", + "bin", + configuration, + "net10.0", + "LockFreeBrokerKeys.dll"); + return File.Exists(path) ? path : throw new FileNotFoundException("Sample assembly was not built.", path); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.IntegrationTests/LockFreeWaitPolicyMatrixIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/LockFreeWaitPolicyMatrixIntegrationTests.cs new file mode 100644 index 0000000..f93f7cd --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/LockFreeWaitPolicyMatrixIntegrationTests.cs @@ -0,0 +1,982 @@ +using System.Buffers; +using System.Diagnostics; +using System.Runtime.InteropServices; +using SharedMemoryStore.IntegrationTests.TestSupport; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class LockFreeWaitPolicyMatrixIntegrationTests +{ + private const int SlotCount = 12; + private const int LeaseRecordCount = 16; + private static readonly TimeSpan BoundaryTimeout = TimeSpan.FromMilliseconds(50); + private static readonly TimeSpan CompletionAllowance = TimeSpan.FromMilliseconds(250); + + [Theory] + [InlineData(WaitPolicyKind.NoWait)] + [InlineData(WaitPolicyKind.Finite)] + [InlineData(WaitPolicyKind.Infinite)] + [Trait("Category", "Integration")] + public void EveryValidWaitPolicyCompletesTheFullPublicSurface(WaitPolicyKind policy) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + StoreWaitOptions wait = Wait(policy); + SharedMemoryStoreOptions options = Options( + $"sms-v2-valid-wait-{policy}-{Guid.NewGuid():N}", + OpenMode.CreateNew); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(options, wait, out MemoryStore? candidate)); + MemoryStore store = Assert.IsType(candidate); + + LockFreeNoOperationLockIntegrationTests.ExerciseCompleteSteadyStateSurface(store, wait); + store.Dispose(); + + Assert.Equal(StoreStatus.StoreDisposed, store.TryPublish(Key(90), [1], default, wait)); + Assert.Equal(StoreStatus.StoreDisposed, store.TryGetDiagnostics(wait, out _)); + } + + [Fact] + [Trait("Category", "Integration")] + public void AlreadyCanceledPolicyWinsAcrossEveryWaitAwareOperationAndLeaksNothing() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var canceled = new StoreWaitOptions(TimeSpan.FromSeconds(5), cancellation.Token); + + SharedMemoryStoreOptions canceledOpen = Options( + $"sms-v2-canceled-open-{Guid.NewGuid():N}", + OpenMode.CreateNew); + Assert.Equal( + StoreOpenStatus.OperationCanceled, + MemoryStore.TryCreateOrOpen(canceledOpen, canceled, out MemoryStore? rejectedOpen)); + Assert.Null(rejectedOpen); + + using MemoryStore store = CreateStore( + $"sms-v2-canceled-surface-{Guid.NewGuid():N}", + OpenMode.CreateNew, + StoreWaitOptions.Default); + + Assert.Equal(StoreStatus.OperationCanceled, store.TryPublish(Key(1), [1], default, canceled)); + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryPublishSegments( + Key(2), + new ReadOnlySequence(new byte[] { 2, 3 }), + default, + canceled, + out long copied)); + Assert.Equal(0, copied); + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryReserve(Key(3), 1, default, canceled, out ValueReservation rejectedReservation)); + Assert.False(rejectedReservation.IsValid); + + Assert.Equal(StoreStatus.Success, store.TryReserve(Key(4), 1, default, out ValueReservation reservation)); + reservation.GetSpan()[0] = 4; + Assert.Equal(StoreStatus.OperationCanceled, reservation.Advance(1, canceled)); + Assert.Equal(0, reservation.BytesWritten); + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + Assert.Equal(StoreStatus.OperationCanceled, reservation.Commit(canceled)); + Assert.True(reservation.IsValid); + Assert.Equal(StoreStatus.OperationCanceled, reservation.Abort(canceled)); + Assert.True(reservation.IsValid); + Assert.Equal(StoreStatus.Success, reservation.Commit()); + + Assert.Equal(StoreStatus.OperationCanceled, store.TryAcquire(Key(4), canceled, out ValueLease rejectedLease)); + Assert.False(rejectedLease.IsValid); + Assert.Equal(StoreStatus.Success, store.TryAcquire(Key(4), out ValueLease lease)); + Assert.Equal(StoreStatus.OperationCanceled, lease.Release(canceled)); + Assert.True(lease.IsValid); + Assert.Equal(StoreStatus.Success, lease.Release()); + + Assert.Equal(StoreStatus.OperationCanceled, store.TryRemove(Key(4), canceled)); + Assert.Equal(StoreStatus.Success, store.TryAcquire(Key(4), out ValueLease preserved)); + Assert.Equal(StoreStatus.Success, preserved.Release()); + + Assert.Equal(StoreStatus.Success, store.TryAcquire(Key(4), out ValueLease recoveryLease)); + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryRecoverLeases(new LeaseRecoveryOptions(true), canceled, out LeaseRecoveryReport leaseReport)); + Assert.Equal(default, leaseReport); + Assert.True(recoveryLease.IsValid); + Assert.Equal(StoreStatus.Success, recoveryLease.Release()); + + Assert.Equal(StoreStatus.Success, store.TryReserve(Key(5), 1, default, out ValueReservation recoveryReservation)); + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + canceled, + out ReservationRecoveryReport reservationReport)); + Assert.Equal(default, reservationReport); + Assert.True(recoveryReservation.IsValid); + Assert.Equal(StoreStatus.Success, recoveryReservation.Abort()); + + Assert.Equal(StoreStatus.OperationCanceled, store.TryGetDiagnostics(canceled, out _)); + + AssertRemovedAndReclaimed(store, Key(4)); + AssertAllValueAndLeaseCapacityReusable(store); + } + + [Fact] + [Trait("Category", "Integration")] + public void LargeCapacityScansHonorNoWaitFiniteAndCancellationBounds() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + const int largeCount = 32_768; + string name = $"sms-v2-large-wait-budget-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: largeCount, + maxValueBytes: 1, + maxDescriptorBytes: 1, + maxKeyBytes: 8, + leaseRecordCount: largeCount, + participantRecordCount: 128, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + var delayCommit = 0; + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + if (entry.Id == LockFreeCheckpointId.CommitBeforePublicationCas + && Volatile.Read(ref delayCommit) != 0) + { + Thread.Sleep(TimeSpan.FromMilliseconds(25)); + } + }); + Assert.Equal( + StoreOpenStatus.Success, + LockFreeInstrumentedStoreFactory.TryCreateOrOpen(options, checkpoint, out MemoryStore? opened)); + using MemoryStore store = Assert.IsType(opened); + + AssertBoundedStatus( + () => store.TryGetDiagnostics(StoreWaitOptions.NoWait, out _), + StoreStatus.StoreBusy, + TimeSpan.Zero); + AssertBoundedStatus( + () => store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + StoreWaitOptions.NoWait, + out _), + StoreStatus.StoreBusy, + TimeSpan.Zero); + AssertBoundedStatus( + () => store.TryRecoverLeases( + new LeaseRecoveryOptions(false), + StoreWaitOptions.NoWait, + out _), + StoreStatus.StoreBusy, + TimeSpan.Zero); + + var finite = new StoreWaitOptions(TimeSpan.FromTicks(1)); + AssertBoundedStatus( + () => store.TryGetDiagnostics(finite, out _), + StoreStatus.StoreBusy, + finite.Timeout); + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var canceledInfinite = new StoreWaitOptions(Timeout.InfiniteTimeSpan, cancellation.Token); + AssertBoundedStatus( + () => store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + canceledInfinite, + out _), + StoreStatus.OperationCanceled, + TimeSpan.Zero); + + var publishDeadline = new StoreWaitOptions(TimeSpan.FromMilliseconds(1)); + Volatile.Write(ref delayCommit, 1); + try + { + AssertBoundedStatus( + () => store.TryPublish([0x41], [0x42], default, publishDeadline), + StoreStatus.StoreBusy, + publishDeadline.Timeout); + } + finally + { + Volatile.Write(ref delayCommit, 0); + } + + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot final)); + Assert.Equal(0, final.InitializingSlotCount); + Assert.Equal(0, final.ReservedSlotCount); + Assert.Equal(0, final.ClaimingLeaseCount); + Assert.Equal(0, final.ReclaimingSlotCount); + Assert.Equal(0, final.PublishedSlotCount); + Assert.Equal(largeCount, final.FreeSlotCount); + } + + [Theory] + [InlineData(BoundaryOperation.Publish)] + [InlineData(BoundaryOperation.SegmentedPublish)] + [InlineData(BoundaryOperation.Reserve)] + [InlineData(BoundaryOperation.Advance)] + [InlineData(BoundaryOperation.Commit)] + [InlineData(BoundaryOperation.Abort)] + [InlineData(BoundaryOperation.Acquire)] + [InlineData(BoundaryOperation.Release)] + [InlineData(BoundaryOperation.Remove)] + [InlineData(BoundaryOperation.Diagnostics)] + [InlineData(BoundaryOperation.LeaseRecovery)] + [InlineData(BoundaryOperation.ReservationRecovery)] + [Trait("Category", "Integration")] + public void CancellationImmediatelyBeforeOrderingReturnsCanceledWithoutOwnerLeak( + BoundaryOperation operation) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var cancellation = new CancellationTokenSource(); + using var controller = BoundaryController.CancelAt(BeforeCheckpoint(operation), cancellation); + using BoundaryContext context = Prepare(operation, controller); + var wait = new StoreWaitOptions(TimeSpan.FromSeconds(5), cancellation.Token); + + StoreStatus status = Invoke(operation, context, wait); + + Assert.True(controller.WasReached, $"{operation} did not reach its pre-ordering checkpoint."); + Assert.Equal(StoreStatus.OperationCanceled, status); + AssertBeforeOrderingStateAndCleanup(operation, context); + AssertNoOwnerControlledLeakage(context.Store); + } + + [Theory] + [InlineData(BoundaryOperation.Publish)] + [InlineData(BoundaryOperation.SegmentedPublish)] + [InlineData(BoundaryOperation.Reserve)] + [InlineData(BoundaryOperation.Advance)] + [InlineData(BoundaryOperation.Commit)] + [InlineData(BoundaryOperation.Abort)] + [InlineData(BoundaryOperation.Acquire)] + [InlineData(BoundaryOperation.Release)] + [InlineData(BoundaryOperation.Remove)] + [InlineData(BoundaryOperation.Diagnostics)] + [InlineData(BoundaryOperation.LeaseRecovery)] + [InlineData(BoundaryOperation.ReservationRecovery)] + [Trait("Category", "Integration")] + public async Task DeadlineImmediatelyBeforeOrderingReturnsBusyWithinLimitAndLeaksNothing( + BoundaryOperation operation) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var controller = BoundaryController.PauseAt(BeforeCheckpoint(operation)); + using BoundaryContext context = Prepare(operation, controller); + var wait = new StoreWaitOptions(BoundaryTimeout); + + Task pending = Task.Run(() => InvokeTimed(operation, context, wait)); + Assert.True( + controller.WaitUntilPaused(TimeSpan.FromSeconds(5)), + $"{operation} did not reach its pre-ordering checkpoint."); + Thread.Sleep(BoundaryTimeout + TimeSpan.FromMilliseconds(30)); + controller.Continue(); + TimedInvocation result = await pending.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.StoreBusy, result.Status); + AssertWithinBound(result.Elapsed, BoundaryTimeout, operation); + AssertBeforeOrderingStateAndCleanup(operation, context); + AssertNoOwnerControlledLeakage(context.Store); + } + + [Theory] + [InlineData(BoundaryOperation.Publish)] + [InlineData(BoundaryOperation.SegmentedPublish)] + [InlineData(BoundaryOperation.Reserve)] + [InlineData(BoundaryOperation.Advance)] + [InlineData(BoundaryOperation.Commit)] + [InlineData(BoundaryOperation.Abort)] + [InlineData(BoundaryOperation.Acquire)] + [InlineData(BoundaryOperation.Release)] + [InlineData(BoundaryOperation.Remove)] + [InlineData(BoundaryOperation.Reclaim)] + [InlineData(BoundaryOperation.Diagnostics)] + [InlineData(BoundaryOperation.LeaseRecovery)] + [InlineData(BoundaryOperation.ReservationRecovery)] + [Trait("Category", "Integration")] + public void CancellationAfterOrderingDoesNotRewriteTheCompletedOutcome(BoundaryOperation operation) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var cancellation = new CancellationTokenSource(); + using var controller = BoundaryController.CancelAt(AfterCheckpoint(operation), cancellation); + using BoundaryContext context = Prepare(operation, controller); + var wait = new StoreWaitOptions(TimeSpan.FromSeconds(5), cancellation.Token); + + StoreStatus status = Invoke(operation, context, wait); + + Assert.True(controller.WasReached, $"{operation} did not reach its post-ordering checkpoint."); + Assert.Contains(status, CompletedOutcomes(operation)); + Assert.NotEqual(StoreStatus.OperationCanceled, status); + AssertAfterOrderingStateAndCleanup(operation, context); + AssertNoOwnerControlledLeakage(context.Store); + } + + [Theory] + [InlineData(BoundaryOperation.Publish)] + [InlineData(BoundaryOperation.SegmentedPublish)] + [InlineData(BoundaryOperation.Reserve)] + [InlineData(BoundaryOperation.Advance)] + [InlineData(BoundaryOperation.Commit)] + [InlineData(BoundaryOperation.Abort)] + [InlineData(BoundaryOperation.Acquire)] + [InlineData(BoundaryOperation.Release)] + [InlineData(BoundaryOperation.Remove)] + [InlineData(BoundaryOperation.Reclaim)] + [InlineData(BoundaryOperation.Diagnostics)] + [InlineData(BoundaryOperation.LeaseRecovery)] + [InlineData(BoundaryOperation.ReservationRecovery)] + [Trait("Category", "Integration")] + public async Task DeadlineAfterOrderingDoesNotRewriteOutcomeAndStillReturnsWithinAllowance( + BoundaryOperation operation) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var controller = BoundaryController.PauseAt(AfterCheckpoint(operation)); + using BoundaryContext context = Prepare(operation, controller); + var wait = new StoreWaitOptions(BoundaryTimeout); + + Task pending = Task.Run(() => InvokeTimed(operation, context, wait)); + Assert.True( + controller.WaitUntilPaused(TimeSpan.FromSeconds(5)), + $"{operation} did not reach its post-ordering checkpoint."); + Thread.Sleep(BoundaryTimeout + TimeSpan.FromMilliseconds(30)); + controller.Continue(); + TimedInvocation result = await pending.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(result.Status, CompletedOutcomes(operation)); + Assert.NotEqual(StoreStatus.StoreBusy, result.Status); + AssertWithinBound(result.Elapsed, BoundaryTimeout, operation); + AssertAfterOrderingStateAndCleanup(operation, context); + AssertNoOwnerControlledLeakage(context.Store); + } + + [Theory] + [InlineData(WaitPolicyKind.NoWait)] + [InlineData(WaitPolicyKind.Finite)] + [Trait("Category", "Integration")] + public void ContendedOpenReturnsBusyWithinTheSelectedBound(WaitPolicyKind policy) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-open-bound-{policy}-{Guid.NewGuid():N}"; + using MemoryStore owner = CreateStore(name, OpenMode.CreateNew, StoreWaitOptions.Default); + using var blocker = new NamedSynchronizationBlocker(name); + StoreWaitOptions wait = policy == WaitPolicyKind.NoWait + ? StoreWaitOptions.NoWait + : new StoreWaitOptions(BoundaryTimeout); + var stopwatch = Stopwatch.StartNew(); + + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting), + wait, + out MemoryStore? rejected); + stopwatch.Stop(); + + Assert.Equal(StoreOpenStatus.StoreBusy, status); + Assert.Null(rejected); + AssertWithinBound(stopwatch.Elapsed, wait.Timeout, BoundaryOperation.Open); + } + + [Fact] + [Trait("Category", "Integration")] + public void ContendedOpenObservesCancellationBeforeItsFiniteTimeout() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-open-cancel-{Guid.NewGuid():N}"; + using MemoryStore owner = CreateStore(name, OpenMode.CreateNew, StoreWaitOptions.Default); + using var blocker = new NamedSynchronizationBlocker(name); + using var cancellation = new CancellationTokenSource(); + cancellation.CancelAfter(TimeSpan.FromMilliseconds(50)); + var stopwatch = Stopwatch.StartNew(); + + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting), + new StoreWaitOptions(TimeSpan.FromSeconds(5), cancellation.Token), + out MemoryStore? rejected); + stopwatch.Stop(); + + Assert.Equal(StoreOpenStatus.OperationCanceled, status); + Assert.Null(rejected); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(1), $"Cancellation took {stopwatch.Elapsed}."); + } + + [Fact] + [Trait("Category", "Integration")] + public async Task InfiniteOpenWaitsUntilTheColdSynchronizationIsReleased() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-open-infinite-{Guid.NewGuid():N}"; + using MemoryStore owner = CreateStore(name, OpenMode.CreateNew, StoreWaitOptions.Default); + var blocker = new NamedSynchronizationBlocker(name); + MemoryStore? opened = null; + Task pending = Task.Run(() => MemoryStore.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting), + StoreWaitOptions.Infinite, + out opened)); + + try + { + await Task.Delay(TimeSpan.FromMilliseconds(75)); + Assert.False(pending.IsCompleted, "Infinite open did not wait for held cold synchronization."); + } + finally + { + // Always release the owner-thread-affine Windows mutex (and the + // equivalent Linux local/file lock) before unwinding the owner. + blocker.Dispose(); + } + + Assert.Equal(StoreOpenStatus.Success, await pending.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.IsType(opened).Dispose(); + } + + private static BoundaryContext Prepare(BoundaryOperation operation, BoundaryController controller) + { + MemoryStore store = CreateInstrumentedStore(controller); + var context = new BoundaryContext(store, Key(1)); + switch (operation) + { + case BoundaryOperation.Advance: + Assert.Equal(StoreStatus.Success, store.TryReserve(context.Key, 1, default, out context.Reservation)); + context.Reservation.GetSpan()[0] = 1; + break; + case BoundaryOperation.Commit: + Assert.Equal(StoreStatus.Success, store.TryReserve(context.Key, 1, default, out context.Reservation)); + context.Reservation.GetSpan()[0] = 1; + Assert.Equal(StoreStatus.Success, context.Reservation.Advance(1)); + break; + case BoundaryOperation.Abort: + Assert.Equal(StoreStatus.Success, store.TryReserve(context.Key, 1, default, out context.Reservation)); + break; + case BoundaryOperation.Acquire: + case BoundaryOperation.Remove: + case BoundaryOperation.Reclaim: + Assert.Equal(StoreStatus.Success, store.TryPublish(context.Key, [1])); + break; + case BoundaryOperation.Release: + case BoundaryOperation.LeaseRecovery: + Assert.Equal(StoreStatus.Success, store.TryPublish(context.Key, [1])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(context.Key, out context.Lease)); + break; + case BoundaryOperation.ReservationRecovery: + Assert.Equal(StoreStatus.Success, store.TryReserve(context.Key, 1, default, out context.Reservation)); + break; + } + + return context; + } + + private static StoreStatus Invoke( + BoundaryOperation operation, + BoundaryContext context, + StoreWaitOptions wait) + { + return operation switch + { + BoundaryOperation.Publish => context.Store.TryPublish(context.Key, [1], default, wait), + BoundaryOperation.SegmentedPublish => context.Store.TryPublishSegments( + context.Key, + new ReadOnlySequence(new byte[] { 1, 2 }), + default, + wait, + out context.CopiedBytes), + BoundaryOperation.Reserve => context.Store.TryReserve( + context.Key, + 1, + default, + wait, + out context.Reservation), + BoundaryOperation.Advance => context.Reservation.Advance(1, wait), + BoundaryOperation.Commit => context.Reservation.Commit(wait), + BoundaryOperation.Abort => context.Reservation.Abort(wait), + BoundaryOperation.Acquire => context.Store.TryAcquire(context.Key, wait, out context.Lease), + BoundaryOperation.Release => context.Lease.Release(wait), + BoundaryOperation.Remove or BoundaryOperation.Reclaim => context.Store.TryRemove(context.Key, wait), + BoundaryOperation.Diagnostics => context.Store.TryGetDiagnostics(wait, out context.Diagnostics), + BoundaryOperation.LeaseRecovery => context.Store.TryRecoverLeases( + new LeaseRecoveryOptions(true), + wait, + out context.LeaseRecoveryReport), + BoundaryOperation.ReservationRecovery => context.Store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + wait, + out context.ReservationRecoveryReport), + _ => throw new ArgumentOutOfRangeException(nameof(operation)) + }; + } + + private static TimedInvocation InvokeTimed( + BoundaryOperation operation, + BoundaryContext context, + StoreWaitOptions wait) + { + var stopwatch = Stopwatch.StartNew(); + StoreStatus status = Invoke(operation, context, wait); + stopwatch.Stop(); + return new TimedInvocation(status, stopwatch.Elapsed); + } + + private static LockFreeCheckpointId BeforeCheckpoint(BoundaryOperation operation) => operation switch + { + BoundaryOperation.Publish or BoundaryOperation.SegmentedPublish => + LockFreeCheckpointId.CommitBeforePublicationCas, + BoundaryOperation.Reserve => + LockFreeCheckpointId.DirectoryBeforeDescriptorPublication, + BoundaryOperation.Advance => LockFreeCheckpointId.AdvanceBeforeBytesAdvancedCas, + BoundaryOperation.Commit => LockFreeCheckpointId.CommitBeforePublicationCas, + BoundaryOperation.Abort => LockFreeCheckpointId.AbortBeforeAbortCas, + BoundaryOperation.Acquire => LockFreeCheckpointId.AcquireBeforeLeaseClaimCas, + BoundaryOperation.Release => LockFreeCheckpointId.ReleaseBeforeActiveReleaseCas, + BoundaryOperation.Remove => LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas, + BoundaryOperation.Diagnostics => LockFreeCheckpointId.DiagnosticsBeforeBoundedScan, + BoundaryOperation.LeaseRecovery or BoundaryOperation.ReservationRecovery => + LockFreeCheckpointId.RecoveryBeforeOwnerClassification, + _ => throw new ArgumentOutOfRangeException(nameof(operation)) + }; + + private static LockFreeCheckpointId AfterCheckpoint(BoundaryOperation operation) => operation switch + { + BoundaryOperation.Publish => LockFreeCheckpointId.PublishAfterCommitPublication, + BoundaryOperation.SegmentedPublish => LockFreeCheckpointId.CommitAfterPublicationCas, + BoundaryOperation.Reserve => LockFreeCheckpointId.ReserveAfterReservationPublication, + BoundaryOperation.Advance => LockFreeCheckpointId.AdvanceAfterBytesAdvancedCas, + BoundaryOperation.Commit => LockFreeCheckpointId.CommitAfterPublicationCas, + BoundaryOperation.Abort => LockFreeCheckpointId.AbortAfterUnlinkCompletion, + BoundaryOperation.Acquire => LockFreeCheckpointId.AcquireAfterPublishedRevalidation, + BoundaryOperation.Release => LockFreeCheckpointId.ReleaseAfterRecordRecycle, + BoundaryOperation.Remove => LockFreeCheckpointId.RemoveAfterLeaseClassification, + BoundaryOperation.Reclaim => LockFreeCheckpointId.ReclaimAfterGenerationAdvance, + BoundaryOperation.Diagnostics => LockFreeCheckpointId.DiagnosticsAfterSnapshotAssembly, + BoundaryOperation.LeaseRecovery or BoundaryOperation.ReservationRecovery => + LockFreeCheckpointId.RecoveryAfterExactRecoveryCas, + _ => throw new ArgumentOutOfRangeException(nameof(operation)) + }; + + private static StoreStatus[] CompletedOutcomes(BoundaryOperation operation) => operation switch + { + BoundaryOperation.Remove or BoundaryOperation.Reclaim => + [StoreStatus.Success, StoreStatus.RemovePending], + _ => [StoreStatus.Success] + }; + + private static void AssertBeforeOrderingStateAndCleanup( + BoundaryOperation operation, + BoundaryContext context) + { + switch (operation) + { + case BoundaryOperation.Publish: + case BoundaryOperation.SegmentedPublish: + case BoundaryOperation.Reserve: + Assert.Equal(StoreStatus.NotFound, context.Store.TryAcquire(context.Key, out _)); + break; + case BoundaryOperation.Advance: + Assert.True(context.Reservation.IsValid); + Assert.Equal(0, context.Reservation.BytesWritten); + Assert.Equal(StoreStatus.Success, context.Reservation.Abort()); + break; + case BoundaryOperation.Commit: + case BoundaryOperation.Abort: + Assert.True(context.Reservation.IsValid); + Assert.Equal(StoreStatus.Success, context.Reservation.Abort()); + break; + case BoundaryOperation.Acquire: + Assert.False(context.Lease.IsValid); + AssertRemovedAndReclaimed(context.Store, context.Key); + break; + case BoundaryOperation.Release: + Assert.True(context.Lease.IsValid); + Assert.Equal(StoreStatus.Success, context.Lease.Release()); + AssertRemovedAndReclaimed(context.Store, context.Key); + break; + case BoundaryOperation.Remove: + Assert.Equal(StoreStatus.Success, context.Store.TryAcquire(context.Key, out ValueLease preserved)); + Assert.Equal(StoreStatus.Success, preserved.Release()); + AssertRemovedAndReclaimed(context.Store, context.Key); + break; + case BoundaryOperation.LeaseRecovery: + Assert.True(context.Lease.IsValid); + Assert.Equal(StoreStatus.Success, context.Lease.Release()); + AssertRemovedAndReclaimed(context.Store, context.Key); + break; + case BoundaryOperation.ReservationRecovery: + Assert.True(context.Reservation.IsValid); + Assert.Equal(StoreStatus.Success, context.Reservation.Abort()); + break; + } + } + + private static void AssertAfterOrderingStateAndCleanup( + BoundaryOperation operation, + BoundaryContext context) + { + switch (operation) + { + case BoundaryOperation.Publish: + case BoundaryOperation.SegmentedPublish: + case BoundaryOperation.Commit: + Assert.Equal(StoreStatus.Success, context.Store.TryAcquire(context.Key, out ValueLease published)); + Assert.Equal(StoreStatus.Success, published.Release()); + AssertRemovedAndReclaimed(context.Store, context.Key); + break; + case BoundaryOperation.Reserve: + Assert.True(context.Reservation.IsValid); + Assert.Equal(StoreStatus.Success, context.Reservation.Abort()); + break; + case BoundaryOperation.Advance: + Assert.True(context.Reservation.IsValid); + Assert.Equal(1, context.Reservation.BytesWritten); + Assert.Equal(StoreStatus.Success, context.Reservation.Abort()); + break; + case BoundaryOperation.Abort: + Assert.False(context.Reservation.IsValid); + break; + case BoundaryOperation.Acquire: + Assert.True(context.Lease.IsValid); + Assert.Equal(StoreStatus.Success, context.Lease.Release()); + AssertRemovedAndReclaimed(context.Store, context.Key); + break; + case BoundaryOperation.Release: + Assert.False(context.Lease.IsValid); + AssertRemovedAndReclaimed(context.Store, context.Key); + break; + case BoundaryOperation.Remove: + case BoundaryOperation.Reclaim: + Assert.Equal(StoreStatus.NotFound, context.Store.TryAcquire(context.Key, out _)); + break; + case BoundaryOperation.LeaseRecovery: + Assert.False(context.Lease.IsValid); + Assert.Equal(1, context.LeaseRecoveryReport.RecoveredLeaseCount); + AssertRemovedAndReclaimed(context.Store, context.Key); + break; + case BoundaryOperation.ReservationRecovery: + Assert.False(context.Reservation.IsValid); + Assert.Equal(1, context.ReservationRecoveryReport.RecoveredReservationCount); + break; + case BoundaryOperation.Diagnostics: + Assert.Equal(StoreProfile.LockFree, context.Diagnostics.Profile); + break; + } + } + + private static void AssertNoOwnerControlledLeakage(MemoryStore store) + { + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out var snapshot)); + Assert.Equal(0, snapshot.ActiveLeaseCount); + Assert.Equal(0, snapshot.ActiveReservationCount); + Assert.Equal(0, snapshot.InitializingSlotCount); + Assert.Equal(0, snapshot.ReservedSlotCount); + Assert.Equal(0, snapshot.ReclaimingSlotCount); + Assert.Equal(0, snapshot.PendingRemovalCount); + Assert.Equal(0, snapshot.PublishedSlotCount); + Assert.Equal(SlotCount, snapshot.FreeSlotCount); + } + + private static void AssertAllValueAndLeaseCapacityReusable(MemoryStore store) + { + byte[] leaseKey = Key(50); + Assert.Equal(StoreStatus.Success, store.TryPublish(leaseKey, [50])); + var leases = new ValueLease[LeaseRecordCount]; + for (var index = 0; index < leases.Length; index++) + { + Assert.Equal(StoreStatus.Success, store.TryAcquire(leaseKey, out leases[index])); + } + + Assert.Equal(StoreStatus.LeaseTableFull, store.TryAcquire(leaseKey, out _)); + foreach (ValueLease lease in leases) + { + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + AssertRemovedAndReclaimed(store, leaseKey); + + for (var index = 0; index < SlotCount; index++) + { + Assert.Equal(StoreStatus.Success, store.TryPublish(Key(100 + index), [(byte)index])); + } + + Assert.Equal(StoreStatus.StoreFull, store.TryPublish(Key(999), [1])); + for (var index = 0; index < SlotCount; index++) + { + AssertRemovedAndReclaimed(store, Key(100 + index)); + } + + AssertNoOwnerControlledLeakage(store); + } + + private static void AssertRemovedAndReclaimed(MemoryStore store, byte[] key) + { + StoreStatus status = store.TryRemove(key, new StoreWaitOptions(TimeSpan.FromSeconds(1))); + Assert.Contains(status, new[] { StoreStatus.Success, StoreStatus.NotFound, StoreStatus.RemovePending }); + if (status == StoreStatus.RemovePending) + { + Assert.Contains( + store.TryRemove(key, new StoreWaitOptions(TimeSpan.FromSeconds(1))), + new[] { StoreStatus.Success, StoreStatus.NotFound }); + } + } + + private static void AssertWithinBound( + TimeSpan elapsed, + TimeSpan limit, + BoundaryOperation operation) + { + TimeSpan maximum = limit + CompletionAllowance; + Assert.True(elapsed <= maximum, $"{operation} took {elapsed} for a {limit} limit (maximum {maximum})."); + } + + private static void AssertBoundedStatus( + Func operation, + StoreStatus expected, + TimeSpan limit) + { + var stopwatch = Stopwatch.StartNew(); + StoreStatus status = operation(); + stopwatch.Stop(); + + Assert.Equal(expected, status); + Assert.True( + stopwatch.Elapsed <= limit + CompletionAllowance, + $"Large-capacity operation took {stopwatch.Elapsed} for a {limit} limit."); + } + + private static MemoryStore CreateStore( + string name, + OpenMode openMode, + StoreWaitOptions wait) + { + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen(Options(name, openMode), wait, out MemoryStore? store)); + return Assert.IsType(store); + } + + private static MemoryStore CreateInstrumentedStore(BoundaryController controller) + { + SharedMemoryStoreOptions options = Options( + $"sms-v2-wait-boundary-{Guid.NewGuid():N}", + OpenMode.CreateNew); + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + LockFreeCheckpointFactory.CreateInstrumented(controller.Observe), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: SlotCount, + maxValueBytes: 64, + maxDescriptorBytes: 16, + maxKeyBytes: 8, + leaseRecordCount: LeaseRecordCount, + participantRecordCount: 4, + openMode, + enableLeaseRecovery: true); + + private static StoreWaitOptions Wait(WaitPolicyKind policy) => policy switch + { + WaitPolicyKind.NoWait => StoreWaitOptions.NoWait, + WaitPolicyKind.Finite => new StoreWaitOptions(TimeSpan.FromMilliseconds(250)), + WaitPolicyKind.Infinite => StoreWaitOptions.Infinite, + _ => throw new ArgumentOutOfRangeException(nameof(policy)) + }; + + private static byte[] Key(int value) => BitConverter.GetBytes(value); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && LayoutV2Constants.IsSupportedArchitecture(RuntimeInformation.ProcessArchitecture); + + public enum WaitPolicyKind + { + NoWait, + Finite, + Infinite + } + + public enum BoundaryOperation + { + Open, + Publish, + SegmentedPublish, + Reserve, + Advance, + Commit, + Abort, + Acquire, + Release, + Remove, + Reclaim, + Diagnostics, + LeaseRecovery, + ReservationRecovery + } + + private readonly record struct TimedInvocation(StoreStatus Status, TimeSpan Elapsed); + + private sealed class BoundaryContext : IDisposable + { + internal BoundaryContext(MemoryStore store, byte[] key) + { + Store = store; + Key = key; + } + + internal MemoryStore Store { get; } + internal byte[] Key { get; } + internal ValueReservation Reservation; + internal ValueLease Lease; + internal long CopiedBytes; + internal DiagnosticsSnapshot Diagnostics; + internal LeaseRecoveryReport LeaseRecoveryReport; + internal ReservationRecoveryReport ReservationRecoveryReport; + + public void Dispose() => Store.Dispose(); + } + + private sealed class BoundaryController : IDisposable + { + private readonly LockFreeCheckpointId _target; + private readonly CancellationTokenSource? _cancellation; + private readonly ManualResetEventSlim _paused = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: false); + private int _reached; + + private BoundaryController( + LockFreeCheckpointId target, + CancellationTokenSource? cancellation) + { + _target = target; + _cancellation = cancellation; + } + + internal bool WasReached => Volatile.Read(ref _reached) != 0; + + internal static BoundaryController CancelAt( + LockFreeCheckpointId target, + CancellationTokenSource cancellation) => new(target, cancellation); + + internal static BoundaryController PauseAt(LockFreeCheckpointId target) => new(target, null); + + internal void Observe(LockFreeCheckpointEntry entry) + { + if (entry.Id != _target || Interlocked.CompareExchange(ref _reached, 1, 0) != 0) + { + return; + } + + if (_cancellation is not null) + { + _cancellation.Cancel(); + return; + } + + _paused.Set(); + if (!_resume.Wait(TimeSpan.FromSeconds(5))) + { + throw new TimeoutException($"Checkpoint {_target} was not resumed."); + } + } + + internal bool WaitUntilPaused(TimeSpan timeout) => _paused.Wait(timeout); + + internal void Continue() => _resume.Set(); + + public void Dispose() + { + _resume.Set(); + _paused.Dispose(); + _resume.Dispose(); + } + } + + private sealed class NamedSynchronizationBlocker : IDisposable + { + private readonly string _storeName; + private readonly ManualResetEventSlim _ready = new(initialState: false); + private readonly ManualResetEventSlim _release = new(initialState: false); + private readonly Thread _thread; + private Exception? _failure; + + internal NamedSynchronizationBlocker(string storeName) + { + _storeName = storeName; + _thread = new Thread(Hold) { IsBackground = true }; + _thread.Start(); + Assert.True(_ready.Wait(TimeSpan.FromSeconds(5)), "The legacy synchronization was not acquired."); + ThrowIfFailed(); + } + + public void Dispose() + { + _release.Set(); + Assert.True(_thread.Join(TimeSpan.FromSeconds(5)), "The legacy synchronization blocker did not stop."); + _ready.Dispose(); + _release.Dispose(); + ThrowIfFailed(); + } + + private void Hold() + { + try + { + using IDisposable synchronization = PlatformCapabilityProbe.HoldStoreSynchronization(_storeName); + _ready.Set(); + _release.Wait(); + } + catch (Exception exception) + { + _failure = exception; + _ready.Set(); + } + } + + private void ThrowIfFailed() + { + if (_failure is not null) + { + throw new Xunit.Sdk.XunitException( + $"The legacy synchronization blocker failed: {_failure}"); + } + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/MappedAtomicLitmusIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/MappedAtomicLitmusIntegrationTests.cs new file mode 100644 index 0000000..c3f9b26 --- /dev/null +++ b/tests/SharedMemoryStore.IntegrationTests/MappedAtomicLitmusIntegrationTests.cs @@ -0,0 +1,648 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Runtime.ExceptionServices; +using SharedMemoryStore.Interop; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.IntegrationTests; + +public sealed class MappedAtomicLitmusIntegrationTests +{ + private const int AgentTimeoutMilliseconds = 120_000; + private const int AgentPollMilliseconds = 100; + + [Fact] + [Trait("Category", "Integration")] + [Trait("Category", "AtomicLitmus")] + public void VolatilePublicationMakesOrdinaryMappedWritesVisibleAcrossProcesses() + { + if (!IsSupportedAtomicHost()) + { + return; + } + + const int iterations = 10_000; + AtomicTestMapping.Execute(mapping => + { + var results = RunAgents( + () => mapping.DescribeWords( + ("sequence", 0), + ("complement", 8), + ("publication", 16), + ("acknowledgement", 24)), + ["atomic-publication-consumer", mapping.Path, iterations.ToString(CultureInfo.InvariantCulture)], + ["atomic-publication-producer", mapping.Path, iterations.ToString(CultureInfo.InvariantCulture)]); + + AssertAgentsSucceeded(results); + Assert.All(results, static result => Assert.Contains("aligned=1", result.StandardOutput, StringComparison.Ordinal)); + Assert.Equal(iterations, mapping.ReadInt64(byteOffset: 16)); + Assert.Equal(iterations, mapping.ReadInt64(byteOffset: 24)); + Assert.Equal(iterations, mapping.ReadInt64(byteOffset: 0)); + Assert.Equal(~(long)iterations, mapping.ReadInt64(byteOffset: 8)); + }); + } + + [Fact] + [Trait("Category", "Integration")] + [Trait("Category", "AtomicLitmus")] + public void CompareExchangeIsAtomicAcrossIndependentMappedViews() + { + if (!IsSupportedAtomicHost()) + { + return; + } + + const int iterationsPerProcess = 50_000; + var iterationArgument = iterationsPerProcess.ToString(CultureInfo.InvariantCulture); + AtomicTestMapping.Execute(mapping => + { + var results = RunAgents( + () => mapping.DescribeWords(("counter", 32)), + ["atomic-cas-worker", mapping.Path, iterationArgument], + ["atomic-cas-worker", mapping.Path, iterationArgument]); + + AssertAgentsSucceeded(results); + Assert.All(results, static result => Assert.Contains("aligned=1", result.StandardOutput, StringComparison.Ordinal)); + Assert.Equal(2L * iterationsPerProcess, mapping.ReadInt64(byteOffset: 32)); + }); + } + + [Fact] + [Trait("Category", "Integration")] + [Trait("Category", "AtomicLitmus")] + public void SequentiallyConsistentRmwForbidsTwoWordDekkerOldOldOutcome() + { + if (!IsSupportedAtomicHost()) + { + return; + } + + const int iterations = 10_000; + var iterationArgument = iterations.ToString(CultureInfo.InvariantCulture); + AtomicTestMapping.Execute(mapping => + { + var results = RunAgents( + () => mapping.DescribeWords( + ("ready0", 0), + ("ready1", 8), + ("phase", 16), + ("done0", 24), + ("done1", 32), + ("word0", 40), + ("word1", 48), + ("seen0", 56), + ("seen1", 64)), + ["atomic-dekker-worker", mapping.Path, iterationArgument, "0"], + ["atomic-dekker-worker", mapping.Path, iterationArgument, "1"], + ["atomic-dekker-coordinator", mapping.Path, iterationArgument]); + + AssertAgentsSucceeded(results); + Assert.All(results, static result => Assert.Contains("aligned=1", result.StandardOutput, StringComparison.Ordinal)); + Assert.Contains("forbidden=0", results[2].StandardOutput, StringComparison.Ordinal); + Assert.Equal(iterations, mapping.ReadInt64(byteOffset: 24)); + Assert.Equal(iterations, mapping.ReadInt64(byteOffset: 32)); + }); + } + + [Fact] + [Trait("Category", "Integration")] + [Trait("Category", "AtomicLitmus")] + public void LockFreeProfileRejectsNonX64BeforeCreatingMappedData() + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || RuntimeInformation.ProcessArchitecture == Architecture.X64) + { + return; + } + + var name = $"sms-non-x64-rejection-{Guid.NewGuid():N}"; + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 4, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + + var status = Store.TryCreateOrOpen(options, out var store); + store?.Dispose(); + + Assert.Equal(StoreOpenStatus.UnsupportedPlatform, status); + Assert.Null(store); + if (OperatingSystem.IsLinux()) + { + var resources = PlatformResourceName.Create(name); + Assert.False(File.Exists(resources.LinuxRegionPath)); + Assert.False(File.Exists(resources.LinuxOwnersPath)); + } + } + + private static bool IsSupportedAtomicHost() + { + return (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + } + + private static AgentResult[] RunAgents(Func failureStateSnapshot, params string[][] commands) + { + var agents = new List(commands.Length); + AgentResult[]? results = null; + Exception? failure = null; + string? stateBeforeStop = null; + string? agentsBeforeStop = null; + var captureFailureDiagnostics = false; + try + { + foreach (var command in commands) + { + agents.Add(StartAgent(command)); + } + + var deadline = Stopwatch.GetTimestamp() + + (long)(AgentTimeoutMilliseconds / 1000d * Stopwatch.Frequency); + while (agents.Any(static agent => !agent.Process.HasExited)) + { + var remaining = deadline - Stopwatch.GetTimestamp(); + if (remaining <= 0) + { + captureFailureDiagnostics = true; + stateBeforeStop = TryDescribeState(failureStateSnapshot); + agentsBeforeStop = DescribeAgentStates(agents); + throw new TimeoutException( + "Mapped atomic agents did not complete within " + + AgentTimeoutMilliseconds.ToString(CultureInfo.InvariantCulture) + + " ms."); + } + + var remainingMilliseconds = (int)Math.Min( + AgentPollMilliseconds, + Math.Max(1, Math.Ceiling(remaining * 1000d / Stopwatch.Frequency))); + Thread.Sleep(remainingMilliseconds); + } + + foreach (var agent in agents) + { + agent.Process.WaitForExit(); + } + + results = agents.Select(ReadAgentResult).ToArray(); + if (results.Any(static result => result.ExitCode != 0)) + { + captureFailureDiagnostics = true; + stateBeforeStop = TryDescribeState(failureStateSnapshot); + agentsBeforeStop = DescribeAgentStates(agents); + throw new InvalidOperationException("One or more mapped atomic agents exited unsuccessfully."); + } + } + catch (Exception exception) + { + failure = exception; + } + + var cleanupFailures = StopAgents(agents); + var stateAfterStop = captureFailureDiagnostics + ? TryDescribeState(failureStateSnapshot) + : null; + var agentDiagnostics = captureFailureDiagnostics + ? DescribeAgentDiagnostics(agents) + : null; + cleanupFailures.AddRange(DisposeAgents(agents)); + + if (failure is not null) + { + if (captureFailureDiagnostics) + { + var diagnosticMessage = failure.Message + + Environment.NewLine + + "state-before-stop: " + stateBeforeStop + + Environment.NewLine + + "state-after-stop: " + stateAfterStop + + Environment.NewLine + + "agents-before-stop: " + agentsBeforeStop + + Environment.NewLine + + "agents-after-stop: " + agentDiagnostics; + failure = failure is TimeoutException + ? new TimeoutException(diagnosticMessage, failure) + : new InvalidOperationException(diagnosticMessage, failure); + } + + if (cleanupFailures.Count != 0) + { + failure = new AggregateException( + failure, + new InvalidOperationException( + "Mapped atomic agent cleanup failed: " + string.Join(" | ", cleanupFailures))); + } + + ExceptionDispatchInfo.Capture(failure).Throw(); + } + + if (cleanupFailures.Count != 0) + { + throw new InvalidOperationException( + "Mapped atomic agent cleanup failed: " + string.Join(" | ", cleanupFailures)); + } + + return results ?? throw new InvalidOperationException("Mapped atomic agents produced no result."); + } + + private static RunningAgent StartAgent(string[] command) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(LocateAgentAssembly()); + foreach (var argument in command) + { + startInfo.ArgumentList.Add(argument); + } + + Process? process = null; + try + { + process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start the lock-free atomic test agent."); + return new RunningAgent( + GetAgentRole(command), + process, + process.StandardOutput.ReadToEndAsync(), + process.StandardError.ReadToEndAsync()); + } + catch (Exception startFailure) + { + var cleanupFailures = new List(); + try + { + if (process is not null && !process.HasExited) + { + process.Kill(entireProcessTree: true); + } + + if (process is not null && !process.WaitForExit(5_000)) + { + cleanupFailures.Add(new TimeoutException( + "Partially started atomic agent did not exit within 5000 ms.")); + } + } + catch (Exception cleanupFailure) + { + cleanupFailures.Add(cleanupFailure); + } + + try + { + process?.Dispose(); + } + catch (Exception disposeFailure) + { + cleanupFailures.Add(disposeFailure); + } + + if (cleanupFailures.Count != 0) + { + throw new AggregateException( + new[] { startFailure }.Concat(cleanupFailures)); + } + + ExceptionDispatchInfo.Capture(startFailure).Throw(); + throw new UnreachableException(); + } + } + + private static string GetAgentRole(string[] command) + { + return command.Length == 4 && command[0] == "atomic-dekker-worker" + ? command[0] + "-" + command[3] + : command[0]; + } + + private static string LocateAgentAssembly() + { + var configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(System.IO.Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + var root = directory?.FullName ?? throw new DirectoryNotFoundException("Repository root not found."); + var path = System.IO.Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + if (!File.Exists(path)) + { + throw new FileNotFoundException("Lock-free atomic test agent was not built.", path); + } + + return path; + } + + private static void AssertAgentsSucceeded(AgentResult[] results) + { + Assert.All(results, static result => Assert.True( + result.ExitCode == 0, + "Agent role: " + + result.Role + + Environment.NewLine + + "exit code: " + + result.ExitCode.ToString(CultureInfo.InvariantCulture) + + Environment.NewLine + + "stdout: " + + result.StandardOutput + + Environment.NewLine + + "stderr: " + + result.StandardError)); + } + + private static AgentResult ReadAgentResult(RunningAgent agent) + { + return new AgentResult( + agent.Role, + agent.Process.ExitCode, + agent.StandardOutput.GetAwaiter().GetResult(), + agent.StandardError.GetAwaiter().GetResult()); + } + + private static string TryDescribeState(Func snapshot) + { + try + { + return snapshot(); + } + catch (Exception exception) + { + return "unavailable(" + exception.GetType().Name + ": " + exception.Message + ")"; + } + } + + private static string DescribeAgentStates(IEnumerable agents) + { + return string.Join("; ", agents.Select(static agent => DescribeAgent(agent, includeOutput: false, 0))); + } + + private static string DescribeAgentDiagnostics(IEnumerable agents) + { + var drainDeadline = Stopwatch.GetTimestamp() + 5L * Stopwatch.Frequency; + return string.Join( + "; ", + agents.Select(agent => DescribeAgent(agent, includeOutput: true, drainDeadline))); + } + + private static string DescribeAgent(RunningAgent agent, bool includeOutput, long drainDeadline) + { + try + { + var description = agent.Role + + "(pid=" + agent.Process.Id.ToString(CultureInfo.InvariantCulture) + + ",state=" + (agent.Process.HasExited + ? "exited:" + agent.Process.ExitCode.ToString(CultureInfo.InvariantCulture) + : "running"); + if (includeOutput) + { + description += ",stdout=" + ReadCompletedOutput(agent.StandardOutput, drainDeadline) + + ",stderr=" + ReadCompletedOutput(agent.StandardError, drainDeadline); + } + + return description + ")"; + } + catch (Exception exception) + { + return agent.Role + "(diagnostic-failed:" + exception.GetType().Name + ":" + exception.Message + ")"; + } + } + + private static string ReadCompletedOutput(Task output, long drainDeadline) + { + try + { + var remaining = drainDeadline - Stopwatch.GetTimestamp(); + if (remaining <= 0) + { + return ""; + } + + var remainingMilliseconds = (int)Math.Min( + int.MaxValue, + Math.Max(1, Math.Ceiling(remaining * 1000d / Stopwatch.Frequency))); + return output.Wait(remainingMilliseconds) + ? (output.Result.Length == 0 ? "" : output.Result.Trim()) + : ""; + } + catch (Exception exception) + { + return ""; + } + } + + private static List StopAgents(IEnumerable agents) + { + var owned = agents.ToArray(); + var failures = new List(); + foreach (var agent in owned) + { + try + { + if (!agent.Process.HasExited) + { + agent.Process.Kill(entireProcessTree: true); + } + } + catch (Exception exception) + { + failures.Add(agent.Role + ": kill " + exception.GetType().Name + ": " + exception.Message); + } + } + + var stopDeadline = Stopwatch.GetTimestamp() + 5L * Stopwatch.Frequency; + foreach (var agent in owned) + { + try + { + var remaining = stopDeadline - Stopwatch.GetTimestamp(); + var remainingMilliseconds = remaining <= 0 + ? 0 + : (int)Math.Min( + int.MaxValue, + Math.Max(1, Math.Ceiling(remaining * 1000d / Stopwatch.Frequency))); + if (!agent.Process.WaitForExit(remainingMilliseconds)) + { + failures.Add(agent.Role + ": did not exit within the shared 5000 ms stop deadline"); + } + } + catch (Exception exception) + { + failures.Add(agent.Role + ": wait " + exception.GetType().Name + ": " + exception.Message); + } + } + + return failures; + } + + private static List DisposeAgents(IEnumerable agents) + { + var failures = new List(); + foreach (var agent in agents) + { + try + { + if (!agent.Process.HasExited) + { + failures.Add(agent.Role + ": process still running when its diagnostic handle was disposed"); + } + + agent.Process.Dispose(); + } + catch (Exception exception) + { + failures.Add(agent.Role + ": dispose " + exception.GetType().Name + ": " + exception.Message); + } + } + + return failures; + } + + private sealed record RunningAgent( + string Role, + Process Process, + Task StandardOutput, + Task StandardError); + + private readonly record struct AgentResult( + string Role, + int ExitCode, + string StandardOutput, + string StandardError); + + private sealed class AtomicTestMapping + { + private AtomicTestMapping(string path) + { + Path = path; + } + + public string Path { get; } + + public static void Execute(Action action) + { + var mapping = Create(); + Exception? failure = null; + try + { + action(mapping); + } + catch (Exception exception) + { + failure = exception; + } + + var cleanupFailure = mapping.TryDelete(); + if (failure is not null) + { + if (cleanupFailure is not null) + { + throw new AggregateException(failure, cleanupFailure); + } + + ExceptionDispatchInfo.Capture(failure).Throw(); + } + + if (cleanupFailure is not null) + { + throw cleanupFailure; + } + } + + public static AtomicTestMapping Create() + { + var path = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + "sms-atomic-" + Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture) + ".map"); + var ownsCreatedFile = false; + try + { + using var stream = new FileStream( + path, + FileMode.CreateNew, + FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete); + ownsCreatedFile = true; + stream.SetLength(4096); + } + catch (Exception creationFailure) + { + if (ownsCreatedFile) + { + try + { + File.Delete(path); + } + catch (Exception cleanupFailure) + { + throw new AggregateException(creationFailure, cleanupFailure); + } + } + + ExceptionDispatchInfo.Capture(creationFailure).Throw(); + throw new UnreachableException(); + } + + return new AtomicTestMapping(path); + } + + public long ReadInt64(int byteOffset) + { + Span bytes = stackalloc byte[sizeof(long)]; + using var stream = new FileStream( + Path, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete); + stream.Position = byteOffset; + stream.ReadExactly(bytes); + return BitConverter.ToInt64(bytes); + } + + public string DescribeWords(params (string Name, int ByteOffset)[] words) + { + return string.Join( + ",", + words.Select(word => word.Name + "=" + ReadInt64(word.ByteOffset).ToString(CultureInfo.InvariantCulture))); + } + + private Exception? TryDelete() + { + Exception? lastFailure = null; + for (var attempt = 0; attempt < 5; attempt++) + { + try + { + File.Delete(Path); + return null; + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + lastFailure = exception; + if (attempt < 4) + { + Thread.Sleep(25 * (attempt + 1)); + } + } + } + + return new IOException("Unable to delete mapped atomic test file after five attempts: " + Path, lastFailure); + } + } +} diff --git a/tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj b/tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj index e28f143..60fbdd1 100644 --- a/tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj +++ b/tests/SharedMemoryStore.IntegrationTests/SharedMemoryStore.IntegrationTests.csproj @@ -5,6 +5,7 @@ enable enable false + true @@ -19,7 +20,12 @@ + + + diff --git a/tests/SharedMemoryStore.InteropTests/LockFreeLayoutRejectionTests.cs b/tests/SharedMemoryStore.InteropTests/LockFreeLayoutRejectionTests.cs new file mode 100644 index 0000000..8e4336a --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/LockFreeLayoutRejectionTests.cs @@ -0,0 +1,202 @@ +using System.Runtime.InteropServices; +using System.Text.Json; +using SharedMemoryStore.InteropTests.TestSupport; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.InteropTests; + +public sealed class LockFreeLayoutRejectionTests +{ + public static TheoryData V12OnlyRuntimes => new() + { + "cpp", + "python" + }; + + [Fact] + public void CompatibilityManifestSeparatesLayoutSupportFromHeaderRecognition() + { + using var document = JsonDocument.Parse( + File.ReadAllText(Path.Combine(RepositoryRoot(), "protocol", "compatibility.json"))); + var root = document.RootElement; + + Assert.Equal(2, root.GetProperty("schema_version").GetInt32()); + var layouts = root.GetProperty("shared_protocol").GetProperty("layouts"); + _ = AssertLayout(layouts, "1.2", major: 1, minor: 2, resourceProtocol: 1, magic: "SMS1"); + JsonElement v2 = AssertLayout( + layouts, + "2.0", + major: 2, + minor: 0, + resourceProtocol: 2, + magic: "SMS2"); + Assert.Equal(7UL, v2.GetProperty("required_features_mask").GetUInt64()); + Assert.Equal( + ["versioned_empty_spill_summary", "publication_intent", "pid_namespace_identity"], + v2.GetProperty("required_features") + .EnumerateArray() + .Select(static feature => feature.GetString() + ?? throw new InvalidDataException("A required feature name cannot be null.")) + .ToArray()); + + var distributions = root.GetProperty("distributions"); + var managed = FindDistribution(distributions, "NuGet"); + Assert.Equal("2.0.0", managed.GetProperty("version").GetString()); + AssertVersions(managed.GetProperty("creates_layouts"), "1.2", "2.0"); + AssertVersions(managed.GetProperty("reads_layouts"), "1.2", "2.0"); + AssertResourceProtocol(managed, "1.2", 1); + AssertResourceProtocol(managed, "2.0", 2); + + AssertV12OnlyDistribution(FindDistribution(distributions, "CMake"), "c_abi"); + AssertV12OnlyDistribution(FindDistribution(distributions, "Python"), "requires_c_abi"); + } + + [Theory] + [MemberData(nameof(V12OnlyRuntimes))] + [Trait("Category", "Integration")] + public async Task V12OnlyClientRejectsSms2BeforePayloadAndLeavesStoreUsable(string runtime) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var definition = AgentDefinition.Resolve(runtime); + if (!definition.IsAvailable()) + { + return; + } + + var name = $"sms-v2-reject-{runtime}-{Guid.NewGuid():N}"; + var options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 8, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + var oversizedV12Bytes = SharedMemoryStoreOptions.CalculateRequiredBytes( + slotCount: 64, + maxValueBytes: 1024, + maxDescriptorBytes: 32, + maxKeyBytes: 32, + leaseRecordCount: 64); + Assert.True(oversizedV12Bytes > options.TotalBytes); + + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(options, out var opened)); + Assert.NotNull(opened); + using var store = opened!; + + byte[] key = [0x71]; + byte[] payload = [0xA5, 0x5A, 0xC3]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, payload)); + + await using var agent = await AgentProcess.StartAsync(definition); + foreach (var (openMode, expectedCode, expectedName) in new[] + { + (openMode: (int)OpenMode.OpenExisting, expectedCode: 4, expectedName: "IncompatibleLayout"), + (openMode: (int)OpenMode.CreateOrOpen, expectedCode: 4, expectedName: "IncompatibleLayout"), + (openMode: (int)OpenMode.CreateNew, expectedCode: 1, expectedName: "AlreadyExists") + }) + { + var rejection = await agent.SendAsync( + "open", + InteropAssertions.OpenArguments( + $"rejected-{openMode}", + name, + openMode, + slotCount: 64, + maxValueBytes: 1024, + maxDescriptorBytes: 32, + maxKeyBytes: 32, + leaseRecordCount: 64)); + + InteropAssertions.Status(rejection, expectedCode, expectedName); + AssertPublishedValue(store, key, payload); + } + + var secondOptions = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 8, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.OpenExisting); + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(secondOptions, out var second)); + using (second) + { + Assert.NotNull(second); + AssertPublishedValue(second!, key, payload); + } + } + + private static void AssertV12OnlyDistribution(JsonElement distribution, string abiProperty) + { + Assert.Equal("1.0", distribution.GetProperty(abiProperty).GetString()); + AssertVersions(distribution.GetProperty("creates_layouts"), "1.2"); + AssertVersions(distribution.GetProperty("reads_layouts"), "1.2"); + AssertVersions(distribution.GetProperty("recognizes_layout_headers"), "1.2", "2.0"); + AssertVersions(distribution.GetProperty("rejects_layouts"), "2.0"); + Assert.Equal("IncompatibleLayout", distribution.GetProperty("rejection_status").GetString()); + Assert.False(distribution.GetProperty("payload_access_before_rejection").GetBoolean()); + AssertResourceProtocol(distribution, "1.2", 1); + } + + private static void AssertPublishedValue(Store store, byte[] key, byte[] expected) + { + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out var lease)); + using (lease) + { + Assert.True(lease.ValueSpan.SequenceEqual(expected)); + } + } + + private static JsonElement AssertLayout( + JsonElement layouts, + string version, + int major, + int minor, + int resourceProtocol, + string magic) + { + var layout = layouts.EnumerateArray().Single( + item => item.GetProperty("version").GetString() == version); + Assert.Equal(major, layout.GetProperty("major").GetInt32()); + Assert.Equal(minor, layout.GetProperty("minor").GetInt32()); + Assert.Equal(resourceProtocol, layout.GetProperty("resource_protocol").GetInt32()); + Assert.Equal(magic, layout.GetProperty("magic").GetString()); + return layout; + } + + private static JsonElement FindDistribution(JsonElement distributions, string ecosystem) => + distributions.EnumerateArray().Single( + item => item.GetProperty("ecosystem").GetString() == ecosystem); + + private static void AssertResourceProtocol(JsonElement distribution, string layout, int expected) => + Assert.Equal( + expected, + distribution.GetProperty("layout_resource_protocols").GetProperty(layout).GetInt32()); + + private static void AssertVersions(JsonElement actual, params string[] expected) => + Assert.Equal(expected, actual.EnumerateArray().Select(static value => value.GetString()).ToArray()); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private static string RepositoryRoot() + { + var current = new DirectoryInfo(AppContext.BaseDirectory); + while (current is not null && !File.Exists(Path.Combine(current.FullName, "SharedMemoryStore.slnx"))) + { + current = current.Parent; + } + + return current?.FullName + ?? throw new DirectoryNotFoundException("Could not locate the SharedMemoryStore repository root."); + } +} diff --git a/tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs b/tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs index 8accae2..fbf19d7 100644 --- a/tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs +++ b/tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs @@ -236,7 +236,7 @@ public async Task ThreeLinuxOwnersCleanOnlyAfterFinalClose() InteropAssertions.Success(await python.SendAsync("close", new { storeId = "python" })); Assert.False(File.Exists(regionPath)); - Assert.False(File.Exists(synchronizationPath)); + Assert.True(File.Exists(synchronizationPath)); Assert.False(File.Exists(ownersPath)); Assert.True(File.Exists(lifecyclePath)); } diff --git a/tests/SharedMemoryStore.InteropTests/TestSupport/AgentProcess.cs b/tests/SharedMemoryStore.InteropTests/TestSupport/AgentProcess.cs index f7b4ced..52fb0bc 100644 --- a/tests/SharedMemoryStore.InteropTests/TestSupport/AgentProcess.cs +++ b/tests/SharedMemoryStore.InteropTests/TestSupport/AgentProcess.cs @@ -83,6 +83,7 @@ private static string RepositoryRoot() internal sealed class AgentProcess : IAsyncDisposable { + private static readonly TimeSpan StartupResponseTimeout = TimeSpan.FromSeconds(30); private readonly AgentDefinition _definition; private readonly Process _process; private readonly Task _stderr; @@ -123,14 +124,25 @@ public static async Task StartAsync(AgentDefinition definition) var process = Process.Start(startInfo) ?? throw new InvalidOperationException($"Could not start the {definition.Runtime} interoperability agent."); var result = new AgentProcess(definition, process); - var ping = await result.SendAsync("ping", arguments: null).ConfigureAwait(false); - if (!ping.Ok || ping.Status.Name != "Success") + try + { + var ping = await result.SendAsync( + "ping", + arguments: null, + StartupResponseTimeout).ConfigureAwait(false); + if (!ping.Ok || ping.Status.Name != "Success") + { + throw new InvalidOperationException( + $"The {definition.Runtime} agent did not answer ping successfully."); + } + + return result; + } + catch { await result.DisposeAsync().ConfigureAwait(false); - throw new InvalidOperationException($"The {definition.Runtime} agent did not answer ping successfully."); + throw; } - - return result; } public async Task SendAsync(string command, T? arguments, TimeSpan? timeout = null) diff --git a/tests/SharedMemoryStore.LinearizabilityTests/AcquireHistoryTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/AcquireHistoryTests.cs new file mode 100644 index 0000000..85b93b5 --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/AcquireHistoryTests.cs @@ -0,0 +1,153 @@ +using System.Runtime.InteropServices; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.LinearizabilityTests; + +public sealed class AcquireHistoryTests +{ + [Fact] + [Trait("Category", "Linearizability")] + public void OverlappingCommitAndAcquireMayLinearizeAsMissingBeforeCommit() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, 1, 3, 8, 9), + Operation(2, ReferenceCommand.Acquire(1, "key"), ReferenceResultCode.NotFound, 2, 4, 5, 6) + }; + + var result = Checker().Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([2, 1], result.Linearization); + } + + [Fact] + [Trait("Category", "Linearizability")] + public void OverlappingCommitAndAcquireMayLinearizeAsAcquireAfterCommit() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, 1, 3, 5, 8), + Operation(2, ReferenceCommand.Acquire(1, "key"), ReferenceResultCode.Success, 2, 4, 6, 7) + }; + + var result = Checker().Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([1, 2], result.Linearization); + } + + [Fact] + [Trait("Category", "Linearizability")] + public void AcquireCannotReturnMissingAfterCommitCompletedBeforeInvocation() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.Acquire(1, "key"), ReferenceResultCode.NotFound, 5, 6, 7, 8) + }; + + var result = Checker().Check(history); + + Assert.False(result.IsLinearizable); + } + + [Fact] + [Trait("Category", "Linearizability")] + public void PublicCommitThenAcquireHistoryMatchesReferenceModel() + { + RequireLeaseRegistry(); + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var store = CreateStore(); + var recorder = new MonotonicHistoryRecorder(); + var publish = recorder.Invoke(1, 1, ReferenceCommand.Publish(1, "key", "value")); + publish.Enter(); + publish.Complete(MapPublish(store.TryPublish([0x41], [0x51]))); + + var acquire = recorder.Invoke(2, 1, ReferenceCommand.Acquire(1, "key")); + acquire.Enter(); + var acquireStatus = store.TryAcquire([0x41], out var lease); + acquire.Complete(MapAcquire(acquireStatus)); + + var result = new LinearizabilityChecker( + participantCapacity: 2, + valueCapacity: 2, + initialParticipants: [1]).Check(recorder.Snapshot()); + try + { + Assert.Equal(StoreStatus.Success, acquireStatus); + Assert.True(lease.IsValid); + Assert.Equal(0x51, lease.ValueSpan[0]); + Assert.True(result.IsLinearizable, result.Failure); + } + finally + { + if (lease.IsValid) + { + Assert.Equal(StoreStatus.Success, lease.Release()); + } + } + } + + private static LinearizabilityChecker Checker() => + new(participantCapacity: 2, valueCapacity: 2, initialParticipants: [1]); + + private static RecordedOperation Operation( + int id, + ReferenceCommand command, + ReferenceResultCode result, + long invocation, + long entry, + long returned, + long response) => + new(id, id, command, result, invocation, entry, returned, response); + + private static ReferenceResultCode MapPublish(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.DuplicateKey => ReferenceResultCode.DuplicateKey, + StoreStatus.StoreFull => ReferenceResultCode.StoreFull, + _ => ReferenceResultCode.Unexpected + }; + + private static ReferenceResultCode MapAcquire(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.NotFound => ReferenceResultCode.NotFound, + _ => ReferenceResultCode.Unexpected + }; + + private static void RequireLeaseRegistry() + { + Assert.True( + typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeLeaseRegistry", + throwOnError: false, + ignoreCase: false) is not null, + "Acquire histories require the missing LockFreeLeaseRegistry implementation."); + } + + private static Store CreateStore() + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-acquire-history-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + var status = Store.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/CheckerSelfTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/CheckerSelfTests.cs new file mode 100644 index 0000000..b494c3b --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/CheckerSelfTests.cs @@ -0,0 +1,943 @@ +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.LinearizabilityTests; + +public sealed class CheckerSelfTests +{ + [Fact] + public void RecorderKeepsInvocationEntryReturnAndResponseDistinctAndMonotonic() + { + var recorder = new MonotonicHistoryRecorder(); + var pending = recorder.Invoke(7, 3, ReferenceCommand.Publish(1, "key", "value")); + + pending.Enter(); + var operation = pending.Complete(ReferenceResultCode.Success); + + Assert.True(operation.HasValidCallEnvelope); + Assert.Equal(7, operation.Id); + Assert.Equal(3, operation.ActorId); + Assert.Equal([operation], recorder.Snapshot()); + } + + [Fact] + public void RecorderDistinguishesRealTimePrecedenceFromOverlap() + { + var recorder = new MonotonicHistoryRecorder(); + var first = recorder.Invoke(1, 1, ReferenceCommand.Publish(1, "a", "1")); + first.Enter(); + var second = recorder.Invoke(2, 2, ReferenceCommand.Publish(1, "b", "2")); + second.Enter(); + var secondCompleted = second.Complete(ReferenceResultCode.Success); + var firstCompleted = first.Complete(ReferenceResultCode.Success); + var third = recorder.Invoke(3, 3, ReferenceCommand.Publish(1, "c", "3")); + third.Enter(); + var thirdCompleted = third.Complete(ReferenceResultCode.Success); + + Assert.True(firstCompleted.Overlaps(secondCompleted)); + Assert.True(secondCompleted.Overlaps(firstCompleted)); + Assert.True(firstCompleted.HappensBefore(thirdCompleted)); + Assert.True(secondCompleted.HappensBefore(thirdCompleted)); + Assert.False(thirdCompleted.Overlaps(firstCompleted)); + } + + [Fact] + public void CheckerAcceptsOneWinnerForOverlappingSameKeyPublications() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "same", "left"), ReferenceResultCode.DuplicateKey, 1, 3, 8, 9), + Operation(2, ReferenceCommand.Publish(1, "same", "right"), ReferenceResultCode.Success, 2, 4, 5, 6) + }; + + var result = Checker(valueCapacity: 2).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([2, 1], result.Linearization); + } + + [Fact] + public void CheckerRejectsDuplicateWhenOverlappingExplicitReserveNeverOrders() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Reserve(1, 10, "same", "owner"), + ReferenceResultCode.StoreBusy, + 1, + 2, + 7, + 8), + Operation( + 2, + ReferenceCommand.Reserve(1, 20, "same", "contender"), + ReferenceResultCode.DuplicateKey, + 3, + 4, + 5, + 6) + }; + + var result = Checker(valueCapacity: 2).Check(history); + + Assert.False(result.IsLinearizable); + Assert.NotNull(result.Failure); + } + + [Fact] + public void CheckerAcceptsDuplicateOnlyWhenOverlappingExplicitReserveOrders() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Reserve(1, 10, "same", "owner"), + ReferenceResultCode.Success, + 1, + 2, + 7, + 8), + Operation( + 2, + ReferenceCommand.Reserve(1, 20, "same", "contender"), + ReferenceResultCode.DuplicateKey, + 3, + 4, + 5, + 6) + }; + + var result = Checker(valueCapacity: 2).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([1, 2], result.Linearization); + } + + [Fact] + public void CheckerAcceptsRetryWhenOverlappingExplicitReserveNeverOrders() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Reserve(1, 10, "same", "owner"), + ReferenceResultCode.StoreBusy, + 1, + 2, + 7, + 8), + Operation( + 2, + ReferenceCommand.Reserve(1, 20, "same", "contender"), + ReferenceResultCode.StoreBusy, + 3, + 4, + 5, + 6) + }; + + var result = Checker(valueCapacity: 2).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + } + + [Fact] + public void CheckerRejectsDuplicateWhenOverlappingConveniencePublishNeverPublishes() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Publish(1, "same", "owner"), + ReferenceResultCode.StoreBusy, + 1, + 2, + 7, + 8), + Operation( + 2, + ReferenceCommand.Reserve(1, 20, "same", "contender"), + ReferenceResultCode.DuplicateKey, + 3, + 4, + 5, + 6) + }; + + var result = Checker(valueCapacity: 2).Check(history); + + Assert.False(result.IsLinearizable); + Assert.NotNull(result.Failure); + } + + [Fact] + public void CheckerAcceptsDuplicateAgainstExplicitReservedKey() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Reserve(1, 10, "same", "owner"), + ReferenceResultCode.Success, + 1, + 2, + 3, + 4), + Operation( + 2, + ReferenceCommand.Publish(1, "same", "contender"), + ReferenceResultCode.DuplicateKey, + 5, + 6, + 7, + 8) + }; + + var result = Checker(valueCapacity: 2).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([1, 2], result.Linearization); + } + + [Fact] + public void CheckerRejectsTwoSuccessfulCurrentGenerationsForOneKey() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "same", "left"), ReferenceResultCode.Success, 1, 3, 8, 9), + Operation(2, ReferenceCommand.Publish(1, "same", "right"), ReferenceResultCode.Success, 2, 4, 5, 6) + }; + + var result = Checker(valueCapacity: 2).Check(history); + + Assert.False(result.IsLinearizable); + Assert.NotNull(result.Failure); + } + + [Fact] + public void CheckerPreservesCompletedBeforeInvokedRealTimeOrder() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "same", "first"), ReferenceResultCode.DuplicateKey, 1, 2, 3, 4), + Operation(2, ReferenceCommand.Publish(1, "same", "second"), ReferenceResultCode.Success, 5, 6, 7, 8) + }; + + var result = Checker(valueCapacity: 2).Check(history); + + Assert.False(result.IsLinearizable); + } + + [Fact] + public void ModelTracksValueCapacityAndRestoresItAfterRemoval() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "a", "1"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.Publish(1, "b", "2"), ReferenceResultCode.StoreFull, 5, 6, 7, 8), + Operation(3, ReferenceCommand.Remove(1, "a"), ReferenceResultCode.Success, 9, 10, 11, 12), + Operation(4, ReferenceCommand.Publish(1, "b", "2"), ReferenceResultCode.Success, 13, 14, 15, 16) + }; + + var result = Checker(valueCapacity: 1).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([1, 2, 3, 4], result.Linearization); + } + + [Fact] + public void ModelTracksParticipantCapacityCloseAndReuse() + { + var history = new[] + { + Operation(1, ReferenceCommand.OpenParticipant(1), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.OpenParticipant(2), ReferenceResultCode.ParticipantTableFull, 5, 6, 7, 8), + Operation(3, ReferenceCommand.CloseParticipant(1), ReferenceResultCode.Success, 9, 10, 11, 12), + Operation(4, ReferenceCommand.OpenParticipant(2), ReferenceResultCode.Success, 13, 14, 15, 16), + Operation(5, ReferenceCommand.Publish(1, "stale", "x"), ReferenceResultCode.ParticipantNotActive, 17, 18, 19, 20) + }; + var checker = new LinearizabilityChecker(participantCapacity: 1, valueCapacity: 1); + + var result = checker.Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + } + + [Fact] + public void InvalidReservationFromReserveWithoutRecoveryEvidenceIsRejected() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Reserve(1, 10, "same", "tentative"), + ReferenceResultCode.InvalidReservation, + 1, + 3, + 6, + 8), + Operation( + 2, + ReferenceCommand.Publish(1, "same", "winner"), + ReferenceResultCode.Success, + 2, + 4, + 5, + 7) + }; + + var result = Checker(valueCapacity: 1).Check(history); + + Assert.False(result.IsLinearizable); + Assert.NotNull(result.Failure); + } + + [Fact] + public void TentativeReservationInvalidatedBeforeOrderingCannotExplainDuplicateKey() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Reserve(1, 10, "same", "tentative"), + ReferenceResultCode.InvalidReservation, + 1, + 3, + 6, + 8), + Operation( + 2, + ReferenceCommand.Publish(1, "same", "candidate"), + ReferenceResultCode.DuplicateKey, + 2, + 4, + 5, + 7) + }; + + var result = Checker(valueCapacity: 1).Check(history); + + Assert.False(result.IsLinearizable); + } + + [Fact] + public void RecoveryAfterReserveOrderingDoesNotRewriteSuccessfulReserve() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Reserve(1, 10, "key", "value"), + ReferenceResultCode.Success, + 1, + 2, + 7, + 8), + Operation( + 2, + ReferenceCommand.RecoverReservation(2, 10), + ReferenceResultCode.Success, + 3, + 4, + 5, + 6), + Operation( + 3, + ReferenceCommand.CommitReservation(1, 10), + ReferenceResultCode.InvalidReservation, + 9, + 10, + 11, + 12) + }; + + var result = new LinearizabilityChecker( + participantCapacity: 2, + valueCapacity: 1, + initialParticipants: [1, 2]).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([1, 2, 3], result.Linearization); + } + + [Fact] + public void CheckerRejectsSuccessfulAcquireWithWrongReturnedBytes() + { + RecordedOperation[] history = + [ + Operation( + 1, + ReferenceCommand.Publish(1, "key", "61"), + ReferenceResultCode.Success, + 1, + 2, + 3, + 4), + Operation( + 2, + ReferenceCommand.AcquireLease(1, 20, "key"), + ReferenceResultCode.Success, + 5, + 6, + 7, + 8, + observedValue: "62", + observedGeneration: 7, + requiresAcquireObservation: true) + ]; + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check(history); + + Assert.False(result.IsLinearizable); + } + + [Fact] + public void CheckerRejectsTwoGenerationsForTheSamePublishedValue() + { + RecordedOperation[] history = + [ + Operation(1, ReferenceCommand.Publish(1, "key", "61"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation( + 2, + ReferenceCommand.AcquireLease(1, 20, "key"), + ReferenceResultCode.Success, + 5, + 6, + 7, + 8, + observedValue: "61", + observedGeneration: 7, + requiresAcquireObservation: true), + Operation(3, ReferenceCommand.ReleaseLease(1, 20), ReferenceResultCode.Success, 9, 10, 11, 12), + Operation( + 4, + ReferenceCommand.AcquireLease(1, 21, "key"), + ReferenceResultCode.Success, + 13, + 14, + 15, + 16, + observedValue: "61", + observedGeneration: 8, + requiresAcquireObservation: true) + ]; + var checker = new LinearizabilityChecker(2, 1, [1], leaseCapacity: 1); + + LinearizabilityCheckResult result = checker.Check(history); + + Assert.False(result.IsLinearizable); + } + + [Fact] + public void RepublishedValueMayBindANewMappedGeneration() + { + RecordedOperation[] history = + [ + Operation(1, ReferenceCommand.Publish(1, "key", "61"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation( + 2, + ReferenceCommand.AcquireLease(1, 20, "key"), + ReferenceResultCode.Success, + 5, + 6, + 7, + 8, + observedValue: "61", + observedGeneration: 7, + requiresAcquireObservation: true), + Operation(3, ReferenceCommand.ReleaseLease(1, 20), ReferenceResultCode.Success, 9, 10, 11, 12), + Operation(4, ReferenceCommand.Remove(1, "key"), ReferenceResultCode.Success, 13, 14, 15, 16), + Operation(5, ReferenceCommand.Publish(1, "key", "62"), ReferenceResultCode.Success, 17, 18, 19, 20), + Operation( + 6, + ReferenceCommand.AcquireLease(1, 21, "key"), + ReferenceResultCode.Success, + 21, + 22, + 23, + 24, + observedValue: "62", + observedGeneration: 8, + requiresAcquireObservation: true) + ]; + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + } + + [Fact] + public void ProductionRaceRejectsInvocationOnlyOverlap() + { + RecordedOperation first = Operation( + 1, + ReferenceCommand.Publish(1, "key", "61"), + ReferenceResultCode.Success, + invocation: 1, + entry: 3, + returned: 4, + response: 9); + RecordedOperation second = Operation( + 2, + ReferenceCommand.Publish(1, "key", "62"), + ReferenceResultCode.DuplicateKey, + invocation: 2, + entry: 5, + returned: 6, + response: 8); + + Assert.False(first.HappensBefore(second)); + Assert.False(second.HappensBefore(first)); + Assert.True(first.Overlaps(second)); + Assert.False(first.ImplementationOverlaps(second)); + Assert.Throws(() => + ProductionGeneratedHistoryTests.AssertProductionRaceOverlap(first, second)); + } + + [Fact] + public void CheckerRejectsStoreBusyFromInfiniteWaitProductionCall() + { + RecordedOperation operation = Operation( + 1, + ReferenceCommand.Publish(1, "key", "61"), + ReferenceResultCode.StoreBusy, + 1, + 2, + 3, + 4, + usesInfiniteWait: true); + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check([operation]); + + Assert.False(result.IsLinearizable); + Assert.Contains("infinite-wait", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void CheckerRejectsUnexpectedProductionResult() + { + RecordedOperation operation = Operation( + 1, + ReferenceCommand.Publish(1, "key", "61"), + ReferenceResultCode.Unexpected, + 1, + 2, + 3, + 4, + usesInfiniteWait: true); + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check([operation]); + + Assert.False(result.IsLinearizable); + Assert.Contains("unexpected production result", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void CheckerRejectsMalformedCallEnvelopeBeforeSearching() + { + var malformed = Operation( + 1, + ReferenceCommand.Publish(1, "key", "value"), + ReferenceResultCode.Success, + invocation: 2, + entry: 1, + returned: 3, + response: 4); + + var result = Checker(valueCapacity: 1).Check([malformed]); + + Assert.False(result.IsLinearizable); + Assert.Contains("invocation < entry", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void PhysicalClaimCanExplainStoreFullWithoutCreatingAbstractKeyOwnership() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Publish(1, "tentative", "value"), + ReferenceResultCode.OperationCanceled, + 1, + 2, + 14, + 15), + Operation( + 2, + ReferenceCommand.Publish(1, "contender", "value"), + ReferenceResultCode.StoreFull, + 4, + 5, + 8, + 9), + Operation( + 3, + ReferenceCommand.Publish(1, "contender", "value"), + ReferenceResultCode.Success, + 16, + 17, + 19, + 20) + }; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.Claim, 0, 1, 3), + new(RecordedSlotResourceKind.StoreFullProof, -1, 1, 6, 7), + new(RecordedSlotResourceKind.Free, 0, 1, 10), + new(RecordedSlotResourceKind.Claim, 0, 2, 18) + ]; + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check(history, resources); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([1, 2, 3], result.Linearization); + } + + [Fact] + public void StoreFullWitnessOutsideImplementationIntervalIsRejected() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Publish(1, "winner", "value"), + ReferenceResultCode.Success, + 1, + 2, + 9, + 10), + Operation( + 2, + ReferenceCommand.Publish(1, "contender", "value"), + ReferenceResultCode.StoreFull, + 3, + 4, + 6, + 7) + }; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.StoreFullProof, -1, 1, 11, 12) + ]; + LinearizabilityChecker checker = Checker(valueCapacity: 1); + + Assert.True(checker.Check(history).IsLinearizable); + LinearizabilityCheckResult result = checker.Check(history, resources); + + Assert.False(result.IsLinearizable); + Assert.Contains("without its own exact double-collect proof", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void StoreFullCandidateInsideCallButConfirmationAfterReturnIsRejected() + { + RecordedOperation[] history = + [ + Operation( + 1, + ReferenceCommand.Publish(1, "contender", "value"), + ReferenceResultCode.StoreFull, + 1, + 2, + 5, + 8) + ]; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.StoreFullProof, -1, 1, 3, 6) + ]; + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check(history, resources); + + Assert.False(result.IsLinearizable); + Assert.Contains("without its own exact double-collect proof", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void DelayedClaimWitnessAloneCannotJustifyStoreFull() + { + RecordedOperation[] history = + [ + Operation( + 1, + ReferenceCommand.Publish(1, "tentative", "value"), + ReferenceResultCode.OperationCanceled, + 1, + 2, + 9, + 10), + Operation( + 2, + ReferenceCommand.Publish(1, "contender", "value"), + ReferenceResultCode.StoreFull, + 3, + 4, + 7, + 8) + ]; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.Claim, 0, 1, 5) + ]; + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check(history, resources); + + Assert.False(result.IsLinearizable); + Assert.Contains("without its own exact double-collect proof", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void OneStoreFullProofCannotJustifyTwoOverlappingCalls() + { + RecordedOperation[] history = + [ + Operation( + 1, + ReferenceCommand.Publish(1, "first", "value"), + ReferenceResultCode.StoreFull, + 1, + 2, + 7, + 8), + Operation( + 2, + ReferenceCommand.Publish(1, "second", "value"), + ReferenceResultCode.StoreFull, + 3, + 4, + 9, + 10) + ]; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.StoreFullProof, -1, 1, 5, 6) + ]; + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check(history, resources); + + Assert.False(result.IsLinearizable); + Assert.Contains("without its own exact double-collect proof", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void TerminalRetireRemainsAFullPhysicalSlotWitness() + { + var history = new[] + { + Operation( + 1, + ReferenceCommand.Publish(1, "contender", "value"), + ReferenceResultCode.StoreFull, + 3, + 4, + 7, + 8) + }; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.Claim, 0, LockFreeSlotTable.TerminalGeneration, 1), + new(RecordedSlotResourceKind.Retire, 0, LockFreeSlotTable.TerminalGeneration, 2), + new(RecordedSlotResourceKind.StoreFullProof, -1, 1, 5, 6) + ]; + + LinearizabilityCheckResult result = Checker(valueCapacity: 1).Check(history, resources); + + Assert.True(result.IsLinearizable, result.Failure); + } + + [Fact] + public void LeaseTableFullRequiresItsOwnExactProofInStrictHistory() + { + RecordedOperation[] history = + [ + Operation( + 1, + ReferenceCommand.Publish(1, "key", "value"), + ReferenceResultCode.Success, + 1, + 2, + 3, + 4), + Operation( + 2, + ReferenceCommand.AcquireLease(1, 20, "key"), + ReferenceResultCode.LeaseTableFull, + 5, + 6, + 9, + 10) + ]; + var checker = new LinearizabilityChecker( + participantCapacity: 2, + valueCapacity: 1, + initialParticipants: [1], + leaseCapacity: 1); + + LinearizabilityCheckResult missing = checker.Check(history, []); + LinearizabilityCheckResult witnessed = checker.Check( + history, + [new(RecordedSlotResourceKind.LeaseTableFullProof, -1, 1, 7, 8)]); + + Assert.False(missing.IsLinearizable); + Assert.Contains("without its own exact double-collect proof", missing.Failure, StringComparison.Ordinal); + Assert.True(witnessed.IsLinearizable, witnessed.Failure); + } + + [Fact] + public void LeaseTableFullProofOutsideImplementationIntervalIsRejected() + { + RecordedOperation[] history = + [ + Operation(1, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.LeaseTableFull, 5, 6, 9, 10) + ]; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.LeaseTableFullProof, -1, 1, 11, 12) + ]; + var checker = new LinearizabilityChecker(2, 1, [1], leaseCapacity: 1); + + LinearizabilityCheckResult result = checker.Check(history, resources); + + Assert.False(result.IsLinearizable); + Assert.Contains("without its own exact double-collect proof", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void LeaseTableFullProofWithWrongConfiguredCapacityIsRejected() + { + RecordedOperation[] history = + [ + Operation(1, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.LeaseTableFull, 5, 6, 9, 10) + ]; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.LeaseTableFullProof, -1, 2, 7, 8) + ]; + var checker = new LinearizabilityChecker(2, 1, [1], leaseCapacity: 1); + + LinearizabilityCheckResult result = checker.Check(history, resources); + + Assert.False(result.IsLinearizable); + Assert.Contains("Invalid LeaseTableFull proof identity", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void OneLeaseTableFullProofCannotJustifyTwoOverlappingCalls() + { + RecordedOperation[] history = + [ + Operation(1, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.LeaseTableFull, 5, 6, 11, 12), + Operation(3, ReferenceCommand.AcquireLease(1, 21, "key"), ReferenceResultCode.LeaseTableFull, 7, 8, 13, 14) + ]; + RecordedSlotResourceWitness[] resources = + [ + new(RecordedSlotResourceKind.LeaseTableFullProof, -1, 1, 9, 10) + ]; + var checker = new LinearizabilityChecker(2, 1, [1], leaseCapacity: 1); + + LinearizabilityCheckResult result = checker.Check(history, resources); + + Assert.False(result.IsLinearizable); + Assert.Contains("without its own exact double-collect proof", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void CombinedReservationLeaseRemovalAndRecoveryLifecycleRestoresCapacity() + { + var history = new[] + { + Operation(1, ReferenceCommand.Reserve(1, 10, "key", "value"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.CommitReservation(1, 10), ReferenceResultCode.Success, 5, 6, 7, 8), + Operation(3, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.Success, 9, 10, 11, 12), + Operation(4, ReferenceCommand.Remove(1, "key"), ReferenceResultCode.RemovePending, 13, 14, 15, 16), + Operation(5, ReferenceCommand.RecoverLease(2, 20), ReferenceResultCode.Success, 17, 18, 19, 20), + Operation(6, ReferenceCommand.ReleaseLease(1, 20), ReferenceResultCode.InvalidLease, 21, 22, 23, 24), + Operation(7, ReferenceCommand.Publish(1, "next", "value"), ReferenceResultCode.Success, 25, 26, 27, 28) + }; + + var result = new LinearizabilityChecker( + participantCapacity: 2, + valueCapacity: 1, + initialParticipants: [1, 2], + leaseCapacity: 1).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([1, 2, 3, 4, 5, 6, 7], result.Linearization); + } + + [Fact] + public void DisposalAtomicallyInvalidatesOwnedReservationsAndLeasesInTheModel() + { + var history = new[] + { + Operation(1, ReferenceCommand.Reserve(1, 10, "reserved", "value"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.Publish(1, "published", "value"), ReferenceResultCode.Success, 5, 6, 7, 8), + Operation(3, ReferenceCommand.AcquireLease(1, 20, "published"), ReferenceResultCode.Success, 9, 10, 11, 12), + Operation(4, ReferenceCommand.Remove(2, "published"), ReferenceResultCode.RemovePending, 13, 14, 15, 16), + Operation(5, ReferenceCommand.DisposeParticipant(1), ReferenceResultCode.Success, 17, 18, 19, 20), + Operation(6, ReferenceCommand.CommitReservation(1, 10), ReferenceResultCode.ParticipantNotActive, 21, 22, 23, 24), + Operation(7, ReferenceCommand.Publish(2, "replacement", "value"), ReferenceResultCode.Success, 25, 26, 27, 28) + }; + + var result = new LinearizabilityChecker( + participantCapacity: 2, + valueCapacity: 2, + initialParticipants: [1, 2], + leaseCapacity: 1).Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + } + + [Fact] + public void FailingHistoryMinimizerIsDeterministicAndDropsIrrelevantOperations() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "irrelevant", "x"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.Remove(1, "irrelevant"), ReferenceResultCode.Success, 5, 6, 7, 8), + Operation(3, ReferenceCommand.Publish(1, "same", "left"), ReferenceResultCode.Success, 9, 11, 16, 17), + Operation(4, ReferenceCommand.Publish(1, "same", "right"), ReferenceResultCode.Success, 10, 12, 13, 14) + }; + LinearizabilityChecker checker = Checker(valueCapacity: 2); + + IReadOnlyList first = checker.MinimizeFailingHistory(history); + IReadOnlyList second = checker.MinimizeFailingHistory(history); + + Assert.Equal(first, second); + Assert.Equal([3, 4], first.Select(static operation => operation.Id)); + Assert.False(checker.Check(first).IsLinearizable); + } + + private static LinearizabilityChecker Checker(int valueCapacity) => + new(participantCapacity: 2, valueCapacity, initialParticipants: [1]); + + private static RecordedOperation Operation( + int id, + ReferenceCommand command, + ReferenceResultCode result, + long invocation, + long entry, + long returned, + long response, + string? observedValue = null, + long observedGeneration = 0, + bool requiresAcquireObservation = false, + bool usesInfiniteWait = false) => + new( + id, + id, + command, + result, + invocation, + entry, + returned, + response, + observedValue, + observedGeneration, + requiresAcquireObservation, + usesInfiniteWait); +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/LeaseCapacityHistoryTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/LeaseCapacityHistoryTests.cs new file mode 100644 index 0000000..3b82617 --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/LeaseCapacityHistoryTests.cs @@ -0,0 +1,278 @@ +using System.Runtime.InteropServices; +using SharedMemoryStore.LockFree; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.LinearizabilityTests; + +public sealed class LeaseCapacityHistoryTests +{ + private static readonly TimeSpan TestBound = TimeSpan.FromSeconds(10); + + [Fact] + [Trait("Category", "Linearizability")] + public void StableLeaseCapacityEmitsOneExactProofInsideReturningAcquire() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var recorder = new MonotonicHistoryRecorder(); + using Store store = CreateStore(recorder, recorder); + CompletePublish(recorder, 1, store, [0x11], "first"); + CompletePublish(recorder, 2, store, [0x12], "second"); + + ValueLease held = default; + try + { + PendingInvocation firstAcquire = recorder.Invoke( + 3, + 1, + ReferenceCommand.AcquireLease(1, 20, "first")); + firstAcquire.Enter(); + StoreStatus firstStatus = store.TryAcquire( + [0x11], + StoreWaitOptions.Infinite, + out held); + Assert.Equal(StoreStatus.Success, firstStatus); + firstAcquire.Complete(ReferenceResultCode.Success); + + PendingInvocation contender = recorder.Invoke( + 4, + 2, + ReferenceCommand.AcquireLease(1, 21, "second")); + contender.Enter(); + StoreStatus contenderStatus = store.TryAcquire( + [0x12], + StoreWaitOptions.Infinite, + out _); + RecordedOperation contenderOperation = contender.Complete( + MapAcquire(contenderStatus)); + + IReadOnlyList resources = + recorder.ResourceSnapshot(); + RecordedSlotResourceWitness proof = Assert.Single( + resources, + static witness => + witness.Kind == RecordedSlotResourceKind.LeaseTableFullProof); + Assert.Equal(StoreStatus.LeaseTableFull, contenderStatus); + Assert.Equal(1, proof.Generation); + Assert.InRange( + proof.Sequence, + contenderOperation.EntrySequence + 1, + contenderOperation.ReturnSequence - 1); + Assert.InRange( + proof.ConfirmationSequence, + proof.Sequence + 1, + contenderOperation.ReturnSequence - 1); + + LinearizabilityCheckResult result = new LinearizabilityChecker( + participantCapacity: 4, + valueCapacity: 2, + initialParticipants: [1], + leaseCapacity: 1).Check(recorder.Snapshot(), resources); + Assert.True(result.IsLinearizable, result.Failure); + } + finally + { + if (held.IsValid) + { + Assert.Equal(StoreStatus.Success, held.Release()); + } + } + } + + [Fact] + [Trait("Category", "Linearizability")] + public async Task LeaseMovementRejectsCandidateAndInfiniteAcquireRetriesToSuccess() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var recorder = new MonotonicHistoryRecorder(); + using var pause = new LeaseTableFullProofPause(recorder); + using Store store = CreateStore(recorder, pause); + CompletePublish(recorder, 1, store, [0x21], "key"); + + ValueLease held = default; + ValueLease replacement = default; + PendingInvocation firstAcquire = recorder.Invoke( + 2, + 1, + ReferenceCommand.AcquireLease(1, 20, "key")); + firstAcquire.Enter(); + StoreStatus firstStatus = store.TryAcquire( + [0x21], + StoreWaitOptions.Infinite, + out held); + Assert.Equal(StoreStatus.Success, firstStatus); + firstAcquire.Complete(ReferenceResultCode.Success); + + PendingInvocation contender = recorder.Invoke( + 3, + 2, + ReferenceCommand.AcquireLease(1, 21, "key")); + StoreStatus contenderStatus = default; + Task contenderTask = Task.Run(() => + { + contender.Enter(); + contenderStatus = store.TryAcquire( + [0x21], + StoreWaitOptions.Infinite, + out replacement); + contender.Complete(MapAcquire(contenderStatus)); + }); + + try + { + pause.WaitUntilReached(); + PendingInvocation release = recorder.Invoke( + 4, + 1, + ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + StoreStatus releaseStatus = held.Release(StoreWaitOptions.Infinite); + Assert.Equal(StoreStatus.Success, releaseStatus); + release.Complete(ReferenceResultCode.Success); + pause.Resume(); + await contenderTask.WaitAsync(TestBound); + + Assert.Equal(StoreStatus.Success, contenderStatus); + Assert.True(replacement.IsValid); + Assert.DoesNotContain( + recorder.ResourceSnapshot(), + static witness => + witness.Kind == RecordedSlotResourceKind.LeaseTableFullProof); + LinearizabilityCheckResult result = new LinearizabilityChecker( + participantCapacity: 4, + valueCapacity: 2, + initialParticipants: [1], + leaseCapacity: 1).Check( + recorder.Snapshot(), + recorder.ResourceSnapshot()); + Assert.True(result.IsLinearizable, result.Failure); + } + finally + { + pause.Resume(); + await contenderTask.WaitAsync(TestBound); + if (held.IsValid) + { + Assert.Equal(StoreStatus.Success, held.Release()); + } + + if (replacement.IsValid) + { + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + } + } + + private static Store CreateStore( + MonotonicHistoryRecorder recorder, + ILockFreeLeaseTableFullProofObserver leaseProofObserver) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-linearizable-lease-full-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 4, + openMode: OpenMode.CreateNew); + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented( + static _ => { }, + recorder.ObserveSlotResource, + recorder, + leaseProofObserver); + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + checkpoint, + out Store? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static void CompletePublish( + MonotonicHistoryRecorder recorder, + int operationId, + Store store, + byte[] key, + string modelKey) + { + PendingInvocation publish = recorder.Invoke( + operationId, + 1, + ReferenceCommand.Publish(1, modelKey, "value")); + publish.Enter(); + StoreStatus status = store.TryPublish(key, [0x31]); + Assert.Equal(StoreStatus.Success, status); + publish.Complete(ReferenceResultCode.Success); + } + + private static ReferenceResultCode MapAcquire(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.NotFound => ReferenceResultCode.NotFound, + StoreStatus.LeaseTableFull => ReferenceResultCode.LeaseTableFull, + StoreStatus.StoreBusy => ReferenceResultCode.StoreBusy, + StoreStatus.OperationCanceled => ReferenceResultCode.OperationCanceled, + _ => ReferenceResultCode.Unexpected + }; + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed class LeaseTableFullProofPause : + ILockFreeLeaseTableFullProofObserver, + IDisposable + { + private readonly ILockFreeLeaseTableFullProofObserver _inner; + private readonly ManualResetEventSlim _reached = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: false); + private int _paused; + + internal LeaseTableFullProofPause( + ILockFreeLeaseTableFullProofObserver inner) + { + _inner = inner; + } + + public long BeginCandidate(int leaseRecordCount) + { + long token = _inner.BeginCandidate(leaseRecordCount); + if (Interlocked.CompareExchange(ref _paused, 1, 0) == 0) + { + _reached.Set(); + if (!_resume.Wait(TestBound)) + { + throw new Xunit.Sdk.XunitException( + "Paused LeaseTableFull proof was not resumed within the test bound."); + } + } + + return token; + } + + public void CompleteCandidate(long token, bool confirmed) => + _inner.CompleteCandidate(token, confirmed); + + internal void WaitUntilReached() => Assert.True( + _reached.Wait(TestBound), + "The LeaseTableFull proof candidate was not reached."); + + internal void Resume() => _resume.Set(); + + public void Dispose() + { + _resume.Set(); + _reached.Dispose(); + _resume.Dispose(); + } + } +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/LinearizabilityChecker.cs b/tests/SharedMemoryStore.LinearizabilityTests/LinearizabilityChecker.cs new file mode 100644 index 0000000..e2a647b --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/LinearizabilityChecker.cs @@ -0,0 +1,513 @@ +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.LinearizabilityTests; + +internal sealed record LinearizabilityCheckResult( + bool IsLinearizable, + IReadOnlyList Linearization, + string? Failure); + +internal sealed class LinearizabilityChecker +{ + private readonly int _participantCapacity; + private readonly int _valueCapacity; + private readonly int _leaseCapacity; + private readonly int[] _initialParticipants; + + public LinearizabilityChecker( + int participantCapacity, + int valueCapacity, + IEnumerable? initialParticipants = null, + int? leaseCapacity = null) + { + _participantCapacity = participantCapacity; + _valueCapacity = valueCapacity; + _leaseCapacity = leaseCapacity ?? Math.Max(1, valueCapacity); + _initialParticipants = initialParticipants?.ToArray() ?? []; + } + + public LinearizabilityCheckResult Check(IReadOnlyList history) + { + return CheckCore( + history, + physicalStoreFullOperations: null, + physicalLeaseTableFullOperations: null); + } + + public LinearizabilityCheckResult Check( + IReadOnlyList history, + IReadOnlyList resourceWitnesses) + { + ArgumentNullException.ThrowIfNull(resourceWitnesses); + if (!TryValidateResourceWitnesses( + history, + resourceWitnesses, + out HashSet? physicalStoreFullOperations, + out HashSet? physicalLeaseTableFullOperations, + out string? failure)) + { + return Failure(failure!); + } + + return CheckCore( + history, + physicalStoreFullOperations, + physicalLeaseTableFullOperations); + } + + private LinearizabilityCheckResult CheckCore( + IReadOnlyList history, + IReadOnlySet? physicalStoreFullOperations, + IReadOnlySet? physicalLeaseTableFullOperations) + { + if (history.Count > 63) + { + return Failure("The bounded checker supports at most 63 operations per history."); + } + + if (history.Select(static operation => operation.Id).Distinct().Count() != history.Count) + { + return Failure("Every operation ID must be unique."); + } + + if (history.Any(static operation => !operation.HasValidCallEnvelope)) + { + return Failure("Every operation requires invocation < entry < return < response."); + } + + if (!TryValidateOperationMetadata(history, out string? metadataFailure)) + { + return Failure(metadataFailure!); + } + + var predecessorMasks = BuildPredecessorMasks(history); + var initial = new ReferenceStoreModel( + _participantCapacity, + _valueCapacity, + _initialParticipants, + _leaseCapacity); + var allMask = history.Count == 0 ? 0UL : (1UL << history.Count) - 1; + var order = new List(history.Count); + var rejectedStates = new HashSet(StringComparer.Ordinal); + if (Search( + history, + predecessorMasks, + allMask, + completedMask: 0, + initial, + order, + rejectedStates, + physicalStoreFullOperations, + physicalLeaseTableFullOperations)) + { + return new LinearizabilityCheckResult(true, order.ToArray(), null); + } + + return Failure("No sequential execution both preserves real-time precedence and explains every result."); + } + + /// + /// Deterministically removes irrelevant operations from a rejected history + /// while preserving at least one non-linearizable witness. + /// + public IReadOnlyList MinimizeFailingHistory( + IReadOnlyList history) + { + if (Check(history).IsLinearizable) + { + throw new ArgumentException("Only a failing history can be minimized.", nameof(history)); + } + + var minimized = history.ToList(); + var index = 0; + while (index < minimized.Count) + { + var candidate = minimized.Where((_, current) => current != index).ToArray(); + if (candidate.Length != 0 && !Check(candidate).IsLinearizable) + { + minimized = candidate.ToList(); + index = 0; + continue; + } + + index++; + } + + return minimized; + } + + public IReadOnlyList MinimizeFailingHistory( + IReadOnlyList history, + IReadOnlyList resourceWitnesses) + { + if (Check(history, resourceWitnesses).IsLinearizable) + { + throw new ArgumentException("Only a failing history can be minimized.", nameof(history)); + } + + var minimized = history.ToList(); + var index = 0; + while (index < minimized.Count) + { + var candidate = minimized.Where((_, current) => current != index).ToArray(); + if (candidate.Length != 0 && !Check(candidate, resourceWitnesses).IsLinearizable) + { + minimized = candidate.ToList(); + index = 0; + continue; + } + + index++; + } + + return minimized; + } + + private static bool Search( + IReadOnlyList history, + ulong[] predecessorMasks, + ulong allMask, + ulong completedMask, + ReferenceStoreModel model, + List order, + HashSet rejectedStates, + IReadOnlySet? physicalStoreFullOperations, + IReadOnlySet? physicalLeaseTableFullOperations) + { + if (completedMask == allMask) + { + return true; + } + + var stateKey = completedMask.ToString(System.Globalization.CultureInfo.InvariantCulture) + + ':' + + model.Fingerprint(); + if (!rejectedStates.Add(stateKey)) + { + return false; + } + + for (var index = 0; index < history.Count; index++) + { + var bit = 1UL << index; + if ((completedMask & bit) != 0 + || (predecessorMasks[index] & ~completedMask) != 0 + || !model.TryApply( + history[index].Command, + history[index].Result, + out var next, + physicalStoreFullOperations?.Contains(history[index].Id) == true, + physicalLeaseTableFullOperations?.Contains(history[index].Id) == true, + history[index].ObservedValue, + history[index].ObservedGeneration, + history[index].RequiresAcquireObservation)) + { + continue; + } + + order.Add(history[index].Id); + if (Search( + history, + predecessorMasks, + allMask, + completedMask | bit, + next!, + order, + rejectedStates, + physicalStoreFullOperations, + physicalLeaseTableFullOperations)) + { + return true; + } + + order.RemoveAt(order.Count - 1); + } + + return false; + } + + private static bool TryValidateOperationMetadata( + IReadOnlyList history, + out string? failure) + { + foreach (RecordedOperation operation in history) + { + if (operation.Result == ReferenceResultCode.Unexpected) + { + failure = $"Operation {operation.Id} returned an unexpected production result."; + return false; + } + + if (operation.UsesInfiniteWait + && operation.Result == ReferenceResultCode.StoreBusy) + { + failure = $"Operation {operation.Id} returned StoreBusy from an infinite-wait production call."; + return false; + } + + bool isAcquire = operation.Command.Kind is ReferenceOperationKind.Acquire + or ReferenceOperationKind.AcquireLease; + bool carriesObservation = operation.ObservedValue is not null + || operation.ObservedGeneration != 0; + if (operation.RequiresAcquireObservation && !isAcquire) + { + failure = $"Operation {operation.Id} requires acquire output metadata but is not an acquire."; + return false; + } + + if (isAcquire && operation.Result == ReferenceResultCode.Success) + { + if (operation.RequiresAcquireObservation + && (operation.ObservedValue is null + || operation.ObservedGeneration is < 1 + or > LockFreeSlotTable.TerminalGeneration)) + { + failure = $"Operation {operation.Id} is a successful acquire without exact returned bytes and generation."; + return false; + } + + if (carriesObservation + && (operation.ObservedValue is null + || operation.ObservedGeneration is < 1 + or > LockFreeSlotTable.TerminalGeneration)) + { + failure = $"Operation {operation.Id} carries a malformed acquire observation."; + return false; + } + } + else if (carriesObservation) + { + failure = $"Operation {operation.Id} carries acquire output metadata for a non-successful acquire."; + return false; + } + } + + failure = null; + return true; + } + + private bool TryValidateResourceWitnesses( + IReadOnlyList history, + IReadOnlyList resourceWitnesses, + out HashSet? physicalStoreFullOperations, + out HashSet? physicalLeaseTableFullOperations, + out string? failure) + { + physicalStoreFullOperations = null; + physicalLeaseTableFullOperations = null; + failure = null; + + var allSequences = new HashSet(); + foreach (RecordedOperation operation in history) + { + long[] envelope = + [ + operation.InvocationSequence, + operation.EntrySequence, + operation.ReturnSequence, + operation.ResponseSequence + ]; + if (envelope.Any(static sequence => sequence <= 0) + || envelope.Any(sequence => !allSequences.Add(sequence))) + { + failure = "Strict histories require positive, globally unique call and resource sequences."; + return false; + } + } + + RecordedSlotResourceWitness[] ordered = resourceWitnesses + .OrderBy(static witness => witness.Sequence) + .ToArray(); + foreach (RecordedSlotResourceWitness witness in ordered) + { + if (witness.Sequence <= 0 || !allSequences.Add(witness.Sequence)) + { + failure = "Strict histories require positive, globally unique call and resource sequences."; + return false; + } + + if (witness.Kind is RecordedSlotResourceKind.StoreFullProof + or RecordedSlotResourceKind.LeaseTableFullProof) + { + if (witness.ConfirmationSequence <= witness.Sequence + || !allSequences.Add(witness.ConfirmationSequence)) + { + failure = "Strict capacity proofs require a unique confirmation sequence after their candidate sequence."; + return false; + } + } + else if (witness.ConfirmationSequence != 0) + { + failure = "Only capacity-proof witnesses may carry a confirmation sequence."; + return false; + } + } + + var states = new SlotResourceState[_valueCapacity]; + var storeFullProofs = new List(); + var leaseTableFullProofs = new List(); + foreach (RecordedSlotResourceWitness witness in ordered) + { + if (witness.Kind == RecordedSlotResourceKind.StoreFullProof) + { + if (witness.SlotIndex != -1 || witness.Generation != _valueCapacity) + { + failure = $"Invalid StoreFull proof identity at sequence {witness.Sequence}."; + return false; + } + + storeFullProofs.Add(witness); + continue; + } + + if (witness.Kind == RecordedSlotResourceKind.LeaseTableFullProof) + { + if (witness.SlotIndex != -1 || witness.Generation != _leaseCapacity) + { + failure = $"Invalid LeaseTableFull proof identity at sequence {witness.Sequence}."; + return false; + } + + leaseTableFullProofs.Add(witness); + continue; + } + + if ((uint)witness.SlotIndex >= (uint)_valueCapacity + || witness.Generation is < 1 or > LockFreeSlotTable.TerminalGeneration) + { + failure = $"Invalid slot resource identity at sequence {witness.Sequence}."; + return false; + } + + ref SlotResourceState state = ref states[witness.SlotIndex]; + switch (witness.Kind) + { + case RecordedSlotResourceKind.Claim: + if (state.Kind != SlotResourceStateKind.Free) + { + failure = $"Slot {witness.SlotIndex} was claimed while already occupied or retired."; + return false; + } + + state = new SlotResourceState(SlotResourceStateKind.Claimed, witness.Generation); + break; + + case RecordedSlotResourceKind.Free: + if (state.Kind != SlotResourceStateKind.Claimed + || state.Generation != witness.Generation) + { + failure = $"Slot {witness.SlotIndex} was freed without its exact generation claim."; + return false; + } + + state = default; + break; + + case RecordedSlotResourceKind.Retire: + if (state.Kind != SlotResourceStateKind.Claimed + || state.Generation != witness.Generation + || witness.Generation != LockFreeSlotTable.TerminalGeneration) + { + failure = $"Slot {witness.SlotIndex} was retired without its exact generation claim."; + return false; + } + + state = new SlotResourceState(SlotResourceStateKind.Retired, witness.Generation); + break; + + default: + failure = $"Unknown slot resource event at sequence {witness.Sequence}."; + return false; + } + } + + physicalStoreFullOperations = []; + var unmatchedProofs = new List(storeFullProofs); + foreach (RecordedOperation operation in history + .Where(static operation => operation.Result == ReferenceResultCode.StoreFull) + .OrderBy(static operation => operation.ReturnSequence)) + { + if (operation.Command.Kind is not ( + ReferenceOperationKind.Publish or ReferenceOperationKind.Reserve)) + { + failure = $"Operation {operation.Id} reports StoreFull for a non-allocating command."; + return false; + } + + int proofIndex = unmatchedProofs.FindIndex(proof => + proof.Sequence > operation.EntrySequence + && proof.ConfirmationSequence > proof.Sequence + && proof.ConfirmationSequence < operation.ReturnSequence); + if (proofIndex < 0) + { + failure = $"Operation {operation.Id} reports StoreFull without its own exact double-collect proof between entry and return."; + return false; + } + + unmatchedProofs.RemoveAt(proofIndex); + physicalStoreFullOperations.Add(operation.Id); + } + + physicalLeaseTableFullOperations = []; + var unmatchedLeaseProofs = new List( + leaseTableFullProofs); + foreach (RecordedOperation operation in history + .Where(static operation => + operation.Result == ReferenceResultCode.LeaseTableFull) + .OrderBy(static operation => operation.ReturnSequence)) + { + if (operation.Command.Kind != ReferenceOperationKind.AcquireLease) + { + failure = $"Operation {operation.Id} reports LeaseTableFull for a non-acquiring command."; + return false; + } + + int proofIndex = unmatchedLeaseProofs.FindIndex(proof => + proof.Sequence > operation.EntrySequence + && proof.ConfirmationSequence > proof.Sequence + && proof.ConfirmationSequence < operation.ReturnSequence); + if (proofIndex < 0) + { + failure = $"Operation {operation.Id} reports LeaseTableFull without its own exact double-collect proof between entry and return."; + return false; + } + + unmatchedLeaseProofs.RemoveAt(proofIndex); + physicalLeaseTableFullOperations.Add(operation.Id); + } + + return true; + } + + private static ulong[] BuildPredecessorMasks(IReadOnlyList history) + { + var masks = new ulong[history.Count]; + for (var current = 0; current < history.Count; current++) + { + for (var candidate = 0; candidate < history.Count; candidate++) + { + if (candidate != current && history[candidate].HappensBefore(history[current])) + { + masks[current] |= 1UL << candidate; + } + } + } + + return masks; + } + + private static LinearizabilityCheckResult Failure(string failure) => + new(false, Array.Empty(), failure); + + private enum SlotResourceStateKind + { + Free, + Claimed, + Retired + } + + private readonly record struct SlotResourceState( + SlotResourceStateKind Kind, + long Generation); + +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/LockFreeHistoryTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/LockFreeHistoryTests.cs new file mode 100644 index 0000000..2a1b4ab --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/LockFreeHistoryTests.cs @@ -0,0 +1,262 @@ +namespace SharedMemoryStore.LinearizabilityTests; + +public sealed class LockFreeHistoryTests +{ + private const int DefaultSeed = 0x5eed_0200; + private readonly Xunit.Abstractions.ITestOutputHelper _output; + + public LockFreeHistoryTests(Xunit.Abstractions.ITestOutputHelper output) + { + _output = output; + } + + public static TheoryData RaceFamilies => new() + { + "publish-publish", + "publish-reserve", + "reserve-reserve", + "commit-acquire", + "acquire-remove", + "release-reclaim", + "recovery-live-action", + "disposal-operation", + "participant-capacity", + "value-capacity", + "lease-capacity", + "cancellation", + "stale-token" + }; + + [Theory] + [MemberData(nameof(RaceFamilies))] + [Trait("Category", "Linearizability")] + public void SeededRaceFamilyHasAReferenceLinearization(string family) + { + HistoryCase testCase = Create(family); + LinearizabilityCheckResult result = testCase.Checker.Check(testCase.History); + + Assert.True(result.IsLinearizable, $"{family}: {result.Failure}"); + Assert.Equal(testCase.History.Count, result.Linearization.Count); + } + + [Fact] + [Trait("Category", "LinearizabilityRandomized")] + public void ConfiguredSeedTierChecksEveryRaceFamilyAndRecordsReproducibleSeed() + { + int repetitions = ConfiguredRepetitions(); + int seed = ConfiguredSeed(); + var random = new Random(seed); + string[] families = + [ + "publish-publish", + "publish-reserve", + "reserve-reserve", + "commit-acquire", + "acquire-remove", + "release-reclaim", + "recovery-live-action", + "disposal-operation", + "participant-capacity", + "value-capacity", + "lease-capacity", + "cancellation", + "stale-token" + ]; + + for (var familyIndex = 0; familyIndex < families.Length; familyIndex++) + { + HistoryCase testCase = Create(families[familyIndex]); + int[] order = new int[testCase.History.Count]; + var checkedHistories = 0; + for (var repetition = 0; repetition < repetitions; repetition++) + { + // Input order is deliberately randomized. Real-time edges live + // in the envelopes, so the checker must not depend on array order. + for (var index = 0; index < order.Length; index++) + { + order[index] = index; + } + + for (var index = order.Length - 1; index > 0; index--) + { + int swap = random.Next(index + 1); + (order[index], order[swap]) = (order[swap], order[index]); + } + + var shuffled = new RecordedOperation[order.Length]; + for (var index = 0; index < order.Length; index++) + { + shuffled[index] = testCase.History[order[index]]; + } + + LinearizabilityCheckResult result = testCase.Checker.Check(shuffled); + Assert.True( + result.IsLinearizable, + $"family={families[familyIndex]}, repetition={repetition}, seed={seed}: {result.Failure}"); + checkedHistories++; + } + + Assert.Equal(repetitions, checkedHistories); + _output.WriteLine( + $"family={families[familyIndex]} seed={seed} " + + $"completedCheckerInvocations={checkedHistories} source=reference-model"); + } + } + + private static HistoryCase Create(string family) => family switch + { + "publish-publish" => Case( + valueCapacity: 2, + operations: + [ + Op(1, ReferenceCommand.Publish(1, "key", "left"), ReferenceResultCode.Success, 1, 3, 8, 9), + Op(2, ReferenceCommand.Publish(1, "key", "right"), ReferenceResultCode.DuplicateKey, 2, 4, 5, 6) + ]), + "publish-reserve" => Case( + valueCapacity: 2, + operations: + [ + Op(1, ReferenceCommand.Publish(1, "key", "left"), ReferenceResultCode.Success, 1, 3, 8, 9), + Op(2, ReferenceCommand.Reserve(1, 10, "key", "right"), ReferenceResultCode.DuplicateKey, 2, 4, 5, 6) + ]), + "reserve-reserve" => Case( + valueCapacity: 2, + operations: + [ + Op(1, ReferenceCommand.Reserve(1, 10, "key", "left"), ReferenceResultCode.Success, 1, 3, 8, 9), + Op(2, ReferenceCommand.Reserve(1, 11, "key", "right"), ReferenceResultCode.DuplicateKey, 2, 4, 5, 6) + ]), + "commit-acquire" => Case( + valueCapacity: 1, + operations: + [ + Setup(60, ReferenceCommand.Reserve(1, 10, "key", "value"), ReferenceResultCode.Success, -8), + Op(1, ReferenceCommand.CommitReservation(1, 10), ReferenceResultCode.Success, 1, 3, 5, 8), + Op(2, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.Success, 2, 4, 6, 7) + ]), + "acquire-remove" => Case( + valueCapacity: 1, + operations: + [ + Setup(60, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, -4), + Op(1, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.Success, 1, 3, 5, 8), + Op(2, ReferenceCommand.Remove(1, "key"), ReferenceResultCode.RemovePending, 2, 4, 6, 7) + ]), + "release-reclaim" => Case( + valueCapacity: 1, + operations: + [ + Setup(60, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, -12), + Setup(61, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.Success, -8), + Setup(62, ReferenceCommand.Remove(1, "key"), ReferenceResultCode.RemovePending, -4), + Op(1, ReferenceCommand.ReleaseLease(1, 20), ReferenceResultCode.Success, 1, 3, 5, 8), + Op(2, ReferenceCommand.Publish(1, "next", "value"), ReferenceResultCode.Success, 2, 4, 6, 7) + ]), + "recovery-live-action" => Case( + valueCapacity: 1, + operations: + [ + Setup(59, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, -8), + Setup(60, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.Success, -4), + Op(1, ReferenceCommand.RecoverLease(1, 20), ReferenceResultCode.InvalidLease, 1, 3, 5, 8), + Op(2, ReferenceCommand.ReleaseLease(1, 20), ReferenceResultCode.Success, 2, 4, 6, 7) + ]), + "disposal-operation" => Case( + valueCapacity: 1, + operations: + [ + Op(1, ReferenceCommand.DisposeParticipant(1), ReferenceResultCode.Success, 1, 3, 5, 8), + Op(2, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.ParticipantNotActive, 2, 4, 6, 7) + ]), + "participant-capacity" => Case( + participantCapacity: 1, + valueCapacity: 1, + operations: + [ + Op(1, ReferenceCommand.OpenParticipant(2), ReferenceResultCode.ParticipantTableFull, 1, 3, 5, 8), + Op(2, ReferenceCommand.CloseParticipant(1), ReferenceResultCode.Success, 2, 4, 6, 7) + ]), + "value-capacity" => Case( + valueCapacity: 1, + operations: + [ + Setup(60, ReferenceCommand.Publish(1, "first", "value"), ReferenceResultCode.Success, -4), + Op(1, ReferenceCommand.Publish(1, "second", "value"), ReferenceResultCode.StoreFull, 1, 3, 5, 8), + Op(2, ReferenceCommand.Remove(1, "first"), ReferenceResultCode.Success, 2, 4, 6, 7) + ]), + "lease-capacity" => Case( + valueCapacity: 2, + leaseCapacity: 1, + operations: + [ + Setup(58, ReferenceCommand.Publish(1, "first", "value"), ReferenceResultCode.Success, -12), + Setup(59, ReferenceCommand.Publish(1, "second", "value"), ReferenceResultCode.Success, -8), + Setup(60, ReferenceCommand.AcquireLease(1, 20, "first"), ReferenceResultCode.Success, -4), + Op(1, ReferenceCommand.AcquireLease(1, 21, "second"), ReferenceResultCode.LeaseTableFull, 1, 3, 5, 8), + Op(2, ReferenceCommand.ReleaseLease(1, 20), ReferenceResultCode.Success, 2, 4, 6, 7) + ]), + "cancellation" => Case( + valueCapacity: 1, + operations: + [ + Op(1, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.OperationCanceled, 1, 3, 5, 8), + Op(2, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, 2, 4, 6, 7) + ]), + "stale-token" => Case( + valueCapacity: 1, + operations: + [ + Setup(59, ReferenceCommand.Publish(1, "key", "value"), ReferenceResultCode.Success, -8), + Setup(60, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.Success, -4), + Op(1, ReferenceCommand.RecoverLease(2, 20), ReferenceResultCode.Success, 1, 3, 5, 8), + Op(2, ReferenceCommand.ReleaseLease(1, 20), ReferenceResultCode.InvalidLease, 2, 4, 6, 7) + ]), + _ => throw new ArgumentOutOfRangeException(nameof(family), family, null) + }; + + private static HistoryCase Case( + int valueCapacity, + IReadOnlyList operations, + int participantCapacity = 2, + int? leaseCapacity = null) => + new( + new LinearizabilityChecker( + participantCapacity, + valueCapacity, + initialParticipants: participantCapacity == 1 ? [1] : [1, 2], + leaseCapacity), + operations); + + private static RecordedOperation Setup( + int id, + ReferenceCommand command, + ReferenceResultCode result, + long invocation) => + Op(id, command, result, invocation, invocation + 1, invocation + 2, invocation + 3); + + private static RecordedOperation Op( + int id, + ReferenceCommand command, + ReferenceResultCode result, + long invocation, + long entry, + long returned, + long response) => + new(id, id, command, result, invocation, entry, returned, response); + + private static int ConfiguredRepetitions() + { + string? configured = Environment.GetEnvironmentVariable("SMS_CHECKER_HISTORY_REPETITIONS"); + return int.TryParse(configured, out int value) && value > 0 ? value : 256; + } + + private static int ConfiguredSeed() + { + string? configured = Environment.GetEnvironmentVariable("SMS_LINEARIZABILITY_SEED"); + return int.TryParse(configured, out int value) ? value : DefaultSeed; + } + + private sealed record HistoryCase( + LinearizabilityChecker Checker, + IReadOnlyList History); +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/ProductionGeneratedHistoryTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/ProductionGeneratedHistoryTests.cs new file mode 100644 index 0000000..a984580 --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/ProductionGeneratedHistoryTests.cs @@ -0,0 +1,1063 @@ +using System.Runtime.InteropServices; +using SharedMemoryStore.LockFree; +using Store = SharedMemoryStore.MemoryStore; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace SharedMemoryStore.LinearizabilityTests; + +/// +/// Captures bounded histories from public MemoryStore calls. These histories +/// complement the high-count outcome stress: they retain real invocation and +/// response envelopes and are checked, and minimized on failure, by the +/// reference-model checker. +/// +public sealed class ProductionGeneratedHistoryTests +{ + private const int DefaultSeed = 0x5eed_0200; + private static readonly byte[] Key = [0x51]; + private static readonly byte[] Left = [0x61]; + private static readonly byte[] Right = [0x62]; + private static readonly byte[] Next = [0x63]; + private const string KeyHex = "51"; + private const string LeftHex = "61"; + private const string RightHex = "62"; + private const string NextHex = "63"; + private readonly ITestOutputHelper _output; + + public ProductionGeneratedHistoryTests(ITestOutputHelper output) + { + _output = output; + } + + public static TheoryData RequiredFamilies => new() + { + "publish-publish", + "publish-reserve", + "reserve-reserve", + "commit-acquire", + "acquire-remove", + "release-reclaim", + "recovery-live-lease", + "disposal-operation" + }; + + [Theory] + [MemberData(nameof(RequiredFamilies))] + [Trait("Category", "ProductionGeneratedHistory")] + public void ConfiguredProductionHistoriesAreLinearizableAndFailureMinimizable(string family) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + int historyCount = ConfiguredHistoryCount(); + int seed = FamilySeed(ConfiguredSeed(), FamilyOrdinal(family)); + var random = new DeterministicRandom(seed); + for (var historyIndex = 0; historyIndex < historyCount; historyIndex++) + { + CapturedHistory captured; + var overlapAttempt = 0; + do + { + overlapAttempt++; + captured = family switch + { + "publish-publish" => CapturePublishPublish(random.NextDelay(), random.NextDelay()), + "publish-reserve" => CapturePublishReserve(random.NextDelay(), random.NextDelay()), + "reserve-reserve" => CaptureReserveReserve(random.NextDelay(), random.NextDelay()), + "commit-acquire" => CaptureCommitAcquire(random.NextDelay(), random.NextDelay()), + "acquire-remove" => CaptureAcquireRemove(random.NextDelay(), random.NextDelay()), + "release-reclaim" => CaptureReleaseReclaim(random.NextDelay(), random.NextDelay()), + "recovery-live-lease" => CaptureRecoveryLiveLease(random.NextDelay(), random.NextDelay()), + "disposal-operation" => CaptureDisposalOperation(random.NextDelay(), random.NextDelay()), + _ => throw new ArgumentOutOfRangeException(nameof(family), family, null) + }; + } + while (!captured.RaceOperations[0].ImplementationOverlaps(captured.RaceOperations[1]) + && overlapAttempt < 128); + + IReadOnlyList history = captured.History; + int actorCount = history.Select(static operation => operation.ActorId).Distinct().Count(); + if (history.Count is < 6 or > 12 || actorCount is < 2 or > 4) + { + throw new XunitException( + $"family={family}, history={historyIndex}, seed={seed}: " + + $"calls={history.Count}, actors={actorCount}; expected 6-12 calls and 2-4 actors."); + } + + AssertProductionRaceOverlap( + captured.RaceOperations[0], + captured.RaceOperations[1], + $"family={family}, history={historyIndex}, seed={seed}, attempts={overlapAttempt}"); + + LinearizabilityCheckResult result = captured.Checker.Check( + history, + captured.ResourceWitnesses); + if (!result.IsLinearizable) + { + IReadOnlyList minimized = captured.Checker.MinimizeFailingHistory( + history, + captured.ResourceWitnesses); + throw new XunitException( + $"family={family}, history={historyIndex}, seed={seed}: {result.Failure}; " + + $"history={Format(history)}; minimized={Format(minimized)}"); + } + } + + _output.WriteLine( + $"family={family} seed={seed} completedHistories={historyCount} " + + "source=production-MemoryStore callsPerHistory=6 actors=2-3 checker=reference minimizer=on-failure"); + } + + private static CapturedHistory CapturePublishPublish(int firstDelay, int secondDelay) + { + var recorder = new MonotonicHistoryRecorder(strictProductionHistory: true); + using Store store = CreateStore("history-publish", slotCount: 2, leaseCount: 2, recorder); + PendingInvocation first = recorder.Invoke(1, 1, ReferenceCommand.Publish(1, KeyHex, LeftHex)); + PendingInvocation second = recorder.Invoke(2, 2, ReferenceCommand.Publish(1, KeyHex, RightHex)); + RunConcurrent( + first, + () => CompletePublish(first, store, Left), + second, + () => CompletePublish(second, store, Right), + firstDelay, + secondDelay); + + PendingInvocation acquire = recorder.Invoke(3, 3, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + acquire.Enter(); + StoreStatus acquireStatus = store.TryAcquire(Key, StoreWaitOptions.Infinite, out ValueLease lease); + CompleteAcquire(acquire, acquireStatus, lease); + + PendingInvocation remove = recorder.Invoke(4, 3, ReferenceCommand.Remove(1, KeyHex)); + remove.Enter(); + remove.Complete(MapRemove(store.TryRemove(Key, StoreWaitOptions.Infinite))); + + PendingInvocation release = recorder.Invoke(5, 3, ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + + PendingInvocation republish = recorder.Invoke(6, 3, ReferenceCommand.Publish(1, KeyHex, NextHex)); + republish.Enter(); + republish.Complete(MapPublish(store.TryPublish(Key, Next, default, StoreWaitOptions.Infinite))); + + IReadOnlyList history = recorder.Snapshot(); + return Captured( + new LinearizabilityChecker(2, 2, initialParticipants: [1], leaseCapacity: 2), + history, + recorder.ResourceSnapshot(), + raceFirstId: 1, + raceSecondId: 2); + } + + private static CapturedHistory CapturePublishReserve(int firstDelay, int secondDelay) + { + var recorder = new MonotonicHistoryRecorder(strictProductionHistory: true); + using Store store = CreateStore("history-publish-reserve", slotCount: 2, leaseCount: 2, recorder); + ValueReservation reservation = default; + StoreStatus publishStatus = default; + StoreStatus reserveStatus = default; + PendingInvocation publish = recorder.Invoke(1, 1, ReferenceCommand.Publish(1, KeyHex, LeftHex)); + PendingInvocation reserve = recorder.Invoke(2, 2, ReferenceCommand.Reserve(1, 10, KeyHex, RightHex)); + RunConcurrent( + publish, + () => + { + publishStatus = store.TryPublish(Key, Left, default, StoreWaitOptions.Infinite); + publish.Complete(MapPublish(publishStatus)); + }, + reserve, + () => + { + reserveStatus = store.TryReserve( + Key, + 1, + default, + StoreWaitOptions.Infinite, + out reservation); + reserve.Complete(MapReserve(reserveStatus)); + }, + firstDelay, + secondDelay); + + if ((publishStatus == StoreStatus.Success ? 1 : 0) + + (reserveStatus == StoreStatus.Success ? 1 : 0) != 1) + { + throw new XunitException( + $"publish/reserve history did not produce one winner: publish={publishStatus}, reserve={reserveStatus}."); + } + + if (reservation.IsValid) + { + PendingInvocation abort = recorder.Invoke(3, 3, ReferenceCommand.AbortReservation(1, 10)); + abort.Enter(); + abort.Complete(MapAbort(reservation.Abort(StoreWaitOptions.Infinite))); + + CompleteSequentialPublish(recorder, 4, actorId: 3, store, Next); + PendingInvocation acquire = recorder.Invoke(5, 3, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + acquire.Enter(); + StoreStatus status = store.TryAcquire(Key, StoreWaitOptions.Infinite, out ValueLease lease); + CompleteAcquire(acquire, status, lease); + + PendingInvocation release = recorder.Invoke(6, 3, ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + } + else + { + PendingInvocation acquire = recorder.Invoke(3, 3, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + acquire.Enter(); + StoreStatus status = store.TryAcquire(Key, StoreWaitOptions.Infinite, out ValueLease lease); + CompleteAcquire(acquire, status, lease); + + PendingInvocation release = recorder.Invoke(4, 3, ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + + PendingInvocation remove = recorder.Invoke(5, 3, ReferenceCommand.Remove(1, KeyHex)); + remove.Enter(); + remove.Complete(MapRemove(store.TryRemove(Key, StoreWaitOptions.Infinite))); + + CompleteSequentialPublish(recorder, 6, actorId: 3, store, Next); + } + + IReadOnlyList history = recorder.Snapshot(); + return Captured( + new LinearizabilityChecker(2, 2, initialParticipants: [1], leaseCapacity: 2), + history, + recorder.ResourceSnapshot(), + raceFirstId: 1, + raceSecondId: 2); + } + + private static CapturedHistory CaptureReserveReserve(int firstDelay, int secondDelay) + { + var recorder = new MonotonicHistoryRecorder(strictProductionHistory: true); + using Store store = CreateStore("history-reserve-reserve", slotCount: 2, leaseCount: 2, recorder); + ValueReservation firstReservation = default; + ValueReservation secondReservation = default; + StoreStatus firstStatus = default; + StoreStatus secondStatus = default; + PendingInvocation first = recorder.Invoke(1, 1, ReferenceCommand.Reserve(1, 10, KeyHex, LeftHex)); + PendingInvocation second = recorder.Invoke(2, 2, ReferenceCommand.Reserve(1, 11, KeyHex, RightHex)); + RunConcurrent( + first, + () => + { + firstStatus = store.TryReserve( + Key, + 1, + default, + StoreWaitOptions.Infinite, + out firstReservation); + first.Complete(MapReserve(firstStatus)); + }, + second, + () => + { + secondStatus = store.TryReserve( + Key, + 1, + default, + StoreWaitOptions.Infinite, + out secondReservation); + second.Complete(MapReserve(secondStatus)); + }, + firstDelay, + secondDelay); + + if ((firstStatus == StoreStatus.Success ? 1 : 0) + + (secondStatus == StoreStatus.Success ? 1 : 0) != 1) + { + throw new XunitException( + $"reserve/reserve history did not produce one winner: first={firstStatus}, second={secondStatus}."); + } + + ValueReservation winner = firstReservation.IsValid ? firstReservation : secondReservation; + int winnerId = firstReservation.IsValid ? 10 : 11; + PendingInvocation abort = recorder.Invoke(3, 3, ReferenceCommand.AbortReservation(1, winnerId)); + abort.Enter(); + abort.Complete(MapAbort(winner.Abort(StoreWaitOptions.Infinite))); + + CompleteSequentialPublish(recorder, 4, actorId: 3, store, Next); + PendingInvocation acquire = recorder.Invoke(5, 3, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + acquire.Enter(); + StoreStatus acquireStatus = store.TryAcquire(Key, StoreWaitOptions.Infinite, out ValueLease lease); + CompleteAcquire(acquire, acquireStatus, lease); + + PendingInvocation release = recorder.Invoke(6, 3, ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + + IReadOnlyList history = recorder.Snapshot(); + return Captured( + new LinearizabilityChecker(2, 2, initialParticipants: [1], leaseCapacity: 2), + history, + recorder.ResourceSnapshot(), + raceFirstId: 1, + raceSecondId: 2); + } + + private static CapturedHistory CaptureCommitAcquire(int firstDelay, int secondDelay) + { + var recorder = new MonotonicHistoryRecorder(strictProductionHistory: true); + using Store store = CreateStore("history-commit", slotCount: 1, leaseCount: 2, recorder); + PendingInvocation reserve = recorder.Invoke(1, 3, ReferenceCommand.Reserve(1, 10, KeyHex, NextHex)); + reserve.Enter(); + StoreStatus reserveStatus = store.TryReserve( + Key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation); + if (reserveStatus == StoreStatus.Success) + { + reservation.GetSpan(1)[0] = Next[0]; + reserveStatus = reservation.Advance(1, StoreWaitOptions.Infinite); + } + reserve.Complete(MapReserve(reserveStatus)); + + ValueLease lease = default; + PendingInvocation commit = recorder.Invoke(2, 1, ReferenceCommand.CommitReservation(1, 10)); + PendingInvocation acquire = recorder.Invoke(3, 2, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + RunConcurrent( + commit, + () => + { + commit.Complete(MapCommit(reservation.Commit(StoreWaitOptions.Infinite))); + }, + acquire, + () => + { + StoreStatus status = store.TryAcquire(Key, StoreWaitOptions.Infinite, out lease); + CompleteAcquire(acquire, status, lease); + }, + firstDelay, + secondDelay); + + if (!lease.IsValid) + { + PendingInvocation acquireAfter = recorder.Invoke(4, 3, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + acquireAfter.Enter(); + StoreStatus acquireAfterStatus = store.TryAcquire(Key, StoreWaitOptions.Infinite, out lease); + CompleteAcquire(acquireAfter, acquireAfterStatus, lease); + + PendingInvocation remove = recorder.Invoke(5, 3, ReferenceCommand.Remove(1, KeyHex)); + remove.Enter(); + remove.Complete(MapRemove(store.TryRemove(Key, StoreWaitOptions.Infinite))); + + PendingInvocation release = recorder.Invoke(6, 3, ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + } + else + { + PendingInvocation remove = recorder.Invoke(4, 3, ReferenceCommand.Remove(1, KeyHex)); + remove.Enter(); + remove.Complete(MapRemove(store.TryRemove(Key, StoreWaitOptions.Infinite))); + + PendingInvocation release = recorder.Invoke(5, 3, ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + + PendingInvocation republish = recorder.Invoke(6, 3, ReferenceCommand.Publish(1, KeyHex, NextHex)); + republish.Enter(); + republish.Complete(MapPublish(store.TryPublish(Key, Next, default, StoreWaitOptions.Infinite))); + } + + IReadOnlyList history = recorder.Snapshot(); + return Captured( + new LinearizabilityChecker(2, 1, initialParticipants: [1], leaseCapacity: 2), + history, + recorder.ResourceSnapshot(), + raceFirstId: 2, + raceSecondId: 3); + } + + private static CapturedHistory CaptureAcquireRemove(int firstDelay, int secondDelay) + { + var recorder = new MonotonicHistoryRecorder(strictProductionHistory: true); + using Store store = CreateStore("history-acquire-remove", slotCount: 1, leaseCount: 2, recorder); + CompleteSequentialPublish(recorder, 1, actorId: 3, store, Left); + + ValueLease lease = default; + StoreStatus acquireStatus = default; + StoreStatus removeStatus = default; + PendingInvocation acquire = recorder.Invoke(2, 1, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + PendingInvocation remove = recorder.Invoke(3, 2, ReferenceCommand.Remove(1, KeyHex)); + RunConcurrent( + acquire, + () => + { + acquireStatus = store.TryAcquire(Key, StoreWaitOptions.Infinite, out lease); + CompleteAcquire(acquire, acquireStatus, lease); + }, + remove, + () => + { + removeStatus = store.TryRemove(Key, StoreWaitOptions.Infinite); + remove.Complete(MapRemove(removeStatus)); + }, + firstDelay, + secondDelay); + + if (acquireStatus == StoreStatus.Success) + { + PendingInvocation release = recorder.Invoke(4, 3, ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + + CompleteSequentialPublish(recorder, 5, actorId: 3, store, Next); + PendingInvocation finalRemove = recorder.Invoke(6, 3, ReferenceCommand.Remove(1, KeyHex)); + finalRemove.Enter(); + finalRemove.Complete(MapRemove(store.TryRemove(Key, StoreWaitOptions.Infinite))); + } + else + { + CompleteSequentialPublish(recorder, 4, actorId: 3, store, Next); + PendingInvocation acquireAfter = recorder.Invoke(5, 3, ReferenceCommand.AcquireLease(1, 21, KeyHex)); + acquireAfter.Enter(); + StoreStatus acquireAfterStatus = store.TryAcquire(Key, StoreWaitOptions.Infinite, out ValueLease current); + CompleteAcquire(acquireAfter, acquireAfterStatus, current); + + PendingInvocation release = recorder.Invoke(6, 3, ReferenceCommand.ReleaseLease(1, 21)); + release.Enter(); + release.Complete(MapRelease(current.Release(StoreWaitOptions.Infinite))); + } + + IReadOnlyList history = recorder.Snapshot(); + return Captured( + new LinearizabilityChecker(2, 1, initialParticipants: [1], leaseCapacity: 2), + history, + recorder.ResourceSnapshot(), + raceFirstId: 2, + raceSecondId: 3); + } + + private static CapturedHistory CaptureReleaseReclaim(int firstDelay, int secondDelay) + { + var recorder = new MonotonicHistoryRecorder(strictProductionHistory: true); + using Store store = CreateStore("history-release", slotCount: 1, leaseCount: 2, recorder); + CompleteSequentialPublish(recorder, 1, actorId: 3, store, Left); + + PendingInvocation acquire = recorder.Invoke(2, 3, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + acquire.Enter(); + StoreStatus acquireStatus = store.TryAcquire(Key, StoreWaitOptions.Infinite, out ValueLease lease); + CompleteAcquire(acquire, acquireStatus, lease); + + PendingInvocation remove = recorder.Invoke(3, 3, ReferenceCommand.Remove(1, KeyHex)); + remove.Enter(); + remove.Complete(MapRemove(store.TryRemove(Key, StoreWaitOptions.Infinite))); + + StoreStatus republishStatus = default; + PendingInvocation release = recorder.Invoke(4, 1, ReferenceCommand.ReleaseLease(1, 20)); + PendingInvocation republish = recorder.Invoke(5, 2, ReferenceCommand.Publish(1, KeyHex, NextHex)); + RunConcurrent( + release, + () => + { + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + }, + republish, + () => + { + republishStatus = store.TryPublish(Key, Next, default, StoreWaitOptions.Infinite); + republish.Complete(MapPublish(republishStatus)); + }, + firstDelay, + secondDelay); + + if (republishStatus == StoreStatus.Success) + { + PendingInvocation verify = recorder.Invoke(6, 3, ReferenceCommand.AcquireLease(1, 21, KeyHex)); + verify.Enter(); + StoreStatus verifyStatus = store.TryAcquire(Key, StoreWaitOptions.Infinite, out ValueLease verifyLease); + CompleteAcquire(verify, verifyStatus, verifyLease); + } + else + { + CompleteSequentialPublish(recorder, 6, actorId: 3, store, Next); + } + + IReadOnlyList history = recorder.Snapshot(); + return Captured( + new LinearizabilityChecker(2, 1, initialParticipants: [1], leaseCapacity: 2), + history, + recorder.ResourceSnapshot(), + raceFirstId: 4, + raceSecondId: 5); + } + + private static CapturedHistory CaptureRecoveryLiveLease(int firstDelay, int secondDelay) + { + var recorder = new MonotonicHistoryRecorder(strictProductionHistory: true); + using Store store = CreateStore( + "history-recovery", + slotCount: 1, + leaseCount: 2, + recorder, + enableLeaseRecovery: true); + CompleteSequentialPublish(recorder, 1, actorId: 3, store, Left); + + PendingInvocation acquire = recorder.Invoke(2, 3, ReferenceCommand.AcquireLease(1, 20, KeyHex)); + acquire.Enter(); + StoreStatus acquireStatus = store.TryAcquire(Key, StoreWaitOptions.Infinite, out ValueLease lease); + CompleteAcquire(acquire, acquireStatus, lease); + + PendingInvocation remove = recorder.Invoke(3, 3, ReferenceCommand.Remove(1, KeyHex)); + remove.Enter(); + remove.Complete(MapRemove(store.TryRemove(Key, StoreWaitOptions.Infinite))); + + PendingInvocation recovery = recorder.Invoke(4, 1, ReferenceCommand.RecoverLease(1, 20)); + PendingInvocation release = recorder.Invoke(5, 2, ReferenceCommand.ReleaseLease(1, 20)); + RunConcurrent( + recovery, + () => + { + StoreStatus status = store.TryRecoverLeases( + new LeaseRecoveryOptions(false), + StoreWaitOptions.Infinite, + out LeaseRecoveryReport report); + recovery.Complete(MapSemanticRecovery(status, report)); + }, + release, + () => + { + release.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + }, + firstDelay, + secondDelay); + + CompleteSequentialPublish(recorder, 6, actorId: 3, store, Next); + IReadOnlyList history = recorder.Snapshot(); + return Captured( + new LinearizabilityChecker(2, 1, initialParticipants: [1], leaseCapacity: 2), + history, + recorder.ResourceSnapshot(), + raceFirstId: 4, + raceSecondId: 5); + } + + private static CapturedHistory CaptureDisposalOperation(int firstDelay, int secondDelay) + { + var recorder = new MonotonicHistoryRecorder(strictProductionHistory: true); + using var overlapGate = new DisposalOperationOverlapGate(); + SharedMemoryStoreOptions options = CreateOptions( + "history-disposal", + slotCount: 1, + leaseCount: 2, + OpenMode.CreateNew); + StoreOpenStatus created = TryCreateInstrumented( + options, + recorder, + out Store? participantOne, + overlapGate.Observe); + if (created != StoreOpenStatus.Success || participantOne is null) + { + throw new XunitException($"Could not create disposal history store: {created}."); + } + + using Store participantTwo = OpenStore(options, OpenMode.OpenExisting, recorder); + try + { + CompleteSequentialPublish(recorder, 1, actorId: 3, participantOne, Left); + overlapGate.Arm(); + + PendingInvocation operation = recorder.Invoke( + 2, + 1, + ReferenceCommand.Publish(1, KeyHex, RightHex)); + PendingInvocation dispose = recorder.Invoke(3, 2, ReferenceCommand.DisposeParticipant(1)); + RunConcurrent( + operation, + () => + { + StoreStatus status = participantOne.TryPublish( + Key, + Right, + default, + StoreWaitOptions.Infinite); + ReferenceResultCode mapped = status switch + { + StoreStatus.DuplicateKey => ReferenceResultCode.DuplicateKey, + StoreStatus.StoreDisposed => ReferenceResultCode.ParticipantNotActive, + StoreStatus.StoreBusy => ReferenceResultCode.Unexpected, + _ => ReferenceResultCode.Unexpected + }; + operation.Complete(mapped); + }, + dispose, + () => + { + participantOne.Dispose(); + dispose.Complete(ReferenceResultCode.Success); + }, + firstDelay, + secondDelay); + overlapGate.AssertSatisfied(); + + PendingInvocation keeperAcquire = recorder.Invoke(4, 3, ReferenceCommand.AcquireLease(2, 21, KeyHex)); + keeperAcquire.Enter(); + StoreStatus keeperAcquireStatus = participantTwo.TryAcquire( + Key, + StoreWaitOptions.Infinite, + out ValueLease lease); + CompleteAcquire(keeperAcquire, keeperAcquireStatus, lease); + + PendingInvocation keeperRelease = recorder.Invoke(5, 3, ReferenceCommand.ReleaseLease(2, 21)); + keeperRelease.Enter(); + keeperRelease.Complete(MapRelease(lease.Release(StoreWaitOptions.Infinite))); + + PendingInvocation remove = recorder.Invoke(6, 3, ReferenceCommand.Remove(2, KeyHex)); + remove.Enter(); + remove.Complete(MapRemove(participantTwo.TryRemove(Key, StoreWaitOptions.Infinite))); + + IReadOnlyList history = recorder.Snapshot(); + return Captured( + new LinearizabilityChecker(2, 1, initialParticipants: [1, 2], leaseCapacity: 2), + history, + recorder.ResourceSnapshot(), + raceFirstId: 2, + raceSecondId: 3); + } + finally + { + participantOne.Dispose(); + } + } + + private static void CompleteSequentialPublish( + MonotonicHistoryRecorder recorder, + int id, + int actorId, + Store store, + byte[] value) + { + PendingInvocation publish = recorder.Invoke( + id, + actorId, + ReferenceCommand.Publish(1, KeyHex, Convert.ToHexString(value))); + publish.Enter(); + publish.Complete(MapPublish(store.TryPublish(Key, value, default, StoreWaitOptions.Infinite))); + } + + private static void CompletePublish(PendingInvocation invocation, Store store, byte[] value) + { + invocation.Complete(MapPublish(store.TryPublish(Key, value, default, StoreWaitOptions.Infinite))); + } + + private static void CompleteAcquire( + PendingInvocation invocation, + StoreStatus status, + in ValueLease lease) => + CompleteAcquire(invocation, MapAcquire(status), lease); + + private static void CompleteAcquire( + PendingInvocation invocation, + ReferenceResultCode result, + in ValueLease lease) + { + if (result != ReferenceResultCode.Success) + { + invocation.Complete(result); + return; + } + + ReadOnlySpan returnedBytes = lease.ValueSpan; + IndexBinding binding = IndexBinding.Decode(lease.HandleForEngine.SlotBinding); + invocation.Complete( + result, + Convert.ToHexString(returnedBytes), + binding.Generation); + } + + private static ReferenceResultCode MapPublish(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.DuplicateKey => ReferenceResultCode.DuplicateKey, + StoreStatus.NotFound => ReferenceResultCode.NotFound, + StoreStatus.StoreFull => ReferenceResultCode.StoreFull, + StoreStatus.StoreBusy => ReferenceResultCode.Unexpected, + _ => ReferenceResultCode.Unexpected + }; + + private static ReferenceResultCode MapReserve(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.DuplicateKey => ReferenceResultCode.DuplicateKey, + StoreStatus.StoreFull => ReferenceResultCode.StoreFull, + StoreStatus.StoreBusy => ReferenceResultCode.Unexpected, + StoreStatus.InvalidReservation => ReferenceResultCode.InvalidReservation, + _ => ReferenceResultCode.Unexpected + }; + + private static ReferenceResultCode MapCommit(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.InvalidReservation => ReferenceResultCode.InvalidReservation, + StoreStatus.ReservationAlreadyCompleted => ReferenceResultCode.ReservationAlreadyCompleted, + StoreStatus.StoreBusy => ReferenceResultCode.Unexpected, + _ => ReferenceResultCode.Unexpected + }; + + private static ReferenceResultCode MapAbort(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.InvalidReservation => ReferenceResultCode.InvalidReservation, + StoreStatus.ReservationAlreadyCompleted => ReferenceResultCode.ReservationAlreadyCompleted, + StoreStatus.StoreBusy => ReferenceResultCode.Unexpected, + _ => ReferenceResultCode.Unexpected + }; + + private static ReferenceResultCode MapAcquire(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.NotFound => ReferenceResultCode.NotFound, + StoreStatus.LeaseTableFull => ReferenceResultCode.LeaseTableFull, + StoreStatus.StoreBusy => ReferenceResultCode.Unexpected, + _ => ReferenceResultCode.Unexpected + }; + + private static ReferenceResultCode MapRemove(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.NotFound => ReferenceResultCode.NotFound, + StoreStatus.RemovePending => ReferenceResultCode.RemovePending, + StoreStatus.StoreBusy => ReferenceResultCode.Unexpected, + _ => ReferenceResultCode.Unexpected + }; + + private static ReferenceResultCode MapRelease(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.InvalidLease => ReferenceResultCode.InvalidLease, + StoreStatus.LeaseAlreadyReleased => ReferenceResultCode.LeaseAlreadyReleased, + StoreStatus.StoreBusy => ReferenceResultCode.Unexpected, + _ => ReferenceResultCode.Unexpected + }; + + private static ReferenceResultCode MapSemanticRecovery( + StoreStatus status, + LeaseRecoveryReport report) + { + if (status == StoreStatus.StoreBusy) + { + return ReferenceResultCode.Unexpected; + } + + if (status != StoreStatus.Success + || report.UnsupportedLeaseCount != 0 + || report.FailedRecoveryCount != 0) + { + return ReferenceResultCode.Unexpected; + } + + return report.RecoveredLeaseCount switch + { + 1 => ReferenceResultCode.Success, + 0 => ReferenceResultCode.InvalidLease, + _ => ReferenceResultCode.Unexpected + }; + } + + private static CapturedHistory Captured( + LinearizabilityChecker checker, + IReadOnlyList history, + IReadOnlyList resourceWitnesses, + int raceFirstId, + int raceSecondId) + { + RecordedOperation first = history.Single(operation => operation.Id == raceFirstId); + RecordedOperation second = history.Single(operation => operation.Id == raceSecondId); + return new CapturedHistory(checker, history, resourceWitnesses, [first, second]); + } + + internal static void AssertProductionRaceOverlap( + RecordedOperation first, + RecordedOperation second, + string context = "production history") + { + if (!first.ImplementationOverlaps(second)) + { + throw new XunitException( + $"{context}: mapped implementation intervals did not overlap " + + $"(first entry/return={first.EntrySequence}/{first.ReturnSequence}, " + + $"second entry/return={second.EntrySequence}/{second.ReturnSequence})."); + } + } + + private static void RunConcurrent( + PendingInvocation firstInvocation, + Action firstBody, + PendingInvocation secondInvocation, + Action secondBody, + int firstDelay, + int secondDelay) + { + using var barrier = new Barrier(3); + Exception? firstFailure = null; + Exception? secondFailure = null; + var firstThread = new Thread(() => + { + try + { + barrier.SignalAndWait(); + Thread.SpinWait(firstDelay); + firstInvocation.Enter(); + firstBody(); + } + catch (Exception error) + { + firstFailure = error; + } + }) + { + IsBackground = true, + Name = "sms-production-history-first" + }; + var secondThread = new Thread(() => + { + try + { + barrier.SignalAndWait(); + Thread.SpinWait(secondDelay); + secondInvocation.Enter(); + secondBody(); + } + catch (Exception error) + { + secondFailure = error; + } + }) + { + IsBackground = true, + Name = "sms-production-history-second" + }; + firstThread.Start(); + secondThread.Start(); + barrier.SignalAndWait(); + if (!firstThread.Join(TimeSpan.FromSeconds(10)) + || !secondThread.Join(TimeSpan.FromSeconds(10))) + { + throw new XunitException("Production history race timed out."); + } + + if (firstFailure is not null || secondFailure is not null) + { + Exception failure = firstFailure ?? secondFailure!; + throw new XunitException( + $"Production history actor failed: {failure.GetType().Name}: {failure.Message}"); + } + } + + private static Store CreateStore( + string prefix, + int slotCount, + int leaseCount, + MonotonicHistoryRecorder recorder, + bool enableLeaseRecovery = false) + { + SharedMemoryStoreOptions options = CreateOptions( + prefix, + slotCount, + leaseCount, + OpenMode.CreateNew, + enableLeaseRecovery); + StoreOpenStatus status = TryCreateInstrumented(options, recorder, out Store? store); + if (status != StoreOpenStatus.Success || store is null) + { + throw new XunitException($"Could not create production history store: {status}."); + } + + return store; + } + + private static Store OpenStore( + SharedMemoryStoreOptions source, + OpenMode openMode, + MonotonicHistoryRecorder recorder) + { + var options = new SharedMemoryStoreOptions + { + Profile = source.Profile, + Name = source.Name, + OpenMode = openMode, + TotalBytes = source.TotalBytes, + SlotCount = source.SlotCount, + MaxValueBytes = source.MaxValueBytes, + MaxDescriptorBytes = source.MaxDescriptorBytes, + MaxKeyBytes = source.MaxKeyBytes, + LeaseRecordCount = source.LeaseRecordCount, + ParticipantRecordCount = source.ParticipantRecordCount, + EnableLeaseRecovery = source.EnableLeaseRecovery + }; + StoreOpenStatus status = TryCreateInstrumented(options, recorder, out Store? store); + if (status != StoreOpenStatus.Success || store is null) + { + throw new XunitException($"Could not open production history store: {status}."); + } + + return store; + } + + private static StoreOpenStatus TryCreateInstrumented( + SharedMemoryStoreOptions options, + MonotonicHistoryRecorder recorder, + out Store? store, + Action? checkpointObserver = null) + { + checkpointObserver ??= static _ => { }; + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented( + checkpointObserver, + recorder.ObserveSlotResource, + recorder, + recorder); + return LockFreeInstrumentedStoreFactory.TryCreateOrOpen(options, checkpoint, out store); + } + + private static SharedMemoryStoreOptions CreateOptions( + string prefix, + int slotCount, + int leaseCount, + OpenMode openMode, + bool enableLeaseRecovery = false) => + SharedMemoryStoreOptions.CreateLockFree( + $"sms-generated-{prefix}-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 8, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: leaseCount, + participantRecordCount: 4, + openMode, + enableLeaseRecovery); + + private static string Format(IReadOnlyList history) => + string.Join( + "; ", + history.Select(static operation => + $"#{operation.Id}/a{operation.ActorId}:{operation.Command.Kind}={operation.Result}")); + + private static int ConfiguredHistoryCount() + { + string? configured = Environment.GetEnvironmentVariable("SMS_PRODUCTION_HISTORY_COUNT"); + return int.TryParse(configured, out int value) && value > 0 ? value : 1; + } + + private static int ConfiguredSeed() + { + string? configured = Environment.GetEnvironmentVariable("SMS_PRODUCTION_HISTORY_SEED"); + return int.TryParse(configured, out int value) ? value : DefaultSeed; + } + + private static int FamilyOrdinal(string family) => family switch + { + "publish-publish" => 1, + "publish-reserve" => 2, + "reserve-reserve" => 3, + "commit-acquire" => 4, + "acquire-remove" => 5, + "release-reclaim" => 6, + "recovery-live-lease" => 7, + "disposal-operation" => 8, + _ => throw new ArgumentOutOfRangeException(nameof(family), family, null) + }; + + private static int FamilySeed(int rootSeed, int family) => + unchecked(rootSeed + family * (int)0x9e37_79b9u); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed record CapturedHistory( + LinearizabilityChecker Checker, + IReadOnlyList History, + IReadOnlyList ResourceWitnesses, + IReadOnlyList RaceOperations); + + private sealed class DisposalOperationOverlapGate : IDisposable + { + private static readonly TimeSpan CheckpointTimeout = TimeSpan.FromSeconds(5); + private readonly ManualResetEventSlim _publishReached = new(); + private readonly ManualResetEventSlim _disposalReached = new(); + private int _armed; + + internal void Arm() + { + if (Interlocked.Exchange(ref _armed, 1) != 0) + { + throw new InvalidOperationException("The disposal overlap gate can be armed only once."); + } + } + + internal void Observe(LockFreeCheckpointEntry entry) + { + if (Volatile.Read(ref _armed) == 0) + { + return; + } + + switch (entry.Id) + { + case LockFreeCheckpointId.PublishBeforeSlotClaim: + _publishReached.Set(); + WaitFor(_disposalReached, "disposal entry"); + break; + case LockFreeCheckpointId.DisposalBeforeLocalGateClose: + _disposalReached.Set(); + WaitFor(_publishReached, "publish entry"); + break; + } + } + + internal void AssertSatisfied() + { + if (!_publishReached.IsSet || !_disposalReached.IsSet) + { + throw new XunitException( + "The disposal-operation history did not reach both deterministic overlap checkpoints."); + } + } + + public void Dispose() + { + _publishReached.Dispose(); + _disposalReached.Dispose(); + } + + private static void WaitFor(ManualResetEventSlim checkpoint, string description) + { + if (!checkpoint.Wait(CheckpointTimeout)) + { + throw new XunitException( + $"Timed out waiting for the deterministic disposal-operation {description} checkpoint."); + } + } + } + + private struct DeterministicRandom + { + private uint _state; + + internal DeterministicRandom(int seed) + { + _state = unchecked((uint)seed); + if (_state == 0) + { + _state = 0x7f4a_7c15u; + } + } + + internal int NextDelay() + { + uint value = _state; + value ^= value << 13; + value ^= value >> 17; + value ^= value << 5; + _state = value; + return (int)(value & 127u); + } + } +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/ProductionRaceStressTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/ProductionRaceStressTests.cs new file mode 100644 index 0000000..238f34f --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/ProductionRaceStressTests.cs @@ -0,0 +1,976 @@ +using System.Runtime.InteropServices; +using SharedMemoryStore.LockFree; +using Store = SharedMemoryStore.MemoryStore; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace SharedMemoryStore.LinearizabilityTests; + +/// +/// Runs the SC-011 repetition count against the real mapped-store operations. +/// The model checker has a separate repetition budget; none of the counts in +/// this test are permutations of a preconstructed history. +/// +public sealed class ProductionRaceStressTests +{ + private const int DefaultSeed = 0x5eed_0200; + private const int DefaultRepetitions = 256; + private static readonly byte[] Key = [0x41]; + private static readonly byte[] LeftValue = [0x11]; + private static readonly byte[] RightValue = [0x22]; + private static readonly byte[] NextValue = [0x33]; + private readonly ITestOutputHelper _output; + + public ProductionRaceStressTests(ITestOutputHelper output) + { + _output = output; + } + + public static TheoryData RequiredFamilies => new() + { + "publish-publish", + "publish-reserve", + "reserve-reserve", + "commit-acquire", + "acquire-remove", + "release-reclaim", + "recovery-live-lease", + "disposal-operation" + }; + + [Theory] + [MemberData(nameof(RequiredFamilies))] + [Trait("Category", "ProductionRaceStress")] + public void ConfiguredRepetitionsExecuteProductionRaceFamily(string family) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + int repetitions = ConfiguredRepetitions(); + int rootSeed = ConfiguredSeed(); + + switch (family) + { + case "publish-publish": + RunPublishPublish(repetitions, FamilySeed(rootSeed, 1)); + break; + case "publish-reserve": + RunPublishReserve(repetitions, FamilySeed(rootSeed, 2)); + break; + case "reserve-reserve": + RunReserveReserve(repetitions, FamilySeed(rootSeed, 3)); + break; + case "commit-acquire": + RunCommitAcquire(repetitions, FamilySeed(rootSeed, 4)); + break; + case "acquire-remove": + RunAcquireRemove(repetitions, FamilySeed(rootSeed, 5)); + break; + case "release-reclaim": + RunReleaseReclaim(repetitions, FamilySeed(rootSeed, 6)); + break; + case "recovery-live-lease": + RunRecoveryLiveLease(repetitions, FamilySeed(rootSeed, 7)); + break; + case "disposal-operation": + RunDisposalOperation(repetitions, FamilySeed(rootSeed, 8)); + break; + default: + throw new ArgumentOutOfRangeException(nameof(family), family, null); + } + } + + private void RunPublishPublish(int repetitions, int seed) + { + const string family = "publish-publish"; + using Store store = CreateStore(family, slotCount: 2, leaseCount: 4); + StoreStatus first = default; + StoreStatus second = default; + using var race = new TwoActorRace( + () => first = store.TryPublish(Key, LeftValue), + () => second = store.TryPublish(Key, RightValue)); + var random = new DeterministicRandom(seed); + + for (var repetition = 0; repetition < repetitions; repetition++) + { + race.Run(random.NextDelay(), random.NextDelay()); + bool firstAllowed = first is StoreStatus.Success + or StoreStatus.DuplicateKey + or StoreStatus.StoreFull + or StoreStatus.StoreBusy; + bool secondAllowed = second is StoreStatus.Success + or StoreStatus.DuplicateKey + or StoreStatus.StoreFull + or StoreStatus.StoreBusy; + int successes = (first == StoreStatus.Success ? 1 : 0) + (second == StoreStatus.Success ? 1 : 0); + if (!firstAllowed || !secondAllowed || successes > 1) + { + Fail(family, repetition, seed, $"first={first}, second={second}, successes={successes}"); + } + + if ((first == StoreStatus.DuplicateKey || second == StoreStatus.DuplicateKey) && successes != 1) + { + Fail(family, repetition, seed, $"duplicate without exactly one winner: first={first}, second={second}"); + } + + if (successes == 1) + { + StoreStatus acquire = store.TryAcquire(Key, out ValueLease lease); + if (acquire != StoreStatus.Success || !lease.IsValid) + { + Fail(family, repetition, seed, $"winner was not readable: acquire={acquire}"); + } + + byte expected = first == StoreStatus.Success ? LeftValue[0] : RightValue[0]; + if (lease.ValueSpan.Length != 1 || lease.ValueSpan[0] != expected) + { + Fail(family, repetition, seed, "winner payload did not match its publication"); + } + + RequireRelease(family, repetition, seed, lease.Release()); + } + + RemoveForReuse(store, family, repetition, seed); + } + + Record(family, repetitions, seed); + } + + private void RunPublishReserve(int repetitions, int seed) + { + const string family = "publish-reserve"; + using Store store = CreateStore(family, slotCount: 2, leaseCount: 4); + StoreStatus publish = default; + StoreStatus reserve = default; + ValueReservation reservation = default; + using var race = new TwoActorRace( + () => publish = store.TryPublish(Key, LeftValue), + () => reserve = store.TryReserve(Key, 1, default, out reservation)); + var random = new DeterministicRandom(seed); + + for (var repetition = 0; repetition < repetitions; repetition++) + { + reservation = default; + race.Run(random.NextDelay(), random.NextDelay()); + + bool publishAllowed = publish is StoreStatus.Success + or StoreStatus.DuplicateKey + or StoreStatus.StoreFull + or StoreStatus.StoreBusy; + bool reserveAllowed = reserve is StoreStatus.Success + or StoreStatus.DuplicateKey + or StoreStatus.StoreFull + or StoreStatus.StoreBusy; + int successes = (publish == StoreStatus.Success ? 1 : 0) + + (reserve == StoreStatus.Success ? 1 : 0); + if (!publishAllowed || !reserveAllowed || successes > 1) + { + Fail(family, repetition, seed, $"publish={publish}, reserve={reserve}, successes={successes}"); + } + + if ((publish == StoreStatus.DuplicateKey || reserve == StoreStatus.DuplicateKey) + && successes != 1) + { + Fail( + family, + repetition, + seed, + $"duplicate without exactly one winner: publish={publish}, reserve={reserve}"); + } + + if (reserve == StoreStatus.Success) + { + if (!reservation.IsValid) + { + Fail(family, repetition, seed, "successful explicit reservation returned an invalid token"); + } + + StoreStatus abort = reservation.Abort(); + if (abort != StoreStatus.Success) + { + Fail(family, repetition, seed, $"reservation cleanup abort={abort}"); + } + } + else if (reservation.IsValid) + { + Fail(family, repetition, seed, $"failed reserve returned a valid token: reserve={reserve}"); + } + + if (publish == StoreStatus.Success) + { + StoreStatus acquire = store.TryAcquire(Key, out ValueLease lease); + if (acquire != StoreStatus.Success + || !lease.IsValid + || lease.ValueSpan.Length != 1 + || lease.ValueSpan[0] != LeftValue[0]) + { + Fail(family, repetition, seed, $"atomic publication was not readable: acquire={acquire}"); + } + + RequireRelease(family, repetition, seed, lease.Release()); + } + + RemoveForReuse(store, family, repetition, seed); + } + + Record(family, repetitions, seed); + } + + private void RunReserveReserve(int repetitions, int seed) + { + const string family = "reserve-reserve"; + using Store store = CreateStore(family, slotCount: 2, leaseCount: 2); + StoreStatus first = default; + StoreStatus second = default; + ValueReservation firstReservation = default; + ValueReservation secondReservation = default; + using var race = new TwoActorRace( + () => first = store.TryReserve(Key, 1, default, out firstReservation), + () => second = store.TryReserve(Key, 1, default, out secondReservation)); + var random = new DeterministicRandom(seed); + + for (var repetition = 0; repetition < repetitions; repetition++) + { + firstReservation = default; + secondReservation = default; + race.Run(random.NextDelay(), random.NextDelay()); + + bool firstAllowed = first is StoreStatus.Success + or StoreStatus.DuplicateKey + or StoreStatus.StoreFull + or StoreStatus.StoreBusy; + bool secondAllowed = second is StoreStatus.Success + or StoreStatus.DuplicateKey + or StoreStatus.StoreFull + or StoreStatus.StoreBusy; + int successes = (first == StoreStatus.Success ? 1 : 0) + + (second == StoreStatus.Success ? 1 : 0); + if (!firstAllowed || !secondAllowed || successes > 1) + { + Fail(family, repetition, seed, $"first={first}, second={second}, successes={successes}"); + } + + if ((first == StoreStatus.DuplicateKey || second == StoreStatus.DuplicateKey) + && successes != 1) + { + Fail( + family, + repetition, + seed, + $"duplicate without exactly one winner: first={first}, second={second}"); + } + + if ((first == StoreStatus.Success) != firstReservation.IsValid + || (second == StoreStatus.Success) != secondReservation.IsValid) + { + Fail( + family, + repetition, + seed, + $"reservation validity mismatch: first={first}/{firstReservation.IsValid}, " + + $"second={second}/{secondReservation.IsValid}"); + } + + if (first == StoreStatus.Success && firstReservation.Abort() != StoreStatus.Success) + { + Fail(family, repetition, seed, "first reservation cleanup did not abort"); + } + + if (second == StoreStatus.Success && secondReservation.Abort() != StoreStatus.Success) + { + Fail(family, repetition, seed, "second reservation cleanup did not abort"); + } + + RemoveForReuse(store, family, repetition, seed); + } + + Record(family, repetitions, seed); + } + + private void RunCommitAcquire(int repetitions, int seed) + { + const string family = "commit-acquire"; + using Store store = CreateStore(family, slotCount: 1, leaseCount: 2); + ValueReservation reservation = default; + ValueLease lease = default; + StoreStatus commit = default; + StoreStatus acquire = default; + using var race = new TwoActorRace( + () => commit = reservation.Commit(), + () => acquire = store.TryAcquire(Key, out lease)); + var random = new DeterministicRandom(seed); + + for (var repetition = 0; repetition < repetitions; repetition++) + { + StoreStatus reserve = ReservePrepared(store, Key, NextValue[0], out reservation); + if (reserve != StoreStatus.Success) + { + Fail(family, repetition, seed, $"setup reserve={reserve}"); + } + + lease = default; + race.Run(random.NextDelay(), random.NextDelay()); + bool commitAllowed = commit is StoreStatus.Success or StoreStatus.StoreBusy; + bool acquireAllowed = acquire is StoreStatus.Success or StoreStatus.NotFound or StoreStatus.StoreBusy; + if (!commitAllowed || !acquireAllowed || (commit != StoreStatus.Success && acquire == StoreStatus.Success)) + { + Fail(family, repetition, seed, $"commit={commit}, acquire={acquire}"); + } + + if (acquire == StoreStatus.Success) + { + if (!lease.IsValid || lease.ValueSpan.Length != 1 || lease.ValueSpan[0] != NextValue[0]) + { + Fail(family, repetition, seed, "successful acquire exposed incomplete or wrong bytes"); + } + + RequireRelease(family, repetition, seed, lease.Release()); + } + + if (commit == StoreStatus.StoreBusy) + { + StoreStatus abort = reservation.Abort(); + if (abort is not (StoreStatus.Success or StoreStatus.ReservationAlreadyCompleted)) + { + Fail(family, repetition, seed, $"cleanup abort={abort}"); + } + } + + RemoveForReuse(store, family, repetition, seed); + } + + Record(family, repetitions, seed); + } + + private void RunAcquireRemove(int repetitions, int seed) + { + const string family = "acquire-remove"; + using Store store = CreateStore(family, slotCount: 1, leaseCount: 2); + ValueLease lease = default; + StoreStatus acquire = default; + StoreStatus remove = default; + using var race = new TwoActorRace( + () => acquire = store.TryAcquire(Key, out lease), + () => remove = store.TryRemove(Key)); + var random = new DeterministicRandom(seed); + + for (var repetition = 0; repetition < repetitions; repetition++) + { + PublishForReuse(store, NextValue, family, repetition, seed); + lease = default; + race.Run(random.NextDelay(), random.NextDelay()); + + bool acquireAllowed = acquire is StoreStatus.Success or StoreStatus.NotFound or StoreStatus.StoreBusy; + bool removeAllowed = remove is StoreStatus.Success or StoreStatus.RemovePending or StoreStatus.StoreBusy; + bool orderingAllowed = acquire switch + { + StoreStatus.Success => remove is StoreStatus.RemovePending or StoreStatus.StoreBusy, + StoreStatus.NotFound => remove is StoreStatus.Success or StoreStatus.RemovePending, + StoreStatus.StoreBusy => removeAllowed, + _ => false + }; + if (!acquireAllowed || !removeAllowed || !orderingAllowed) + { + Fail(family, repetition, seed, $"acquire={acquire}, remove={remove}"); + } + + if (acquire == StoreStatus.Success) + { + if (!lease.IsValid || lease.ValueSpan.Length != 1 || lease.ValueSpan[0] != NextValue[0]) + { + Fail(family, repetition, seed, "successful acquire exposed wrong bytes"); + } + + RequireRelease(family, repetition, seed, lease.Release()); + } + + RemoveForReuse(store, family, repetition, seed); + } + + Record(family, repetitions, seed); + } + + private void RunReleaseReclaim(int repetitions, int seed) + { + const string family = "release-reclaim"; + using Store store = CreateStore(family, slotCount: 1, leaseCount: 2); + ValueLease lease = default; + StoreStatus release = default; + StoreStatus publish = default; + using var race = new TwoActorRace( + () => release = lease.Release(), + () => publish = store.TryPublish(Key, NextValue)); + var random = new DeterministicRandom(seed); + + for (var repetition = 0; repetition < repetitions; repetition++) + { + PublishForReuse(store, LeftValue, family, repetition, seed); + StoreStatus acquire = store.TryAcquire(Key, out lease); + if (acquire != StoreStatus.Success) + { + Fail(family, repetition, seed, $"setup acquire={acquire}"); + } + + StoreStatus remove = store.TryRemove(Key); + if (remove != StoreStatus.RemovePending) + { + Fail(family, repetition, seed, $"setup remove={remove}"); + } + + race.Run(random.NextDelay(), random.NextDelay()); + if (release != StoreStatus.Success + || publish is not (StoreStatus.Success or StoreStatus.DuplicateKey or StoreStatus.StoreBusy)) + { + Fail(family, repetition, seed, $"release={release}, publish={publish}"); + } + + if (publish != StoreStatus.Success) + { + PublishForReuse(store, NextValue, family, repetition, seed); + } + + StoreStatus verify = store.TryAcquire(Key, out ValueLease current); + if (verify != StoreStatus.Success + || current.ValueSpan.Length != 1 + || current.ValueSpan[0] != NextValue[0]) + { + Fail(family, repetition, seed, $"reused generation verify={verify}"); + } + + RequireRelease(family, repetition, seed, current.Release()); + RemoveForReuse(store, family, repetition, seed); + } + + Record(family, repetitions, seed); + } + + private void RunRecoveryLiveLease(int repetitions, int seed) + { + const string family = "recovery-live-lease"; + using Store store = CreateStore(family, slotCount: 1, leaseCount: 4, enableLeaseRecovery: true); + ValueLease lease = default; + ValueLease guardLease = default; + StoreStatus recovery = default; + LeaseRecoveryReport report = default; + StoreStatus release = default; + long recoverySuccesses = 0; + long recoveryBusy = 0; + long liveActiveWitnesses = 0; + using var race = new TwoActorRace( + () => recovery = store.TryRecoverLeases(new LeaseRecoveryOptions(false), out report), + () => release = lease.Release()); + var random = new DeterministicRandom(seed); + + for (var repetition = 0; repetition < repetitions; repetition++) + { + PublishForReuse(store, LeftValue, family, repetition, seed); + StoreStatus acquire = store.TryAcquire(Key, out lease); + if (acquire != StoreStatus.Success) + { + Fail(family, repetition, seed, $"setup acquire={acquire}"); + } + + StoreStatus guardAcquire = store.TryAcquire(Key, out guardLease); + if (guardAcquire != StoreStatus.Success) + { + Fail(family, repetition, seed, $"setup guard acquire={guardAcquire}"); + } + + StoreStatus remove = store.TryRemove(Key); + if (remove != StoreStatus.RemovePending) + { + Fail(family, repetition, seed, $"setup remove={remove}"); + } + + report = default; + race.Run(random.NextDelay(), random.NextDelay()); + if (recovery is not (StoreStatus.Success or StoreStatus.StoreBusy) + || release != StoreStatus.Success) + { + Fail(family, repetition, seed, $"recovery={recovery}, release={release}"); + } + + if (recovery == StoreStatus.Success) + { + recoverySuccesses++; + if (report.RecoveredLeaseCount != 0 + || report.UnsupportedLeaseCount != 0 + || report.FailedRecoveryCount != 0 + || report.ActiveLeaseCount < 1) + { + Fail( + family, + repetition, + seed, + $"recovered={report.RecoveredLeaseCount}, active={report.ActiveLeaseCount}, " + + $"unsupported={report.UnsupportedLeaseCount}, failed={report.FailedRecoveryCount}, release={release}"); + } + + liveActiveWitnesses++; + } + else + { + recoveryBusy++; + } + + RequireRelease(family, repetition, seed, guardLease.Release()); + PublishForReuse(store, NextValue, family, repetition, seed); + RemoveForReuse(store, family, repetition, seed); + } + + if (recoverySuccesses == 0) + { + throw new XunitException("Normal recovery returned StoreBusy for every configured live-lease race."); + } + + Record( + family, + repetitions, + seed, + $"recoverySuccesses={recoverySuccesses} recoveryBusy={recoveryBusy} " + + $"liveActiveWitnesses={liveActiveWitnesses}"); + } + + private void RunDisposalOperation(int repetitions, int seed) + { + const string family = "disposal-operation"; + SharedMemoryStoreOptions create = CreateOptions(family, slotCount: 1, leaseCount: 2, OpenMode.CreateNew); + using Store keeper = OpenStore(create, OpenMode.CreateNew); + if (keeper.TryPublish(Key, LeftValue) != StoreStatus.Success) + { + throw new XunitException("Could not publish the disposal race fixture."); + } + + Store? participant = null; + StoreStatus acquire = default; + ValueLease lease = default; + int racedDisposeCompleted = 0; + long operationWins = 0; + long disposalWins = 0; + using var race = new TwoActorRace( + () => acquire = participant!.TryAcquire(Key, out lease), + () => + { + participant!.Dispose(); + Volatile.Write(ref racedDisposeCompleted, 1); + }); + var random = new DeterministicRandom(seed); + + for (var repetition = 0; repetition < repetitions; repetition++) + { + participant = OpenStore(create, OpenMode.OpenExisting); + acquire = default; + lease = default; + Volatile.Write(ref racedDisposeCompleted, 0); + try + { + race.Run(random.NextDelay(), random.NextDelay()); + if (acquire == StoreStatus.Success) + { + operationWins++; + if (lease.HandleForEngine.IsDefault) + { + Fail(family, repetition, seed, "successful acquire returned a default lease token"); + } + + if (lease.IsValid) + { + Fail(family, repetition, seed, "completed disposal did not invalidate its successful lease"); + } + + // A successful lease may already have been invalidated and + // reclaimed by this exact handle's completed Dispose. Its + // successful return is the operation-before-disposal outcome. + } + else if (acquire == StoreStatus.StoreDisposed) + { + disposalWins++; + if (lease.IsValid) + { + Fail(family, repetition, seed, "disposed-first acquire returned a valid lease"); + } + } + else + { + Fail(family, repetition, seed, $"acquire={acquire}"); + } + + StoreStatus keeperAcquire = keeper.TryAcquire(Key, out ValueLease keeperLease); + if (keeperAcquire != StoreStatus.Success + || !keeperLease.IsValid + || keeperLease.ValueSpan.Length != 1 + || keeperLease.ValueSpan[0] != LeftValue[0]) + { + Fail( + family, + repetition, + seed, + $"disposing one handle affected the keeper: acquire={keeperAcquire}"); + } + + RequireRelease(family, repetition, seed, keeperLease.Release()); + } + finally + { + // A passing repetition executes exactly the one Dispose call in + // the raced actor. Cleanup invokes Dispose only when the actor + // failed before completing, in which case no completion marker + // is emitted or credited. + if (Volatile.Read(ref racedDisposeCompleted) == 0) + { + participant!.Dispose(); + } + + participant = null; + } + } + + if (repetitions >= 1_000 && (operationWins == 0 || disposalWins == 0)) + { + throw new XunitException( + $"Seeded disposal race did not reach both documented orderings: " + + $"operationWins={operationWins}, disposalWins={disposalWins}."); + } + + _output.WriteLine( + $"family={family} seed={seed} completed={repetitions} " + + $"productionOperationRaces={repetitions} disposeCalls={repetitions} " + + $"freshHandles={repetitions} operationWins={operationWins} disposalWins={disposalWins} " + + "control=persistent-two-phase-barrier"); + } + + private static StoreStatus ReservePrepared( + Store store, + byte[] key, + byte value, + out ValueReservation reservation) + { + StoreStatus status = store.TryReserve(key, payloadLength: 1, descriptor: default, out reservation); + if (status != StoreStatus.Success) + { + return status; + } + + Span destination = reservation.GetSpan(1); + if (destination.Length < 1) + { + return StoreStatus.UnknownFailure; + } + + destination[0] = value; + return reservation.Advance(1); + } + + private static void PublishForReuse( + Store store, + byte[] value, + string family, + int repetition, + int seed) + { + for (var attempt = 0; attempt < 1_024; attempt++) + { + StoreStatus status = store.TryPublish(Key, value); + if (status == StoreStatus.Success) + { + return; + } + + if (status is not (StoreStatus.DuplicateKey or StoreStatus.StoreBusy)) + { + Fail(family, repetition, seed, $"reuse publish={status}, attempt={attempt}"); + } + + Thread.SpinWait(16 + (attempt & 31)); + } + + Fail(family, repetition, seed, "reuse publication did not converge in 1024 attempts"); + } + + private static void RemoveForReuse(Store store, string family, int repetition, int seed) + { + for (var attempt = 0; attempt < 1_024; attempt++) + { + StoreStatus status = store.TryRemove(Key); + if (status is StoreStatus.Success or StoreStatus.NotFound) + { + return; + } + + if (status is not (StoreStatus.RemovePending or StoreStatus.StoreBusy)) + { + Fail(family, repetition, seed, $"cleanup remove={status}, attempt={attempt}"); + } + + Thread.SpinWait(16 + (attempt & 31)); + } + + Fail(family, repetition, seed, "cleanup removal did not converge in 1024 attempts"); + } + + private static void RequireRelease(string family, int repetition, int seed, StoreStatus status) + { + if (status != StoreStatus.Success) + { + Fail(family, repetition, seed, $"lease release={status}"); + } + } + + private void Record(string family, int repetitions, int seed, string? details = null) + { + _output.WriteLine( + $"family={family} seed={seed} completed={repetitions} " + + $"productionOperationRaces={repetitions} control=persistent-two-phase-barrier" + + (details is null ? string.Empty : " " + details)); + } + + private static void Fail(string family, int repetition, int seed, string details) + { + throw new XunitException($"family={family}, repetition={repetition}, seed={seed}: {details}"); + } + + private static Store CreateStore( + string family, + int slotCount, + int leaseCount, + bool enableLeaseRecovery = false) + { + SharedMemoryStoreOptions options = CreateOptions( + family, + slotCount, + leaseCount, + OpenMode.CreateNew, + enableLeaseRecovery); + StoreOpenStatus status = Store.TryCreateOrOpen(options, out Store? store); + if (status != StoreOpenStatus.Success || store is null) + { + throw new XunitException($"Could not create {family} production-race store: {status}."); + } + + return store; + } + + private static Store OpenStore(SharedMemoryStoreOptions source, OpenMode openMode) + { + var options = new SharedMemoryStoreOptions + { + Profile = source.Profile, + Name = source.Name, + OpenMode = openMode, + TotalBytes = source.TotalBytes, + SlotCount = source.SlotCount, + MaxValueBytes = source.MaxValueBytes, + MaxDescriptorBytes = source.MaxDescriptorBytes, + MaxKeyBytes = source.MaxKeyBytes, + LeaseRecordCount = source.LeaseRecordCount, + ParticipantRecordCount = source.ParticipantRecordCount, + EnableLeaseRecovery = source.EnableLeaseRecovery + }; + StoreOpenStatus status = Store.TryCreateOrOpen(options, out Store? store); + if (status != StoreOpenStatus.Success || store is null) + { + throw new XunitException($"Could not open production-race keeper: {status}."); + } + + return store; + } + + private static SharedMemoryStoreOptions CreateOptions( + string family, + int slotCount, + int leaseCount, + OpenMode openMode, + bool enableLeaseRecovery = false) => + SharedMemoryStoreOptions.CreateLockFree( + $"sms-sc011-{family}-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 8, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: leaseCount, + participantRecordCount: 4, + openMode, + enableLeaseRecovery); + + private static int ConfiguredRepetitions() + { + string? configured = Environment.GetEnvironmentVariable("SMS_PRODUCTION_RACE_REPETITIONS"); + return int.TryParse(configured, out int value) && value > 0 ? value : DefaultRepetitions; + } + + private static int ConfiguredSeed() + { + string? configured = Environment.GetEnvironmentVariable("SMS_PRODUCTION_RACE_SEED"); + return int.TryParse(configured, out int value) ? value : DefaultSeed; + } + + private static int FamilySeed(int rootSeed, int family) => + unchecked(rootSeed + family * (int)0x9e37_79b9u); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed class TwoActorRace : IDisposable + { + private readonly Action _first; + private readonly Action _second; + private readonly Thread _firstThread; + private readonly Thread _secondThread; + private int _epoch; + private int _goEpoch; + private int _ready; + private int _completed; + private int _firstDelay; + private int _secondDelay; + private int _stop; + private Exception? _failure; + + internal TwoActorRace(Action first, Action second) + { + _first = first; + _second = second; + _firstThread = StartThread(firstActor: true); + _secondThread = StartThread(firstActor: false); + } + + internal void Run(int firstDelay, int secondDelay) + { + Volatile.Write(ref _firstDelay, firstDelay); + Volatile.Write(ref _secondDelay, secondDelay); + Volatile.Write(ref _ready, 0); + Volatile.Write(ref _completed, 0); + int epoch = unchecked(Volatile.Read(ref _epoch) + 1); + Volatile.Write(ref _epoch, epoch); + + var waits = 0; + while (Volatile.Read(ref _ready) != 2) + { + Thread.SpinWait(32); + if ((++waits & 4_095) == 0) + { + Thread.Yield(); + } + } + + Volatile.Write(ref _goEpoch, epoch); + waits = 0; + while (Volatile.Read(ref _completed) != 2) + { + Thread.SpinWait(32); + if ((++waits & 4_095) == 0) + { + Thread.Yield(); + } + } + + if (_failure is not null) + { + throw new XunitException($"Production race worker failed: {_failure.GetType().Name}: {_failure.Message}"); + } + } + + public void Dispose() + { + Volatile.Write(ref _stop, 1); + Volatile.Write(ref _epoch, unchecked(Volatile.Read(ref _epoch) + 1)); + if (!_firstThread.Join(TimeSpan.FromSeconds(10)) || !_secondThread.Join(TimeSpan.FromSeconds(10))) + { + throw new XunitException("Production race workers did not stop."); + } + } + + private Thread StartThread(bool firstActor) + { + var thread = new Thread(() => Worker(firstActor)) + { + IsBackground = true, + Name = firstActor ? "sms-sc011-first" : "sms-sc011-second" + }; + thread.Start(); + return thread; + } + + private void Worker(bool firstActor) + { + int observedEpoch = 0; + while (true) + { + int epoch; + var waits = 0; + while ((epoch = Volatile.Read(ref _epoch)) == observedEpoch) + { + Thread.SpinWait(32); + if ((++waits & 4_095) == 0) + { + Thread.Yield(); + } + } + + observedEpoch = epoch; + if (Volatile.Read(ref _stop) != 0) + { + return; + } + + try + { + Interlocked.Increment(ref _ready); + waits = 0; + while (Volatile.Read(ref _goEpoch) != epoch) + { + Thread.SpinWait(32); + if ((++waits & 4_095) == 0) + { + Thread.Yield(); + } + } + + Thread.SpinWait(firstActor ? Volatile.Read(ref _firstDelay) : Volatile.Read(ref _secondDelay)); + if (firstActor) + { + _first(); + } + else + { + _second(); + } + } + catch (Exception error) + { + Interlocked.CompareExchange(ref _failure, error, null); + } + finally + { + Interlocked.Increment(ref _completed); + } + } + } + } + + private struct DeterministicRandom + { + private uint _state; + + internal DeterministicRandom(int seed) + { + _state = unchecked((uint)seed); + if (_state == 0) + { + _state = 0xa341_316cu; + } + } + + internal int NextDelay() + { + uint value = _state; + value ^= value << 13; + value ^= value >> 17; + value ^= value << 5; + _state = value; + return (int)(value & 63u); + } + } +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/ProjectScaffoldTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/ProjectScaffoldTests.cs new file mode 100644 index 0000000..424fff7 --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/ProjectScaffoldTests.cs @@ -0,0 +1,10 @@ +namespace SharedMemoryStore.LinearizabilityTests; + +public sealed class ProjectScaffoldTests +{ + [Fact] + public void TestProjectIsDiscoverable() + { + Assert.True(true); + } +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/PublicationHistoryTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/PublicationHistoryTests.cs new file mode 100644 index 0000000..502b26c --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/PublicationHistoryTests.cs @@ -0,0 +1,383 @@ +using System.Runtime.InteropServices; +using SharedMemoryStore.LockFree; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.LinearizabilityTests; + +public sealed class PublicationHistoryTests +{ + [Fact] + [Trait("Category", "Linearizability")] + public void ConcurrentSameKeyPublishHistoryHasExactlyOneWinner() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var recorder = new MonotonicHistoryRecorder(); + using var store = CreateLockFreeStore(slotCount: 2, recorder); + var first = recorder.Invoke(1, 1, ReferenceCommand.Publish(1, "same", "left")); + var second = recorder.Invoke(2, 2, ReferenceCommand.Publish(1, "same", "right")); + + RunConcurrent( + () => CompletePublish(first, store, [0x51], [0x11]), + () => CompletePublish(second, store, [0x51], [0x22])); + + var history = recorder.Snapshot(); + var result = new LinearizabilityChecker( + participantCapacity: 4, + valueCapacity: 2, + initialParticipants: [1]).Check(history, recorder.ResourceSnapshot()); + + Assert.All(history, static operation => Assert.True(operation.HasValidCallEnvelope)); + Assert.True(history[0].Overlaps(history[1])); + Assert.Single(history, static operation => operation.Result == ReferenceResultCode.Success); + Assert.Single(history, static operation => operation.Result == ReferenceResultCode.DuplicateKey); + Assert.True(result.IsLinearizable, result.Failure); + AssertLockFree(store); + } + + [Fact] + [Trait("Category", "Linearizability")] + public void ConcurrentDifferentKeyPublishHistoryRespectsOneSlotGlobalCapacity() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var recorder = new MonotonicHistoryRecorder(); + using var store = CreateLockFreeStore(slotCount: 1, recorder); + var first = recorder.Invoke(1, 1, ReferenceCommand.Publish(1, "left-key", "left")); + var second = recorder.Invoke(2, 2, ReferenceCommand.Publish(1, "right-key", "right")); + + RunConcurrent( + () => CompletePublish(first, store, [0x61], [0x11]), + () => CompletePublish(second, store, [0x62], [0x22])); + + var history = recorder.Snapshot(); + var result = new LinearizabilityChecker( + participantCapacity: 4, + valueCapacity: 1, + initialParticipants: [1]).Check(history, recorder.ResourceSnapshot()); + + Assert.True(history[0].Overlaps(history[1])); + Assert.Single(history, static operation => operation.Result == ReferenceResultCode.Success); + Assert.Single(history, static operation => operation.Result == ReferenceResultCode.StoreFull); + Assert.True(result.IsLinearizable, result.Failure); + AssertLockFree(store); + } + + [Fact] + [Trait("Category", "Linearizability")] + public async Task StoreFullWhileOneSlotClaimIsPausedHasAnExactPhysicalWitness() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var recorder = new MonotonicHistoryRecorder(); + using var pause = new ClaimPause(); + using var cancellation = new CancellationTokenSource(); + using Store store = CreateLockFreeStore(slotCount: 1, recorder, pause.Observe); + PendingInvocation tentative = recorder.Invoke( + 1, + 1, + ReferenceCommand.Publish(1, "tentative", "value")); + StoreStatus tentativeStatus = default; + Task tentativeTask = Task.Run(() => + { + tentative.Enter(); + tentativeStatus = store.TryPublish( + [0x71], + [0x11], + default, + new StoreWaitOptions(Timeout.InfiniteTimeSpan, cancellation.Token)); + tentative.Complete(MapPublish(tentativeStatus)); + }); + + try + { + pause.WaitUntilReached(); + PendingInvocation contender = recorder.Invoke( + 2, + 2, + ReferenceCommand.Publish(1, "contender", "value")); + contender.Enter(); + StoreStatus contenderStatus = store.TryPublish( + [0x72], + [0x22], + default, + StoreWaitOptions.Infinite); + RecordedOperation contenderOperation = contender.Complete(MapPublish(contenderStatus)); + + cancellation.Cancel(); + pause.Resume(); + await tentativeTask.WaitAsync(TimeSpan.FromSeconds(10)); + + IReadOnlyList history = recorder.Snapshot(); + IReadOnlyList resources = recorder.ResourceSnapshot(); + LinearizabilityCheckResult result = new LinearizabilityChecker( + participantCapacity: 4, + valueCapacity: 1, + initialParticipants: [1]).Check(history, resources); + + Assert.Equal(StoreStatus.StoreFull, contenderStatus); + Assert.Equal(StoreStatus.OperationCanceled, tentativeStatus); + RecordedSlotResourceWitness claim = Assert.Single( + resources, + static witness => witness.Kind == RecordedSlotResourceKind.Claim); + RecordedSlotResourceWitness freed = Assert.Single( + resources, + static witness => witness.Kind == RecordedSlotResourceKind.Free); + Assert.True(claim.Sequence < contenderOperation.ReturnSequence); + Assert.True(freed.Sequence > contenderOperation.ReturnSequence); + Assert.True(result.IsLinearizable, result.Failure); + } + finally + { + cancellation.Cancel(); + pause.Resume(); + await tentativeTask.WaitAsync(TimeSpan.FromSeconds(10)); + } + } + + [Fact] + [Trait("Category", "Linearizability")] + public void StableTwoSlotCapacityEmitsOneExactStoreFullProof() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var recorder = new MonotonicHistoryRecorder(); + using Store store = CreateLockFreeStore(slotCount: 2, recorder); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x41], [0x11])); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x42], [0x22])); + + PendingInvocation contender = recorder.Invoke( + 1, + 1, + ReferenceCommand.Publish(1, "contender", "value")); + contender.Enter(); + StoreStatus status = store.TryPublish([0x43], [0x33], default, StoreWaitOptions.Infinite); + RecordedOperation operation = contender.Complete(MapPublish(status)); + IReadOnlyList resources = recorder.ResourceSnapshot(); + + Assert.Equal(StoreStatus.StoreFull, status); + RecordedSlotResourceWitness proof = Assert.Single( + resources, + static witness => witness.Kind == RecordedSlotResourceKind.StoreFullProof); + Assert.InRange(proof.Sequence, operation.EntrySequence + 1, operation.ReturnSequence - 1); + LinearizabilityCheckResult result = new LinearizabilityChecker( + participantCapacity: 4, + valueCapacity: 2, + initialParticipants: [1]).Check([operation], resources); + Assert.True(result.IsLinearizable, result.Failure); + } + + [Fact] + [Trait("Category", "Linearizability")] + public async Task SlotMovementBetweenCollectsMakesInfiniteCallerRetryAndClaimWithoutAFullProof() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var recorder = new MonotonicHistoryRecorder(); + using var pause = new StoreFullProofPause(); + using Store store = CreateLockFreeStore(slotCount: 2, recorder, pause.Observe); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x51], [0x11])); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x52], [0x22])); + + PendingInvocation contender = recorder.Invoke( + 1, + 1, + ReferenceCommand.Publish(1, "contender", "value")); + StoreStatus contenderStatus = default; + Task contenderTask = Task.Run(() => + { + contender.Enter(); + contenderStatus = store.TryPublish( + [0x53], + [0x33], + default, + StoreWaitOptions.Infinite); + contender.Complete(MapPublish(contenderStatus)); + }); + + try + { + pause.WaitUntilReached(); + Assert.Equal(StoreStatus.Success, store.TryRemove([0x51], StoreWaitOptions.Infinite)); + pause.Resume(); + await contenderTask.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(StoreStatus.Success, contenderStatus); + Assert.DoesNotContain( + recorder.ResourceSnapshot(), + static witness => witness.Kind == RecordedSlotResourceKind.StoreFullProof); + Assert.Equal(StoreStatus.StoreFull, store.TryPublish([0x54], [0x44])); + } + finally + { + pause.Resume(); + await contenderTask.WaitAsync(TimeSpan.FromSeconds(10)); + } + } + + private static Store CreateLockFreeStore( + int slotCount, + MonotonicHistoryRecorder recorder, + Action? checkpointObserver = null) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-linearizable-publish-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 4, + participantRecordCount: 4, + openMode: OpenMode.CreateNew); + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented( + entry => checkpointObserver?.Invoke(entry), + recorder.ObserveSlotResource, + recorder, + recorder); + var status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen(options, checkpoint, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static void CompletePublish( + PendingInvocation invocation, + Store store, + byte[] key, + byte[] value) + { + invocation.Enter(); + var status = store.TryPublish(key, value); + invocation.Complete(status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.DuplicateKey => ReferenceResultCode.DuplicateKey, + StoreStatus.StoreFull => ReferenceResultCode.StoreFull, + StoreStatus.OperationCanceled => ReferenceResultCode.OperationCanceled, + StoreStatus.StoreBusy => ReferenceResultCode.StoreBusy, + _ => ReferenceResultCode.Unexpected + }); + } + + private static ReferenceResultCode MapPublish(StoreStatus status) => status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.DuplicateKey => ReferenceResultCode.DuplicateKey, + StoreStatus.StoreFull => ReferenceResultCode.StoreFull, + StoreStatus.OperationCanceled => ReferenceResultCode.OperationCanceled, + StoreStatus.StoreBusy => ReferenceResultCode.StoreBusy, + _ => ReferenceResultCode.Unexpected + }; + + private static void RunConcurrent(Action first, Action second) + { + using var start = new Barrier(participantCount: 3); + var firstTask = Task.Run(() => + { + start.SignalAndWait(); + first(); + }); + var secondTask = Task.Run(() => + { + start.SignalAndWait(); + second(); + }); + start.SignalAndWait(); + Assert.True(Task.WaitAll([firstTask, secondTask], TimeSpan.FromSeconds(10))); + } + + private static void AssertLockFree(Store store) + { + Assert.Equal(StoreProfile.LockFree, store.Profile); + Assert.Equal(StoreProfile.LockFree, store.ProtocolInfo.Profile); + Assert.Equal(2, store.ProtocolInfo.LayoutMajorVersion); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed class ClaimPause : IDisposable + { + private readonly ManualResetEventSlim _reached = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: false); + private int _paused; + + internal void Observe(LockFreeCheckpointEntry entry) + { + if (entry.Id != LockFreeCheckpointId.SlotClaimAfterParticipantRecheck + || Interlocked.Exchange(ref _paused, 1) != 0) + { + return; + } + + _reached.Set(); + if (!_resume.Wait(TimeSpan.FromSeconds(10))) + { + throw new Xunit.Sdk.XunitException("Paused claim was not resumed within the test bound."); + } + } + + internal void WaitUntilReached() + { + Assert.True(_reached.Wait(TimeSpan.FromSeconds(10)), "The slot claim checkpoint was not reached."); + } + + internal void Resume() => _resume.Set(); + + public void Dispose() + { + _resume.Set(); + _reached.Dispose(); + _resume.Dispose(); + } + } + + private sealed class StoreFullProofPause : IDisposable + { + private readonly ManualResetEventSlim _reached = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: false); + private int _paused; + + internal void Observe(LockFreeCheckpointEntry entry) + { + if (entry.Id != LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification + || Interlocked.Exchange(ref _paused, 1) != 0) + { + return; + } + + _reached.Set(); + if (!_resume.Wait(TimeSpan.FromSeconds(10))) + { + throw new Xunit.Sdk.XunitException("Paused StoreFull proof was not resumed within the test bound."); + } + } + + internal void WaitUntilReached() => + Assert.True(_reached.Wait(TimeSpan.FromSeconds(10)), "The StoreFull proof checkpoint was not reached."); + + internal void Resume() => _resume.Set(); + + public void Dispose() + { + _resume.Set(); + _reached.Dispose(); + _resume.Dispose(); + } + } +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/ReferenceStoreModel.cs b/tests/SharedMemoryStore.LinearizabilityTests/ReferenceStoreModel.cs new file mode 100644 index 0000000..e75da0a --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/ReferenceStoreModel.cs @@ -0,0 +1,901 @@ +using System.Collections.Concurrent; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.LinearizabilityTests; + +internal enum ReferenceOperationKind +{ + OpenParticipant, + CloseParticipant, + Publish, + Reserve, + CommitReservation, + AbortReservation, + Acquire, + AcquireLease, + ReleaseLease, + Remove, + RecoverReservation, + RecoverLease, + DisposeParticipant +} + +internal enum ReferenceResultCode +{ + Success, + DuplicateKey, + NotFound, + StoreFull, + ParticipantTableFull, + ParticipantNotActive, + LeaseTableFull, + RemovePending, + InvalidLease, + LeaseAlreadyReleased, + InvalidReservation, + ReservationAlreadyCompleted, + StoreDisposed, + OperationCanceled, + StoreBusy, + Unexpected +} + +internal readonly record struct ReferenceCommand( + ReferenceOperationKind Kind, + int ParticipantId, + string Key, + string Value, + int TokenId, + int ByteCount) +{ + public static ReferenceCommand OpenParticipant(int participantId) => + new(ReferenceOperationKind.OpenParticipant, participantId, string.Empty, string.Empty, 0, 0); + + public static ReferenceCommand CloseParticipant(int participantId) => + new(ReferenceOperationKind.CloseParticipant, participantId, string.Empty, string.Empty, 0, 0); + + public static ReferenceCommand Publish(int participantId, string key, string value) => + new(ReferenceOperationKind.Publish, participantId, key, value, 0, 0); + + public static ReferenceCommand Reserve( + int participantId, + int reservationId, + string key, + string value) => + new(ReferenceOperationKind.Reserve, participantId, key, value, reservationId, value.Length); + + public static ReferenceCommand CommitReservation(int participantId, int reservationId) => + new(ReferenceOperationKind.CommitReservation, participantId, string.Empty, string.Empty, reservationId, 0); + + public static ReferenceCommand AbortReservation(int participantId, int reservationId) => + new(ReferenceOperationKind.AbortReservation, participantId, string.Empty, string.Empty, reservationId, 0); + + public static ReferenceCommand Acquire(int participantId, string key) => + new(ReferenceOperationKind.Acquire, participantId, key, string.Empty, 0, 0); + + public static ReferenceCommand AcquireLease(int participantId, int leaseId, string key) => + new(ReferenceOperationKind.AcquireLease, participantId, key, string.Empty, leaseId, 0); + + public static ReferenceCommand ReleaseLease(int participantId, int leaseId) => + new(ReferenceOperationKind.ReleaseLease, participantId, string.Empty, string.Empty, leaseId, 0); + + public static ReferenceCommand Remove(int participantId, string key) => + new(ReferenceOperationKind.Remove, participantId, key, string.Empty, 0, 0); + + public static ReferenceCommand RecoverReservation(int participantId, int reservationId) => + new(ReferenceOperationKind.RecoverReservation, participantId, string.Empty, string.Empty, reservationId, 0); + + public static ReferenceCommand RecoverLease(int participantId, int leaseId) => + new(ReferenceOperationKind.RecoverLease, participantId, string.Empty, string.Empty, leaseId, 0); + + public static ReferenceCommand DisposeParticipant(int participantId) => + new(ReferenceOperationKind.DisposeParticipant, participantId, string.Empty, string.Empty, 0, 0); +} + +internal sealed record RecordedOperation( + int Id, + int ActorId, + ReferenceCommand Command, + ReferenceResultCode Result, + long InvocationSequence, + long EntrySequence, + long ReturnSequence, + long ResponseSequence, + string? ObservedValue = null, + long ObservedGeneration = 0, + bool RequiresAcquireObservation = false, + bool UsesInfiniteWait = false) +{ + public bool HasValidCallEnvelope => + InvocationSequence < EntrySequence + && EntrySequence < ReturnSequence + && ReturnSequence < ResponseSequence; + + public bool HappensBefore(RecordedOperation other) => + ResponseSequence < other.InvocationSequence; + + public bool Overlaps(RecordedOperation other) => + !HappensBefore(other) && !other.HappensBefore(this); + + /// + /// Returns whether the two calls were simultaneously inside the mapped + /// implementation. Invocation/response overlap alone proves no mapped + /// race because callers can create both invocation records in advance. + /// + public bool ImplementationOverlaps(RecordedOperation other) => + EntrySequence < other.ReturnSequence + && other.EntrySequence < ReturnSequence; +} + +internal enum RecordedSlotResourceKind +{ + Claim, + Free, + Retire, + StoreFullProof, + LeaseTableFullProof +} + +internal readonly record struct RecordedSlotResourceWitness( + RecordedSlotResourceKind Kind, + int SlotIndex, + long Generation, + long Sequence, + long ConfirmationSequence = 0); + +internal sealed class MonotonicHistoryRecorder : + ILockFreeStoreFullProofObserver, + ILockFreeLeaseTableFullProofObserver +{ + private readonly ConcurrentDictionary _operations = new(); + private readonly ConcurrentQueue _slotResources = new(); + private readonly ConcurrentDictionary _storeFullCandidates = new(); + private readonly ConcurrentDictionary _leaseTableFullCandidates = new(); + private readonly bool _strictProductionHistory; + private long _sequence; + + internal MonotonicHistoryRecorder(bool strictProductionHistory = false) + { + _strictProductionHistory = strictProductionHistory; + } + + public PendingInvocation Invoke(int id, int actorId, ReferenceCommand command) + { + return new PendingInvocation( + this, + id, + actorId, + command, + NextSequence(), + _strictProductionHistory); + } + + public IReadOnlyList Snapshot() + { + return _operations.Values.OrderBy(static operation => operation.Id).ToArray(); + } + + public IReadOnlyList ResourceSnapshot() + { + return _slotResources.OrderBy(static witness => witness.Sequence).ToArray(); + } + + internal void ObserveSlotResource(LockFreeSlotResourceEvent resourceEvent) + { + RecordedSlotResourceKind kind = resourceEvent.Kind switch + { + LockFreeSlotResourceEventKind.Claim => RecordedSlotResourceKind.Claim, + LockFreeSlotResourceEventKind.Free => RecordedSlotResourceKind.Free, + LockFreeSlotResourceEventKind.Retire => RecordedSlotResourceKind.Retire, + _ => throw new ArgumentOutOfRangeException(nameof(resourceEvent)) + }; + _slotResources.Enqueue(new RecordedSlotResourceWitness( + kind, + resourceEvent.SlotIndex, + resourceEvent.Generation, + NextSequence())); + } + + public long BeginCandidate(int slotCount) + { + if (slotCount <= 0) + { + throw new ArgumentOutOfRangeException(nameof(slotCount)); + } + + long candidateSequence = NextSequence(); + if (!_storeFullCandidates.TryAdd(candidateSequence, slotCount)) + { + throw new InvalidOperationException( + $"StoreFull proof candidate sequence {candidateSequence} was observed twice."); + } + + return candidateSequence; + } + + public void CompleteCandidate(long token, bool confirmed) + { + if (token <= 0 || !_storeFullCandidates.TryRemove(token, out int slotCount)) + { + throw new InvalidOperationException( + $"StoreFull proof token {token} completed without its candidate."); + } + + if (!confirmed) + { + return; + } + + // The second collect validates the earlier common instant. The witness + // therefore carries the candidate's sequence plus its later distinct + // confirmation sequence. + _slotResources.Enqueue(new RecordedSlotResourceWitness( + RecordedSlotResourceKind.StoreFullProof, + SlotIndex: -1, + Generation: slotCount, + Sequence: token, + ConfirmationSequence: NextSequence())); + } + + long ILockFreeLeaseTableFullProofObserver.BeginCandidate(int leaseRecordCount) + { + if (leaseRecordCount <= 0) + { + throw new ArgumentOutOfRangeException(nameof(leaseRecordCount)); + } + + long candidateSequence = NextSequence(); + if (!_leaseTableFullCandidates.TryAdd(candidateSequence, leaseRecordCount)) + { + throw new InvalidOperationException( + $"LeaseTableFull proof candidate sequence {candidateSequence} was observed twice."); + } + + return candidateSequence; + } + + void ILockFreeLeaseTableFullProofObserver.CompleteCandidate( + long token, + bool confirmed) + { + if (token <= 0 + || !_leaseTableFullCandidates.TryRemove(token, out int leaseRecordCount)) + { + throw new InvalidOperationException( + $"LeaseTableFull proof token {token} completed without its candidate."); + } + + if (!confirmed) + { + return; + } + + _slotResources.Enqueue(new RecordedSlotResourceWitness( + RecordedSlotResourceKind.LeaseTableFullProof, + SlotIndex: -1, + Generation: leaseRecordCount, + Sequence: token, + ConfirmationSequence: NextSequence())); + } + + internal long NextSequence() => Interlocked.Increment(ref _sequence); + + internal void Add(RecordedOperation operation) + { + if (!_operations.TryAdd(operation.Id, operation)) + { + throw new InvalidOperationException($"Operation ID {operation.Id} was recorded more than once."); + } + } +} + +internal sealed class PendingInvocation +{ + private readonly MonotonicHistoryRecorder _recorder; + private readonly int _id; + private readonly int _actorId; + private readonly ReferenceCommand _command; + private readonly long _invocationSequence; + private readonly bool _strictProductionHistory; + private long _entrySequence; + private int _completed; + + internal PendingInvocation( + MonotonicHistoryRecorder recorder, + int id, + int actorId, + ReferenceCommand command, + long invocationSequence, + bool strictProductionHistory) + { + _recorder = recorder; + _id = id; + _actorId = actorId; + _command = command; + _invocationSequence = invocationSequence; + _strictProductionHistory = strictProductionHistory; + } + + public void Enter() + { + if (Interlocked.CompareExchange(ref _entrySequence, _recorder.NextSequence(), 0) != 0) + { + throw new InvalidOperationException("An invocation can enter the implementation only once."); + } + } + + public RecordedOperation Complete( + ReferenceResultCode result, + string? observedValue = null, + long observedGeneration = 0) + { + if (Volatile.Read(ref _entrySequence) == 0) + { + throw new InvalidOperationException("An invocation must enter before it returns."); + } + + if (Interlocked.Exchange(ref _completed, 1) != 0) + { + throw new InvalidOperationException("An invocation can complete only once."); + } + + bool isAcquire = _command.Kind is ReferenceOperationKind.Acquire + or ReferenceOperationKind.AcquireLease; + if (_strictProductionHistory + && isAcquire + && result == ReferenceResultCode.Success + && (observedValue is null || observedGeneration <= 0)) + { + throw new InvalidOperationException( + "A successful production acquire must record its returned bytes and exact nonzero slot generation."); + } + + if ((!isAcquire || result != ReferenceResultCode.Success) + && (observedValue is not null || observedGeneration != 0)) + { + throw new InvalidOperationException( + "Only a successful acquire may carry a returned value observation."); + } + + var returnSequence = _recorder.NextSequence(); + var responseSequence = _recorder.NextSequence(); + var operation = new RecordedOperation( + _id, + _actorId, + _command, + result, + _invocationSequence, + Volatile.Read(ref _entrySequence), + returnSequence, + responseSequence, + observedValue, + observedGeneration, + RequiresAcquireObservation: _strictProductionHistory && isAcquire, + UsesInfiniteWait: _strictProductionHistory); + _recorder.Add(operation); + return operation; + } +} + +internal sealed class ReferenceStoreModel +{ + private readonly HashSet _participants; + private readonly Dictionary _values; + private readonly Dictionary _reservations; + private readonly Dictionary _leases; + + public ReferenceStoreModel( + int participantCapacity, + int valueCapacity, + IEnumerable? initialParticipants = null, + int? leaseCapacity = null) + { + ArgumentOutOfRangeException.ThrowIfLessThan(participantCapacity, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(valueCapacity, 1); + ParticipantCapacity = participantCapacity; + ValueCapacity = valueCapacity; + LeaseCapacity = leaseCapacity ?? Math.Max(1, valueCapacity); + ArgumentOutOfRangeException.ThrowIfLessThan(LeaseCapacity, 1); + _participants = initialParticipants?.ToHashSet() ?? []; + if (_participants.Count > participantCapacity || _participants.Any(static participant => participant <= 0)) + { + throw new ArgumentException("Initial participants must be positive and fit the configured capacity.", nameof(initialParticipants)); + } + + _values = new Dictionary(StringComparer.Ordinal); + _reservations = []; + _leases = []; + } + + private ReferenceStoreModel(ReferenceStoreModel source) + { + ParticipantCapacity = source.ParticipantCapacity; + ValueCapacity = source.ValueCapacity; + LeaseCapacity = source.LeaseCapacity; + _participants = new HashSet(source._participants); + _values = new Dictionary(source._values, StringComparer.Ordinal); + _reservations = new Dictionary(source._reservations); + _leases = new Dictionary(source._leases); + } + + public int ParticipantCapacity { get; } + + public int ValueCapacity { get; } + + public int LeaseCapacity { get; } + + public int ParticipantCount => _participants.Count; + + public int ValueCount => _values.Count + _reservations.Count; + + public bool TryGetValue(string key, out string? value) + { + if (_values.TryGetValue(key, out ValueEntry entry) + && entry.State == ReferenceValueState.Published) + { + value = entry.Value; + return true; + } + + value = null; + return false; + } + + public bool TryApply( + ReferenceCommand command, + ReferenceResultCode observedResult, + out ReferenceStoreModel? next, + bool physicalStoreFullWitnessed = false, + bool physicalLeaseTableFullWitnessed = false, + string? observedValue = null, + long observedGeneration = 0, + bool requiresAcquireObservation = false) + { + next = new ReferenceStoreModel(this); + + // A claimed-but-not-yet-ordered lifecycle consumes a physical slot but + // owns no abstract key. The strict checker admits StoreFull at that + // physical point without adding a value or reservation to this model. + if (observedResult == ReferenceResultCode.StoreFull && physicalStoreFullWitnessed) + { + bool validPhysicalResult = _participants.Contains(command.ParticipantId) + && command.Kind is ReferenceOperationKind.Publish or ReferenceOperationKind.Reserve + && (command.Kind != ReferenceOperationKind.Reserve + || (command.TokenId > 0 && !_reservations.ContainsKey(command.TokenId))); + if (validPhysicalResult) + { + return true; + } + + next = null; + return false; + } + + // A Claiming lease record consumes physical capacity before the acquire + // becomes abstractly successful. A strict exact proof may therefore + // justify LeaseTableFull even when this bounded model has no public + // lease token for every occupied physical record. + if (observedResult == ReferenceResultCode.LeaseTableFull + && physicalLeaseTableFullWitnessed) + { + bool validPhysicalResult = _participants.Contains(command.ParticipantId) + && command.Kind == ReferenceOperationKind.AcquireLease + && command.TokenId > 0 + && !_leases.ContainsKey(command.TokenId) + && _values.TryGetValue(command.Key, out ValueEntry value) + && value.State == ReferenceValueState.Published; + if (validPhysicalResult) + { + return true; + } + + next = null; + return false; + } + + if (observedResult is ReferenceResultCode.OperationCanceled or ReferenceResultCode.StoreBusy) + { + return true; + } + + var expectedResult = next.Apply(command); + if (expectedResult == observedResult) + { + if (requiresAcquireObservation + || observedValue is not null + || observedGeneration != 0) + { + if (!next.TryBindAcquireObservation( + command, + observedResult, + observedValue, + observedGeneration)) + { + next = null; + return false; + } + } + + return true; + } + + // The production contract permits RemovePending after the logical + // removal ordering point even when no protecting lease remains, when + // bounded unlink/reclaim work is still incomplete. Its abstract state + // transition is the same successful removal modeled above. + if (command.Kind == ReferenceOperationKind.Remove + && observedResult == ReferenceResultCode.RemovePending + && expectedResult == ReferenceResultCode.Success) + { + return true; + } + + next = null; + return false; + } + + public string Fingerprint() + { + var participants = string.Join(',', _participants.Order()); + var values = string.Join( + '|', + _values.OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .Select(static pair => + pair.Key + + "=" + + pair.Value.Value + + ":" + + pair.Value.State + + ":g" + + pair.Value.MappedGeneration)); + var reservations = string.Join( + '|', + _reservations.OrderBy(static pair => pair.Key) + .Select(static pair => pair.Key + "=" + pair.Value.ParticipantId + ":" + pair.Value.Key)); + var leases = string.Join( + '|', + _leases.OrderBy(static pair => pair.Key) + .Select(static pair => pair.Key + "=" + pair.Value.ParticipantId + ":" + pair.Value.Key)); + return participants + ";" + values + ";" + reservations + ";" + leases; + } + + private ReferenceResultCode Apply(ReferenceCommand command) + { + return command.Kind switch + { + ReferenceOperationKind.OpenParticipant => OpenParticipant(command.ParticipantId), + ReferenceOperationKind.CloseParticipant => CloseParticipant(command.ParticipantId), + ReferenceOperationKind.Publish => Publish(command.ParticipantId, command.Key, command.Value), + ReferenceOperationKind.Reserve => Reserve(command), + ReferenceOperationKind.CommitReservation => CommitReservation(command), + ReferenceOperationKind.AbortReservation => AbortReservation(command), + ReferenceOperationKind.Acquire => Acquire(command.ParticipantId, command.Key), + ReferenceOperationKind.AcquireLease => AcquireLease(command), + ReferenceOperationKind.ReleaseLease => ReleaseLease(command), + ReferenceOperationKind.Remove => Remove(command.ParticipantId, command.Key), + ReferenceOperationKind.RecoverReservation => RecoverReservation( + command.ParticipantId, + command.TokenId), + ReferenceOperationKind.RecoverLease => RecoverLease( + command.ParticipantId, + command.TokenId), + ReferenceOperationKind.DisposeParticipant => DisposeParticipant(command.ParticipantId), + _ => ReferenceResultCode.Unexpected + }; + } + + private bool TryBindAcquireObservation( + ReferenceCommand command, + ReferenceResultCode result, + string? observedValue, + long observedGeneration) + { + if (command.Kind is not ( + ReferenceOperationKind.Acquire or ReferenceOperationKind.AcquireLease)) + { + return false; + } + + if (result != ReferenceResultCode.Success) + { + return observedValue is null && observedGeneration == 0; + } + + if (observedValue is null + || observedGeneration is < 1 or > LockFreeSlotTable.TerminalGeneration + || !_values.TryGetValue(command.Key, out ValueEntry value) + || value.State != ReferenceValueState.Published + || !string.Equals(value.Value, observedValue, StringComparison.Ordinal)) + { + return false; + } + + if (value.MappedGeneration != 0 + && value.MappedGeneration != observedGeneration) + { + return false; + } + + if (value.MappedGeneration == 0) + { + _values[command.Key] = value with { MappedGeneration = observedGeneration }; + } + + return true; + } + + private ReferenceResultCode OpenParticipant(int participantId) + { + if (participantId <= 0 || _participants.Contains(participantId)) + { + return ReferenceResultCode.ParticipantNotActive; + } + + if (_participants.Count == ParticipantCapacity) + { + return ReferenceResultCode.ParticipantTableFull; + } + + _participants.Add(participantId); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode CloseParticipant(int participantId) + { + return _participants.Remove(participantId) + ? ReferenceResultCode.Success + : ReferenceResultCode.ParticipantNotActive; + } + + private ReferenceResultCode Publish(int participantId, string key, string value) + { + if (!_participants.Contains(participantId)) + { + return ReferenceResultCode.ParticipantNotActive; + } + + if (ContainsKey(key)) + { + return ReferenceResultCode.DuplicateKey; + } + + if (ValueCount == ValueCapacity) + { + return ReferenceResultCode.StoreFull; + } + + _values.Add(key, new ValueEntry(value, ReferenceValueState.Published, MappedGeneration: 0)); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode Reserve(ReferenceCommand command) + { + if (!_participants.Contains(command.ParticipantId)) + { + return ReferenceResultCode.ParticipantNotActive; + } + + if (command.TokenId <= 0 || _reservations.ContainsKey(command.TokenId)) + { + return ReferenceResultCode.InvalidReservation; + } + + if (ContainsKey(command.Key)) + { + return ReferenceResultCode.DuplicateKey; + } + + if (ValueCount == ValueCapacity) + { + return ReferenceResultCode.StoreFull; + } + + _reservations.Add( + command.TokenId, + new ReservationEntry(command.ParticipantId, command.Key, command.Value)); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode CommitReservation(ReferenceCommand command) + { + if (!_participants.Contains(command.ParticipantId)) + { + return ReferenceResultCode.ParticipantNotActive; + } + + if (!_reservations.Remove(command.TokenId, out ReservationEntry reservation) + || reservation.ParticipantId != command.ParticipantId) + { + return ReferenceResultCode.InvalidReservation; + } + + _values.Add( + reservation.Key, + new ValueEntry(reservation.Value, ReferenceValueState.Published, MappedGeneration: 0)); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode AbortReservation(ReferenceCommand command) + { + if (!_participants.Contains(command.ParticipantId)) + { + return ReferenceResultCode.ParticipantNotActive; + } + + return _reservations.Remove(command.TokenId, out ReservationEntry reservation) + && reservation.ParticipantId == command.ParticipantId + ? ReferenceResultCode.Success + : ReferenceResultCode.InvalidReservation; + } + + private ReferenceResultCode Remove(int participantId, string key) + { + if (!_participants.Contains(participantId)) + { + return ReferenceResultCode.ParticipantNotActive; + } + + if (!_values.TryGetValue(key, out ValueEntry value)) + { + return ReferenceResultCode.NotFound; + } + + bool isProtected = _leases.Values.Any( + lease => string.Equals(lease.Key, key, StringComparison.Ordinal)); + if (value.State == ReferenceValueState.RemoveRequested) + { + // Logical absence rejects new acquires, but the exact generation + // remains discoverable to a retrying remove until reclamation. + if (isProtected) + { + return ReferenceResultCode.RemovePending; + } + + _values.Remove(key); + return ReferenceResultCode.Success; + } + + if (isProtected) + { + _values[key] = value with { State = ReferenceValueState.RemoveRequested }; + return ReferenceResultCode.RemovePending; + } + + _values.Remove(key); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode Acquire(int participantId, string key) + { + if (!_participants.Contains(participantId)) + { + return ReferenceResultCode.ParticipantNotActive; + } + + return _values.TryGetValue(key, out ValueEntry value) + && value.State == ReferenceValueState.Published + ? ReferenceResultCode.Success + : ReferenceResultCode.NotFound; + } + + private ReferenceResultCode AcquireLease(ReferenceCommand command) + { + if (!_participants.Contains(command.ParticipantId)) + { + return ReferenceResultCode.ParticipantNotActive; + } + + if (!_values.TryGetValue(command.Key, out ValueEntry value) + || value.State != ReferenceValueState.Published) + { + return ReferenceResultCode.NotFound; + } + + if (command.TokenId <= 0 || _leases.ContainsKey(command.TokenId)) + { + return ReferenceResultCode.InvalidLease; + } + + if (_leases.Count == LeaseCapacity) + { + return ReferenceResultCode.LeaseTableFull; + } + + _leases.Add(command.TokenId, new LeaseEntry(command.ParticipantId, command.Key)); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode ReleaseLease(ReferenceCommand command) + { + if (!_leases.Remove(command.TokenId, out LeaseEntry lease) + || lease.ParticipantId != command.ParticipantId) + { + return ReferenceResultCode.InvalidLease; + } + + ReclaimIfUnprotected(lease.Key); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode RecoverReservation(int recoveringParticipantId, int reservationId) + { + if (!_reservations.TryGetValue(reservationId, out ReservationEntry reservation) + || reservation.ParticipantId == recoveringParticipantId) + { + return ReferenceResultCode.InvalidReservation; + } + + _reservations.Remove(reservationId); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode RecoverLease(int recoveringParticipantId, int leaseId) + { + if (!_leases.TryGetValue(leaseId, out LeaseEntry lease) + || lease.ParticipantId == recoveringParticipantId) + { + return ReferenceResultCode.InvalidLease; + } + + _leases.Remove(leaseId); + ReclaimIfUnprotected(lease.Key); + return ReferenceResultCode.Success; + } + + private ReferenceResultCode DisposeParticipant(int participantId) + { + if (!_participants.Remove(participantId)) + { + return ReferenceResultCode.StoreDisposed; + } + + foreach (int reservationId in _reservations + .Where(pair => pair.Value.ParticipantId == participantId) + .Select(static pair => pair.Key) + .ToArray()) + { + _reservations.Remove(reservationId); + } + + foreach (int leaseId in _leases + .Where(pair => pair.Value.ParticipantId == participantId) + .Select(static pair => pair.Key) + .ToArray()) + { + LeaseEntry lease = _leases[leaseId]; + _leases.Remove(leaseId); + ReclaimIfUnprotected(lease.Key); + } + + return ReferenceResultCode.Success; + } + + private bool ContainsKey(string key) => + _values.ContainsKey(key) + || _reservations.Values.Any(reservation => string.Equals(reservation.Key, key, StringComparison.Ordinal)); + + private void ReclaimIfUnprotected(string key) + { + if (_values.TryGetValue(key, out ValueEntry value) + && value.State == ReferenceValueState.RemoveRequested + && !_leases.Values.Any(lease => string.Equals(lease.Key, key, StringComparison.Ordinal))) + { + _values.Remove(key); + } + } + + private enum ReferenceValueState + { + Published, + RemoveRequested + } + + private readonly record struct ValueEntry( + string Value, + ReferenceValueState State, + long MappedGeneration); + + private readonly record struct ReservationEntry(int ParticipantId, string Key, string Value); + + private readonly record struct LeaseEntry(int ParticipantId, string Key); +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/RemoveHistoryTests.cs b/tests/SharedMemoryStore.LinearizabilityTests/RemoveHistoryTests.cs new file mode 100644 index 0000000..d81dfcc --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/RemoveHistoryTests.cs @@ -0,0 +1,283 @@ +using System.Runtime.InteropServices; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.LinearizabilityTests; + +public sealed class RemoveHistoryTests +{ + [Fact] + [Trait("Category", "Linearizability")] + public void OverlappingAcquireSuccessAndRemovePendingLinearizeAcquireFirst() + { + var history = new[] + { + Operation(1, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.Success, 1, 3, 5, 8), + Operation(2, ReferenceCommand.Remove(1, "key"), ReferenceResultCode.RemovePending, 2, 4, 6, 7) + }; + + var result = CreateChecker().Check(WithPublishedSetup(history)); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([63, 1, 2], result.Linearization); + } + + [Fact] + [Trait("Category", "Linearizability")] + public void OverlappingRemoveAndNotFoundAcquireLinearizeRemoveFirst() + { + var history = new[] + { + Operation(1, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.NotFound, 1, 3, 7, 8), + Operation(2, ReferenceCommand.Remove(1, "key"), ReferenceResultCode.Success, 2, 4, 5, 6) + }; + + var result = CreateChecker().Check(WithPublishedSetup(history)); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([63, 2, 1], result.Linearization); + } + + [Fact] + [Trait("Category", "Linearizability")] + public void AcquireCannotSucceedWhenInvokedAfterCompletedRemoval() + { + var history = new[] + { + Operation(1, ReferenceCommand.Remove(1, "key"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.AcquireLease(1, 20, "key"), ReferenceResultCode.Success, 5, 6, 7, 8) + }; + + var result = CreateChecker().Check(WithPublishedSetup(history)); + + Assert.False(result.IsLinearizable); + } + + [Fact] + [Trait("Category", "Linearizability")] + public void RemovePendingWithoutALeaseRepresentsCompletedLogicalRemovalWithBoundedWorkRemaining() + { + var history = new[] + { + Operation(1, ReferenceCommand.Publish(1, "key", "first"), ReferenceResultCode.Success, 1, 2, 3, 4), + Operation(2, ReferenceCommand.Remove(1, "key"), ReferenceResultCode.RemovePending, 5, 6, 7, 8), + Operation(3, ReferenceCommand.Publish(1, "key", "second"), ReferenceResultCode.Success, 9, 10, 11, 12) + }; + + var result = CreateChecker().Check(history); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([1, 2, 3], result.Linearization); + } + + [Fact] + [Trait("Category", "Linearizability")] + public void SequentialInfiniteRemoveRetriesRemainPendingUntilProtectingLeaseIsReleased() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var store = CreateStore(); + Assert.Equal( + StoreStatus.Success, + store.TryPublish([1], [9], default, StoreWaitOptions.Infinite)); + var recorder = new MonotonicHistoryRecorder(); + ValueLease lease = default; + + try + { + PendingInvocation acquire = recorder.Invoke( + 1, + actorId: 1, + ReferenceCommand.AcquireLease(1, 20, "key")); + acquire.Enter(); + StoreStatus acquireStatus = store.TryAcquire( + [1], + StoreWaitOptions.Infinite, + out lease); + Assert.Equal(StoreStatus.Success, acquireStatus); + acquire.Complete(ReferenceResultCode.Success); + + PendingInvocation firstRemove = recorder.Invoke( + 2, + actorId: 2, + ReferenceCommand.Remove(1, "key")); + firstRemove.Enter(); + StoreStatus firstRemoveStatus = store.TryRemove([1], StoreWaitOptions.Infinite); + Assert.Equal(StoreStatus.RemovePending, firstRemoveStatus); + firstRemove.Complete(ReferenceResultCode.RemovePending); + + PendingInvocation secondRemove = recorder.Invoke( + 3, + actorId: 2, + ReferenceCommand.Remove(1, "key")); + secondRemove.Enter(); + StoreStatus secondRemoveStatus = store.TryRemove([1], StoreWaitOptions.Infinite); + Assert.Equal(StoreStatus.RemovePending, secondRemoveStatus); + secondRemove.Complete(ReferenceResultCode.RemovePending); + + PendingInvocation release = recorder.Invoke( + 4, + actorId: 1, + ReferenceCommand.ReleaseLease(1, 20)); + release.Enter(); + StoreStatus releaseStatus = lease.Release(StoreWaitOptions.Infinite); + Assert.Equal(StoreStatus.Success, releaseStatus); + release.Complete(ReferenceResultCode.Success); + + Assert.Equal( + StoreStatus.Success, + store.TryGetDiagnostics(StoreWaitOptions.Infinite, out var diagnostics)); + Assert.Equal(0, diagnostics.PendingRemovalCount); + Assert.Equal(2, diagnostics.FreeSlotCount); + + PendingInvocation finalRemove = recorder.Invoke( + 5, + actorId: 2, + ReferenceCommand.Remove(1, "key")); + finalRemove.Enter(); + StoreStatus finalRemoveStatus = store.TryRemove([1], StoreWaitOptions.Infinite); + Assert.Equal(StoreStatus.NotFound, finalRemoveStatus); + finalRemove.Complete(ReferenceResultCode.NotFound); + + LinearizabilityCheckResult result = CreateChecker().Check( + WithPublishedSetup(recorder.Snapshot())); + + Assert.True(result.IsLinearizable, result.Failure); + Assert.Equal([63, 1, 2, 3, 4, 5], result.Linearization); + } + finally + { + if (lease.IsValid) + { + _ = lease.Release(StoreWaitOptions.Infinite); + } + } + } + + [Fact] + [Trait("Category", "Linearizability")] + public async Task PublicOverlappingAcquireRemoveHistoryMatchesReferenceOrdering() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var store = CreateStore(); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [9])); + var recorder = new MonotonicHistoryRecorder(); + var acquireInvocation = recorder.Invoke(1, 1, ReferenceCommand.AcquireLease(1, 20, "key")); + var removeInvocation = recorder.Invoke(2, 2, ReferenceCommand.Remove(1, "key")); + ValueLease lease = default; + + await RunConcurrent( + () => + { + acquireInvocation.Enter(); + var status = store.TryAcquire([1], out lease); + acquireInvocation.Complete(status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.NotFound => ReferenceResultCode.NotFound, + _ => ReferenceResultCode.Unexpected + }); + }, + () => + { + removeInvocation.Enter(); + var status = store.TryRemove([1]); + removeInvocation.Complete(status switch + { + StoreStatus.Success => ReferenceResultCode.Success, + StoreStatus.RemovePending => ReferenceResultCode.RemovePending, + StoreStatus.NotFound => ReferenceResultCode.NotFound, + _ => ReferenceResultCode.Unexpected + }); + }); + + var result = CreateChecker().Check(WithPublishedSetup(recorder.Snapshot())); + try + { + Assert.True(result.IsLinearizable, result.Failure); + } + finally + { + if (lease.IsValid) + { + Assert.Equal(StoreStatus.Success, lease.Release()); + } + } + } + + private static LinearizabilityChecker CreateChecker() => + new(2, 2, initialParticipants: [1]); + + private static IReadOnlyList WithPublishedSetup( + IReadOnlyList operations) + { + var history = new RecordedOperation[operations.Count + 1]; + history[0] = new RecordedOperation( + 63, + 1, + ReferenceCommand.Publish(1, "key", "value"), + ReferenceResultCode.Success, + -4, + -3, + -2, + -1); + for (var index = 0; index < operations.Count; index++) + { + history[index + 1] = operations[index]; + } + + return history; + } + + private static RecordedOperation Operation( + int id, + ReferenceCommand command, + ReferenceResultCode result, + long invocation, + long entry, + long returned, + long response) => + new(id, id, command, result, invocation, entry, returned, response); + + private static Store CreateStore() + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-remove-history-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + Assert.Equal(StoreOpenStatus.Success, Store.TryCreateOrOpen(options, out var store)); + return Assert.IsType(store); + } + + private static async Task RunConcurrent(Action first, Action second) + { + using var barrier = new Barrier(3); + var firstTask = Task.Run(() => + { + barrier.SignalAndWait(); + first(); + }); + var secondTask = Task.Run(() => + { + barrier.SignalAndWait(); + second(); + }); + barrier.SignalAndWait(); + await Task.WhenAll(firstTask, secondTask).WaitAsync(TimeSpan.FromSeconds(5)); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.LinearizabilityTests/SharedMemoryStore.LinearizabilityTests.csproj b/tests/SharedMemoryStore.LinearizabilityTests/SharedMemoryStore.LinearizabilityTests.csproj new file mode 100644 index 0000000..34b7033 --- /dev/null +++ b/tests/SharedMemoryStore.LinearizabilityTests/SharedMemoryStore.LinearizabilityTests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + diff --git a/tests/SharedMemoryStore.LockFreeAgent/CheckpointCatalogCommands.cs b/tests/SharedMemoryStore.LockFreeAgent/CheckpointCatalogCommands.cs new file mode 100644 index 0000000..50ccb2a --- /dev/null +++ b/tests/SharedMemoryStore.LockFreeAgent/CheckpointCatalogCommands.cs @@ -0,0 +1,46 @@ +using System.Text.Json; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.LockFreeAgent; + +/// +/// Emits the production checkpoint inventory for external qualification tools. +/// Keeping the catalog in one place prevents a newly appended checkpoint from +/// silently disappearing from cross-process pause qualification. +/// +internal static class CheckpointCatalogCommands +{ + internal static int Run(string[] arguments) + { + if (arguments.Length != 1) + { + return 64; + } + + CheckpointCatalogEntry[] entries = LockFreeCheckpointCatalog.Entries + .Select(static entry => new CheckpointCatalogEntry( + (int)entry.Id, + entry.Id.ToString(), + entry.Family.ToString(), + entry.Position.ToString(), + entry.Pause.ToString(), + entry.Crash.ToString(), + entry.Race.ToString(), + entry.IsPublicOrderingPoint, + entry.Description)) + .ToArray(); + Console.WriteLine(JsonSerializer.Serialize(entries)); + return 0; + } + + private readonly record struct CheckpointCatalogEntry( + int Id, + string Name, + string Family, + string Position, + string Pause, + string Crash, + string Race, + bool IsPublicOrderingPoint, + string Description); +} diff --git a/tests/SharedMemoryStore.LockFreeAgent/CheckpointCrashCommands.cs b/tests/SharedMemoryStore.LockFreeAgent/CheckpointCrashCommands.cs new file mode 100644 index 0000000..e109b24 --- /dev/null +++ b/tests/SharedMemoryStore.LockFreeAgent/CheckpointCrashCommands.cs @@ -0,0 +1,928 @@ +using System.Diagnostics; +using System.Globalization; +using System.Reflection; +using System.Text.Json; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.LockFreeAgent; + +/// +/// Cross-process deterministic checkpoint participant. The command deliberately +/// blocks inside the production checkpoint callback so the controller can +/// terminate the process at the exact protocol transition. +/// +internal static class CheckpointCrashCommands +{ + private const int InvalidArgumentsExitCode = 64; + private const int OperationFailureExitCode = 66; + private const int CheckpointNotReachedExitCode = 68; + + internal static int Run(string[] arguments) + { + if (!Arguments.TryParse(arguments, out Arguments parsed)) + { + return InvalidArgumentsExitCode; + } + + bool suspensionProbe = string.Equals(arguments[0], "checkpoint-pause", StringComparison.Ordinal); + LockFreeCheckpointEntry target; + try + { + target = LockFreeCheckpointCatalog.Get(parsed.Checkpoint); + } + catch (ArgumentOutOfRangeException) + { + return InvalidArgumentsExitCode; + } + + var armed = target.Family == LockFreeCheckpointFamily.Participant; + var reached = 0; + Action? beforePause = null; + Action? afterContinue = null; + CheckpointPreparation? preparation = null; + LeaseHandle capturedLease = default; + var checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + preparation?.Observe(entry.Id); + if (!armed || entry.Id != target.Id || Interlocked.Exchange(ref reached, 1) != 0) + { + return; + } + + // A suspension probe measures healthy-process progress while this + // participant is stopped. Test-only corruption used to reach a + // validation checkpoint must therefore be repaired before the + // controller starts that measurement window. Crash probes retain + // the injected word so recovery can verify the repair path. + beforePause?.Invoke(); + + var signal = new CheckpointSignal( + (int)entry.Id, + entry.Id.ToString(), + entry.Family.ToString(), + entry.Position.ToString(), + entry.Crash.ToString(), + Environment.ProcessId, + capturedLease.StoreId, + capturedLease.ParticipantToken, + capturedLease.SlotBinding, + capturedLease.LeaseToken); + Console.WriteLine("CHECKPOINT " + JsonSerializer.Serialize(signal)); + Console.Out.Flush(); + + string? command = Console.ReadLine(); + if (!string.Equals(command, "CONTINUE", StringComparison.Ordinal)) + { + Environment.Exit(CheckpointNotReachedExitCode); + } + + afterContinue?.Invoke(); + }); + + if (target.Id == LockFreeCheckpointId.ParticipantAfterRecoveryFenceBeforeReferenceScan + && !CreateDefiniteStaleParticipant(parsed.Options)) + { + return OperationFailureExitCode; + } + + StoreOpenStatus open = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + parsed.Options, + checkpoint, + out MemoryStore? store); + if (open != StoreOpenStatus.Success || store is null) + { + Console.Error.WriteLine("Open failed: " + open); + return OperationFailureExitCode; + } + + using (store) + { + if (target.Family != LockFreeCheckpointFamily.Participant) + { + StoreStatus capture = store.TryAcquire(parsed.TokenKey, out ValueLease lease); + if (capture != StoreStatus.Success) + { + Console.Error.WriteLine("Token lease acquire failed: " + capture); + return OperationFailureExitCode; + } + + capturedLease = lease.HandleForEngine; + armed = true; + } + + StoreStatus status = ExecuteTarget( + store, + target.Id, + parsed, + suspensionProbe, + (id, action) => preparation = new CheckpointPreparation(id, action), + continuation => beforePause = continuation, + continuation => afterContinue = continuation); + if (Volatile.Read(ref reached) == 0) + { + Console.Error.WriteLine("Target checkpoint was not reached: " + target.Id); + return CheckpointNotReachedExitCode; + } + + bool expectedStoreFull = status == StoreStatus.StoreFull + && target.Id is LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification + or LockFreeCheckpointId.StoreFullAfterExactDoubleCollect; + bool canceledInsertSuspension = suspensionProbe + && target.Id == LockFreeCheckpointId.DirectoryAfterCancelLocationClearBeforeDescriptorRejection; + bool expectedStatus = canceledInsertSuspension + ? status == StoreStatus.InvalidReservation + : status is StoreStatus.Success + or StoreStatus.RemovePending + or StoreStatus.DuplicateKey + || expectedStoreFull + || (suspensionProbe && status == StoreStatus.StoreBusy); + if (!expectedStatus) + { + Console.Error.WriteLine("Target operation failed after continue: " + status); + return OperationFailureExitCode; + } + + Console.WriteLine("OK " + (suspensionProbe ? "checkpoint-pause " : "checkpoint-crash ") + target.Id); + return 0; + } + } + + private static StoreStatus ExecuteTarget( + MemoryStore store, + LockFreeCheckpointId checkpoint, + in Arguments parsed, + bool suspensionProbe, + Action setPreparation, + Action setBeforePause, + Action setAfterContinue) + { + switch (checkpoint) + { + case LockFreeCheckpointId.PublishBeforeSlotClaim: + case LockFreeCheckpointId.PublishAfterCommitPublication: + return store.TryPublish(parsed.OperationKey, parsed.Value, parsed.Descriptor); + + case LockFreeCheckpointId.ReserveBeforeSlotClaim: + case LockFreeCheckpointId.ReserveAfterReservationPublication: + case LockFreeCheckpointId.SlotClaimAfterParticipantRecheck: + return store.TryReserve( + parsed.OperationKey, + parsed.Value.Length, + parsed.Descriptor, + out _); + + case LockFreeCheckpointId.ReserveAfterExistingLookup: + return store.TryReserve( + parsed.ExistingKey, + parsed.Value.Length, + parsed.Descriptor, + out _); + + case LockFreeCheckpointId.CommitBeforePublicationCas: + case LockFreeCheckpointId.CommitAfterPublicationCas: + { + StoreStatus reserve = PrepareReservation(store, parsed.OperationKey, parsed, out ValueReservation reservation); + return reserve == StoreStatus.Success ? reservation.Commit() : reserve; + } + + case LockFreeCheckpointId.AdvanceBeforeBytesAdvancedCas: + case LockFreeCheckpointId.AdvanceAfterBytesAdvancedCas: + { + StoreStatus reserve = store.TryReserve( + parsed.OperationKey, + parsed.Value.Length, + parsed.Descriptor, + out ValueReservation reservation); + if (reserve != StoreStatus.Success) + { + return reserve; + } + + parsed.Value.CopyTo(reservation.GetSpan(parsed.Value.Length)); + return reservation.Advance(parsed.Value.Length); + } + + case LockFreeCheckpointId.AbortBeforeAbortCas: + case LockFreeCheckpointId.AbortAfterOwnershipReleaseCas: + case LockFreeCheckpointId.AbortAfterUnlinkCompletion: + { + StoreStatus reserve = store.TryReserve( + parsed.OperationKey, + parsed.Value.Length, + parsed.Descriptor, + out ValueReservation reservation); + return reserve == StoreStatus.Success ? reservation.Abort() : reserve; + } + + case LockFreeCheckpointId.AcquireBeforeLeaseClaimCas: + case LockFreeCheckpointId.AcquireAfterLeaseActivationBeforeFinalLookup: + case LockFreeCheckpointId.AcquireAfterPublishedRevalidation: + return store.TryAcquire(parsed.ExistingKey, out _); + + case LockFreeCheckpointId.ProjectBeforeHandleValidation: + case LockFreeCheckpointId.ProjectAfterMetadataReadBeforeControlRevalidation: + case LockFreeCheckpointId.ProjectAfterSpanProjection: + { + StoreStatus acquire = store.TryAcquire(parsed.ExistingKey, out ValueLease lease); + if (acquire != StoreStatus.Success) + { + return acquire; + } + + _ = lease.ValueSpan.Length; + return lease.Release(); + } + + case LockFreeCheckpointId.ReleaseBeforeActiveReleaseCas: + case LockFreeCheckpointId.ReleaseAfterOwnershipReleaseCas: + case LockFreeCheckpointId.ReleaseAfterRecordRecycle: + { + StoreStatus acquire = store.TryAcquire(parsed.ExistingKey, out ValueLease lease); + return acquire == StoreStatus.Success ? lease.Release() : acquire; + } + + case LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas: + case LockFreeCheckpointId.RemoveAfterLeaseClassification: + case LockFreeCheckpointId.ReclaimBeforeOwnershipCas: + case LockFreeCheckpointId.ReclaimAfterGenerationAdvance: + case LockFreeCheckpointId.ReclaimAfterLeaseScanBeforeOwnershipCas: + case LockFreeCheckpointId.DirectoryAfterLocationValidation: + case LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead: + case LockFreeCheckpointId.DirectoryAfterUnlinkDescriptorClearBeforeGenerationAdvance: + case LockFreeCheckpointId.ReclaimAfterMetadataValidation: + return store.TryRemove(parsed.ExistingKey); + + case LockFreeCheckpointId.DirectoryBeforeDescriptorPublication: + case LockFreeCheckpointId.DirectoryAfterDescriptorClear: + case LockFreeCheckpointId.DirectoryAfterOperationValidation: + case LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation: + case LockFreeCheckpointId.DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas: + case LockFreeCheckpointId.DirectoryAfterLocationPublicationBeforeSourceRevalidation: + case LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch: + case LockFreeCheckpointId.DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication: + case LockFreeCheckpointId.DirectoryAfterInsertCompletionStateValidationBeforeLocationRead: + case LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck: + case LockFreeCheckpointId.ReserveAfterDirectoryInsertBeforePendingClassification: + { + StoreStatus reserve = store.TryReserve( + parsed.OperationKey, + parsed.Value.Length, + parsed.Descriptor, + out ValueReservation reservation); + if (reserve != StoreStatus.Success) + { + return reserve; + } + + return reservation.Abort(); + } + + case LockFreeCheckpointId.DirectoryAfterCancelLocationClearBeforeDescriptorRejection: + { + // This checkpoint describes a cancellation racing the insert + // helper, rather than a state reached by a single ordinary + // reserve/abort call. Turn the exact reservation Aborting at + // the helper's freshly revalidated dispatch boundary. The + // same production helper then enters CancelInsert and leaves + // a genuine cross-process crash-recovery state. + byte[] operationKey = parsed.OperationKey; + setPreparation( + LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch, + () => BeginAbortInFlightReservation(store, operationKey)); + StoreStatus reserve = store.TryReserve( + parsed.OperationKey, + parsed.Value.Length, + parsed.Descriptor, + out ValueReservation reservation); + if (reserve != StoreStatus.Success) + { + return reserve; + } + + // Preserve the primary reserve outcome for the suspension + // oracle. Cleanup must not transform an unexpected Success + // into the expected InvalidReservation and hide a regression. + StoreStatus cleanup = reservation.Abort(); + return cleanup is StoreStatus.Success or StoreStatus.InvalidReservation + ? StoreStatus.Success + : cleanup; + } + + case LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas: + case LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication: + case LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan: + case LockFreeCheckpointId.DirectoryAfterSpillSummaryClear: + return ExecuteSpillSummaryTarget( + store, + checkpoint, + parsed.Options.SlotCount, + parsed.SpillFirstBucket, + parsed.SpillSecondBucket); + + case LockFreeCheckpointId.DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation: + return ExecuteInvalidReferenceTarget( + store, + parsed.TokenKey, + parsed.Options.SlotCount, + parsed.Value, + parsed.Descriptor, + suspensionProbe, + setPreparation, + setBeforePause, + setAfterContinue); + + case LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification: + case LockFreeCheckpointId.StoreFullAfterExactDoubleCollect: + return ExecuteStoreFullTarget(store, checkpoint, parsed, setAfterContinue); + + case LockFreeCheckpointId.DiagnosticsBeforeBoundedScan: + case LockFreeCheckpointId.DiagnosticsAfterSnapshotAssembly: + return store.TryGetDiagnostics(out _); + + case LockFreeCheckpointId.RecoveryBeforeOwnerClassification: + case LockFreeCheckpointId.RecoveryAfterExactRecoveryCas: + { + StoreStatus reserve = store.TryReserve( + parsed.RecoveryKey, + parsed.Value.Length, + parsed.Descriptor, + out _); + if (reserve != StoreStatus.Success) + { + return reserve; + } + + return store.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: true), + out _); + } + + case LockFreeCheckpointId.ParticipantAfterRecoveryFenceBeforeReferenceScan: + return store.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: false), + out _); + + case LockFreeCheckpointId.DisposalBeforeLocalGateClose: + case LockFreeCheckpointId.DisposalAfterParticipantClosingPublication: + case LockFreeCheckpointId.DisposalAfterParticipantRelease: + case LockFreeCheckpointId.ParticipantBeforeReclaimGenerationAdvanceCas: + store.Dispose(); + return StoreStatus.Success; + + case LockFreeCheckpointId.ParticipantBeforeRegisteringCas: + case LockFreeCheckpointId.ParticipantAfterIdentityKindWrite: + case LockFreeCheckpointId.ParticipantAfterReservedWrite: + case LockFreeCheckpointId.ParticipantAfterProcessStartWrite: + case LockFreeCheckpointId.ParticipantAfterPidNamespaceWrite: + case LockFreeCheckpointId.ParticipantAfterOpenSequenceWrite: + case LockFreeCheckpointId.ParticipantAfterActivePublication: + case LockFreeCheckpointId.ParticipantAfterRegistrationBeforeEngineConstruction: + return StoreStatus.Success; + + default: + return StoreStatus.UnknownFailure; + } + } + + private static StoreStatus ExecuteInvalidReferenceTarget( + MemoryStore store, + byte[] key, + int slotCount, + byte[] value, + byte[] descriptor, + bool suspensionProbe, + Action setPreparation, + Action setBeforePause, + Action setAfterContinue) + { + DirectoryReferenceMutation mutation = DirectoryReferenceMutation.Capture(store, key); + setPreparation( + LockFreeCheckpointId.ReserveBeforeSlotClaim, + mutation.InjectInvalidReference); + if (suspensionProbe) + { + setBeforePause(mutation.RestoreExactReference); + } + + setAfterContinue(mutation.RestoreExactReference); + byte[] collisionKey = GenerateBucketPairMate(key, slotCount); + StoreStatus publish = store.TryPublish(collisionKey, value, descriptor); + return publish == StoreStatus.Success ? store.TryRemove(collisionKey) : publish; + } + + private static StoreStatus ExecuteStoreFullTarget( + MemoryStore store, + LockFreeCheckpointId checkpoint, + in Arguments parsed, + Action setAfterContinue) + { + var resumed = 0; + setAfterContinue(() => Volatile.Write(ref resumed, 1)); + for (var index = 0; index < parsed.Options.SlotCount; index++) + { + byte[] key = CreateStoreFullFillerKey(checkpoint, index); + StoreStatus publish = store.TryPublish(key, parsed.Value, parsed.Descriptor); + if (Volatile.Read(ref resumed) != 0) + { + if (publish == StoreStatus.Success) + { + return store.TryRemove(key); + } + + return publish; + } + + if (publish != StoreStatus.Success) + { + return publish; + } + } + + return store.TryPublish(parsed.OperationKey, parsed.Value, parsed.Descriptor); + } + + private static byte[] CreateStoreFullFillerKey( + LockFreeCheckpointId checkpoint, + int index) => + BitConverter.GetBytes( + 0x6f00_0000_0000_0000UL + | ((ulong)(byte)checkpoint << 48) + | checked((uint)(index + 1))); + + private static bool CreateDefiniteStaleParticipant(SharedMemoryStoreOptions options) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(typeof(CheckpointCrashCommands).Assembly.Location); + foreach (string argument in new[] + { + "participant-orphan", + options.Name, + options.SlotCount.ToString(CultureInfo.InvariantCulture), + options.MaxValueBytes.ToString(CultureInfo.InvariantCulture), + options.MaxDescriptorBytes.ToString(CultureInfo.InvariantCulture), + options.MaxKeyBytes.ToString(CultureInfo.InvariantCulture), + options.LeaseRecordCount.ToString(CultureInfo.InvariantCulture), + options.ParticipantRecordCount.ToString(CultureInfo.InvariantCulture) + }) + { + startInfo.ArgumentList.Add(argument); + } + + using Process process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start stale-participant helper."); + if (!process.WaitForExit(10_000) || process.ExitCode != 0) + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + + Console.Error.WriteLine( + "Stale-participant helper failed: exit=" + + (process.HasExited ? process.ExitCode.ToString(CultureInfo.InvariantCulture) : "timeout") + + " stderr=" + process.StandardError.ReadToEnd()); + return false; + } + + return true; + } + + private static StoreStatus ExecuteSpillSummaryTarget( + MemoryStore store, + LockFreeCheckpointId checkpoint, + int slotCount, + int firstBucket, + int secondBucket) + { + byte[][] keys = GenerateBucketPairCollisions( + count: 17, + slotCount: slotCount, + firstBucket: firstBucket, + secondBucket: secondBucket); + for (var index = 0; index < 16; index++) + { + StoreStatus seed = store.TryPublish(keys[index], [unchecked((byte)index)]); + if (seed != StoreStatus.Success) + { + return seed; + } + } + + StoreStatus publish = store.TryPublish(keys[16], [0xA5]); + if (publish != StoreStatus.Success + || checkpoint is LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas + or LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication) + { + return publish; + } + + return store.TryRemove(keys[16]); + } + + private static byte[][] GenerateBucketPairCollisions( + int count, + int slotCount, + int firstBucket, + int secondBucket) + { + var keys = new List(count); + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + uint bucketMask = checked((uint)((primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + for (long candidate = 1; keys.Count < count; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + ulong hash = StoreKey.Hash(key); + int first = (int)(Mix(hash) & bucketMask); + int second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & bucketMask); + if (second == first) + { + second = (first + 1) & (int)bucketMask; + } + + if (first == firstBucket && second == secondBucket) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static byte[] GenerateBucketPairMate(byte[] anchorKey, int slotCount) + { + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + uint bucketMask = checked((uint)((primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + GetBuckets(StoreKey.Hash(anchorKey), bucketMask, out int anchorFirst, out int anchorSecond); + for (long candidate = 1; ; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + if (key.AsSpan().SequenceEqual(anchorKey)) + { + continue; + } + + GetBuckets(StoreKey.Hash(key), bucketMask, out int first, out int second); + if (first == anchorFirst && second == anchorSecond) + { + return key; + } + } + } + + private static void GetBuckets(ulong hash, uint bucketMask, out int first, out int second) + { + first = (int)(Mix(hash) & bucketMask); + second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & bucketMask); + if (second == first) + { + second = (first + 1) & (int)bucketMask; + } + } + + private static int NextPowerOfTwo(int value) + { + var result = 1; + while (result < value) + { + result <<= 1; + } + + return result; + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static StoreStatus PrepareReservation( + MemoryStore store, + byte[] key, + in Arguments parsed, + out ValueReservation reservation) + { + StoreStatus reserve = store.TryReserve( + key, + parsed.Value.Length, + parsed.Descriptor, + out reservation); + if (reserve != StoreStatus.Success) + { + return reserve; + } + + parsed.Value.CopyTo(reservation.GetSpan(parsed.Value.Length)); + return reservation.Advance(parsed.Value.Length); + } + + private static void BeginAbortInFlightReservation(MemoryStore store, byte[] key) + { + object engine = ReadPrivate(store, "_engine"); + LockFreeSlotTable slots = ReadPrivate(engine, "_slots"); + StoreLayoutV2 layout = ReadPrivate(engine, "_layout"); + ulong storeId = ReadPrivate(slots, "_storeId"); + ulong keyHash = StoreKey.Hash(key); + for (var slotIndex = 0; slotIndex < layout.SlotCount; slotIndex++) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + if ((unchecked((ulong)control) & 0x7UL) != LockFreeSlotTable.InitializingState + || Volatile.Read(ref slot.KeyHash) != keyHash + || Volatile.Read(ref slot.KeyLength) != key.Length) + { + continue; + } + + ulong binding = Volatile.Read(ref slot.DirectoryBinding); + IndexBinding decoded = IndexBinding.Decode(binding); + if (decoded.SlotIndex != slotIndex) + { + continue; + } + + var handle = new ReservationHandle( + storeId, + unchecked((ulong)control) >> 36, + binding, + Volatile.Read(ref slot.ValueLength)); + if (!slots.GetInitializingKeySpan(handle).SequenceEqual(key)) + { + continue; + } + + StoreStatus beginAbort = slots.TryBeginAbort(handle); + if (beginAbort != StoreStatus.Success) + { + throw new InvalidOperationException( + "Unable to begin the deterministic insert cancellation: " + beginAbort); + } + + return; + } + + throw new InvalidOperationException( + "Unable to locate the in-flight insert before cancellation."); + } + + private static T ReadPrivate(object owner, string fieldName) + { + FieldInfo field = owner.GetType().GetField( + fieldName, + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + "Missing field " + owner.GetType().FullName + "." + fieldName + "."); + return field.GetValue(owner) is T value + ? value + : throw new InvalidOperationException( + "Unexpected value in " + owner.GetType().FullName + "." + fieldName + "."); + } + + private sealed class DirectoryReferenceMutation + { + private readonly MemoryMappedStoreRegion _region; + private readonly StoreLayoutV2 _layout; + private readonly DirectoryLocation _location; + private readonly ulong _exactBinding; + private readonly ulong _invalidBinding; + + private DirectoryReferenceMutation( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + DirectoryLocation location, + ulong exactBinding) + { + _region = region; + _layout = layout; + _location = location; + _exactBinding = exactBinding; + IndexBinding decoded = IndexBinding.Decode(exactBinding); + _invalidBinding = IndexBinding.Encode( + decoded.SlotIndex, + checked(decoded.Generation + 1)); + } + + internal static DirectoryReferenceMutation Capture(MemoryStore store, byte[] key) + { + object engine = ReadPrivate(store, "_engine"); + LockFreeKeyDirectory directory = ReadPrivate(engine, "_directory"); + MemoryMappedStoreRegion region = ReadPrivate(engine, "_region"); + StoreLayoutV2 layout = ReadPrivate(engine, "_layout"); + StoreStatus lookup = directory.TryLookup( + key, + StoreKey.Hash(key), + out ulong binding, + out DirectoryLocation location); + if (lookup != StoreStatus.Success) + { + throw new InvalidOperationException("Unable to capture directory reference: " + lookup); + } + + return new DirectoryReferenceMutation(region, layout, location, binding); + } + + internal void InjectInvalidReference() + { + long observed = AtomicControlWord.CompareExchange( + ref Cell(), + unchecked((long)_invalidBinding), + unchecked((long)_exactBinding)); + if (unchecked((ulong)observed) != _exactBinding) + { + throw new InvalidOperationException("The directory reference changed before invalid-reference injection."); + } + } + + internal void RestoreExactReference() + { + long observed = AtomicControlWord.CompareExchange( + ref Cell(), + unchecked((long)_exactBinding), + unchecked((long)_invalidBinding)); + ulong raw = unchecked((ulong)observed); + if (raw != _invalidBinding && raw != _exactBinding) + { + throw new InvalidOperationException("The injected directory reference changed before restoration."); + } + } + + private unsafe ref long Cell() + { + long offset = _location.Kind switch + { + 1 => PrimaryCellOffset(_layout, checked((int)_location.Index)), + 2 => _layout.OverflowDirectoryOffset + (_location.Index * _layout.OverflowStride), + _ => throw new InvalidOperationException("The captured directory location is invalid.") + }; + return ref *(long*)(_region.Pointer + offset); + } + + private static long PrimaryCellOffset(StoreLayoutV2 layout, int absoluteCellIndex) + { + int bucket = absoluteCellIndex / LayoutV2Constants.PrimaryLanesPerBucket; + int lane = absoluteCellIndex % LayoutV2Constants.PrimaryLanesPerBucket; + return layout.PrimaryDirectoryOffset + + ((long)bucket * layout.PrimaryBucketStride) + + 16 + + (lane * sizeof(long)); + } + + } + + private sealed class CheckpointPreparation + { + private readonly LockFreeCheckpointId _checkpoint; + private readonly Action _action; + private int _applied; + + internal CheckpointPreparation(LockFreeCheckpointId checkpoint, Action action) + { + _checkpoint = checkpoint; + _action = action; + } + + internal void Observe(LockFreeCheckpointId checkpoint) + { + if (checkpoint == _checkpoint + && Interlocked.CompareExchange(ref _applied, 1, 0) == 0) + { + _action(); + } + } + } + + private readonly record struct CheckpointSignal( + int Id, + string Name, + string Family, + string Position, + string Crash, + int ProcessId, + ulong StoreId, + ulong ParticipantToken, + ulong SlotBinding, + ulong LeaseToken); + + private readonly record struct Arguments( + SharedMemoryStoreOptions Options, + LockFreeCheckpointId Checkpoint, + byte[] TokenKey, + byte[] ExistingKey, + byte[] OperationKey, + byte[] RecoveryKey, + byte[] Value, + byte[] Descriptor, + int SpillFirstBucket, + int SpillSecondBucket) + { + internal static bool TryParse(string[] values, out Arguments parsed) + { + parsed = default; + bool pauseProtocol = values.Length > 0 + && string.Equals(values[0], "checkpoint-pause", StringComparison.Ordinal); + int expectedLength = pauseProtocol ? 18 : 16; + int versionIndex = pauseProtocol ? 17 : 15; + if (values.Length != expectedLength + || string.IsNullOrWhiteSpace(values[1]) + || !TryPositive(values[2], out int slotCount) + || !TryPositive(values[3], out int maxValueBytes) + || !TryNonNegative(values[4], out int maxDescriptorBytes) + || !TryPositive(values[5], out int maxKeyBytes) + || !TryPositive(values[6], out int leaseRecordCount) + || !TryPositive(values[7], out int participantRecordCount) + || !int.TryParse(values[8], NumberStyles.None, CultureInfo.InvariantCulture, out int checkpointValue) + || !Enum.IsDefined(typeof(LockFreeCheckpointId), checkpointValue) + || !TryDecode(values[9], out byte[] tokenKey) + || !TryDecode(values[10], out byte[] existingKey) + || !TryDecode(values[11], out byte[] operationKey) + || !TryDecode(values[12], out byte[] recoveryKey) + || !TryDecode(values[13], out byte[] value) + || !TryDecode(values[14], out byte[] descriptor) + || values[versionIndex] != (pauseProtocol ? "v2" : "v1") + || tokenKey.Length is 0 + || existingKey.Length is 0 + || operationKey.Length is 0 + || recoveryKey.Length is 0 + || tokenKey.Length > maxKeyBytes + || existingKey.Length > maxKeyBytes + || operationKey.Length > maxKeyBytes + || recoveryKey.Length > maxKeyBytes + || value.Length > maxValueBytes + || descriptor.Length > maxDescriptorBytes) + { + return false; + } + + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + int primaryBucketCount = primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket; + int spillFirstBucket = 0; + int spillSecondBucket = 1; + if (pauseProtocol + && (!TryNonNegative(values[15], out spillFirstBucket) + || !TryNonNegative(values[16], out spillSecondBucket) + || spillFirstBucket == spillSecondBucket + || spillFirstBucket >= primaryBucketCount + || spillSecondBucket >= primaryBucketCount)) + { + return false; + } + + var options = SharedMemoryStoreOptions.CreateLockFree( + values[1], + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount, + OpenMode.OpenExisting, + enableLeaseRecovery: true); + parsed = new Arguments( + options, + (LockFreeCheckpointId)checkpointValue, + tokenKey, + existingKey, + operationKey, + recoveryKey, + value, + descriptor, + spillFirstBucket, + spillSecondBucket); + return true; + } + + private static bool TryDecode(string text, out byte[] bytes) + { + try + { + bytes = Convert.FromHexString(text); + return true; + } + catch (FormatException) + { + bytes = []; + return false; + } + } + + private static bool TryPositive(string text, out int value) => + int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value) && value > 0; + + private static bool TryNonNegative(string text, out int value) => + int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value) && value >= 0; + } +} diff --git a/tests/SharedMemoryStore.LockFreeAgent/LinuxFileLockCommands.cs b/tests/SharedMemoryStore.LockFreeAgent/LinuxFileLockCommands.cs new file mode 100644 index 0000000..ca697fc --- /dev/null +++ b/tests/SharedMemoryStore.LockFreeAgent/LinuxFileLockCommands.cs @@ -0,0 +1,33 @@ +using SharedMemoryStore.Interop; + +namespace SharedMemoryStore.LockFreeAgent; + +internal static class LinuxFileLockCommands +{ + private const int InvalidArgumentsExitCode = 64; + private const int UnsupportedPlatformExitCode = 65; + + public static int Run(string[] arguments) + { + if (arguments.Length != 2 || !Path.IsPathFullyQualified(arguments[1])) + { + return InvalidArgumentsExitCode; + } + + if (!OperatingSystem.IsLinux()) + { + return UnsupportedPlatformExitCode; + } + + StoreStatus status = LinuxFileLock.TryAcquire( + arguments[1], + StoreWaitOptions.NoWait, + out LinuxFileLock? fileLock); + using (fileLock) + { + Console.WriteLine("RESULT " + status); + } + + return 0; + } +} diff --git a/tests/SharedMemoryStore.LockFreeAgent/ParticipantOrphanCommands.cs b/tests/SharedMemoryStore.LockFreeAgent/ParticipantOrphanCommands.cs new file mode 100644 index 0000000..4d4dab2 --- /dev/null +++ b/tests/SharedMemoryStore.LockFreeAgent/ParticipantOrphanCommands.cs @@ -0,0 +1,51 @@ +using System.Globalization; + +namespace SharedMemoryStore.LockFreeAgent; + +/// Creates one Active participant record and exits without Dispose. +internal static class ParticipantOrphanCommands +{ + internal static int Run(string[] arguments) + { + if (arguments.Length != 8 + || string.IsNullOrWhiteSpace(arguments[1]) + || !TryPositive(arguments[2], out int slotCount) + || !TryPositive(arguments[3], out int maxValueBytes) + || !TryNonNegative(arguments[4], out int maxDescriptorBytes) + || !TryPositive(arguments[5], out int maxKeyBytes) + || !TryPositive(arguments[6], out int leaseRecordCount) + || !TryPositive(arguments[7], out int participantRecordCount)) + { + return 64; + } + + var options = SharedMemoryStoreOptions.CreateLockFree( + arguments[1], + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount, + OpenMode.OpenExisting, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + if (status != StoreOpenStatus.Success || store is null) + { + Console.Error.WriteLine("Orphan open failed: " + status); + return 66; + } + + // Intentionally do not Dispose: process termination is the crash being + // modeled, and the shared Active participant record must remain fenced + // only by a later explicit recovery sweep. + GC.KeepAlive(store); + return 0; + } + + private static bool TryPositive(string text, out int value) => + int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value) && value > 0; + + private static bool TryNonNegative(string text, out int value) => + int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value) && value >= 0; +} diff --git a/tests/SharedMemoryStore.LockFreeAgent/Program.cs b/tests/SharedMemoryStore.LockFreeAgent/Program.cs new file mode 100644 index 0000000..dce162f --- /dev/null +++ b/tests/SharedMemoryStore.LockFreeAgent/Program.cs @@ -0,0 +1,915 @@ +using System.Diagnostics; +using System.Globalization; +using System.IO.MemoryMappedFiles; +using System.Runtime.InteropServices; +using System.Text.Json; +using SharedMemoryStore; +using SharedMemoryStore.LockFreeAgent; + +const int invalidArgumentsExitCode = 64; +const int unsupportedPlatformExitCode = 65; +const int unhandledFailureExitCode = 69; + +if (args.Length == 1 && string.Equals(args[0], "probe", StringComparison.OrdinalIgnoreCase)) +{ + Console.WriteLine("READY " + + Environment.ProcessId.ToString(CultureInfo.InvariantCulture) + + " " + + (OperatingSystem.IsWindows() ? "windows" : OperatingSystem.IsLinux() ? "linux" : "unsupported")); + return 0; +} + +if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || RuntimeInformation.ProcessArchitecture != Architecture.X64) +{ + return unsupportedPlatformExitCode; +} + +try +{ + return args.Length == 0 + ? invalidArgumentsExitCode + : args[0] switch + { + "atomic-publication-producer" => AtomicCommands.RunPublicationProducer(args), + "atomic-publication-consumer" => AtomicCommands.RunPublicationConsumer(args), + "atomic-cas-worker" => AtomicCommands.RunCasWorker(args), + "atomic-dekker-worker" => AtomicCommands.RunDekkerWorker(args), + "atomic-dekker-coordinator" => AtomicCommands.RunDekkerCoordinator(args), + "lease-read" => LeaseCommands.RunRead(args), + "lease-hold" => LeaseCommands.RunHold(args), + "churn-worker" => ChurnCommands.Run(args), + "checkpoint-catalog" => CheckpointCatalogCommands.Run(args), + "checkpoint-pause" => CheckpointCrashCommands.Run(args), + "checkpoint-crash" => CheckpointCrashCommands.Run(args), + "participant-orphan" => ParticipantOrphanCommands.Run(args), + "raw-visibility-publisher" => RawVisibilityCommands.RunPublisher(args), + "raw-visibility-reader" => RawVisibilityCommands.RunReader(args), + "raw-visibility-remover" => RawVisibilityCommands.RunRemover(args), + "steady-no-lock" => SteadyNoLockCommands.Run(args), + "linux-file-lock-probe" => LinuxFileLockCommands.Run(args), + _ => invalidArgumentsExitCode + }; +} +catch (Exception exception) +{ + Console.Error.WriteLine(exception.GetType().Name + ": " + exception.Message); + return unhandledFailureExitCode; +} + +internal static class ChurnCommands +{ + private const int OperationFailureExitCode = 66; + private const int ContentMismatchExitCode = 67; + private const int TimeoutExitCode = 68; + private const int WarmupIterations = 128; + private const int SampleWindow = 256; + private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(30); + + public static int Run(string[] arguments) + { + if (arguments.Length != 13 + || !TryCreateOptions(arguments, out var options) + || !int.TryParse(arguments[8], NumberStyles.None, CultureInfo.InvariantCulture, out var workerId) + || workerId < 0 + || !int.TryParse(arguments[9], NumberStyles.None, CultureInfo.InvariantCulture, out var iterations) + || iterations < SampleWindow * 2 + || iterations > 100_000_000 + || !TryParseKeys(arguments[10], options.MaxKeyBytes, out var keys)) + { + return 64; + } + + StoreOpenStatus openStatus = MemoryStore.TryCreateOrOpen(options, out var store); + if (openStatus != StoreOpenStatus.Success || store is null) + { + Console.Error.WriteLine("Open failed: " + openStatus); + return OperationFailureExitCode; + } + + using (store) + { + File.WriteAllText(arguments[11], Environment.ProcessId.ToString(CultureInfo.InvariantCulture)); + if (!WaitForFile(arguments[12])) + { + return TimeoutExitCode; + } + + var earlyPublish = new long[SampleWindow]; + var earlyMissing = new long[SampleWindow]; + var latePublish = new long[SampleWindow]; + var lateMissing = new long[SampleWindow]; + var value = new byte[8]; + var removePendingCount = 0; + for (var operation = -WarmupIterations; operation < iterations; operation++) + { + byte[] key = keys[(operation + WarmupIterations) % keys.Length]; + BitConverter.TryWriteBytes(value, ((long)workerId << 32) | (uint)(operation + WarmupIterations)); + + long started = Stopwatch.GetTimestamp(); + StoreStatus publish = store.TryPublish(key, value); + long publishTicks = Stopwatch.GetTimestamp() - started; + if (publish != StoreStatus.Success) + { + Console.Error.WriteLine( + "Publish failed: " + publish + + " worker=" + workerId.ToString(CultureInfo.InvariantCulture) + + " operation=" + operation.ToString(CultureInfo.InvariantCulture) + + " key=" + Convert.ToHexString(key)); + return OperationFailureExitCode; + } + + StoreStatus acquire = store.TryAcquire(key, out var lease); + if (acquire != StoreStatus.Success || !lease.ValueSpan.SequenceEqual(value)) + { + _ = lease.Release(); + return ContentMismatchExitCode; + } + + if (lease.Release() != StoreStatus.Success) + { + return OperationFailureExitCode; + } + + StoreStatus remove = store.TryRemove(key); + if (remove is not (StoreStatus.Success or StoreStatus.RemovePending)) + { + Console.Error.WriteLine( + "Remove failed: " + remove + + " worker=" + workerId.ToString(CultureInfo.InvariantCulture) + + " operation=" + operation.ToString(CultureInfo.InvariantCulture) + + " key=" + Convert.ToHexString(key)); + return OperationFailureExitCode; + } + + // The default one-second operation policy permits + // RemovePending after the logical removal ordering point when + // bounded classification/reclaim work is incomplete. The + // following NotFound check proves logical absence, and the + // controller's final full-capacity fill proves eventual exact + // reclamation. Preserve the count as qualification evidence. + if (remove == StoreStatus.RemovePending && operation >= 0) + { + removePendingCount++; + } + + started = Stopwatch.GetTimestamp(); + StoreStatus missing = store.TryAcquire(key, out _); + long missingTicks = Stopwatch.GetTimestamp() - started; + if (missing != StoreStatus.NotFound) + { + Console.Error.WriteLine("Missing lookup failed: " + missing); + return OperationFailureExitCode; + } + + if (operation is >= 0 and < SampleWindow) + { + earlyPublish[operation] = publishTicks; + earlyMissing[operation] = missingTicks; + } + else if (operation >= iterations - SampleWindow) + { + int sample = operation - (iterations - SampleWindow); + latePublish[sample] = publishTicks; + lateMissing[sample] = missingTicks; + } + } + + var result = new ChurnResult( + workerId, + iterations, + keys.Length, + removePendingCount, + Percentile99(earlyPublish), + Percentile99(latePublish), + Percentile99(earlyMissing), + Percentile99(lateMissing)); + Console.WriteLine("RESULT " + JsonSerializer.Serialize(result)); + return 0; + } + } + + private static bool TryCreateOptions(string[] arguments, out SharedMemoryStoreOptions options) + { + options = null!; + if (string.IsNullOrWhiteSpace(arguments[1]) + || !TryPositive(arguments[2], out var slotCount) + || !TryPositive(arguments[3], out var maxValueBytes) + || !TryNonNegative(arguments[4], out var maxDescriptorBytes) + || !TryPositive(arguments[5], out var maxKeyBytes) + || !TryPositive(arguments[6], out var leaseRecordCount) + || !TryPositive(arguments[7], out var participantRecordCount)) + { + return false; + } + + options = SharedMemoryStoreOptions.CreateLockFree( + arguments[1], + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount, + OpenMode.OpenExisting, + enableLeaseRecovery: true); + return true; + } + + private static bool TryParseKeys(string text, int maxKeyBytes, out byte[][] keys) + { + keys = []; + try + { + keys = text.Split(';', StringSplitOptions.RemoveEmptyEntries) + .Select(Convert.FromHexString) + .ToArray(); + return keys.Length > 0 && keys.All(key => key.Length is > 0 && key.Length <= maxKeyBytes); + } + catch (FormatException) + { + keys = []; + return false; + } + } + + private static long Percentile99(long[] samples) + { + Array.Sort(samples); + int index = Math.Min(samples.Length - 1, (int)Math.Ceiling(samples.Length * 0.99d) - 1); + return samples[index]; + } + + private static bool WaitForFile(string path) + { + long deadline = Stopwatch.GetTimestamp() + (long)(WaitTimeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (!File.Exists(path)) + { + if (Stopwatch.GetTimestamp() >= deadline) + { + return false; + } + + spin.SpinOnce(); + } + + return true; + } + + private static bool TryPositive(string value, out int result) => + int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out result) && result > 0; + + private static bool TryNonNegative(string value, out int result) => + int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out result) && result >= 0; + + private readonly record struct ChurnResult( + int WorkerId, + int Iterations, + int CollisionKeyCount, + int RemovePendingCount, + long EarlyPublishP99Ticks, + long LatePublishP99Ticks, + long EarlyMissingP99Ticks, + long LateMissingP99Ticks); +} + +internal static class LeaseCommands +{ + private const int OperationFailureExitCode = 66; + private const int ContentMismatchExitCode = 67; + private const int TimeoutExitCode = 68; + private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(30); + + public static int RunRead(string[] arguments) + { + if (!StoreArguments.TryParse(arguments, expectedLength: 14, out var parsed) + || !TryDecode(arguments[8], out var key) + || key.Length == 0 + || !TryDecode(arguments[9], out var expectedValue) + || !TryDecode(arguments[10], out var expectedDescriptor) + || !int.TryParse(arguments[11], NumberStyles.None, CultureInfo.InvariantCulture, out var iterations) + || iterations is < 1 or > 1_000_000) + { + return 64; + } + + StoreOpenStatus openStatus = MemoryStore.TryCreateOrOpen(parsed.Options, out var store); + if (openStatus != StoreOpenStatus.Success || store is null) + { + Console.Error.WriteLine("Open failed: " + openStatus); + return OperationFailureExitCode; + } + + using (store) + { + File.WriteAllText(arguments[13], Environment.ProcessId.ToString(CultureInfo.InvariantCulture)); + if (!WaitForFile(arguments[12])) + { + return TimeoutExitCode; + } + + ulong checksum = 0; + for (var iteration = 0; iteration < iterations; iteration++) + { + StoreStatus acquireStatus = store.TryAcquire(key, out var lease); + if (acquireStatus != StoreStatus.Success) + { + Console.Error.WriteLine("Acquire failed: " + acquireStatus); + return OperationFailureExitCode; + } + + if (!lease.ValueSpan.SequenceEqual(expectedValue) + || !lease.DescriptorSpan.SequenceEqual(expectedDescriptor) + || lease.ValueLength != expectedValue.Length + || lease.DescriptorLength != expectedDescriptor.Length) + { + _ = lease.Release(); + return ContentMismatchExitCode; + } + + checksum = AddChecksum(checksum, lease.ValueSpan); + checksum = AddChecksum(checksum, lease.DescriptorSpan); + StoreStatus releaseStatus = lease.Release(); + if (releaseStatus != StoreStatus.Success) + { + Console.Error.WriteLine("Release failed: " + releaseStatus); + return OperationFailureExitCode; + } + } + + Console.WriteLine( + "OK lease-read iterations=" + + iterations.ToString(CultureInfo.InvariantCulture) + + " checksum=" + + checksum.ToString(CultureInfo.InvariantCulture)); + return 0; + } + } + + public static int RunHold(string[] arguments) + { + if (!StoreArguments.TryParse(arguments, expectedLength: 13, out var parsed) + || !TryDecode(arguments[8], out var key) + || key.Length == 0 + || !TryDecode(arguments[9], out var expectedValue) + || !TryDecode(arguments[10], out var expectedDescriptor)) + { + return 64; + } + + StoreOpenStatus openStatus = MemoryStore.TryCreateOrOpen(parsed.Options, out var store); + if (openStatus != StoreOpenStatus.Success || store is null) + { + Console.Error.WriteLine("Open failed: " + openStatus); + return OperationFailureExitCode; + } + + using (store) + { + StoreStatus acquireStatus = store.TryAcquire(key, out var lease); + if (acquireStatus != StoreStatus.Success) + { + Console.Error.WriteLine("Acquire failed: " + acquireStatus); + return OperationFailureExitCode; + } + + if (!lease.ValueSpan.SequenceEqual(expectedValue) + || !lease.DescriptorSpan.SequenceEqual(expectedDescriptor)) + { + _ = lease.Release(); + return ContentMismatchExitCode; + } + + File.WriteAllText(arguments[11], Environment.ProcessId.ToString(CultureInfo.InvariantCulture)); + if (!WaitForFile(arguments[12])) + { + _ = lease.Release(); + return TimeoutExitCode; + } + + // The observer keeps using the borrowed spans after an arbitrary + // pause; another reader must not own its progress or lifetime. + if (!lease.ValueSpan.SequenceEqual(expectedValue) + || !lease.DescriptorSpan.SequenceEqual(expectedDescriptor)) + { + _ = lease.Release(); + return ContentMismatchExitCode; + } + + StoreStatus releaseStatus = lease.Release(); + if (releaseStatus != StoreStatus.Success) + { + Console.Error.WriteLine("Release failed: " + releaseStatus); + return OperationFailureExitCode; + } + + Console.WriteLine("OK lease-hold released=1"); + return 0; + } + } + + private static bool TryDecode(string value, out byte[] bytes) + { + bytes = []; + if (value == "-") + { + return true; + } + + try + { + bytes = Convert.FromHexString(value); + return true; + } + catch (FormatException) + { + return false; + } + } + + private static bool WaitForFile(string path) + { + long deadline = Stopwatch.GetTimestamp() + (long)(WaitTimeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (!File.Exists(path)) + { + if (Stopwatch.GetTimestamp() >= deadline) + { + return false; + } + + spin.SpinOnce(); + } + + return true; + } + + private static ulong AddChecksum(ulong checksum, ReadOnlySpan bytes) + { + foreach (byte value in bytes) + { + checksum = unchecked((checksum ^ value) * 1_099_511_628_211UL); + } + + return checksum; + } + + private readonly record struct StoreArguments(SharedMemoryStoreOptions Options) + { + public static bool TryParse(string[] arguments, int expectedLength, out StoreArguments parsed) + { + parsed = default; + if (arguments.Length != expectedLength + || string.IsNullOrWhiteSpace(arguments[1]) + || !TryPositive(arguments[2], out var slotCount) + || !TryPositive(arguments[3], out var maxValueBytes) + || !TryNonNegative(arguments[4], out var maxDescriptorBytes) + || !TryPositive(arguments[5], out var maxKeyBytes) + || !TryPositive(arguments[6], out var leaseRecordCount) + || !TryPositive(arguments[7], out var participantRecordCount)) + { + return false; + } + + parsed = new StoreArguments(SharedMemoryStoreOptions.CreateLockFree( + arguments[1], + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount, + OpenMode.OpenExisting, + enableLeaseRecovery: true)); + return true; + } + + private static bool TryPositive(string value, out int result) => + int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out result) && result > 0; + + private static bool TryNonNegative(string value, out int result) => + int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out result) && result >= 0; + } +} + +internal static class AtomicCommands +{ + private const int AtomicFailureExitCode = 66; + private const int VisibilityFailureExitCode = 67; + private const int TimeoutExitCode = 68; + + private const int SequenceOffset = 0; + private const int ComplementOffset = 8; + private const int PublicationOffset = 16; + private const int AcknowledgementOffset = 24; + private const int CasCounterOffset = 32; + + private const int DekkerReady0Offset = 0; + private const int DekkerReady1Offset = 8; + private const int DekkerPhaseOffset = 16; + private const int DekkerDone0Offset = 24; + private const int DekkerDone1Offset = 32; + private const int DekkerWord0Offset = 40; + private const int DekkerWord1Offset = 48; + private const int DekkerSeen0Offset = 56; + private const int DekkerSeen1Offset = 64; + + private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(30); + + public static int RunPublicationProducer(string[] arguments) + { + if (!TryParseArguments(arguments, expectedLength: 3, out var path, out var iterations)) + { + return 64; + } + + using var words = MappedAtomicWords.Open(path); + if (!words.AreAligned(SequenceOffset, ComplementOffset, PublicationOffset, AcknowledgementOffset)) + { + return AtomicFailureExitCode; + } + + ref var sequence = ref words[SequenceOffset]; + ref var complement = ref words[ComplementOffset]; + ref var publication = ref words[PublicationOffset]; + ref var acknowledgement = ref words[AcknowledgementOffset]; + for (long iteration = 1; iteration <= iterations; iteration++) + { + if (!WaitForExact(ref acknowledgement, iteration - 1, "publication-producer/acknowledgement")) + { + return TimeoutExitCode; + } + + sequence = iteration; + complement = ~iteration; + Volatile.Write(ref publication, iteration); + } + + Console.WriteLine("OK publication-producer " + iterations.ToString(CultureInfo.InvariantCulture) + " aligned=1"); + return 0; + } + + public static int RunPublicationConsumer(string[] arguments) + { + if (!TryParseArguments(arguments, expectedLength: 3, out var path, out var iterations)) + { + return 64; + } + + using var words = MappedAtomicWords.Open(path); + if (!words.AreAligned(SequenceOffset, ComplementOffset, PublicationOffset, AcknowledgementOffset)) + { + return AtomicFailureExitCode; + } + + ref var sequence = ref words[SequenceOffset]; + ref var complement = ref words[ComplementOffset]; + ref var publication = ref words[PublicationOffset]; + ref var acknowledgement = ref words[AcknowledgementOffset]; + for (long iteration = 1; iteration <= iterations; iteration++) + { + if (!WaitForExact(ref publication, iteration, "publication-consumer/publication")) + { + return TimeoutExitCode; + } + + var observedSequence = sequence; + var observedComplement = complement; + if (observedSequence != iteration || observedComplement != ~iteration) + { + Console.Error.WriteLine( + "Publication mismatch at " + + iteration.ToString(CultureInfo.InvariantCulture) + + ": sequence=" + + observedSequence.ToString(CultureInfo.InvariantCulture) + + ", complement=" + + observedComplement.ToString(CultureInfo.InvariantCulture)); + return VisibilityFailureExitCode; + } + + Volatile.Write(ref acknowledgement, iteration); + } + + Console.WriteLine("OK publication-consumer " + iterations.ToString(CultureInfo.InvariantCulture) + " aligned=1"); + return 0; + } + + public static int RunCasWorker(string[] arguments) + { + if (!TryParseArguments(arguments, expectedLength: 3, out var path, out var iterations)) + { + return 64; + } + + using var words = MappedAtomicWords.Open(path); + if (!words.AreAligned(CasCounterOffset)) + { + return AtomicFailureExitCode; + } + + ref var counter = ref words[CasCounterOffset]; + for (var iteration = 0; iteration < iterations; iteration++) + { + var spin = new SpinWait(); + while (true) + { + var observed = Volatile.Read(ref counter); + if (Interlocked.CompareExchange(ref counter, checked(observed + 1), observed) == observed) + { + break; + } + + spin.SpinOnce(); + } + } + + Console.WriteLine("OK cas-worker " + iterations.ToString(CultureInfo.InvariantCulture) + " aligned=1"); + return 0; + } + + public static int RunDekkerWorker(string[] arguments) + { + if (arguments.Length != 4 + || !TryParsePositiveIterations(arguments[2], out var iterations) + || !int.TryParse(arguments[3], NumberStyles.None, CultureInfo.InvariantCulture, out var role) + || role is < 0 or > 1) + { + return 64; + } + + using var words = MappedAtomicWords.Open(arguments[1]); + if (!words.AreAligned( + DekkerReady0Offset, + DekkerReady1Offset, + DekkerPhaseOffset, + DekkerDone0Offset, + DekkerDone1Offset, + DekkerWord0Offset, + DekkerWord1Offset, + DekkerSeen0Offset, + DekkerSeen1Offset)) + { + return AtomicFailureExitCode; + } + + ref var ready = ref words[role == 0 ? DekkerReady0Offset : DekkerReady1Offset]; + ref var phase = ref words[DekkerPhaseOffset]; + ref var done = ref words[role == 0 ? DekkerDone0Offset : DekkerDone1Offset]; + ref var ownWord = ref words[role == 0 ? DekkerWord0Offset : DekkerWord1Offset]; + ref var otherWord = ref words[role == 0 ? DekkerWord1Offset : DekkerWord0Offset]; + ref var seen = ref words[role == 0 ? DekkerSeen0Offset : DekkerSeen1Offset]; + + Volatile.Write(ref ready, 1); + for (long iteration = 1; iteration <= iterations; iteration++) + { + if (!WaitForExact( + ref phase, + iteration, + role == 0 ? "dekker-worker-0/phase" : "dekker-worker-1/phase")) + { + return TimeoutExitCode; + } + + Interlocked.Exchange(ref ownWord, 1); + Volatile.Write(ref seen, Volatile.Read(ref otherWord)); + Volatile.Write(ref done, iteration); + } + + Console.WriteLine( + "OK dekker-worker role=" + + role.ToString(CultureInfo.InvariantCulture) + + " iterations=" + + iterations.ToString(CultureInfo.InvariantCulture) + + " aligned=1"); + return 0; + } + + public static int RunDekkerCoordinator(string[] arguments) + { + if (!TryParseArguments(arguments, expectedLength: 3, out var path, out var iterations)) + { + return 64; + } + + using var words = MappedAtomicWords.Open(path); + if (!words.AreAligned( + DekkerReady0Offset, + DekkerReady1Offset, + DekkerPhaseOffset, + DekkerDone0Offset, + DekkerDone1Offset, + DekkerWord0Offset, + DekkerWord1Offset, + DekkerSeen0Offset, + DekkerSeen1Offset)) + { + return AtomicFailureExitCode; + } + + ref var ready0 = ref words[DekkerReady0Offset]; + ref var ready1 = ref words[DekkerReady1Offset]; + ref var phase = ref words[DekkerPhaseOffset]; + ref var done0 = ref words[DekkerDone0Offset]; + ref var done1 = ref words[DekkerDone1Offset]; + ref var word0 = ref words[DekkerWord0Offset]; + ref var word1 = ref words[DekkerWord1Offset]; + ref var seen0 = ref words[DekkerSeen0Offset]; + ref var seen1 = ref words[DekkerSeen1Offset]; + + if (!WaitForExact(ref ready0, 1, "dekker-coordinator/ready0") + || !WaitForExact(ref ready1, 1, "dekker-coordinator/ready1")) + { + return TimeoutExitCode; + } + + for (long iteration = 1; iteration <= iterations; iteration++) + { + if (!WaitForExact(ref done0, iteration - 1, "dekker-coordinator/done0-previous") + || !WaitForExact(ref done1, iteration - 1, "dekker-coordinator/done1-previous")) + { + return TimeoutExitCode; + } + + Volatile.Write(ref word0, 0); + Volatile.Write(ref word1, 0); + Volatile.Write(ref seen0, -1); + Volatile.Write(ref seen1, -1); + Volatile.Write(ref phase, iteration); + + if (!WaitForExact(ref done0, iteration, "dekker-coordinator/done0-current") + || !WaitForExact(ref done1, iteration, "dekker-coordinator/done1-current")) + { + return TimeoutExitCode; + } + + if (Volatile.Read(ref seen0) == 0 && Volatile.Read(ref seen1) == 0) + { + Console.Error.WriteLine("Forbidden Dekker outcome at iteration " + iteration.ToString(CultureInfo.InvariantCulture)); + return VisibilityFailureExitCode; + } + } + + Console.WriteLine("OK dekker-coordinator " + iterations.ToString(CultureInfo.InvariantCulture) + " forbidden=0 aligned=1"); + return 0; + } + + private static bool TryParseArguments( + string[] arguments, + int expectedLength, + out string path, + out int iterations) + { + path = string.Empty; + iterations = 0; + if (arguments.Length != expectedLength || !TryParsePositiveIterations(arguments[2], out iterations)) + { + return false; + } + + path = arguments[1]; + return path.Length != 0; + } + + private static bool TryParsePositiveIterations(string value, out int iterations) + { + return int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out iterations) + && iterations is > 0 and <= 1_000_000; + } + + private static bool WaitForExact(ref long word, long expected, string context) + { + var deadline = Stopwatch.GetTimestamp() + (long)(WaitTimeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (true) + { + var observed = Volatile.Read(ref word); + if (observed == expected) + { + return true; + } + + if (Stopwatch.GetTimestamp() >= deadline) + { + Console.Error.WriteLine( + "Atomic wait timeout: context=" + context + + "; expected=" + expected.ToString(CultureInfo.InvariantCulture) + + "; observed=" + observed.ToString(CultureInfo.InvariantCulture) + + "."); + return false; + } + + spin.SpinOnce(); + } + } +} + +internal sealed unsafe class MappedAtomicWords : IDisposable +{ + public const int Length = 4096; + + private readonly MemoryMappedFile _mapping; + private readonly MemoryMappedViewAccessor _accessor; + private byte* _pointer; + private bool _disposed; + + private MappedAtomicWords(MemoryMappedFile mapping, MemoryMappedViewAccessor accessor, byte* pointer) + { + _mapping = mapping; + _accessor = accessor; + _pointer = pointer; + } + + public ref long this[int byteOffset] + { + get + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (byteOffset < 0 || byteOffset > Length - sizeof(long)) + { + throw new ArgumentOutOfRangeException(nameof(byteOffset)); + } + + return ref *(long*)(_pointer + byteOffset); + } + } + + public static MappedAtomicWords Open(string path) + { + var stream = new FileStream( + path, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete); + MemoryMappedFile? mapping = null; + MemoryMappedViewAccessor? accessor = null; + byte* pointer = null; + try + { + if (stream.Length < Length) + { + throw new InvalidDataException("Atomic test mapping is shorter than the required control block."); + } + + mapping = MemoryMappedFile.CreateFromFile( + stream, + mapName: null, + capacity: Length, + MemoryMappedFileAccess.ReadWrite, + HandleInheritability.None, + leaveOpen: false); + accessor = mapping.CreateViewAccessor(0, Length, MemoryMappedFileAccess.ReadWrite); + accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer); + pointer += accessor.PointerOffset; + var result = new MappedAtomicWords(mapping, accessor, pointer); + mapping = null; + accessor = null; + return result; + } + catch + { + if (pointer is not null && accessor is not null) + { + accessor.SafeMemoryMappedViewHandle.ReleasePointer(); + } + + accessor?.Dispose(); + mapping?.Dispose(); + stream.Dispose(); + throw; + } + } + + public bool AreAligned(params int[] byteOffsets) + { + foreach (var byteOffset in byteOffsets) + { + if (byteOffset < 0 + || byteOffset > Length - sizeof(long) + || ((nuint)(_pointer + byteOffset) & (sizeof(long) - 1)) != 0) + { + return false; + } + } + + return true; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + if (_pointer is not null) + { + _accessor.SafeMemoryMappedViewHandle.ReleasePointer(); + _pointer = null; + } + + _accessor.Dispose(); + _mapping.Dispose(); + } +} diff --git a/tests/SharedMemoryStore.LockFreeAgent/RawVisibilityCommands.cs b/tests/SharedMemoryStore.LockFreeAgent/RawVisibilityCommands.cs new file mode 100644 index 0000000..351bd56 --- /dev/null +++ b/tests/SharedMemoryStore.LockFreeAgent/RawVisibilityCommands.cs @@ -0,0 +1,629 @@ +using System.Buffers.Binary; +using System.Diagnostics; +using System.Globalization; +using System.Text.Json; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.LockFreeAgent; + +/// +/// Production-no-op full-protocol visibility workers. The workers deliberately +/// use no checkpoint callback, diagnostics counter, file signal, or console +/// write between their common clock rendezvous and the terminal store value. +/// +internal static class RawVisibilityCommands +{ + private const int InvalidArgumentsExitCode = 64; + private const int OperationFailureExitCode = 66; + private const int ContentMismatchExitCode = 67; + private const int TimeoutExitCode = 68; + private const int PayloadHeaderLength = 96; + private const int DescriptorLength = 48; + private const ulong PayloadMagic = 0x3257_4152_534d_5301UL; + private const ulong DescriptorMagic = 0x3252_4353_4544_5301UL; + private static readonly TimeSpan WorkloadTimeout = TimeSpan.FromSeconds(60); + private static readonly TimeSpan FirstRemovalDelay = TimeSpan.FromMilliseconds(500); + + internal static int RunPublisher(string[] arguments) + { + if (!Arguments.TryParse(arguments, out Arguments parsed)) + { + return InvalidArgumentsExitCode; + } + + StoreOpenStatus open = MemoryStore.TryCreateOrOpen(parsed.Options, out MemoryStore? store); + if (open != StoreOpenStatus.Success || store is null) + { + return OperationFailureExitCode; + } + + using (store) + { + WaitForCommonStart(parsed.StartUtcTicks); + long deadline = Deadline(); + var spin = new SpinWait(); + long minimumGeneration = long.MaxValue; + long maximumGeneration = 0; + byte[][] keys = CreateDataKeys(parsed.Seed, parsed.KeyCount); + var descriptor = new byte[DescriptorLength]; + for (var sequence = 1; sequence <= parsed.Iterations; sequence++) + { + int keyIndex = (sequence - 1) % parsed.KeyCount; + byte[] key = keys[keyIndex]; + Pattern.FillDescriptor(descriptor, (ulong)sequence, key, keyIndex); + while (true) + { + StoreStatus reserve = store.TryReserve( + key, + parsed.PayloadLength, + descriptor, + out ValueReservation reservation); + if (reserve == StoreStatus.Success) + { + long generation = IndexBinding.Decode( + reservation.HandleForEngine.SlotBinding).Generation; + minimumGeneration = Math.Min(minimumGeneration, generation); + maximumGeneration = Math.Max(maximumGeneration, generation); + Span payload = reservation.GetSpan(parsed.PayloadLength); + if (payload.Length != parsed.PayloadLength) + { + _ = reservation.Abort(); + return ContentMismatchExitCode; + } + + Pattern.FillPayload(payload, (ulong)sequence, generation, key, keyIndex); + if (reservation.Advance(parsed.PayloadLength) != StoreStatus.Success + || reservation.Commit() != StoreStatus.Success) + { + return OperationFailureExitCode; + } + + break; + } + + if (!IsRetryablePublish(reserve)) + { + return OperationFailureExitCode; + } + + if (Expired(deadline)) + { + return TimeoutExitCode; + } + + spin.SpinOnce(); + } + } + + byte[] terminalKey = Pattern.CreateKey(parsed.Seed, Pattern.TerminalKeyIndex); + Pattern.FillDescriptor( + descriptor, + checked((ulong)parsed.Iterations + 1), + terminalKey, + Pattern.TerminalKeyIndex); + while (true) + { + StoreStatus reserve = store.TryReserve( + terminalKey, + parsed.PayloadLength, + descriptor, + out ValueReservation reservation); + if (reserve == StoreStatus.Success) + { + long generation = IndexBinding.Decode( + reservation.HandleForEngine.SlotBinding).Generation; + Span payload = reservation.GetSpan(parsed.PayloadLength); + if (payload.Length != parsed.PayloadLength) + { + _ = reservation.Abort(); + return ContentMismatchExitCode; + } + + Pattern.FillPayload( + payload, + checked((ulong)parsed.Iterations + 1), + generation, + terminalKey, + Pattern.TerminalKeyIndex); + if (reservation.Advance(parsed.PayloadLength) != StoreStatus.Success + || reservation.Commit() != StoreStatus.Success) + { + return OperationFailureExitCode; + } + + break; + } + + if (!IsRetryablePublish(reserve)) + { + return OperationFailureExitCode; + } + + if (Expired(deadline)) + { + return TimeoutExitCode; + } + + spin.SpinOnce(); + } + + WriteResult(new Result( + "publisher", + parsed.Iterations, + parsed.Iterations, + 0, + minimumGeneration, + maximumGeneration)); + return 0; + } + } + + internal static int RunReader(string[] arguments) + { + if (!Arguments.TryParse(arguments, out Arguments parsed)) + { + return InvalidArgumentsExitCode; + } + + StoreOpenStatus open = MemoryStore.TryCreateOrOpen(parsed.Options, out MemoryStore? store); + if (open != StoreOpenStatus.Success || store is null) + { + return OperationFailureExitCode; + } + + using (store) + { + WaitForCommonStart(parsed.StartUtcTicks); + long deadline = Deadline(); + long observations = 0; + ulong checksum = 14_695_981_039_346_656_037UL; + long minimumGeneration = long.MaxValue; + long maximumGeneration = 0; + var spin = new SpinWait(); + var scanStart = 0; + byte[][] keys = CreateDataKeys(parsed.Seed, parsed.KeyCount); + byte[] terminalKey = Pattern.CreateKey(parsed.Seed, Pattern.TerminalKeyIndex); + while (true) + { + for (var offset = 0; offset < parsed.KeyCount; offset++) + { + int keyIndex = (scanStart + offset) % parsed.KeyCount; + byte[] key = keys[keyIndex]; + StoreStatus acquire = store.TryAcquire(key, out ValueLease lease); + if (acquire == StoreStatus.Success) + { + if (!Pattern.ValidateLease( + lease, + key, + keyIndex, + parsed.KeyCount, + parsed.PayloadLength, + parsed.Iterations, + out ulong sequence)) + { + _ = lease.Release(); + return ContentMismatchExitCode; + } + + checksum = unchecked((checksum ^ sequence) * 1_099_511_628_211UL); + long generation = IndexBinding.Decode(lease.HandleForEngine.SlotBinding).Generation; + minimumGeneration = Math.Min(minimumGeneration, generation); + maximumGeneration = Math.Max(maximumGeneration, generation); + observations++; + if (lease.Release() != StoreStatus.Success) + { + return OperationFailureExitCode; + } + } + else if (!IsRetryableAcquire(acquire)) + { + return OperationFailureExitCode; + } + } + + StoreStatus terminalAcquire = store.TryAcquire(terminalKey, out ValueLease terminalLease); + if (terminalAcquire == StoreStatus.Success) + { + bool valid = Pattern.ValidateLease( + terminalLease, + terminalKey, + Pattern.TerminalKeyIndex, + parsed.KeyCount, + parsed.PayloadLength, + parsed.Iterations, + out ulong sequence); + StoreStatus release = terminalLease.Release(); + if (!valid + || sequence != checked((ulong)parsed.Iterations + 1) + || release != StoreStatus.Success) + { + return ContentMismatchExitCode; + } + + if (observations == 0) + { + return ContentMismatchExitCode; + } + + WriteResult(new Result( + "reader", + 1, + observations, + checksum, + minimumGeneration, + maximumGeneration)); + return 0; + } + + if (!IsRetryableAcquire(terminalAcquire)) + { + return OperationFailureExitCode; + } + + if (Expired(deadline)) + { + return TimeoutExitCode; + } + + scanStart = (scanStart + 1) % parsed.KeyCount; + spin.SpinOnce(); + } + } + } + + internal static int RunRemover(string[] arguments) + { + if (!Arguments.TryParse(arguments, out Arguments parsed)) + { + return InvalidArgumentsExitCode; + } + + StoreOpenStatus open = MemoryStore.TryCreateOrOpen(parsed.Options, out MemoryStore? store); + if (open != StoreOpenStatus.Success || store is null) + { + return OperationFailureExitCode; + } + + using (store) + { + WaitForCommonStart(parsed.StartUtcTicks); + WaitUntilUtc(checked(parsed.StartUtcTicks + FirstRemovalDelay.Ticks)); + long deadline = Deadline(); + var removed = new bool[parsed.Iterations + 1]; + var spin = new SpinWait(); + var scanStart = 0; + var removedCount = 0; + ulong checksum = 14_695_981_039_346_656_037UL; + long minimumGeneration = long.MaxValue; + long maximumGeneration = 0; + byte[][] keys = CreateDataKeys(parsed.Seed, parsed.KeyCount); + while (removedCount < parsed.Iterations) + { + for (var offset = 0; offset < parsed.KeyCount && removedCount < parsed.Iterations; offset++) + { + int keyIndex = (scanStart + offset) % parsed.KeyCount; + byte[] key = keys[keyIndex]; + StoreStatus acquire = store.TryAcquire(key, out ValueLease lease); + if (acquire == StoreStatus.Success) + { + if (!Pattern.ValidateLease( + lease, + key, + keyIndex, + parsed.KeyCount, + parsed.PayloadLength, + parsed.Iterations, + out ulong sequence) + || sequence == 0 + || sequence > (ulong)parsed.Iterations + || removed[checked((int)sequence)]) + { + _ = lease.Release(); + return ContentMismatchExitCode; + } + + if (lease.Release() != StoreStatus.Success) + { + return OperationFailureExitCode; + } + + StoreStatus remove = store.TryRemove(key); + if (remove is StoreStatus.Success or StoreStatus.RemovePending) + { + long generation = IndexBinding.Decode(lease.HandleForEngine.SlotBinding).Generation; + minimumGeneration = Math.Min(minimumGeneration, generation); + maximumGeneration = Math.Max(maximumGeneration, generation); + removed[checked((int)sequence)] = true; + removedCount++; + checksum = unchecked((checksum ^ sequence) * 1_099_511_628_211UL); + } + else if (remove != StoreStatus.StoreBusy) + { + return OperationFailureExitCode; + } + } + else if (!IsRetryableAcquire(acquire)) + { + return OperationFailureExitCode; + } + } + + if (Expired(deadline)) + { + return TimeoutExitCode; + } + + scanStart = (scanStart + 1) % parsed.KeyCount; + spin.SpinOnce(); + } + + WriteResult(new Result( + "remover", + parsed.Iterations, + removedCount, + checksum, + minimumGeneration, + maximumGeneration)); + return 0; + } + } + + private static bool IsRetryablePublish(StoreStatus status) => + status is StoreStatus.DuplicateKey or StoreStatus.StoreFull or StoreStatus.StoreBusy; + + private static bool IsRetryableAcquire(StoreStatus status) => + status is StoreStatus.NotFound or StoreStatus.StoreBusy or StoreStatus.LeaseTableFull; + + private static byte[][] CreateDataKeys(int seed, int keyCount) + { + var keys = new byte[keyCount][]; + for (var keyIndex = 0; keyIndex < keys.Length; keyIndex++) + { + keys[keyIndex] = Pattern.CreateKey(seed, keyIndex); + } + + return keys; + } + + private static long Deadline() => + Stopwatch.GetTimestamp() + (long)(WorkloadTimeout.TotalSeconds * Stopwatch.Frequency); + + private static bool Expired(long deadline) => Stopwatch.GetTimestamp() >= deadline; + + private static void WaitForCommonStart(long utcTicks) => WaitUntilUtc(utcTicks); + + private static void WaitUntilUtc(long utcTicks) + { + var spin = new SpinWait(); + while (DateTime.UtcNow.Ticks < utcTicks) + { + spin.SpinOnce(); + } + } + + private static void WriteResult(in Result result) => + Console.WriteLine("RESULT " + JsonSerializer.Serialize(result)); + + private readonly record struct Result( + string Role, + int Completed, + long Observations, + ulong Checksum, + long MinimumGeneration, + long MaximumGeneration); + + private readonly record struct Arguments( + SharedMemoryStoreOptions Options, + int Iterations, + int KeyCount, + int PayloadLength, + int Seed, + long StartUtcTicks) + { + internal static bool TryParse(string[] arguments, out Arguments parsed) + { + parsed = default; + if (arguments.Length != 13 + || string.IsNullOrWhiteSpace(arguments[1]) + || !TryPositive(arguments[2], out int slotCount) + || !TryPositive(arguments[3], out int maxValueBytes) + || !TryNonNegative(arguments[4], out int maxDescriptorBytes) + || !TryPositive(arguments[5], out int maxKeyBytes) + || !TryPositive(arguments[6], out int leaseRecordCount) + || !TryPositive(arguments[7], out int participantRecordCount) + || !TryPositive(arguments[8], out int iterations) + || !TryPositive(arguments[9], out int keyCount) + || !TryPositive(arguments[10], out int payloadLength) + || !int.TryParse(arguments[11], NumberStyles.Integer, CultureInfo.InvariantCulture, out int seed) + || !long.TryParse(arguments[12], NumberStyles.None, CultureInfo.InvariantCulture, out long startUtcTicks) + || slotCount > 1_048_575 + || keyCount > 1_024 + || iterations > 1_000_000 + || payloadLength < PayloadHeaderLength + || payloadLength > maxValueBytes + || maxDescriptorBytes < DescriptorLength + || maxKeyBytes < Pattern.KeyLength + || startUtcTicks <= 0) + { + return false; + } + + parsed = new Arguments( + SharedMemoryStoreOptions.CreateLockFree( + arguments[1], + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount, + OpenMode.OpenExisting, + enableLeaseRecovery: true), + iterations, + keyCount, + payloadLength, + seed, + startUtcTicks); + return true; + } + + private static bool TryPositive(string text, out int value) => + int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value) && value > 0; + + private static bool TryNonNegative(string text, out int value) => + int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value) && value >= 0; + } + + private static class Pattern + { + internal const int KeyLength = 16; + internal const int TerminalKeyIndex = -1; + + internal static byte[] CreateKey(int seed, int keyIndex) + { + var key = new byte[KeyLength]; + BinaryPrimitives.WriteInt32LittleEndian(key, seed); + BinaryPrimitives.WriteInt32LittleEndian(key.AsSpan(4), keyIndex); + ulong identity = Mix(unchecked((uint)seed) | ((ulong)unchecked((uint)keyIndex) << 32)); + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8), identity); + return key; + } + + internal static void FillDescriptor( + Span descriptor, + ulong sequence, + ReadOnlySpan key, + int keyIndex) + { + ulong signature = KeySignature(key); + BinaryPrimitives.WriteUInt64LittleEndian(descriptor, DescriptorMagic); + BinaryPrimitives.WriteUInt64LittleEndian(descriptor[8..], sequence); + BinaryPrimitives.WriteUInt64LittleEndian(descriptor[16..], ~sequence); + BinaryPrimitives.WriteUInt64LittleEndian(descriptor[24..], signature); + BinaryPrimitives.WriteUInt64LittleEndian(descriptor[32..], ~signature); + BinaryPrimitives.WriteInt32LittleEndian(descriptor[40..], keyIndex); + BinaryPrimitives.WriteInt32LittleEndian(descriptor[44..], ~keyIndex); + } + + internal static void FillPayload( + Span payload, + ulong sequence, + long generation, + ReadOnlySpan key, + int keyIndex) + { + ulong signature = KeySignature(key); + BinaryPrimitives.WriteUInt64LittleEndian(payload, PayloadMagic); + BinaryPrimitives.WriteUInt64LittleEndian(payload[8..], sequence); + BinaryPrimitives.WriteUInt64LittleEndian(payload[16..], ~sequence); + BinaryPrimitives.WriteInt64LittleEndian(payload[24..], generation); + BinaryPrimitives.WriteInt64LittleEndian(payload[32..], ~generation); + BinaryPrimitives.WriteUInt64LittleEndian(payload[40..], signature); + BinaryPrimitives.WriteUInt64LittleEndian(payload[48..], ~signature); + BinaryPrimitives.WriteInt64LittleEndian(payload[56..], payload.Length); + BinaryPrimitives.WriteInt64LittleEndian(payload[64..], ~((long)payload.Length)); + BinaryPrimitives.WriteInt32LittleEndian(payload[72..], keyIndex); + BinaryPrimitives.WriteInt32LittleEndian(payload[76..], ~keyIndex); + key.CopyTo(payload[80..(80 + KeyLength)]); + for (var offset = PayloadHeaderLength; offset < payload.Length; offset++) + { + payload[offset] = PayloadByte(sequence, generation, signature, offset); + } + } + + internal static bool ValidateLease( + in ValueLease lease, + ReadOnlySpan expectedKey, + int expectedKeyIndex, + int keyCount, + int expectedPayloadLength, + int dataIterations, + out ulong sequence) + { + sequence = 0; + ReadOnlySpan payload = lease.ValueSpan; + if (lease.ValueLength != expectedPayloadLength + || payload.Length != expectedPayloadLength + || lease.DescriptorLength != DescriptorLength + || payload.Length < PayloadHeaderLength + || BinaryPrimitives.ReadUInt64LittleEndian(payload) != PayloadMagic) + { + return false; + } + + sequence = BinaryPrimitives.ReadUInt64LittleEndian(payload[8..]); + long generation = BinaryPrimitives.ReadInt64LittleEndian(payload[24..]); + ulong signature = KeySignature(expectedKey); + if (sequence == 0 + || sequence > checked((ulong)dataIterations + 1) + || BinaryPrimitives.ReadUInt64LittleEndian(payload[16..]) != ~sequence + || generation <= 0 + || BinaryPrimitives.ReadInt64LittleEndian(payload[32..]) != ~generation + || BinaryPrimitives.ReadUInt64LittleEndian(payload[40..]) != signature + || BinaryPrimitives.ReadUInt64LittleEndian(payload[48..]) != ~signature + || BinaryPrimitives.ReadInt64LittleEndian(payload[56..]) != expectedPayloadLength + || BinaryPrimitives.ReadInt64LittleEndian(payload[64..]) != ~((long)expectedPayloadLength) + || BinaryPrimitives.ReadInt32LittleEndian(payload[72..]) != expectedKeyIndex + || BinaryPrimitives.ReadInt32LittleEndian(payload[76..]) != ~expectedKeyIndex + || !payload[80..(80 + KeyLength)].SequenceEqual(expectedKey) + || IndexBinding.Decode(lease.HandleForEngine.SlotBinding).Generation != generation) + { + return false; + } + + ReadOnlySpan descriptor = lease.DescriptorSpan; + if (BinaryPrimitives.ReadUInt64LittleEndian(descriptor) != DescriptorMagic + || BinaryPrimitives.ReadUInt64LittleEndian(descriptor[8..]) != sequence + || BinaryPrimitives.ReadUInt64LittleEndian(descriptor[16..]) != ~sequence + || BinaryPrimitives.ReadUInt64LittleEndian(descriptor[24..]) != signature + || BinaryPrimitives.ReadUInt64LittleEndian(descriptor[32..]) != ~signature + || BinaryPrimitives.ReadInt32LittleEndian(descriptor[40..]) != expectedKeyIndex + || BinaryPrimitives.ReadInt32LittleEndian(descriptor[44..]) != ~expectedKeyIndex) + { + return false; + } + + for (var offset = PayloadHeaderLength; offset < payload.Length; offset++) + { + if (payload[offset] != PayloadByte(sequence, generation, signature, offset)) + { + return false; + } + } + + return expectedKeyIndex == TerminalKeyIndex + ? sequence == checked((ulong)dataIterations + 1) + : sequence <= (ulong)dataIterations + && expectedKeyIndex == checked((int)((sequence - 1) % (ulong)keyCount)); + } + + private static byte PayloadByte(ulong sequence, long generation, ulong signature, int offset) + { + ulong value = sequence + ^ unchecked((ulong)generation * 0x9e37_79b9_7f4a_7c15UL) + ^ signature + ^ unchecked((ulong)offset * 0xd6e8_feb8_6659_fd93UL); + return (byte)(Mix(value) >> ((offset & 7) * 8)); + } + + private static ulong KeySignature(ReadOnlySpan key) + { + ulong hash = 14_695_981_039_346_656_037UL; + foreach (byte value in key) + { + hash = unchecked((hash ^ value) * 1_099_511_628_211UL); + } + + return hash; + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + } +} diff --git a/tests/SharedMemoryStore.LockFreeAgent/SharedMemoryStore.LockFreeAgent.csproj b/tests/SharedMemoryStore.LockFreeAgent/SharedMemoryStore.LockFreeAgent.csproj new file mode 100644 index 0000000..17fe480 --- /dev/null +++ b/tests/SharedMemoryStore.LockFreeAgent/SharedMemoryStore.LockFreeAgent.csproj @@ -0,0 +1,16 @@ + + + + Exe + net10.0 + enable + enable + false + true + + + + + + + diff --git a/tests/SharedMemoryStore.LockFreeAgent/SteadyNoLockCommands.cs b/tests/SharedMemoryStore.LockFreeAgent/SteadyNoLockCommands.cs new file mode 100644 index 0000000..59bd806 --- /dev/null +++ b/tests/SharedMemoryStore.LockFreeAgent/SteadyNoLockCommands.cs @@ -0,0 +1,259 @@ +using System.Diagnostics; +using System.Globalization; + +namespace SharedMemoryStore.LockFreeAgent; + +/// +/// Emits no console output between the external go and done file markers so a +/// syscall tracer can isolate only the warmed layout-v2 steady-state interval. +/// +internal static class SteadyNoLockCommands +{ + private const int InvalidArgumentsExitCode = 64; + private const int OperationFailureExitCode = 66; + private const int TimeoutExitCode = 68; + private const int WarmupIterations = 64; + private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(60); + + internal static int Run(string[] arguments) + { + if (!Arguments.TryParse(arguments, out Arguments parsed)) + { + return InvalidArgumentsExitCode; + } + + StoreOpenStatus open = MemoryStore.TryCreateOrOpen(parsed.Options, out MemoryStore? store); + if (open != StoreOpenStatus.Success || store is null) + { + return OperationFailureExitCode; + } + + using (store) + { + byte[] key = CreateKey(); + for (var iteration = 0; iteration < WarmupIterations; iteration++) + { + if (!RunCycle(store, key, iteration)) + { + return OperationFailureExitCode; + } + } + + PublishMarker(parsed.ReadyPath, Environment.ProcessId.ToString(CultureInfo.InvariantCulture)); + if (!WaitForFile(parsed.GoPath)) + { + return TimeoutExitCode; + } + + var succeeded = true; + for (var iteration = 0; iteration < parsed.Iterations; iteration++) + { + if (!RunCycle(store, key, checked(WarmupIterations + iteration))) + { + succeeded = false; + break; + } + } + + // This write is the end marker. Do not add console/logging calls + // above it: the parent traces the warmed go-to-done interval. + PublishMarker(parsed.DonePath, succeeded ? "ok" : "failed"); + if (!succeeded) + { + Console.Error.WriteLine("A steady-no-lock operation returned an unexpected status."); + return OperationFailureExitCode; + } + + return 0; + } + } + + private static bool RunCycle(MemoryStore store, byte[] key, int iteration) + { + Span value = stackalloc byte[16]; + BitConverter.TryWriteBytes(value, iteration); + BitConverter.TryWriteBytes(value[8..], ~((long)iteration)); + Span descriptor = stackalloc byte[8]; + BitConverter.TryWriteBytes(descriptor, unchecked(iteration * 31 + 7)); + + if (!TryPublishEventually(store, key, value, descriptor) + || store.TryAcquire(key, out ValueLease lease) != StoreStatus.Success) + { + return false; + } + + bool contentMatches = lease.ValueSpan.SequenceEqual(value) + && lease.DescriptorSpan.SequenceEqual(descriptor); + if (lease.Release() != StoreStatus.Success || !contentMatches) + { + return false; + } + + StoreStatus remove = store.TryRemove(key); + if (remove is not (StoreStatus.Success or StoreStatus.RemovePending)) + { + return false; + } + + if (store.TryRecoverLeases( + new LeaseRecoveryOptions(true), + out LeaseRecoveryReport leaseReport) != StoreStatus.Success + || leaseReport.RecoveredLeaseCount != 0 + || store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + out ReservationRecoveryReport reservationReport) != StoreStatus.Success + || reservationReport.RecoveredReservationCount != 0 + || store.TryGetDiagnostics(out DiagnosticsSnapshot diagnostics) != StoreStatus.Success + || diagnostics.Profile != StoreProfile.LockFree) + { + return false; + } + + return WaitUntilAbsent(store, key); + } + + private static bool TryPublishEventually( + MemoryStore store, + byte[] key, + ReadOnlySpan value, + ReadOnlySpan descriptor) + { + long deadline = Stopwatch.GetTimestamp() + (long)(WaitTimeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (true) + { + StoreStatus publish = store.TryPublish(key, value, descriptor); + if (publish == StoreStatus.Success) + { + return true; + } + + if (publish is not (StoreStatus.DuplicateKey or StoreStatus.StoreFull or StoreStatus.StoreBusy)) + { + return false; + } + + _ = store.TryRemove(key); + if (Stopwatch.GetTimestamp() >= deadline) + { + return false; + } + + spin.SpinOnce(); + } + } + + private static bool WaitUntilAbsent(MemoryStore store, byte[] key) + { + long deadline = Stopwatch.GetTimestamp() + (long)(WaitTimeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (true) + { + StoreStatus acquire = store.TryAcquire(key, out ValueLease lease); + if (acquire == StoreStatus.NotFound) + { + return true; + } + + if (acquire == StoreStatus.Success) + { + _ = lease.Release(); + } + else if (acquire is not (StoreStatus.StoreBusy or StoreStatus.LeaseTableFull)) + { + return false; + } + + if (Stopwatch.GetTimestamp() >= deadline) + { + return false; + } + + spin.SpinOnce(); + } + } + + private static byte[] CreateKey() => [0x53, 0x4d, 0x53, 0x32, 0x4e, 0x4f, 0x4c, 0x4b]; + + private static void PublishMarker(string path, string content) + { + // File.WriteAllText publishes the destination name before its payload + // is complete. The parent synchronizes on name visibility, so publish + // a closed file with a same-directory atomic rename instead. + string temporaryPath = $"{path}.{Environment.ProcessId.ToString(CultureInfo.InvariantCulture)}.tmp"; + File.WriteAllText(temporaryPath, content); + File.Move(temporaryPath, path); + } + + private static bool WaitForFile(string path) + { + long deadline = Stopwatch.GetTimestamp() + (long)(WaitTimeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (!File.Exists(path)) + { + if (Stopwatch.GetTimestamp() >= deadline) + { + return false; + } + + spin.SpinOnce(); + } + + return true; + } + + private readonly record struct Arguments( + SharedMemoryStoreOptions Options, + int Iterations, + string ReadyPath, + string GoPath, + string DonePath) + { + internal static bool TryParse(string[] arguments, out Arguments parsed) + { + parsed = default; + if (arguments.Length != 12 + || string.IsNullOrWhiteSpace(arguments[1]) + || !TryPositive(arguments[2], out int slotCount) + || !TryPositive(arguments[3], out int maxValueBytes) + || !TryNonNegative(arguments[4], out int maxDescriptorBytes) + || !TryPositive(arguments[5], out int maxKeyBytes) + || !TryPositive(arguments[6], out int leaseRecordCount) + || !TryPositive(arguments[7], out int participantRecordCount) + || !TryPositive(arguments[8], out int iterations) + || maxValueBytes < 16 + || maxDescriptorBytes < 8 + || maxKeyBytes < 8 + || iterations > 1_000_000 + || string.IsNullOrWhiteSpace(arguments[9]) + || string.IsNullOrWhiteSpace(arguments[10]) + || string.IsNullOrWhiteSpace(arguments[11])) + { + return false; + } + + parsed = new Arguments( + SharedMemoryStoreOptions.CreateLockFree( + arguments[1], + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + participantRecordCount, + OpenMode.OpenExisting, + enableLeaseRecovery: true), + iterations, + arguments[9], + arguments[10], + arguments[11]); + return true; + } + + private static bool TryPositive(string text, out int value) => + int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value) && value > 0; + + private static bool TryNonNegative(string text, out int value) => + int.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value) && value >= 0; + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LeaseOwnerClassifierCrossPlatformTests.cs b/tests/SharedMemoryStore.UnitTests/LeaseOwnerClassifierCrossPlatformTests.cs index 00dbe18..47caf42 100644 --- a/tests/SharedMemoryStore.UnitTests/LeaseOwnerClassifierCrossPlatformTests.cs +++ b/tests/SharedMemoryStore.UnitTests/LeaseOwnerClassifierCrossPlatformTests.cs @@ -1,5 +1,7 @@ using System.Diagnostics; +using SharedMemoryStore.LayoutV2; using SharedMemoryStore.Leasing; +using SharedMemoryStore.LockFree; namespace SharedMemoryStore.UnitTests; @@ -51,6 +53,96 @@ public void LiveProcessIsClassifiedAsActiveOnSupportedHosts() } } + [Fact] + public void ExactParticipantIdentityDistinguishesPidReuseAndUnknownIdentity() + { + if (!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + { + return; + } + + Assert.True(LeaseOwnerClassifier.TryCaptureCurrentProcessIdentity( + out int identityKind, + out long processStartValue, + out ulong pidNamespaceId)); + + var current = new ParticipantIncarnation( + RecordIndex: 0, + Generation: 1, + Token: 1, + State: LayoutV2Constants.ParticipantActive, + ProcessId: Environment.ProcessId, + IdentityKind: identityKind, + ProcessStartValue: processStartValue, + OpenSequence: 1, + PidNamespaceId: pidNamespaceId, + ReservedValue: 0, + Control: 0); + + Assert.Equal(LeaseOwnerKind.CurrentProcess, LeaseOwnerClassifier.Classify(current).Kind); + + long reusedStartValue = processStartValue == long.MaxValue + ? processStartValue - 1 + : processStartValue + 1; + Assert.Equal( + LeaseOwnerKind.StaleProcess, + LeaseOwnerClassifier.Classify(current with { ProcessStartValue = reusedStartValue }).Kind); + Assert.Equal( + LeaseOwnerKind.Unsupported, + LeaseOwnerClassifier.Classify(current with + { + IdentityKind = LayoutV2Constants.IdentityUnknown, + ProcessStartValue = 0 + }).Kind); + Assert.Equal( + LeaseOwnerKind.UnsafeRecord, + LeaseOwnerClassifier.Classify(current with { ProcessId = 0 }).Kind); + } + + [Fact] + public void LinuxParticipantClassificationRequiresExactCurrentPidNamespaceBeforePidLookup() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + Assert.True(LeaseOwnerClassifier.TryCaptureCurrentProcessIdentity( + out int identityKind, + out long processStartValue, + out ulong pidNamespaceId)); + Assert.NotEqual(0UL, pidNamespaceId); + + var current = new ParticipantIncarnation( + RecordIndex: 0, + Generation: 1, + Token: 1, + State: LayoutV2Constants.ParticipantActive, + ProcessId: Environment.ProcessId, + IdentityKind: identityKind, + ProcessStartValue: processStartValue, + OpenSequence: 1, + PidNamespaceId: pidNamespaceId, + ReservedValue: 0, + Control: 0); + + Assert.Equal(LeaseOwnerKind.CurrentProcess, LeaseOwnerClassifier.Classify(current).Kind); + Assert.Equal( + LeaseOwnerKind.Unsupported, + LeaseOwnerClassifier.Classify(current with { PidNamespaceId = 0 }).Kind); + + ulong differentNamespace = pidNamespaceId == ulong.MaxValue + ? pidNamespaceId - 1 + : pidNamespaceId + 1; + Assert.Equal( + LeaseOwnerKind.Unsupported, + LeaseOwnerClassifier.Classify(current with + { + ProcessId = int.MaxValue, + PidNamespaceId = differentNamespace + }).Kind); + } + private static Process StartLiveProcess() { var startInfo = OperatingSystem.IsWindows() diff --git a/tests/SharedMemoryStore.UnitTests/LinuxOwnerAnchorTests.cs b/tests/SharedMemoryStore.UnitTests/LinuxOwnerAnchorTests.cs new file mode 100644 index 0000000..5839ecc --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LinuxOwnerAnchorTests.cs @@ -0,0 +1,258 @@ +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using SharedMemoryStore.Interop; + +namespace SharedMemoryStore.UnitTests; + +[SupportedOSPlatform("linux")] +public sealed class LinuxOwnerAnchorTests +{ + [Fact] + public async Task HeldAnchorIsLiveToLocalAndSeparateDescriptorProbesAndDisposesCrossThread() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var directory = new TemporaryDirectory(); + string ownersPath = Path.Combine(directory.Path, "store.owners"); + Guid token = Guid.NewGuid(); + LinuxOwnerAnchor anchor = await Task.Run(() => LinuxOwnerAnchor.Create(ownersPath, token)); + string anchorPath = LinuxOwnerAnchor.GetPath(ownersPath, token); + try + { + Assert.True(File.Exists(anchorPath)); + Assert.Equal( + LinuxSharedMemoryDirectory.PrivateFileMode, + File.GetUnixFileMode(anchorPath)); + Assert.Equal(LinuxOwnerAnchorState.Locked, LinuxOwnerAnchor.Probe(ownersPath, token)); + Assert.Equal( + LinuxOwnerAnchorState.Locked, + LinuxOwnerAnchor.Probe(ownersPath, token, honorLocalRegistry: false)); + } + finally + { + await Task.Run(anchor.Dispose); + } + + Assert.Equal(LinuxOwnerAnchorState.Missing, LinuxOwnerAnchor.Probe(ownersPath, token)); + Assert.False(File.Exists(anchorPath)); + } + + [Fact] + public void PresentUnlockedAnchorIsStaleAndMissingAnchorIsLegacyFallbackCandidate() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var directory = new TemporaryDirectory(); + string ownersPath = Path.Combine(directory.Path, "store.owners"); + Guid token = Guid.NewGuid(); + string anchorPath = LinuxOwnerAnchor.GetPath(ownersPath, token); + + Assert.Equal(LinuxOwnerAnchorState.Missing, LinuxOwnerAnchor.Probe(ownersPath, token)); + File.WriteAllBytes(anchorPath, []); + File.SetUnixFileMode(anchorPath, LinuxSharedMemoryDirectory.PrivateFileMode); + + Assert.Equal(LinuxOwnerAnchorState.Unlocked, LinuxOwnerAnchor.Probe(ownersPath, token)); + } + + [Fact] + public void DuplicateCreateDoesNotDeleteTheExistingLockedAnchor() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var directory = new TemporaryDirectory(); + string ownersPath = Path.Combine(directory.Path, "store.owners"); + Guid token = Guid.NewGuid(); + using LinuxOwnerAnchor first = LinuxOwnerAnchor.Create(ownersPath, token); + + Assert.Throws(() => LinuxOwnerAnchor.Create(ownersPath, token)); + + Assert.True(File.Exists(first.AnchorPath)); + Assert.Equal( + LinuxOwnerAnchorState.Locked, + LinuxOwnerAnchor.Probe(ownersPath, token, honorLocalRegistry: false)); + } + + [Fact] + public void SymbolicLinkAnchorIsRetainedConservatively() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var directory = new TemporaryDirectory(); + string ownersPath = Path.Combine(directory.Path, "store.owners"); + Guid token = Guid.NewGuid(); + string targetPath = Path.Combine(directory.Path, "target"); + string anchorPath = LinuxOwnerAnchor.GetPath(ownersPath, token); + File.WriteAllBytes(targetPath, []); + File.CreateSymbolicLink(anchorPath, targetPath); + + Assert.Equal(LinuxOwnerAnchorState.Ambiguous, LinuxOwnerAnchor.Probe(ownersPath, token)); + } + + [Fact] + public void DirectoryAndDanglingLinkAnchorsAreRetainedConservatively() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var directory = new TemporaryDirectory(); + string ownersPath = Path.Combine(directory.Path, "store.owners"); + Guid directoryToken = Guid.NewGuid(); + string directoryAnchor = LinuxOwnerAnchor.GetPath(ownersPath, directoryToken); + Directory.CreateDirectory(directoryAnchor); + Assert.Equal( + LinuxOwnerAnchorState.Ambiguous, + LinuxOwnerAnchor.Probe(ownersPath, directoryToken)); + + Guid linkToken = Guid.NewGuid(); + string linkAnchor = LinuxOwnerAnchor.GetPath(ownersPath, linkToken); + File.CreateSymbolicLink(linkAnchor, Path.Combine(directory.Path, "missing-target")); + Assert.Equal( + LinuxOwnerAnchorState.Ambiguous, + LinuxOwnerAnchor.Probe(ownersPath, linkToken)); + } + + [Fact] + public void ReferencedFifoProbeIsAmbiguousAndRetained() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var directory = new TemporaryDirectory(); + string ownersPath = Path.Combine(directory.Path, "store.owners"); + Guid token = Guid.NewGuid(); + string fifoPath = CreateFifoAnchor(ownersPath, token); + File.WriteAllText( + ownersPath, + $"{Environment.ProcessId}:proc-test:{token:N}{Environment.NewLine}"); + + Assert.Equal( + LinuxOwnerAnchorState.Ambiguous, + LinuxOwnerAnchor.Probe(ownersPath, token, honorLocalRegistry: false)); + Assert.True(File.Exists(fifoPath)); + } + + [Fact] + public void SweepRetainsUnreferencedFifoArtifact() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var directory = new TemporaryDirectory(); + string ownersPath = Path.Combine(directory.Path, "store.owners"); + Guid token = Guid.NewGuid(); + string fifoPath = CreateFifoAnchor(ownersPath, token); + + LinuxOwnerAnchor.SweepUnreferencedArtifacts(ownersPath, new HashSet()); + + Assert.True(File.Exists(fifoPath)); + Assert.Equal( + LinuxOwnerAnchorState.Ambiguous, + LinuxOwnerAnchor.Probe(ownersPath, token, honorLocalRegistry: false)); + } + + [Fact] + public void SweepDeletesOnlyWellFormedUnlockedUnreferencedArtifacts() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var directory = new TemporaryDirectory(); + string ownersPath = Path.Combine(directory.Path, "store.owners"); + Guid staleToken = Guid.NewGuid(); + Guid referencedToken = Guid.NewGuid(); + Guid lockedToken = Guid.NewGuid(); + Guid ambiguousToken = Guid.NewGuid(); + string stalePath = CreateUnlockedAnchor(ownersPath, staleToken); + string referencedPath = CreateUnlockedAnchor(ownersPath, referencedToken); + string ambiguousPath = LinuxOwnerAnchor.GetPath(ownersPath, ambiguousToken); + string ambiguousTarget = Path.Combine(directory.Path, "ambiguous-target"); + File.WriteAllBytes(ambiguousTarget, []); + File.CreateSymbolicLink(ambiguousPath, ambiguousTarget); + string malformedPath = ownersPath + ".anchor.not-a-valid-owner-token"; + File.WriteAllBytes(malformedPath, []); + using LinuxOwnerAnchor locked = LinuxOwnerAnchor.Create(ownersPath, lockedToken); + string lockedPath = locked.AnchorPath; + + LinuxOwnerAnchor.SweepUnreferencedArtifacts( + ownersPath, + new HashSet { referencedToken }); + + Assert.False(File.Exists(stalePath)); + Assert.True(File.Exists(referencedPath)); + Assert.True(File.Exists(lockedPath)); + Assert.True(File.Exists(ambiguousPath)); + Assert.True(File.Exists(ambiguousTarget)); + Assert.True(File.Exists(malformedPath)); + Assert.Equal( + LinuxOwnerAnchorState.Locked, + LinuxOwnerAnchor.Probe(ownersPath, lockedToken, honorLocalRegistry: false)); + Assert.Equal( + LinuxOwnerAnchorState.Ambiguous, + LinuxOwnerAnchor.Probe(ownersPath, ambiguousToken, honorLocalRegistry: false)); + } + + private static string CreateUnlockedAnchor(string ownersPath, Guid token) + { + string path = LinuxOwnerAnchor.GetPath(ownersPath, token); + File.WriteAllBytes(path, []); + File.SetUnixFileMode(path, LinuxSharedMemoryDirectory.PrivateFileMode); + return path; + } + + private static string CreateFifoAnchor(string ownersPath, Guid token) + { + string path = LinuxOwnerAnchor.GetPath(ownersPath, token); + int result = MkFifo(path, 0x180); // 0600 + Assert.True(result == 0, $"mkfifo failed with errno {Marshal.GetLastPInvokeError()}."); + return path; + } + + [DllImport("libc", EntryPoint = "mkfifo", SetLastError = true)] + private static extern int MkFifo( + [MarshalAs(UnmanagedType.LPUTF8Str)] string path, + uint mode); + + private sealed class TemporaryDirectory : IDisposable + { + internal TemporaryDirectory() + { + Path = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + "sms-owner-anchor-tests-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + internal string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + catch + { + } + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeAcquireAllocationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeAcquireAllocationTests.cs new file mode 100644 index 0000000..ab0cf5d --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeAcquireAllocationTests.cs @@ -0,0 +1,134 @@ +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeAcquireAllocationTests +{ + private const int MeasuredAcquireCycles = 1_000_000; + private const int TieredPgoWarmupCycles = 50_000; + + [Fact] + public void OneMillionWarmedAcquireProjectReleaseCyclesAllocateZeroBytes() + { + using var store = CreateLockFreeStore(leaseRecordCount: 2); + var key = new byte[] { 1, 0, 2 }; + var value = new byte[] { 3, 0, 4, 5 }; + var descriptor = new byte[] { 6, 0, 7 }; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, value, descriptor)); + + // Fifty thousand cycles are intentionally outside the measured million. + // This lets tiered PGO promote the facade, directory, lease-registry, and + // projection paths before allocation accounting begins. + RunSuccessfulCycles(store, key, TieredPgoWarmupCycles); + CollectGarbage(); + + var before = GC.GetAllocatedBytesForCurrentThread(); + RunSuccessfulCycles(store, key, MeasuredAcquireCycles); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(0, allocated); + } + + [Fact] + public void WarmExpectedMissAndLeaseTableFullPathsAllocateZeroBytes() + { + const int MeasuredFailureCycles = 100_000; + using var store = CreateLockFreeStore(leaseRecordCount: 1); + var publishedKey = new byte[] { 1 }; + var missingKey = new byte[] { 2 }; + Assert.Equal(StoreStatus.Success, store.TryPublish(publishedKey, [9], [8])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(publishedKey, out var held)); + + RunExpectedFailures( + store, + publishedKey, + missingKey, + TieredPgoWarmupCycles); + CollectGarbage(); + + var before = GC.GetAllocatedBytesForCurrentThread(); + RunExpectedFailures(store, publishedKey, missingKey, MeasuredFailureCycles); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(0, allocated); + Assert.Equal(StoreStatus.Success, held.Release()); + } + + private static void RunSuccessfulCycles(MemoryStore store, byte[] key, int cycles) + { + for (var index = 0; index < cycles; index++) + { + StoreStatus acquire = store.TryAcquire(key, out var lease); + if (acquire != StoreStatus.Success + || !lease.IsValid + || lease.ValueLength != 4 + || lease.DescriptorLength != 3) + { + throw new InvalidOperationException( + $"Acquire/projection metadata failed at cycle {index}: {acquire}."); + } + + ReadOnlySpan value = lease.ValueSpan; + ReadOnlySpan descriptor = lease.DescriptorSpan; + if (value.Length != 4 + || value[0] + value[1] + value[2] + value[3] != 12 + || descriptor.Length != 3 + || descriptor[0] + descriptor[1] + descriptor[2] != 13) + { + throw new InvalidOperationException($"Projected bytes failed at cycle {index}."); + } + + StoreStatus release = lease.Release(); + if (release != StoreStatus.Success || lease.IsValid) + { + throw new InvalidOperationException( + $"Lease release failed at cycle {index}: {release}."); + } + } + } + + private static void RunExpectedFailures( + MemoryStore store, + byte[] publishedKey, + byte[] missingKey, + int cycles) + { + for (var index = 0; index < cycles; index++) + { + StoreStatus missing = store.TryAcquire(missingKey, out var missingLease); + StoreStatus full = store.TryAcquire(publishedKey, out var fullLease); + if (missing != StoreStatus.NotFound + || missingLease.IsValid + || full != StoreStatus.LeaseTableFull + || fullLease.IsValid) + { + throw new InvalidOperationException( + $"Expected miss/full failed at cycle {index}: miss={missing}, full={full}."); + } + } + } + + private static MemoryStore CreateLockFreeStore(int leaseRecordCount) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-acquire-allocation-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 1, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + var status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + var result = Assert.IsType(store); + Assert.Equal(StoreProfile.LockFree, result.Profile); + return result; + } + + private static void CollectGarbage() + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeAcquireCleanupTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeAcquireCleanupTests.cs new file mode 100644 index 0000000..a21999d --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeAcquireCleanupTests.cs @@ -0,0 +1,182 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Interop; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeAcquireCleanupTests +{ + [Fact] + public async Task DeadlineAfterLeaseActivationReturnsBusyAndRecyclesProvisionalLease() + { + if (!IsSupportedHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateStore(scheduler); + byte[] key = [1]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [7])); + scheduler.PauseAt(LockFreeCheckpointId.AcquireAfterLeaseActivationBeforeFinalLookup); + + TimeSpan operationTimeout = TimeSpan.FromSeconds(2); + ValueLease observed = default; + Task acquire = StartLongRunning(() => store.TryAcquire( + key, + new StoreWaitOptions(operationTimeout), + out observed)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + await Task.Delay(operationTimeout + TimeSpan.FromMilliseconds(200)); + scheduler.Continue(); + + Assert.Equal(StoreStatus.StoreBusy, await acquire.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.False(observed.IsValid); + AssertLeaseRecordWasRecycled(store); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease replacement)); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task CancellationAfterLeaseActivationReturnsCanceledAndRecyclesProvisionalLease() + { + if (!IsSupportedHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateStore(scheduler); + byte[] key = [2]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [8])); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.AcquireAfterLeaseActivationBeforeFinalLookup); + + ValueLease observed = default; + Task acquire = StartLongRunning(() => store.TryAcquire( + key, + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token), + out observed)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + cancellation.Cancel(); + scheduler.Continue(); + + Assert.Equal(StoreStatus.OperationCanceled, await acquire.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.False(observed.IsValid); + AssertLeaseRecordWasRecycled(store); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease replacement)); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task CorruptExactDirectoryCellAfterLeaseActivationPropagatesCorruptionAndRecyclesLease() + { + if (!IsSupportedHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateStore(scheduler); + byte[] key = [3]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [9])); + DirectoryLocation location = LocateExactDirectoryCell(store, key); + scheduler.PauseAt(LockFreeCheckpointId.AcquireAfterLeaseActivationBeforeFinalLookup); + + ValueLease observed = default; + Task acquire = StartLongRunning(() => store.TryAcquire( + key, + StoreWaitOptions.Infinite, + out observed)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + CorruptDirectoryCell(store, location); + scheduler.Continue(); + + Assert.Equal(StoreStatus.CorruptStore, await acquire.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.False(observed.IsValid); + AssertLeaseRecordWasRecycled(store); + } + + private static void AssertLeaseRecordWasRecycled(MemoryStore store) + { + object engine = ReadPrivate(store, "_engine"); + LockFreeLeaseRegistry leases = ReadPrivate(engine, "_leases"); + long control = AtomicControlWord.LoadAcquire(ref leases.Record(0).Control); + Assert.Equal(LockFreeLeaseRegistry.FreeState, (int)((ulong)control & 0x7UL)); + } + + private static Task StartLongRunning(Func operation) => + Task.Factory.StartNew( + operation, + CancellationToken.None, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); + + private static DirectoryLocation LocateExactDirectoryCell(MemoryStore store, byte[] key) + { + object engine = ReadPrivate(store, "_engine"); + LockFreeKeyDirectory directory = ReadPrivate(engine, "_directory"); + Assert.Equal( + StoreStatus.Success, + directory.TryLookup(key, StoreKey.Hash(key), out _, out DirectoryLocation location)); + return location; + } + + private static unsafe void CorruptDirectoryCell(MemoryStore store, DirectoryLocation location) + { + object engine = ReadPrivate(store, "_engine"); + MemoryMappedStoreRegion region = ReadPrivate(engine, "_region"); + StoreLayoutV2 layout = ReadPrivate(engine, "_layout"); + long offset = location.Kind switch + { + 1 => layout.PrimaryDirectoryOffset + + ((location.Index / LayoutV2Constants.PrimaryLanesPerBucket) * layout.PrimaryBucketStride) + + 16 + + ((location.Index % LayoutV2Constants.PrimaryLanesPerBucket) * sizeof(long)), + 2 => layout.OverflowDirectoryOffset + (location.Index * layout.OverflowStride), + _ => throw new InvalidOperationException("The exact binding has an invalid directory kind.") + }; + + ref long cell = ref *(long*)(region.Pointer + offset); + // Generation one with a zero encoded index is structurally impossible; + // it must be reported as corruption, not treated as a stale binding. + AtomicControlWord.StoreRelease(ref cell, unchecked((long)(1UL << 31))); + } + + private static T ReadPrivate(object owner, string fieldName) + { + FieldInfo field = owner.GetType().GetField( + fieldName, + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Missing field {owner.GetType().FullName}.{fieldName}."); + return Assert.IsAssignableFrom(field.GetValue(owner)); + } + + private static MemoryStore CreateStore(ControlledLockFreeScheduler scheduler) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-acquire-cleanup-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + scheduler.CreateInstrumentedCheckpoint(), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeAcquireStateTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeAcquireStateTests.cs new file mode 100644 index 0000000..c7f43fb --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeAcquireStateTests.cs @@ -0,0 +1,233 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeAcquireStateTests +{ + [Fact] + public void MissingExactKeyReturnsNotFoundWithoutConsumingLeaseCapacity() + { + RequireLeaseRegistry(); + using var store = CreateStore(leaseRecordCount: 1); + + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([0x44], out var missing)); + Assert.False(missing.IsValid); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x45], [0x55])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0x45], out var present)); + Assert.Equal(StoreStatus.Success, present.Release()); + } + + [Fact] + public void CommitThenAcquireProjectsOnlyTheExactCommittedDescriptorAndPayload() + { + RequireLeaseRegistry(); + using var store = CreateStore(leaseRecordCount: 2); + Assert.Equal(StoreStatus.Success, store.TryReserve([0x31], payloadLength: 3, [0x71, 0x72], out var reservation)); + new byte[] { 0x11, 0x12, 0x13 }.CopyTo(reservation.GetSpan(3)); + Assert.Equal(StoreStatus.Success, reservation.Advance(3)); + Assert.Equal(StoreStatus.Success, reservation.Commit()); + + Assert.Equal(StoreStatus.Success, store.TryAcquire([0x31], out var lease)); + Assert.Equal([0x11, 0x12, 0x13], lease.ValueSpan.ToArray()); + Assert.Equal([0x71, 0x72], lease.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public void LookupRequiresStableCellAndSlotControlDoubleValidation() + { + const ulong binding = 0x0000_0002_8000_0001UL; + var stable = LookupOracle.Evaluate(new LookupObservation( + FirstCell: binding, + FirstControl: 0x13, + BindingGenerationMatches: true, + HashMatches: true, + KeyBytesMatch: true, + SecondControl: 0x13, + SecondCell: binding)); + var changedCell = LookupOracle.Evaluate(new LookupObservation( + binding, 0x13, true, true, true, 0x13, 0)); + var changedControl = LookupOracle.Evaluate(new LookupObservation( + binding, 0x13, true, true, true, 0x1B, binding)); + + Assert.Equal(LookupOutcome.Found, stable); + Assert.Equal(LookupOutcome.Retry, changedCell); + Assert.Equal(LookupOutcome.Retry, changedControl); + } + + [Fact] + public void EqualHashWithDifferentExactKeyIsNotAMatch() + { + const ulong binding = 0x0000_0002_8000_0001UL; + var observation = new LookupObservation( + FirstCell: binding, + FirstControl: 0x13, + BindingGenerationMatches: true, + HashMatches: true, + KeyBytesMatch: false, + SecondControl: 0x13, + SecondCell: binding); + + Assert.Equal(LookupOutcome.NotFound, LookupOracle.Evaluate(observation)); + } + + [Fact] + public void StaleGenerationIsClearedOnlyByExactBindingCas() + { + const ulong stale = 0x0000_0002_8000_0001UL; + const ulong replacement = 0x0000_0003_0000_0001UL; + var staleObservation = new LookupObservation( + FirstCell: stale, + FirstControl: 0x1B, + BindingGenerationMatches: false, + HashMatches: true, + KeyBytesMatch: true, + SecondControl: 0x1B, + SecondCell: stale); + + Assert.Equal(LookupOutcome.Stale, LookupOracle.Evaluate(staleObservation)); + Assert.Equal(replacement, LookupOracle.ClearStale(currentCell: replacement, expectedStale: stale)); + Assert.Equal(0UL, LookupOracle.ClearStale(currentCell: stale, expectedStale: stale)); + } + + [Fact] + public void BoundedCleanupSchedulesEventuallyClearEveryObservedStaleCellWithoutClearingCurrentOnes() + { + var cells = new[] + { + new CleanupCell(11, IsStale: true), + new CleanupCell(12, IsStale: false), + new CleanupCell(13, IsStale: true), + new CleanupCell(14, IsStale: true) + }; + var cursor = 0; + for (var invocation = 0; invocation < cells.Length; invocation++) + { + cursor = LookupOracle.HelpStale(cells, cursor, budget: 1); + } + + Assert.Equal(0UL, cells[0].Binding); + Assert.Equal(12UL, cells[1].Binding); + Assert.Equal(0UL, cells[2].Binding); + Assert.Equal(0UL, cells[3].Binding); + } + + [Fact] + public void ProductionLookupAcceptsHashAndExactKeyAndProvidesAHelpingPath() + { + var directory = typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeKeyDirectory", + throwOnError: false, + ignoreCase: false); + Assert.True(directory is not null, "LockFreeKeyDirectory is required."); + var methods = directory!.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + MethodInfo[] lookups = methods.Where(method => method.Name == "TryLookup").ToArray(); + Assert.Contains( + lookups, + lookup => lookup.GetParameters().Any(parameter => parameter.ParameterType == typeof(ulong)) + && lookup.GetParameters().Any( + parameter => parameter.ParameterType == typeof(ReadOnlySpan))); + Assert.Contains(methods, static method => method.Name.Contains("Help", StringComparison.Ordinal)); + } + + private static void RequireLeaseRegistry() + { + Assert.True( + typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeLeaseRegistry", + throwOnError: false, + ignoreCase: false) is not null, + "Acquire requires the missing LockFreeLeaseRegistry implementation."); + } + + private static Store CreateStore(int leaseRecordCount) + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + throw new PlatformNotSupportedException("The lock-free profile is qualified only on Windows/Linux x64."); + } + + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-acquire-state-{Guid.NewGuid():N}", + slotCount: 3, + maxValueBytes: 16, + maxDescriptorBytes: 8, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + var status = Store.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private enum LookupOutcome + { + Found, + NotFound, + Retry, + Stale + } + + private readonly record struct LookupObservation( + ulong FirstCell, + ulong FirstControl, + bool BindingGenerationMatches, + bool HashMatches, + bool KeyBytesMatch, + ulong SecondControl, + ulong SecondCell); + + private sealed class CleanupCell(ulong binding, bool IsStale) + { + public ulong Binding { get; set; } = binding; + + public bool IsStale { get; } = IsStale; + } + + private static class LookupOracle + { + public static LookupOutcome Evaluate(LookupObservation observation) + { + if (observation.FirstCell == 0) + { + return LookupOutcome.NotFound; + } + + if (!observation.BindingGenerationMatches) + { + return LookupOutcome.Stale; + } + + if (!observation.HashMatches || !observation.KeyBytesMatch) + { + return LookupOutcome.NotFound; + } + + return observation.FirstControl == observation.SecondControl + && observation.FirstCell == observation.SecondCell + ? LookupOutcome.Found + : LookupOutcome.Retry; + } + + public static ulong ClearStale(ulong currentCell, ulong expectedStale) => + currentCell == expectedStale ? 0 : currentCell; + + public static int HelpStale(CleanupCell[] cells, int cursor, int budget) + { + for (var visited = 0; visited < budget; visited++) + { + var index = cursor++ % cells.Length; + if (cells[index].IsStale) + { + cells[index].Binding = 0; + } + } + + return cursor % cells.Length; + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeCheckpointCoverageTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeCheckpointCoverageTests.cs new file mode 100644 index 0000000..f38aa47 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeCheckpointCoverageTests.cs @@ -0,0 +1,521 @@ +using System.Collections; +using System.Reflection; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeCheckpointCoverageTests +{ + private static readonly string[] RequiredAbaWindowCheckpoints = + [ + "DirectoryAfterOperationValidation", + "DirectoryAfterLocationValidation", + "DirectoryAfterUnlinkOperationValidationBeforeLocationRead", + "DirectoryAfterLocationPublisherBindingValidation", + "DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas", + "DirectoryAfterLocationPublicationBeforeSourceRevalidation", + "DirectoryAfterCurrentOperationRevalidationBeforeDispatch", + "DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication", + "DirectoryAfterInsertCompletionStateValidationBeforeLocationRead", + "DirectoryAfterUnlinkDescriptorClearBeforeGenerationAdvance", + "DirectoryAfterCancelLocationClearBeforeDescriptorRejection", + "DirectoryBeforeSpillSummaryPublicationCas", + "DirectoryAfterSpillSummaryPublication", + "DirectoryAfterEmptySpillSummaryScan", + "DirectoryAfterSpillSummaryClear", + "ReclaimAfterMetadataValidation" + ]; + + private static readonly string[] RequiredInsertCancellationWindowCheckpoints = + [ + "DirectoryAfterCurrentOperationRevalidationBeforeDispatch", + "DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication", + "DirectoryAfterInsertCompletionStateValidationBeforeLocationRead", + "DirectoryAfterCancelLocationClearBeforeDescriptorRejection", + "ReserveAfterDirectoryInsertBeforePendingClassification", + "DirectoryBeforeInsertOuterLoopBudgetCheck" + ]; + + private static readonly string[] RequiredFamilies = + [ + "Publish", + "Reserve", + "Commit", + "Abort", + "Acquire", + "Project", + "Release", + "Remove", + "Reclaim", + "Directory", + "Diagnostics", + "Recovery", + "Disposal", + "Participant", + "Advance" + ]; + + [Fact] + public void RequiredTransitionFamiliesAndCheckpointCatalogCoverEachOther() + { + var entries = ReadCheckpointEntries(); + var catalogFamilies = entries + .Select(entry => ReadRequiredMember(entry, "Family")) + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToArray(); + + Assert.Equal(RequiredFamilies.Order(StringComparer.Ordinal), catalogFamilies); + foreach (var family in RequiredFamilies) + { + var positions = entries + .Where(entry => ReadRequiredMember(entry, "Family") == family) + .Select(entry => ReadRequiredMember(entry, "Position")) + .ToHashSet(StringComparer.Ordinal); + Assert.Contains("Before", positions); + Assert.Contains("After", positions); + } + } + + [Fact] + public void EveryCheckpointHasUniqueIdentityAndPauseCrashRaceClassifications() + { + var entries = ReadCheckpointEntries(); + var ids = new HashSet(StringComparer.Ordinal); + foreach (var entry in entries) + { + Assert.True(ids.Add(ReadRequiredMember(entry, "Id")), "Checkpoint IDs must be stable and unique."); + AssertClassified(entry, "Pause"); + AssertClassified(entry, "Crash"); + AssertClassified(entry, "Race"); + } + } + + [Fact] + public void CatalogIncludesPostValidationAbaWindowsForDirectoryAndReclaimHelpers() + { + string[] ids = ReadCheckpointEntries() + .Select(entry => ReadRequiredMember(entry, "Id")) + .ToArray(); + + foreach (string required in RequiredAbaWindowCheckpoints) + { + Assert.Contains(required, ids); + } + } + + [Fact] + public void CatalogIncludesEveryInsertCancellationPostValidationWindow() + { + string[] ids = ReadCheckpointEntries() + .Select(entry => ReadRequiredMember(entry, "Id")) + .ToArray(); + + foreach (string required in RequiredInsertCancellationWindowCheckpoints) + { + Assert.Contains(required, ids); + } + } + + [Fact] + public void DirectoryOperationAndLocationWordsCarryTheValidatedSlotGeneration() + { + var assembly = typeof(MemoryStore).Assembly; + Type? operationType = assembly.GetType( + "SharedMemoryStore.LockFree.DirectoryOperation", + throwOnError: false, + ignoreCase: false); + Type? locationType = assembly.GetType( + "SharedMemoryStore.LockFree.DirectoryLocation", + throwOnError: false, + ignoreCase: false); + Assert.NotNull(operationType); + Assert.NotNull(locationType); + Type operation = operationType!; + Type location = locationType!; + + Assert.Contains( + operation.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic), + member => member.Name.Contains("Generation", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + location.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic), + member => member.Name.Contains("Generation", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + operation.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .Where(method => method.Name.Contains("Encode", StringComparison.OrdinalIgnoreCase)) + .SelectMany(method => method.GetParameters()), + parameter => parameter.Name?.Contains("generation", StringComparison.OrdinalIgnoreCase) == true); + Assert.Contains( + location.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .Where(method => method.Name.Contains("Encode", StringComparison.OrdinalIgnoreCase)) + .SelectMany(method => method.GetParameters()), + parameter => parameter.Name?.Contains("generation", StringComparison.OrdinalIgnoreCase) == true); + } + + [Fact] + public void HelperConvergesFromEveryInsertPauseWithoutDuplicatingTheBinding() + { + var pauses = InsertOracle.NormalInsertPauses(binding: 0x0000_0002_8000_0001UL).ToArray(); + Assert.NotEmpty(pauses); + + foreach (var paused in pauses) + { + var completed = InsertOracle.HelpToQuiescence(paused); + + Assert.Equal(SlotControl.Reserved, completed.Control); + Assert.Equal(DirectoryPhase.Complete, completed.Phase); + Assert.False(completed.DescriptorPublished); + Assert.Equal(paused.CandidateBinding, completed.CellBinding); + Assert.Equal(1, completed.BindingInstallCount); + } + } + + [Fact] + public void StaleDescriptorAtEveryInsertPhaseIsClearedWithoutTouchingUnrelatedBinding() + { + const ulong unrelatedBinding = 0x0000_0003_0000_0002UL; + foreach (var phase in Enum.GetValues().Where(static phase => phase != DirectoryPhase.None)) + { + var paused = new InsertState( + SlotControl.Initializing, + phase, + DescriptorPublished: true, + DescriptorGenerationMatches: false, + CandidateBinding: 0x0000_0002_8000_0001UL, + CellBinding: unrelatedBinding, + BindingInstallCount: 0, + TargetConflictRemaining: false); + + var completed = InsertOracle.HelpToQuiescence(paused); + + Assert.False(completed.DescriptorPublished); + Assert.Equal(unrelatedBinding, completed.CellBinding); + Assert.Equal(0, completed.BindingInstallCount); + } + } + + [Fact] + public void ConflictingTargetSelectionRestartsAndRemainsHelpable() + { + var paused = new InsertState( + SlotControl.Initializing, + DirectoryPhase.TargetSelected, + DescriptorPublished: true, + DescriptorGenerationMatches: true, + CandidateBinding: 0x0000_0002_8000_0001UL, + CellBinding: 0x0000_0003_0000_0002UL, + BindingInstallCount: 0, + TargetConflictRemaining: true); + + var completed = InsertOracle.HelpToQuiescence(paused); + + Assert.Equal(SlotControl.Reserved, completed.Control); + Assert.Equal(DirectoryPhase.Complete, completed.Phase); + Assert.Equal(paused.CandidateBinding, completed.CellBinding); + Assert.Equal(1, completed.BindingInstallCount); + Assert.False(completed.DescriptorPublished); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void DelayedValidatedDirectoryHelperCannotMutateReusedGeneration(bool locationWasValidated) + { + const long oldGeneration = 41; + const long newGeneration = 42; + ulong oldBinding = Binding(oldGeneration); + ulong newBinding = Binding(newGeneration); + var old = new TaggedDirectoryState( + oldGeneration, + TaggedControl.Reclaiming, + oldBinding, + TaggedWord(oldGeneration, 1), + TaggedWord(oldGeneration, 2), + oldBinding, + KeyHash: 0x1111, + KeyLength: 1, + DescriptorLength: 1, + ValueLength: 2, + PayloadMarker: 0x11); + var validated = ValidatedDirectoryHelper.Capture(old, locationWasValidated); + + var reused = new TaggedDirectoryState( + newGeneration, + TaggedControl.Initializing, + newBinding, + TaggedWord(newGeneration, 3), + TaggedWord(newGeneration, 4), + newBinding, + KeyHash: 0x2222, + KeyLength: 3, + DescriptorLength: 2, + ValueLength: 7, + PayloadMarker: 0x22); + + TaggedDirectoryState afterDelayedResume = TaggedDirectoryOracle.ResumeValidatedHelper(reused, validated); + + Assert.Equal(reused, afterDelayedResume); + } + + [Fact] + public void DelayedHelperRollsBackOnlyItsExactOldBindingResidueAfterSlotReuse() + { + const long oldGeneration = 51; + const long newGeneration = 52; + ulong oldBinding = Binding(oldGeneration); + ulong newBinding = Binding(newGeneration); + var validated = ValidatedDirectoryHelper.Capture( + new TaggedDirectoryState( + oldGeneration, + TaggedControl.Reclaiming, + oldBinding, + TaggedWord(oldGeneration, 1), + TaggedWord(oldGeneration, 2), + oldBinding, + KeyHash: 0x3333, + KeyLength: 1, + DescriptorLength: 1, + ValueLength: 1, + PayloadMarker: 0x33), + locationWasValidated: true); + var reusedWithOldResidue = new TaggedDirectoryState( + newGeneration, + TaggedControl.Initializing, + newBinding, + TaggedWord(newGeneration, 3), + TaggedWord(newGeneration, 4), + oldBinding, + KeyHash: 0x4444, + KeyLength: 4, + DescriptorLength: 3, + ValueLength: 8, + PayloadMarker: 0x44); + + TaggedDirectoryState cleaned = TaggedDirectoryOracle.ResumeValidatedHelper( + reusedWithOldResidue, + validated); + + Assert.Equal(0UL, cleaned.TargetCellBinding); + Assert.Equal(reusedWithOldResidue with { TargetCellBinding = 0 }, cleaned); + + var newBindingInTarget = reusedWithOldResidue with { TargetCellBinding = newBinding }; + Assert.Equal( + newBindingInTarget, + TaggedDirectoryOracle.ResumeValidatedHelper(newBindingInTarget, validated)); + } + + private static object[] ReadCheckpointEntries() + { + var assembly = typeof(MemoryStore).Assembly; + var catalogType = assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeCheckpointCatalog", + throwOnError: false, + ignoreCase: false); + Assert.True( + catalogType is not null, + "The lock-free implementation must expose an internal canonical LockFreeCheckpointCatalog for deterministic validation."); + var entriesMember = catalogType!.GetMember( + "Entries", + BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic).SingleOrDefault(); + Assert.True(entriesMember is not null, "LockFreeCheckpointCatalog must expose a static Entries member."); + var value = entriesMember switch + { + PropertyInfo property => property.GetValue(null), + FieldInfo field => field.GetValue(null), + _ => null + }; + var enumerable = Assert.IsAssignableFrom(value); + var entries = enumerable.Cast().ToArray(); + Assert.NotEmpty(entries); + return entries; + } + + private static string ReadRequiredMember(object entry, string name) + { + var flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.IgnoreCase; + var type = entry.GetType(); + var value = type.GetProperty(name, flags)?.GetValue(entry) + ?? type.GetField(name, flags)?.GetValue(entry); + Assert.True(value is not null, $"Checkpoint entry '{type.FullName}' must expose {name}."); + var text = value!.ToString(); + Assert.False(string.IsNullOrWhiteSpace(text)); + return text!; + } + + private static void AssertClassified(object entry, string classification) + { + var text = ReadRequiredMember(entry, classification); + Assert.False(string.Equals("Unspecified", text, StringComparison.OrdinalIgnoreCase)); + } + + private enum SlotControl + { + Initializing, + Reserved, + Aborting + } + + private enum DirectoryPhase + { + None, + Prepared, + TargetSelected, + BindingChanged, + Rejected, + Complete + } + + private enum TaggedControl + { + Reclaiming, + Free, + Initializing + } + + private readonly record struct InsertState( + SlotControl Control, + DirectoryPhase Phase, + bool DescriptorPublished, + bool DescriptorGenerationMatches, + ulong CandidateBinding, + ulong CellBinding, + int BindingInstallCount, + bool TargetConflictRemaining); + + private readonly record struct TaggedDirectoryState( + long Generation, + TaggedControl Control, + ulong SlotBinding, + ulong OperationWord, + ulong LocationWord, + ulong TargetCellBinding, + ulong KeyHash, + int KeyLength, + int DescriptorLength, + int ValueLength, + long PayloadMarker); + + private readonly record struct ValidatedDirectoryHelper( + long Generation, + ulong Binding, + ulong OperationWord, + ulong LocationWord, + bool LocationWasValidated) + { + internal static ValidatedDirectoryHelper Capture( + TaggedDirectoryState state, + bool locationWasValidated) => + new( + state.Generation, + state.SlotBinding, + state.OperationWord, + state.LocationWord, + locationWasValidated); + } + + private static class TaggedDirectoryOracle + { + internal static TaggedDirectoryState ResumeValidatedHelper( + TaggedDirectoryState current, + ValidatedDirectoryHelper delayed) + { + // Any stale target installed by this exact old-generation helper is + // independently removable. Every slot-local write remains fenced by + // the complete generation-tagged operation/location/control tuple. + ulong target = current.TargetCellBinding == delayed.Binding + ? 0 + : current.TargetCellBinding; + if (current.Generation != delayed.Generation + || current.SlotBinding != delayed.Binding + || current.OperationWord != delayed.OperationWord + || (delayed.LocationWasValidated && current.LocationWord != delayed.LocationWord)) + { + return current with { TargetCellBinding = target }; + } + + return current with { TargetCellBinding = target }; + } + } + + private static ulong Binding(long generation) => + ((ulong)generation << 31) | 1UL; + + private static ulong TaggedWord(long generation, byte payload) => + ((ulong)generation << 16) | payload; + + private static class InsertOracle + { + public static IEnumerable NormalInsertPauses(ulong binding) + { + var state = new InsertState( + SlotControl.Initializing, + DirectoryPhase.Prepared, + DescriptorPublished: true, + DescriptorGenerationMatches: true, + CandidateBinding: binding, + CellBinding: 0, + BindingInstallCount: 0, + TargetConflictRemaining: false); + yield return state; + while (state.DescriptorPublished) + { + state = HelpOne(state); + yield return state; + } + } + + public static InsertState HelpToQuiescence(InsertState state) + { + for (var step = 0; state.DescriptorPublished && step < 16; step++) + { + state = HelpOne(state); + } + + Assert.False(state.DescriptorPublished, "Every published insert descriptor must converge under helping."); + return state; + } + + private static InsertState HelpOne(InsertState state) + { + if (!state.DescriptorPublished) + { + return state; + } + + if (!state.DescriptorGenerationMatches) + { + return state with { DescriptorPublished = false }; + } + + return state.Phase switch + { + DirectoryPhase.Prepared => state with { Phase = DirectoryPhase.TargetSelected }, + DirectoryPhase.TargetSelected when state.TargetConflictRemaining => state with + { + Phase = DirectoryPhase.Prepared, + CellBinding = 0, + TargetConflictRemaining = false + }, + DirectoryPhase.TargetSelected when state.CellBinding == 0 => state with + { + Phase = DirectoryPhase.BindingChanged, + CellBinding = state.CandidateBinding, + BindingInstallCount = state.BindingInstallCount + 1 + }, + DirectoryPhase.TargetSelected when state.CellBinding == state.CandidateBinding => + state with { Phase = DirectoryPhase.BindingChanged }, + DirectoryPhase.BindingChanged => state with + { + Control = SlotControl.Reserved, + Phase = DirectoryPhase.Complete + }, + DirectoryPhase.Complete => state with { DescriptorPublished = false }, + DirectoryPhase.Rejected => state with + { + Control = SlotControl.Aborting, + DescriptorPublished = false + }, + _ => throw new InvalidOperationException($"Unsafe insert state: {state}") + }; + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeCheckpointSpecializationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeCheckpointSpecializationTests.cs new file mode 100644 index 0000000..009d2b4 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeCheckpointSpecializationTests.cs @@ -0,0 +1,190 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using System.Collections.Concurrent; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeCheckpointSpecializationTests +{ + [Fact] + public void OrdinaryAndInstrumentedFactoriesCloseTheSameEngineOverStaticStrategies() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore ordinary = OpenOrdinary(); + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore instrumented = OpenInstrumented(scheduler); + + object ordinaryEngine = ReadEngine(ordinary); + object instrumentedEngine = ReadEngine(instrumented); + Type ordinaryType = ordinaryEngine.GetType(); + Type instrumentedType = instrumentedEngine.GetType(); + + Assert.True(ordinaryType.IsConstructedGenericType); + Assert.True(instrumentedType.IsConstructedGenericType); + Assert.Equal(typeof(LockFreeStoreEngine<>), ordinaryType.GetGenericTypeDefinition()); + Assert.Equal(typeof(LockFreeStoreEngine<>), instrumentedType.GetGenericTypeDefinition()); + Assert.Equal(typeof(NoOpLockFreeCheckpoint), Assert.Single(ordinaryType.GetGenericArguments())); + Assert.Equal(typeof(InstrumentedLockFreeCheckpoint), Assert.Single(instrumentedType.GetGenericArguments())); + + Assert.Equal( + typeof(NoOpLockFreeCheckpoint), + RequiredField(ordinaryType, "_checkpoint").FieldType); + Assert.Equal( + typeof(InstrumentedLockFreeCheckpoint), + RequiredField(instrumentedType, "_checkpoint").FieldType); + + Assert.True(typeof(LockFreeStoreEngine).IsAbstract && typeof(LockFreeStoreEngine).IsSealed); + Assert.Empty(typeof(LockFreeStoreEngine).GetFields( + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)); + } + + [Fact] + public void ProtocolComponentsCarryNoRuntimeCheckpointCallbacksOrEmitterFields() + { + Type[] components = + [ + typeof(LockFreeStoreEngine<>), + typeof(LockFreeParticipantRegistry), + typeof(LockFreeSlotTable), + typeof(LockFreeKeyDirectory), + typeof(LockFreeLeaseRegistry), + typeof(LockFreeReclaimer), + typeof(LockFreeRecovery), + typeof(LockFreeDiagnostics) + ]; + + foreach (Type component in components) + { + FieldInfo[] fields = component.GetFields( + BindingFlags.Instance | BindingFlags.Static | + BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain(fields, field => IsAction(field.FieldType)); + Assert.DoesNotContain(fields, field => + typeof(ILockFreeCheckpointEmitter).IsAssignableFrom(field.FieldType)); + Assert.DoesNotContain(fields, field => + Nullable.GetUnderlyingType(field.FieldType) == typeof(InstrumentedLockFreeCheckpoint)); + } + + FieldInfo[] engineFields = typeof(LockFreeStoreEngine<>).GetFields( + BindingFlags.Instance | BindingFlags.NonPublic); + FieldInfo checkpoint = Assert.Single(engineFields, field => field.Name == "_checkpoint"); + Assert.True(checkpoint.FieldType.IsGenericParameter); + } + + [Fact] + public void InstrumentedOperationsObserveOnlyCanonicalCatalogEntries() + { + if (!IsSupportedHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = OpenInstrumented(scheduler); + scheduler.ClearObservations(); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out ValueLease lease)); + Assert.Equal(7, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, store.TryRemove([1])); + + ControlledLockFreeScheduler.Observation[] observations = scheduler.Snapshot().ToArray(); + Assert.NotEmpty(observations); + foreach (ControlledLockFreeScheduler.Observation observation in observations) + { + Assert.Equal( + LockFreeCheckpointCatalog.Get(observation.Entry.Id), + observation.Entry); + } + } + + [Fact] + public void ResourceObserverPairsExactClaimAndReleaseWithoutChangingCheckpointCallbacks() + { + if (!IsSupportedHost()) + { + return; + } + + var checkpoints = new ConcurrentQueue(); + var resources = new ConcurrentQueue(); + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented( + checkpoints.Enqueue, + resources.Enqueue); + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options($"sms-v2-resource-observer-{Guid.NewGuid():N}"), + checkpoint, + out MemoryStore? candidate); + Assert.Equal(StoreOpenStatus.Success, status); + using MemoryStore store = Assert.IsType(candidate); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryRemove([1], StoreWaitOptions.Infinite)); + + Assert.NotEmpty(checkpoints); + LockFreeSlotResourceEvent[] observed = resources.ToArray(); + LockFreeSlotResourceEvent claim = Assert.Single( + observed, + static item => item.Kind == LockFreeSlotResourceEventKind.Claim); + LockFreeSlotResourceEvent released = Assert.Single( + observed, + static item => item.Kind is LockFreeSlotResourceEventKind.Free + or LockFreeSlotResourceEventKind.Retire); + Assert.Equal(claim.SlotIndex, released.SlotIndex); + Assert.Equal(claim.Generation, released.Generation); + } + + private static bool IsAction(Type type) => + type == typeof(Action) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Action<>)); + + private static FieldInfo RequiredField(Type type, string name) => + type.GetField(name, BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Missing required field {type.FullName}.{name}."); + + private static object ReadEngine(MemoryStore store) => + RequiredField(typeof(MemoryStore), "_engine").GetValue(store) + ?? throw new InvalidOperationException("The lock-free facade has no engine."); + + private static MemoryStore OpenOrdinary() + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options($"sms-v2-checkpoint-noop-{Guid.NewGuid():N}"), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore OpenInstrumented(ControlledLockFreeScheduler scheduler) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options($"sms-v2-checkpoint-instrumented-{Guid.NewGuid():N}"), + scheduler.CreateInstrumentedCheckpoint(), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options(string name) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeConflictCleanupAndAtomicAbortTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeConflictCleanupAndAtomicAbortTests.cs new file mode 100644 index 0000000..9e413c8 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeConflictCleanupAndAtomicAbortTests.cs @@ -0,0 +1,472 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Engines; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeConflictCleanupAndAtomicAbortTests +{ + [Fact] + public void NoWaitStopsBeforeFreshLookupAfterSuccessfullyReclaimingRemovedGeneration() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = CreateStore("fresh-lookup-budget"); + byte[] key = [0x41]; + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation publisher)); + + ReservationHandle removedHandle = publisher.HandleForEngine; + IndexBinding removedBinding = IndexBinding.Decode(removedHandle.SlotBinding); + publisher.GetSpan(1)[0] = 0x91; + Assert.Equal(StoreStatus.Success, publisher.Advance(1, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, publisher.Commit(StoreWaitOptions.Infinite)); + + // NoWait orders logical removal but intentionally leaves its physical + // generation for a later helper. + Assert.Equal(StoreStatus.RemovePending, store.TryRemove(key, StoreWaitOptions.NoWait)); + + StoreStatus noWait = store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.NoWait, + out ValueReservation rejected); + + Assert.Equal(StoreStatus.StoreBusy, noWait); + Assert.False(rejected.IsValid); + + // The create call did successfully reclaim the old generation. Its + // StoreBusy result is specifically the operation-wide retry gate before + // a fresh structural lookup, not an incomplete cleanup result. + LockFreeSlotTable slots = ReadSlots(store); + long reclaimed = AtomicControlWord.LoadAcquire( + ref slots.Slot(removedBinding.SlotIndex).Control); + Assert.Equal(LockFreeSlotTable.FreeState, SlotState(reclaimed)); + Assert.Equal(removedBinding.Generation + 1, SlotGeneration(reclaimed)); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation replacement)); + Assert.True(replacement.IsValid); + Assert.Equal(StoreStatus.Success, replacement.Abort(StoreWaitOptions.Infinite)); + } + + [Theory] + [InlineData(LockFreeSlotTable.AbortingState)] + [InlineData(LockFreeSlotTable.ReclaimingState)] + public void NoWaitStopsBeforeFreshLookupAfterCleaningHelpableGeneration( + int helpableState) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = CreateStore($"helpable-{helpableState}"); + byte[] key = [0x43]; + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation owner)); + + ReservationHandle oldHandle = owner.HandleForEngine; + IndexBinding oldBinding = IndexBinding.Decode(oldHandle.SlotBinding); + LockFreeSlotTable slots = ReadSlots(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(oldBinding.SlotIndex); + + if (helpableState == LockFreeSlotTable.AbortingState) + { + long reserved = SlotControl( + LockFreeSlotTable.ReservedState, + oldBinding.Generation, + checked((int)oldHandle.ParticipantToken)); + long aborting = SlotControl( + helpableState, + oldBinding.Generation, + participantToken: 0); + Assert.Equal( + reserved, + AtomicControlWord.CompareExchange(ref slot.Control, aborting, reserved)); + } + else + { + owner.GetSpan(1)[0] = 0x93; + Assert.Equal(StoreStatus.Success, owner.Advance(1, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, owner.Commit(StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.RemovePending, store.TryRemove(key, StoreWaitOptions.NoWait)); + + long removeRequested = SlotControl( + LockFreeSlotTable.RemoveRequestedState, + oldBinding.Generation, + participantToken: 0); + long reclaiming = SlotControl( + LockFreeSlotTable.ReclaimingState, + oldBinding.Generation, + participantToken: 0); + Assert.Equal( + removeRequested, + AtomicControlWord.CompareExchange( + ref slot.Control, + reclaiming, + removeRequested)); + } + + Assert.Equal( + StoreStatus.StoreBusy, + store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.NoWait, + out ValueReservation rejected)); + Assert.False(rejected.IsValid); + + long reclaimed = AtomicControlWord.LoadAcquire(ref slot.Control); + Assert.Equal(LockFreeSlotTable.FreeState, SlotState(reclaimed)); + Assert.Equal(oldBinding.Generation + 1, SlotGeneration(reclaimed)); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation replacement)); + Assert.Equal(StoreStatus.Success, replacement.Abort(StoreWaitOptions.Infinite)); + } + + [Theory] + [InlineData(AtomicAbortScenario.OwnInitializing)] + [InlineData(AtomicAbortScenario.OwnReserved)] + [InlineData(AtomicAbortScenario.Aborting)] + [InlineData(AtomicAbortScenario.LaterGeneration)] + [InlineData(AtomicAbortScenario.Reclaiming)] + [InlineData(AtomicAbortScenario.TerminalRetired)] + [InlineData(AtomicAbortScenario.LowerGeneration)] + [InlineData(AtomicAbortScenario.Published)] + [InlineData(AtomicAbortScenario.RemoveRequested)] + [InlineData(AtomicAbortScenario.Free)] + [InlineData(AtomicAbortScenario.NonterminalRetired)] + [InlineData(AtomicAbortScenario.WrongInitializingOwner)] + [InlineData(AtomicAbortScenario.WrongReservedOwner)] + [InlineData(AtomicAbortScenario.OwnedAborting)] + [InlineData(AtomicAbortScenario.WrongDirectoryBinding)] + [InlineData(AtomicAbortScenario.ExplicitPublicationIntent)] + [InlineData(AtomicAbortScenario.UnknownPublicationIntent)] + public void AtomicCandidateAbortStrictlyClassifiesStableSlotState( + AtomicAbortScenario scenario) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = CreateStore($"atomic-abort-{scenario}"); + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + [0x42], + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + + ReservationHandle originalHandle = reservation.HandleForEngine; + IndexBinding originalBinding = IndexBinding.Decode(originalHandle.SlotBinding); + LockFreeSlotTable slots = ReadSlots(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(originalBinding.SlotIndex); + long originalControl = AtomicControlWord.LoadAcquire(ref slot.Control); + Assert.Equal(LockFreeSlotTable.ReservedState, SlotState(originalControl)); + + ReservationHandle targetHandle = originalHandle; + ulong targetBinding = originalHandle.SlotBinding; + long targetGeneration = originalBinding.Generation; + int targetIntent = (int)SlotPublicationIntent.AtomicPublication; + int participant = checked((int)originalHandle.ParticipantToken); + int wrongParticipant = participant == 1 ? 2 : 1; + TentativeReservationAbortResult expected = scenario switch + { + AtomicAbortScenario.OwnInitializing + or AtomicAbortScenario.OwnReserved + or AtomicAbortScenario.Aborting => TentativeReservationAbortResult.Aborted, + AtomicAbortScenario.LaterGeneration + or AtomicAbortScenario.Reclaiming + or AtomicAbortScenario.TerminalRetired => TentativeReservationAbortResult.Invalid, + _ => TentativeReservationAbortResult.Corrupt, + }; + long targetControl = scenario switch + { + AtomicAbortScenario.OwnInitializing => + SlotControl(LockFreeSlotTable.InitializingState, targetGeneration, participant), + AtomicAbortScenario.OwnReserved => + SlotControl(LockFreeSlotTable.ReservedState, targetGeneration, participant), + AtomicAbortScenario.Aborting => + SlotControl(LockFreeSlotTable.AbortingState, targetGeneration, participantToken: 0), + AtomicAbortScenario.LaterGeneration => + SlotControl(LockFreeSlotTable.FreeState, targetGeneration + 1, participantToken: 0), + AtomicAbortScenario.Reclaiming => + SlotControl(LockFreeSlotTable.ReclaimingState, targetGeneration, participantToken: 0), + AtomicAbortScenario.TerminalRetired => + Retarget( + originalHandle, + originalBinding.SlotIndex, + LockFreeSlotTable.TerminalGeneration, + LockFreeSlotTable.TerminalGeneration, + ref targetHandle, + ref targetBinding, + ref targetGeneration, + LockFreeSlotTable.RetiredState, + participantToken: 0), + AtomicAbortScenario.LowerGeneration => + Retarget( + originalHandle, + originalBinding.SlotIndex, + originalBinding.Generation + 1, + originalBinding.Generation, + ref targetHandle, + ref targetBinding, + ref targetGeneration, + LockFreeSlotTable.ReservedState, + participant), + AtomicAbortScenario.Published => + SlotControl(LockFreeSlotTable.PublishedState, targetGeneration, participantToken: 0), + AtomicAbortScenario.RemoveRequested => + SlotControl(LockFreeSlotTable.RemoveRequestedState, targetGeneration, participantToken: 0), + AtomicAbortScenario.Free => + SlotControl(LockFreeSlotTable.FreeState, targetGeneration, participantToken: 0), + AtomicAbortScenario.NonterminalRetired => + SlotControl(LockFreeSlotTable.RetiredState, targetGeneration, participantToken: 0), + AtomicAbortScenario.WrongInitializingOwner => + SlotControl(LockFreeSlotTable.InitializingState, targetGeneration, wrongParticipant), + AtomicAbortScenario.WrongReservedOwner => + SlotControl(LockFreeSlotTable.ReservedState, targetGeneration, wrongParticipant), + AtomicAbortScenario.OwnedAborting => + SlotControl(LockFreeSlotTable.AbortingState, targetGeneration, participant), + AtomicAbortScenario.WrongDirectoryBinding => + SlotControl(LockFreeSlotTable.ReservedState, targetGeneration, participant), + AtomicAbortScenario.ExplicitPublicationIntent => + SlotControl(LockFreeSlotTable.ReservedState, targetGeneration, participant), + AtomicAbortScenario.UnknownPublicationIntent => + SlotControl(LockFreeSlotTable.ReservedState, targetGeneration, participant), + _ => throw new ArgumentOutOfRangeException(nameof(scenario)), + }; + + if (scenario == AtomicAbortScenario.WrongDirectoryBinding) + { + targetBinding = IndexBinding.Encode( + originalBinding.SlotIndex, + originalBinding.Generation + 1); + } + + if (scenario == AtomicAbortScenario.ExplicitPublicationIntent) + { + targetIntent = (int)SlotPublicationIntent.ExplicitReservation; + } + else if (scenario == AtomicAbortScenario.UnknownPublicationIntent) + { + targetIntent = 3; + } + + Volatile.Write(ref slot.DirectoryBinding, targetBinding); + Volatile.Write(ref slot.PublicationIntent, targetIntent); + AtomicControlWord.StoreRelease(ref slot.Control, targetControl); + try + { + Assert.Equal(expected, slots.TryBeginAtomicCandidateAbort(targetHandle)); + long observed = AtomicControlWord.LoadAcquire(ref slot.Control); + if (expected == TentativeReservationAbortResult.Aborted) + { + Assert.Equal(LockFreeSlotTable.AbortingState, SlotState(observed)); + Assert.Equal(targetGeneration, SlotGeneration(observed)); + Assert.Equal(0UL, SlotParticipant(observed)); + } + else + { + Assert.Equal(targetControl, observed); + } + } + finally + { + Volatile.Write(ref slot.DirectoryBinding, originalHandle.SlotBinding); + Volatile.Write( + ref slot.PublicationIntent, + (int)SlotPublicationIntent.ExplicitReservation); + AtomicControlWord.StoreRelease(ref slot.Control, originalControl); + } + + bool structurallyMalformed = scenario is + AtomicAbortScenario.NonterminalRetired + or AtomicAbortScenario.WrongInitializingOwner + or AtomicAbortScenario.WrongReservedOwner + or AtomicAbortScenario.OwnedAborting; + Assert.Equal( + structurallyMalformed ? StoreStatus.CorruptStore : StoreStatus.Success, + reservation.Abort(StoreWaitOptions.Infinite)); + } + + [Fact] + public void ExactDirectoryBindingRejectsOwnedControlWithOutOfRangeParticipantIndex() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = CreateStore("directory-participant-token"); + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + [0x51], + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + + ReservationHandle handle = reservation.HandleForEngine; + IndexBinding binding = IndexBinding.Decode(handle.SlotBinding); + LockFreeSlotTable slots = ReadSlots(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + long originalControl = AtomicControlWord.LoadAcquire(ref slot.Control); + const int malformedParticipant = 7; // generation 1, index-plus-one 3 for count 2 + Assert.False(ParticipantToken.IsStructurallyValid(malformedParticipant, 2)); + AtomicControlWord.StoreRelease( + ref slot.Control, + SlotControl( + LockFreeSlotTable.ReservedState, + binding.Generation, + malformedParticipant)); + + try + { + StoreStatus status = slots.ClassifyDirectoryBinding( + handle.SlotBinding, + out int state, + out SlotPublicationIntent intent); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Equal(LockFreeSlotTable.FreeState, state); + Assert.Equal(SlotPublicationIntent.None, intent); + } + finally + { + AtomicControlWord.StoreRelease(ref slot.Control, originalControl); + } + + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + } + + private static long Retarget( + in ReservationHandle original, + int slotIndex, + long handleGeneration, + long controlGeneration, + ref ReservationHandle targetHandle, + ref ulong targetBinding, + ref long targetGeneration, + int controlState, + int participantToken) + { + targetBinding = IndexBinding.Encode(slotIndex, handleGeneration); + targetGeneration = handleGeneration; + targetHandle = new ReservationHandle( + original.StoreId, + original.ParticipantToken, + targetBinding, + original.PayloadLength); + + return SlotControl(controlState, controlGeneration, participantToken); + } + + private static MemoryStore CreateStore(string suffix) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-conflict-abort-{suffix}-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static LockFreeSlotTable ReadSlots(MemoryStore store) + { + object engine = ReadPrivate(store, "_engine"); + return ReadPrivate(engine, "_slots"); + } + + private static T ReadPrivate(object owner, string fieldName) + { + FieldInfo field = owner.GetType().GetField( + fieldName, + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + $"Missing field {owner.GetType().FullName}.{fieldName}."); + return Assert.IsAssignableFrom(field.GetValue(owner)); + } + + private static long SlotControl(int state, long generation, int participantToken) => + unchecked((long)AtomicControlWord.EncodeSlot(state, generation, participantToken)); + + private static int SlotState(long control) => + (int)(unchecked((ulong)control) & 0x7UL); + + private static long SlotGeneration(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static ulong SlotParticipant(long control) => + (unchecked((ulong)control) >> 36) & 0x0fff_ffffUL; + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + public enum AtomicAbortScenario + { + OwnInitializing, + OwnReserved, + Aborting, + LaterGeneration, + Reclaiming, + TerminalRetired, + LowerGeneration, + Published, + RemoveRequested, + Free, + NonterminalRetired, + WrongInitializingOwner, + WrongReservedOwner, + OwnedAborting, + WrongDirectoryBinding, + ExplicitPublicationIntent, + UnknownPublicationIntent, + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeCorruptionScannerTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeCorruptionScannerTests.cs new file mode 100644 index 0000000..5a31942 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeCorruptionScannerTests.cs @@ -0,0 +1,314 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed unsafe class LockFreeCorruptionScannerTests +{ + [Fact] + public void PublishAllocationScanLatchesMalformedSlotControl() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open("publish-slot-scan"); + EngineInternals internals = ReadInternals(store); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(0); + long malformed = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.FreeState, + generation: 1, + participantToken: 1)); + AtomicControlWord.StoreRelease(ref slot.Control, malformed); + + Assert.Equal(StoreStatus.CorruptStore, store.TryPublish([0x11], [0x21])); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref slot.Control)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void AcquireAllocationScanLatchesMalformedLeaseControl() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open("acquire-lease-scan"); + byte[] key = [0x12]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x22])); + EngineInternals internals = ReadInternals(store); + ref LeaseRecordV2 lease = ref internals.Leases.Record(0); + long malformed = unchecked((long)AtomicControlWord.EncodeLease( + LockFreeLeaseRegistry.RetiredState, + generation: 1, + participantToken: 0)); + AtomicControlWord.StoreRelease(ref lease.Control, malformed); + + Assert.Equal(StoreStatus.CorruptStore, store.TryAcquire(key, out _)); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref lease.Control)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void LeaseRecoveryScanLatchesMalformedControlInsteadOfCountingFailure() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open("lease-recovery-scan"); + EngineInternals internals = ReadInternals(store); + ref LeaseRecordV2 lease = ref internals.Leases.Record(0); + long malformed = unchecked((long)AtomicControlWord.EncodeLease( + LockFreeLeaseRegistry.RetiredState, + generation: 1, + participantToken: 0)); + AtomicControlWord.StoreRelease(ref lease.Control, malformed); + + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverLeases(new LeaseRecoveryOptions(false), out _)); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref lease.Control)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void ReservationRecoveryScanLatchesMalformedControlInsteadOfSkippingIt() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open("reservation-recovery-scan"); + EngineInternals internals = ReadInternals(store); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(0); + long malformed = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.RetiredState, + generation: 1, + participantToken: 0)); + AtomicControlWord.StoreRelease(ref slot.Control, malformed); + + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations(new ReservationRecoveryOptions(false), out _)); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref slot.Control)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void ParticipantReferenceProofLatchesMalformedResourceControl() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open("participant-reference-scan"); + EngineInternals internals = ReadInternals(store); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(0); + long malformed = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.RetiredState, + generation: 1, + participantToken: 0)); + AtomicControlWord.StoreRelease(ref slot.Control, malformed); + + StoreStatus status = internals.Participants.HasParticipantReferences( + internals.Registration.Token, + LockFreeOperationBudget.StructuralAttempt, + out bool hasReferences); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.True(hasReferences); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref slot.Control)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void HotOperationLatchesMalformedLocalParticipantControl() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open("local-participant-control"); + EngineInternals internals = ReadInternals(store); + ref ParticipantRecordV2 participant = ref *(ParticipantRecordV2*)( + internals.Region.Pointer + + internals.Layout.ParticipantOffset + + ((long)internals.Registration.RecordIndex * internals.Layout.ParticipantStride)); + long malformed = unchecked((long)AtomicControlWord.EncodeParticipant( + LayoutV2Constants.ParticipantFree, + internals.Registration.Generation, + pid: Environment.ProcessId)); + AtomicControlWord.StoreRelease(ref participant.Control, malformed); + + Assert.Equal(StoreStatus.CorruptStore, store.TryPublish([0x13], [0x23])); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref participant.Control)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void ReservationRecoveryExactReferenceScanLatchesMalformedUnrelatedDirectoryWord() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open("reservation-directory-reference-scan"); + Assert.Equal(StoreStatus.Success, store.TryReserve([0x14], 1, default, out var reservation)); + EngineInternals internals = ReadInternals(store); + int slotIndex = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding).SlotIndex; + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(slotIndex); + Assert.Equal(StoreStatus.Success, internals.Slots.TryBeginAbort(reservation.HandleForEngine)); + + ulong locationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryLocation)); + DirectoryLocation location = DirectoryLocation.Decode(locationRaw); + ref long exactCell = ref DirectoryCell(internals, location); + Assert.Equal( + unchecked((long)slot.DirectoryBinding), + AtomicControlWord.LoadAcquire(ref exactCell)); + AtomicControlWord.StoreRelease(ref exactCell, 0); + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, 0); + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, 0); + + long malformed = unchecked((long)IndexBinding.Encode( + internals.Layout.SlotCount, + generation: 1)); + ref long mutation = ref DirectoryWord(internals, DirectoryWordKind.BucketMutation); + AtomicControlWord.StoreRelease(ref mutation, malformed); + + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations(new ReservationRecoveryOptions(false), out _)); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref mutation)); + AssertCorrupt(internals.Region); + } + + [Theory] + [InlineData(DirectoryWordKind.BucketMutation)] + [InlineData(DirectoryWordKind.PrimaryCell)] + [InlineData(DirectoryWordKind.OverflowCell)] + public void DiagnosticsLatchesMalformedDirectoryWordWithoutRewritingIt( + DirectoryWordKind wordKind) + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open($"diagnostics-{wordKind}"); + EngineInternals internals = ReadInternals(store); + long malformed = unchecked((long)IndexBinding.Encode( + internals.Layout.SlotCount, + generation: 1)); + ref long word = ref DirectoryWord(internals, wordKind); + Assert.Equal(0, AtomicControlWord.LoadAcquire(ref word)); + AtomicControlWord.StoreRelease(ref word, malformed); + + Assert.Equal(StoreStatus.CorruptStore, store.TryGetDiagnostics(out _)); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref word)); + AssertCorrupt(internals.Region); + } + + private static MemoryStore Open(string purpose) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-corruption-{purpose}-{Guid.NewGuid():N}", + slotCount: 4, + maxValueBytes: 8, + maxDescriptorBytes: 4, + maxKeyBytes: 4, + leaseRecordCount: 4, + participantRecordCount: 4, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static EngineInternals ReadInternals(MemoryStore store) + { + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + return new EngineInternals( + ReadField(engine, "_slots"), + ReadField(engine, "_leases"), + ReadField(engine, "_participants"), + ReadField(engine, "_registration"), + ReadField(engine, "_region"), + ReadField(engine, "_layout")); + } + + private static T ReadField(object owner, string name) => + Assert.IsType(owner.GetType() + .GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(owner)); + + private static void AssertCorrupt(MemoryMappedStoreRegion region) => + Assert.Equal( + LayoutV2Constants.StoreCorrupt, + AtomicControlWord.LoadAcquire(ref ((StoreHeaderV2*)region.Pointer)->Control)); + + private static ref long DirectoryWord( + in EngineInternals internals, + DirectoryWordKind wordKind) + { + long offset = wordKind switch + { + DirectoryWordKind.BucketMutation => internals.Layout.PrimaryDirectoryOffset + 8, + DirectoryWordKind.PrimaryCell => internals.Layout.PrimaryDirectoryOffset + 16, + DirectoryWordKind.OverflowCell => internals.Layout.OverflowDirectoryOffset, + _ => throw new ArgumentOutOfRangeException(nameof(wordKind)), + }; + return ref *(long*)(internals.Region.Pointer + offset); + } + + private static ref long DirectoryCell( + in EngineInternals internals, + DirectoryLocation location) + { + long offset = location.Kind switch + { + 1 => internals.Layout.PrimaryDirectoryOffset + + ((location.Index / LayoutV2Constants.PrimaryLanesPerBucket) + * internals.Layout.PrimaryBucketStride) + + 16 + + ((location.Index % LayoutV2Constants.PrimaryLanesPerBucket) * sizeof(long)), + 2 => internals.Layout.OverflowDirectoryOffset + + (location.Index * internals.Layout.OverflowStride), + _ => throw new ArgumentOutOfRangeException(nameof(location)), + }; + return ref *(long*)(internals.Region.Pointer + offset); + } + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private readonly record struct EngineInternals( + LockFreeSlotTable Slots, + LockFreeLeaseRegistry Leases, + LockFreeParticipantRegistry Participants, + LockFreeParticipantRegistry.Registration Registration, + MemoryMappedStoreRegion Region, + StoreLayoutV2 Layout); + + public enum DirectoryWordKind + { + BucketMutation, + PrimaryCell, + OverflowCell, + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeDedicatedThreadAllocationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeDedicatedThreadAllocationTests.cs new file mode 100644 index 0000000..9febdb3 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeDedicatedThreadAllocationTests.cs @@ -0,0 +1,170 @@ +using System.Buffers; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeDedicatedThreadAllocationTests +{ + private const int WarmupCycles = 50_000; + private const int MeasuredCycles = 1_000_000; + + [Fact] + public void MillionSegmentedPublishRemoveCyclesAllocateZeroOnWarmedDedicatedThread() + { + using MemoryStore store = CreateStore(slotCount: 1, leaseCount: 1); + byte[] key = [0x31]; + ReadOnlySequence payload = TwoSegmentSequence([1, 2], [3, 4]); + + AllocationRun result = RunDedicated(() => + { + StoreStatus publish = store.TryPublishSegments(key, payload, [], out long copied); + if (publish != StoreStatus.Success || copied != 4) + { + return publish == StoreStatus.Success ? StoreStatus.UnknownFailure : publish; + } + + return store.TryRemove(key); + }); + + Assert.Null(result.Exception); + Assert.Equal(StoreStatus.Success, result.LastStatus); + Assert.Equal(MeasuredCycles, result.CompletedCycles); + Assert.Equal(0, result.AllocatedBytes); + } + + [Fact] + public void MillionExpectedFailureSetsAllocateZeroOnWarmedDedicatedThread() + { + using MemoryStore store = CreateStore(slotCount: 1, leaseCount: 1); + byte[] publishedKey = [0x41]; + byte[] capacityKey = [0x42]; + byte[] missingKey = [0x43]; + byte[] value = [0x51]; + Assert.Equal(StoreStatus.Success, store.TryPublish(publishedKey, value)); + Assert.Equal(StoreStatus.Success, store.TryAcquire(publishedKey, out ValueLease held)); + + AllocationRun result; + try + { + result = RunDedicated(() => + { + if (store.TryPublish(publishedKey, value) != StoreStatus.DuplicateKey + || store.TryReserve(publishedKey, 1, [], out _) != StoreStatus.DuplicateKey + || store.TryPublish(capacityKey, value) != StoreStatus.StoreFull + || store.TryAcquire(publishedKey, out _) != StoreStatus.LeaseTableFull + || store.TryAcquire(missingKey, out _) != StoreStatus.NotFound) + { + return StoreStatus.UnknownFailure; + } + + return StoreStatus.Success; + }); + } + finally + { + Assert.Equal(StoreStatus.Success, held.Release()); + } + + Assert.Null(result.Exception); + Assert.Equal(StoreStatus.Success, result.LastStatus); + Assert.Equal(MeasuredCycles, result.CompletedCycles); + Assert.Equal(0, result.AllocatedBytes); + } + + private static AllocationRun RunDedicated(Func cycle) + { + AllocationRun result = default; + var thread = new Thread(() => + { + try + { + for (var index = 0; index < WarmupCycles; index++) + { + StoreStatus status = cycle(); + if (status != StoreStatus.Success) + { + result = new AllocationRun(-1, index, status, null); + return; + } + } + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + long before = GC.GetAllocatedBytesForCurrentThread(); + StoreStatus last = StoreStatus.Success; + var completed = 0; + for (; completed < MeasuredCycles; completed++) + { + last = cycle(); + if (last != StoreStatus.Success) + { + break; + } + } + + long allocated = GC.GetAllocatedBytesForCurrentThread() - before; + result = new AllocationRun(allocated, completed, last, null); + } + catch (Exception exception) + { + result = new AllocationRun(-1, 0, StoreStatus.UnknownFailure, exception); + } + }) + { + IsBackground = true, + Name = "SharedMemoryStore allocation qualification" + }; + + thread.Start(); + Assert.True(thread.Join(TimeSpan.FromMinutes(5)), "Dedicated allocation run timed out."); + return result; + } + + private static ReadOnlySequence TwoSegmentSequence(byte[] first, byte[] second) + { + var start = new Segment(first); + Segment end = start.Append(second); + return new ReadOnlySequence(start, 0, end, end.Memory.Length); + } + + private static MemoryStore CreateStore(int slotCount, int leaseCount) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-dedicated-allocation-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 8, + maxDescriptorBytes: 0, + maxKeyBytes: 4, + leaseRecordCount: leaseCount, + participantRecordCount: 1, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(options, out MemoryStore? store)); + return Assert.IsType(store); + } + + private sealed class Segment : ReadOnlySequenceSegment + { + internal Segment(ReadOnlyMemory memory) + { + Memory = memory; + } + + internal Segment Append(ReadOnlyMemory memory) + { + var segment = new Segment(memory) + { + RunningIndex = RunningIndex + Memory.Length + }; + Next = segment; + return segment; + } + } + + private readonly record struct AllocationRun( + long AllocatedBytes, + int CompletedCycles, + StoreStatus LastStatus, + Exception? Exception); +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryCleanupCorruptionTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryCleanupCorruptionTests.cs new file mode 100644 index 0000000..00c32d2 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryCleanupCorruptionTests.cs @@ -0,0 +1,518 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeDirectoryCleanupCorruptionTests +{ + private const int IntentInsert = 1; + private const int IntentUnlink = 2; + private const int PhaseTargetSelected = 2; + private const int TargetPrimary = 1; + private const long Generation = 17; + private const ulong KeyHash = 0xd6e8_feb8_6659_fd93UL; + + [Fact] + public void UnlinkCellCleanupCasLossLatchesStableMalformedWinner() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore(out EngineInternals internals); + SeedTargetSelectedOperation(internals, IntentUnlink, LockFreeSlotTable.ReclaimingState); + ref long cell = ref PrimaryCell(internals); + long malformed = MalformedBinding(internals.Layout); + AtomicControlWord.StoreRelease(ref cell, malformed); + + Assert.Equal( + StoreStatus.CorruptStore, + internals.Directory.HelpMutation(CanonicalBucket(internals.Layout), maxSteps: 1)); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref cell)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void CanceledInsertLocationCleanupCasLossLatchesStableMalformedWinner() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore(out EngineInternals internals); + SeedTargetSelectedOperation(internals, IntentInsert, LockFreeSlotTable.AbortingState); + AtomicControlWord.StoreRelease( + ref PrimaryCell(internals), + unchecked((long)IndexBinding.Encode(slotIndex: 0, Generation))); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(0); + long malformed = unchecked((long)ulong.MaxValue); + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, malformed); + + Assert.Equal( + StoreStatus.CorruptStore, + internals.Directory.HelpMutation(CanonicalBucket(internals.Layout), maxSteps: 1)); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void MutationCleanupCasLossLatchesStableMalformedWinner() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore(out EngineInternals internals); + int bucket = CanonicalBucket(internals.Layout); + ulong expected = IndexBinding.Encode(slotIndex: 0, Generation); + ref long mutation = ref BucketMutation(internals, bucket); + long malformed = MalformedBinding(internals.Layout); + AtomicControlWord.StoreRelease(ref mutation, malformed); + + MethodInfo cleanup = typeof(LockFreeKeyDirectory).GetMethod( + "TryClearMutationWord", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Missing mutation cleanup helper."); + StoreStatus status = Assert.IsType(cleanup.Invoke( + internals.Directory, + [bucket, expected])); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref mutation)); + AssertCorrupt(internals.Region); + } + + [Fact] + public void LiveReservationRecoveryLatchesStableMalformedCompletedLocation() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore(out EngineInternals internals); + Assert.Equal(StoreStatus.Success, store.TryReserve([0x31], 1, default, out var reservation)); + int slotIndex = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding).SlotIndex; + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(slotIndex); + long malformed = unchecked((long)ulong.MaxValue); + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, malformed); + + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations(new ReservationRecoveryOptions(false), out _)); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + AssertCorrupt(internals.Region); + } + + [Fact] + public async Task CanceledCompletedInsertWithStableMalformedLocationReturnsCorrupt() + { + if (!IsSupportedHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore(scheduler, out EngineInternals internals); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck); + Task<(StoreStatus Status, ValueReservation Reservation)> reserve = Task.Run(() => + { + StoreStatus status = store.TryReserve([0x32], 1, default, out var reservation); + return (status, reservation); + }); + + bool resumed = false; + try + { + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + (int bucket, ulong binding) = FindMutation(internals.Directory, internals.Layout); + Assert.Equal( + StoreStatus.StoreBusy, + internals.Directory.HelpMutation(bucket, maxSteps: 3)); + IndexBinding decoded = IndexBinding.Decode(binding); + DirectoryOperation operation = ReadOperation(internals.Slots, decoded.SlotIndex); + Assert.Equal(IntentInsert, operation.Intent); + Assert.Equal(5, operation.Phase); + long control = ReadSlotControl(internals.Slots, decoded.SlotIndex); + Assert.Equal(LockFreeSlotTable.ReservedState, (int)((ulong)control & 0x7UL)); + var handle = new ReservationHandle( + ReadField(internals.Slots, "_storeId"), + unchecked((ulong)control) >> 36, + binding, + ReadValueLength(internals.Slots, decoded.SlotIndex)); + Assert.Equal(StoreStatus.Success, internals.Slots.TryBeginAbort(handle)); + + long malformed = unchecked((long)ulong.MaxValue); + WriteLocation(internals.Slots, decoded.SlotIndex, malformed); + scheduler.Continue(); + resumed = true; + + (StoreStatus status, ValueReservation reservation) = + await reserve.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.False(reservation.IsValid); + Assert.Equal(malformed, ReadLocation(internals.Slots, decoded.SlotIndex)); + AssertCorrupt(internals.Region); + } + finally + { + if (!resumed) + { + scheduler.Continue(); + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CanceledInsertZeroLocationWindowRemainsLegalForOutcomeAndRecovery( + bool recoverBeforeOwnerResumes) + { + if (!IsSupportedHost()) + { + return; + } + + using var ownerScheduler = new ControlledLockFreeScheduler(); + using var cancelScheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore(ownerScheduler, out EngineInternals internals); + ownerScheduler.PauseAt(LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck); + Task<(StoreStatus Status, ValueReservation Reservation)> reserve = Task.Run(() => + { + StoreStatus status = store.TryReserve([0x33], 1, default, out var reservation); + return (status, reservation); + }); + + bool ownerResumed = false; + bool cancelResumed = false; + Task? cancel = null; + try + { + Assert.True(ownerScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + (int bucket, ulong binding) = FindMutation(internals.Directory, internals.Layout); + Assert.Equal( + StoreStatus.StoreBusy, + internals.Directory.HelpMutation(bucket, maxSteps: 3)); + IndexBinding decoded = IndexBinding.Decode(binding); + DirectoryOperation completed = ReadOperation(internals.Slots, decoded.SlotIndex); + Assert.Equal(IntentInsert, completed.Intent); + Assert.Equal(5, completed.Phase); + long control = ReadSlotControl(internals.Slots, decoded.SlotIndex); + var handle = new ReservationHandle( + ReadField(internals.Slots, "_storeId"), + unchecked((ulong)control) >> 36, + binding, + ReadValueLength(internals.Slots, decoded.SlotIndex)); + Assert.Equal(StoreStatus.Success, internals.Slots.TryBeginAbort(handle)); + + cancelScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterCancelLocationClearBeforeDescriptorRejection); + cancel = Task.Run(() => + { + InstrumentedLockFreeCheckpoint checkpoint = + cancelScheduler.CreateInstrumentedCheckpoint(); + return internals.Directory.HelpMutation( + bucket, + LockFreeOperationBudget.UnboundedScan, + ref checkpoint, + maxSteps: 128); + }); + Assert.True(cancelScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal(0, ReadLocation(internals.Slots, decoded.SlotIndex)); + DirectoryOperation pausedOperation = ReadOperation(internals.Slots, decoded.SlotIndex); + Assert.Equal(IntentInsert, pausedOperation.Intent); + Assert.Equal(5, pausedOperation.Phase); + Assert.Equal( + LayoutV2Constants.StoreReady, + ReadStoreControl(internals.Region)); + + if (recoverBeforeOwnerResumes) + { + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations(new ReservationRecoveryOptions(false), out _)); + Assert.Equal( + LayoutV2Constants.StoreReady, + ReadStoreControl(internals.Region)); + } + + ownerScheduler.Continue(); + ownerResumed = true; + (StoreStatus ownerStatus, ValueReservation reservation) = + await reserve.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.InvalidReservation, ownerStatus); + Assert.False(reservation.IsValid); + + cancelScheduler.Continue(); + cancelResumed = true; + Assert.NotEqual( + StoreStatus.CorruptStore, + await cancel.WaitAsync(TimeSpan.FromSeconds(5))); + } + finally + { + if (!ownerResumed) + { + ownerScheduler.Continue(); + } + + if (cancel is not null && !cancelResumed) + { + cancelScheduler.Continue(); + } + } + } + + [Fact] + public void SpillSummaryCasLossLatchesOnlyStableMalformedObservation() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore transientStore = CreateStore(out EngineInternals transient); + ulong binding = IndexBinding.Encode(slotIndex: 0, generation: 1); + ulong validReplacement = SpillSummary.EncodeEmpty(binding); + ulong desired = SpillSummary.EncodePresent(binding); + ulong malformed = ulong.MaxValue; + Assert.Equal( + StoreStatus.Success, + InvokeSummaryObservationValidation( + transient.Directory, + validReplacement, + expected: 0, + desired, + malformed)); + Assert.Equal( + LayoutV2Constants.StoreReady, + ReadStoreControl(transient.Region)); + + using MemoryStore stableStore = CreateStore(out EngineInternals stable); + Assert.Equal( + StoreStatus.CorruptStore, + InvokeSummaryObservationValidation( + stable.Directory, + malformed, + expected: 0, + desired, + malformed)); + AssertCorrupt(stable.Region); + } + + private static void SeedTargetSelectedOperation( + in EngineInternals internals, + int intent, + int slotState) + { + int bucket = CanonicalBucket(internals.Layout); + int targetIndex = bucket * LayoutV2Constants.PrimaryLanesPerBucket; + ulong binding = IndexBinding.Encode(slotIndex: 0, Generation); + ulong operation = DirectoryOperation.Encode( + intent, + PhaseTargetSelected, + TargetPrimary, + targetIndex, + Generation); + ulong location = DirectoryLocation.Encode(TargetPrimary, targetIndex, Generation); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(0); + slot.DirectoryBinding = binding; + slot.KeyHash = KeyHash; + Volatile.Write( + ref slot.PublicationIntent, + (int)SlotPublicationIntent.ExplicitReservation); + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, unchecked((long)location)); + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, unchecked((long)operation)); + AtomicControlWord.StoreRelease( + ref slot.Control, + unchecked((long)AtomicControlWord.EncodeSlot( + slotState, + Generation, + participantToken: 0))); + AtomicControlWord.StoreRelease(ref BucketMutation(internals, bucket), unchecked((long)binding)); + } + + private static unsafe ref long PrimaryCell(in EngineInternals internals) + { + int bucket = CanonicalBucket(internals.Layout); + long offset = internals.Layout.PrimaryDirectoryOffset + + ((long)bucket * internals.Layout.PrimaryBucketStride) + + 16; + return ref *(long*)(internals.Region.Pointer + offset); + } + + private static unsafe ref long BucketMutation(in EngineInternals internals, int bucket) + { + long offset = internals.Layout.PrimaryDirectoryOffset + + ((long)bucket * internals.Layout.PrimaryBucketStride) + + 8; + return ref *(long*)(internals.Region.Pointer + offset); + } + + private static DirectoryOperation ReadOperation(LockFreeSlotTable slots, int slotIndex) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + return DirectoryOperation.Decode( + unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation))); + } + + private static long ReadSlotControl(LockFreeSlotTable slots, int slotIndex) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + return AtomicControlWord.LoadAcquire(ref slot.Control); + } + + private static int ReadValueLength(LockFreeSlotTable slots, int slotIndex) => + slots.Slot(slotIndex).ValueLength; + + private static long ReadLocation(LockFreeSlotTable slots, int slotIndex) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + return AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation); + } + + private static void WriteLocation(LockFreeSlotTable slots, int slotIndex, long value) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, value); + } + + private static long MalformedBinding(StoreLayoutV2 layout) => + unchecked((long)IndexBinding.Encode(layout.SlotCount, generation: 1)); + + private static int CanonicalBucket(StoreLayoutV2 layout) => + (int)(Mix(KeyHash) & checked((uint)(layout.PrimaryBucketCount - 1))); + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static MemoryStore CreateStore(out EngineInternals internals) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-cleanup-corruption-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + MemoryStore result = Assert.IsType(store); + object engine = ReadField(result, "_engine"); + internals = new EngineInternals( + ReadField(engine, "_directory"), + ReadField(engine, "_slots"), + ReadField(engine, "_region"), + ReadField(engine, "_layout")); + return result; + } + + private static MemoryStore CreateInstrumentedStore( + ControlledLockFreeScheduler scheduler, + out EngineInternals internals) + { + SharedMemoryStoreOptions options = Options( + $"sms-v2-cleanup-corruption-instrumented-{Guid.NewGuid():N}"); + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + scheduler.CreateInstrumentedCheckpoint(), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + MemoryStore result = Assert.IsType(store); + object engine = ReadField(result, "_engine"); + internals = new EngineInternals( + ReadField(engine, "_directory"), + ReadField(engine, "_slots"), + ReadField(engine, "_region"), + ReadField(engine, "_layout")); + return result; + } + + private static SharedMemoryStoreOptions Options(string name) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 2, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + + private static (int Bucket, ulong Binding) FindMutation( + LockFreeKeyDirectory directory, + StoreLayoutV2 layout) + { + for (var bucket = 0; bucket < layout.PrimaryBucketCount; bucket++) + { + ulong binding = directory.ReadCanonicalMutation(bucket); + if (binding != 0) + { + return (bucket, binding); + } + } + + throw new InvalidOperationException("No active canonical mutation was found."); + } + + private static StoreStatus InvokeSummaryObservationValidation( + LockFreeKeyDirectory directory, + ulong current, + ulong expected, + ulong desired, + ulong observed) + { + MethodInfo validation = typeof(LockFreeKeyDirectory).GetMethod( + "ValidateSpillSummaryCasObservation", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Missing spill-summary CAS validator."); + object?[] arguments = + [ + unchecked((long)current), + expected, + desired, + observed, + ]; + return Assert.IsType(validation.Invoke(directory, arguments)); + } + + private static T ReadField(object owner, string name) => + Assert.IsAssignableFrom(owner.GetType() + .GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(owner)); + + private static void AssertCorrupt(MemoryMappedStoreRegion region) => + Assert.Equal(LayoutV2Constants.StoreCorrupt, ReadStoreControl(region)); + + private static unsafe long ReadStoreControl(MemoryMappedStoreRegion region) => + AtomicControlWord.LoadAcquire(ref ((StoreHeaderV2*)region.Pointer)->Control); + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private readonly record struct EngineInternals( + LockFreeKeyDirectory Directory, + LockFreeSlotTable Slots, + MemoryMappedStoreRegion Region, + StoreLayoutV2 Layout); +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryCollisionTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryCollisionTests.cs new file mode 100644 index 0000000..d691242 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryCollisionTests.cs @@ -0,0 +1,234 @@ +using System.Reflection; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeDirectoryCollisionTests +{ + private const ulong ExactCollisionHash = 0x0123_4567_89AB_CDEFUL; + + [Fact] + public void ExactHashCollisionsUseBothPrimaryChoicesThenPreserveAllSlotCapacityInOverflow() + { + const int slotCount = 24; + var directory = new CollisionDirectoryOracle(slotCount); + + for (var index = 0; index < slotCount; index++) + { + Assert.Equal( + InsertResult.Success, + directory.Insert(Key(index), ExactCollisionHash, Binding(index))); + } + + Assert.Equal(8, directory.FirstBucketOccupancy); + Assert.Equal(8, directory.SecondBucketOccupancy); + Assert.Equal(slotCount - 16, directory.OverflowOccupancy); + Assert.True(directory.HasSpill); + Assert.Equal(InsertResult.StoreFull, directory.Insert(Key(slotCount), ExactCollisionHash, Binding(slotCount))); + Assert.Equal(slotCount, directory.Count); + for (var index = 0; index < slotCount; index++) + { + Assert.Equal(Binding(index), directory.Lookup(Key(index), ExactCollisionHash)); + } + } + + [Fact] + public void ExactUnlinkNeverClearsAnotherCollidingKeyAndRestoresOneAdmission() + { + const int slotCount = 20; + var directory = new CollisionDirectoryOracle(slotCount); + for (var index = 0; index < slotCount; index++) + { + Assert.Equal(InsertResult.Success, directory.Insert(Key(index), ExactCollisionHash, Binding(index))); + } + + Assert.False(directory.Unlink(Key(18), ExactCollisionHash, Binding(17))); + Assert.Equal(Binding(18), directory.Lookup(Key(18), ExactCollisionHash)); + Assert.True(directory.Unlink(Key(18), ExactCollisionHash, Binding(18))); + Assert.Null(directory.Lookup(Key(18), ExactCollisionHash)); + Assert.Equal(slotCount - 1, directory.Count); + + Assert.Equal(InsertResult.Success, directory.Insert(Key(100), ExactCollisionHash, Binding(100))); + Assert.Equal(slotCount, directory.Count); + for (var index = 0; index < slotCount; index++) + { + if (index != 18) + { + Assert.Equal(Binding(index), directory.Lookup(Key(index), ExactCollisionHash)); + } + } + } + + [Fact] + public void SpillSummarySkipsOverflowWhenEmptyAndClearsAfterLastOverflowUnlink() + { + var directory = new CollisionDirectoryOracle(slotCount: 20); + for (var index = 0; index < 16; index++) + { + Assert.Equal(InsertResult.Success, directory.Insert(Key(index), ExactCollisionHash, Binding(index))); + } + + Assert.False(directory.HasSpill); + Assert.Null(directory.Lookup(Key(99), ExactCollisionHash)); + Assert.Equal(0, directory.OverflowScanCount); + + Assert.Equal(InsertResult.Success, directory.Insert(Key(16), ExactCollisionHash, Binding(16))); + Assert.True(directory.HasSpill); + Assert.Equal(Binding(16), directory.Lookup(Key(16), ExactCollisionHash)); + Assert.True(directory.OverflowScanCount > 0); + Assert.True(directory.Unlink(Key(16), ExactCollisionHash, Binding(16))); + Assert.False(directory.HasSpill); + } + + [Fact] + public void ProductionDirectoryContractProvidesCapacityPreservingOverflowAndExactOperations() + { + var type = typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeKeyDirectory", + throwOnError: false, + ignoreCase: false); + Assert.True(type is not null, "The lock-free profile must implement LockFreeKeyDirectory."); + + var methods = type!.GetMethods( + BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + Assert.Contains(methods, static method => method.Name.Contains("Lookup", StringComparison.Ordinal)); + Assert.Contains(methods, static method => method.Name.Contains("Insert", StringComparison.Ordinal)); + Assert.Contains(methods, static method => method.Name.Contains("Unlink", StringComparison.Ordinal)); + Assert.Contains(methods, static method => method.Name.Contains("Overflow", StringComparison.Ordinal) + || method.Name.Contains("Spill", StringComparison.Ordinal)); + } + + private static string Key(int index) => "collision-key-" + index.ToString(System.Globalization.CultureInfo.InvariantCulture); + + private static ulong Binding(int index) => ((ulong)(index + 1) << 31) | (uint)(index + 1); + + private enum InsertResult + { + Success, + DuplicateKey, + StoreFull + } + + private sealed class CollisionDirectoryOracle + { + private readonly Entry?[] _firstBucket = new Entry?[8]; + private readonly Entry?[] _secondBucket = new Entry?[8]; + private readonly Entry?[] _overflow; + private readonly int _slotCount; + + public CollisionDirectoryOracle(int slotCount) + { + _slotCount = slotCount; + _overflow = new Entry?[slotCount]; + } + + public int Count { get; private set; } + + public bool HasSpill { get; private set; } + + public int OverflowScanCount { get; private set; } + + public int FirstBucketOccupancy => _firstBucket.Count(static entry => entry is not null); + + public int SecondBucketOccupancy => _secondBucket.Count(static entry => entry is not null); + + public int OverflowOccupancy => _overflow.Count(static entry => entry is not null); + + public InsertResult Insert(string key, ulong hash, ulong binding) + { + if (Lookup(key, hash) is not null) + { + return InsertResult.DuplicateKey; + } + + if (Count == _slotCount) + { + return InsertResult.StoreFull; + } + + var entry = new Entry(key, hash, binding); + if (TryPlace(_firstBucket, entry) || TryPlace(_secondBucket, entry)) + { + Count++; + return InsertResult.Success; + } + + HasSpill = true; + Assert.True(TryPlace(_overflow, entry), "OverflowCount == SlotCount must preserve admission for every owned slot."); + Count++; + return InsertResult.Success; + } + + public ulong? Lookup(string key, ulong hash) + { + var primary = Find(_firstBucket, key, hash) ?? Find(_secondBucket, key, hash); + if (primary is not null) + { + return primary.Binding; + } + + if (!HasSpill) + { + return null; + } + + OverflowScanCount++; + return Find(_overflow, key, hash)?.Binding; + } + + public bool Unlink(string key, ulong hash, ulong exactBinding) + { + if (TryUnlink(_firstBucket, key, hash, exactBinding) + || TryUnlink(_secondBucket, key, hash, exactBinding) + || (HasSpill && TryUnlink(_overflow, key, hash, exactBinding))) + { + Count--; + HasSpill = _overflow.Any(static entry => entry is not null); + return true; + } + + return false; + } + + private static bool TryPlace(Entry?[] cells, Entry entry) + { + for (var index = 0; index < cells.Length; index++) + { + if (cells[index] is null) + { + cells[index] = entry; + return true; + } + } + + return false; + } + + private static Entry? Find(Entry?[] cells, string key, ulong hash) + { + return cells.FirstOrDefault(entry => + entry is not null + && entry.Hash == hash + && string.Equals(entry.Key, key, StringComparison.Ordinal)); + } + + private static bool TryUnlink(Entry?[] cells, string key, ulong hash, ulong exactBinding) + { + for (var index = 0; index < cells.Length; index++) + { + var entry = cells[index]; + if (entry is not null + && entry.Hash == hash + && entry.Binding == exactBinding + && string.Equals(entry.Key, key, StringComparison.Ordinal)) + { + cells[index] = null; + return true; + } + } + + return false; + } + + private sealed record Entry(string Key, ulong Hash, ulong Binding); + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryGenerationStressTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryGenerationStressTests.cs new file mode 100644 index 0000000..bc51300 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryGenerationStressTests.cs @@ -0,0 +1,1441 @@ +using System.Globalization; +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace SharedMemoryStore.UnitTests; + +/// +/// Configured SC-017 evidence over the real mapped directory protocol. The +/// ordinary test default executes every transition once; release qualification +/// raises the total through SMS_DIRECTORY_GENERATION_STRESS_REPETITIONS. +/// +public sealed class LockFreeDirectoryGenerationStressTests +{ + private const string RepetitionsVariable = "SMS_DIRECTORY_GENERATION_STRESS_REPETITIONS"; + private const string SeedVariable = "SMS_DIRECTORY_GENERATION_STRESS_SEED"; + private const ulong DefaultSeed = 0x5C01_700D_D1CE_7001UL; + private const int QualificationTransitionCount = 50; + private const int CanonicalBucket = 0; + private const int PrimarySlotCount = 1; + private const int OverflowSlotCount = 17; + private const int OverflowAnchorCount = 16; + + private static readonly TransitionCase[] TransitionCases = + [ + Operation("insert-primary-prepared", DelayedOperation.Insert, DirectoryTarget.Primary, 1, phase: 1), + Operation("insert-primary-target-selected", DelayedOperation.Insert, DirectoryTarget.Primary, 2, phase: 2), + Operation("insert-primary-binding-changed", DelayedOperation.Insert, DirectoryTarget.Primary, 3, phase: 3), + Operation("insert-primary-complete", DelayedOperation.Insert, DirectoryTarget.Primary, 4, phase: 5), + Operation("insert-overflow-prepared", DelayedOperation.Insert, DirectoryTarget.Overflow, 1, phase: 1), + Operation("insert-overflow-target-selected", DelayedOperation.Insert, DirectoryTarget.Overflow, 2, phase: 2), + Operation("insert-overflow-binding-changed", DelayedOperation.Insert, DirectoryTarget.Overflow, 3, phase: 3), + Operation("insert-overflow-complete", DelayedOperation.Insert, DirectoryTarget.Overflow, 4, phase: 5), + + Operation("unlink-primary-prepared", DelayedOperation.Unlink, DirectoryTarget.Primary, 1, phase: 1), + Operation("unlink-primary-target-selected", DelayedOperation.Unlink, DirectoryTarget.Primary, 2, phase: 2), + Operation("unlink-primary-binding-changed", DelayedOperation.Unlink, DirectoryTarget.Primary, 3, phase: 3), + Operation("unlink-primary-complete", DelayedOperation.Unlink, DirectoryTarget.Primary, 4, phase: 5), + Operation("unlink-overflow-prepared", DelayedOperation.Unlink, DirectoryTarget.Overflow, 1, phase: 1), + Operation("unlink-overflow-target-selected", DelayedOperation.Unlink, DirectoryTarget.Overflow, 2, phase: 2), + Operation("unlink-overflow-binding-changed", DelayedOperation.Unlink, DirectoryTarget.Overflow, 3, phase: 3), + Operation("unlink-overflow-complete", DelayedOperation.Unlink, DirectoryTarget.Overflow, 4, phase: 5), + + OperationRevalidation( + "insert-primary-revalidated-prepared-before-dispatch", + DelayedOperation.Insert, + DirectoryTarget.Primary, + 1, + phase: 1), + OperationRevalidation( + "insert-primary-revalidated-target-selected-before-dispatch", + DelayedOperation.Insert, + DirectoryTarget.Primary, + 2, + phase: 2), + OperationRevalidation( + "insert-primary-revalidated-binding-changed-before-dispatch", + DelayedOperation.Insert, + DirectoryTarget.Primary, + 3, + phase: 3), + OperationRevalidation( + "insert-primary-revalidated-complete-before-dispatch", + DelayedOperation.Insert, + DirectoryTarget.Primary, + 4, + phase: 5), + OperationRevalidation( + "insert-overflow-revalidated-prepared-before-dispatch", + DelayedOperation.Insert, + DirectoryTarget.Overflow, + 1, + phase: 1), + OperationRevalidation( + "insert-overflow-revalidated-target-selected-before-dispatch", + DelayedOperation.Insert, + DirectoryTarget.Overflow, + 2, + phase: 2), + OperationRevalidation( + "insert-overflow-revalidated-binding-changed-before-dispatch", + DelayedOperation.Insert, + DirectoryTarget.Overflow, + 3, + phase: 3), + OperationRevalidation( + "insert-overflow-revalidated-complete-before-dispatch", + DelayedOperation.Insert, + DirectoryTarget.Overflow, + 4, + phase: 5), + + OperationRevalidation( + "unlink-primary-revalidated-prepared-before-dispatch", + DelayedOperation.Unlink, + DirectoryTarget.Primary, + 1, + phase: 1), + OperationRevalidation( + "unlink-primary-revalidated-target-selected-before-dispatch", + DelayedOperation.Unlink, + DirectoryTarget.Primary, + 2, + phase: 2), + OperationRevalidation( + "unlink-primary-revalidated-binding-changed-before-dispatch", + DelayedOperation.Unlink, + DirectoryTarget.Primary, + 3, + phase: 3), + OperationRevalidation( + "unlink-primary-revalidated-complete-before-dispatch", + DelayedOperation.Unlink, + DirectoryTarget.Primary, + 4, + phase: 5), + OperationRevalidation( + "unlink-overflow-revalidated-prepared-before-dispatch", + DelayedOperation.Unlink, + DirectoryTarget.Overflow, + 1, + phase: 1), + OperationRevalidation( + "unlink-overflow-revalidated-target-selected-before-dispatch", + DelayedOperation.Unlink, + DirectoryTarget.Overflow, + 2, + phase: 2), + OperationRevalidation( + "unlink-overflow-revalidated-binding-changed-before-dispatch", + DelayedOperation.Unlink, + DirectoryTarget.Overflow, + 3, + phase: 3), + OperationRevalidation( + "unlink-overflow-revalidated-complete-before-dispatch", + DelayedOperation.Unlink, + DirectoryTarget.Overflow, + 4, + phase: 5), + + InsertBindingChangedStateValidation( + "insert-primary-binding-changed-state-validated-before-reserved", + DirectoryTarget.Primary), + InsertBindingChangedStateValidation( + "insert-overflow-binding-changed-state-validated-before-reserved", + DirectoryTarget.Overflow), + + Location("unlink-primary-prepared-location", DirectoryTarget.Primary, 1, phase: 1), + Location("unlink-primary-target-location", DirectoryTarget.Primary, 2, phase: 2), + Location("unlink-overflow-prepared-location", DirectoryTarget.Overflow, 1, phase: 1), + Location("unlink-overflow-target-location", DirectoryTarget.Overflow, 2, phase: 2), + + UnlinkLocationRead( + "unlink-primary-after-operation-validation-before-location-read", + DirectoryTarget.Primary), + UnlinkLocationRead( + "unlink-overflow-after-operation-validation-before-location-read", + DirectoryTarget.Overflow), + LocationPublication( + "insert-primary-after-binding-validation-before-location-publication", + DirectoryTarget.Primary), + LocationPublication( + "insert-overflow-after-binding-validation-before-location-publication", + DirectoryTarget.Overflow), + LocationPublication( + "insert-primary-after-empty-location-source-revalidation-before-publication-cas", + DirectoryTarget.Primary, + LockFreeCheckpointId.DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas), + LocationPublication( + "insert-overflow-after-empty-location-source-revalidation-before-publication-cas", + DirectoryTarget.Overflow, + LockFreeCheckpointId.DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas), + LocationPublication( + "insert-primary-after-location-publication-before-source-revalidation", + DirectoryTarget.Primary, + LockFreeCheckpointId.DirectoryAfterLocationPublicationBeforeSourceRevalidation), + LocationPublication( + "insert-overflow-after-location-publication-before-source-revalidation", + DirectoryTarget.Overflow, + LockFreeCheckpointId.DirectoryAfterLocationPublicationBeforeSourceRevalidation), + + Spill( + "insert-overflow-before-present-cas", + DelayedOperation.Insert, + LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas, + phase: 2), + Spill( + "insert-overflow-after-present-publication", + DelayedOperation.Insert, + LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication, + phase: 2), + Spill( + "unlink-overflow-after-empty-scan", + DelayedOperation.Unlink, + LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan, + phase: 5), + Spill( + "unlink-overflow-after-versioned-clear", + DelayedOperation.Unlink, + LockFreeCheckpointId.DirectoryAfterSpillSummaryClear, + phase: 5) + ]; + + // SC-017 is specifically a stale directory-helper-after-validation test. + // These are every Directory-family checkpoint at which such a helper can + // still perform a generation-fenced side effect. + private static readonly LockFreeCheckpointId[] Sc017MutationCheckpoints = + [ + LockFreeCheckpointId.DirectoryAfterOperationValidation, + LockFreeCheckpointId.DirectoryAfterLocationValidation, + LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead, + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation, + LockFreeCheckpointId.DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas, + LockFreeCheckpointId.DirectoryAfterLocationPublicationBeforeSourceRevalidation, + LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch, + LockFreeCheckpointId.DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication, + LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas, + LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication, + LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan, + LockFreeCheckpointId.DirectoryAfterSpillSummaryClear + ]; + + // These catalog entries bracket the directory protocol but are not + // directory-helper mutation windows. BeforeDescriptorPublication runs + // before a canonical mutation/operation is visible, so another participant + // has no exact descriptor to validate or complete. AfterDescriptorClear + // runs only after TryInsert has observed completion and the canonical + // mutation is zero, so no delayed directory side effect remains. + // AfterInsertCompletionStateValidationBeforeLocationRead is likewise a + // caller-side terminal classification window after helping is complete. + // BeforeInsertOuterLoopBudgetCheck is a caller retry boundary before any + // helper dispatch in that iteration. + // AfterInvalidReferenceConfirmationBeforeBindingRevalidation is a + // read-only correction window: the helper has not acquired mutation + // ownership and can only revalidate or restart the lookup. + // AfterUnlinkDescriptorClearBeforeGenerationAdvance follows the unlink + // helper's final exact directory write; only slot generation advance + // remains, so no delayed directory side effect can cross reuse. + // AfterCancelLocationClearBeforeDescriptorRejection is a cancellation-only + // observability seam. Its remaining exact descriptor/mutation release is + // covered by the deterministic canceled-outcome and recovery tests rather + // than the ordinary insert/unlink reuse matrix. + // Their owner/recovery schedules are covered by reservation tests, not + // SC-017. + private static readonly LockFreeCheckpointId[] NonMutationDirectoryBrackets = + [ + LockFreeCheckpointId.DirectoryBeforeDescriptorPublication, + LockFreeCheckpointId.DirectoryAfterDescriptorClear, + LockFreeCheckpointId.DirectoryAfterInsertCompletionStateValidationBeforeLocationRead, + LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck, + LockFreeCheckpointId.DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation, + LockFreeCheckpointId.DirectoryAfterUnlinkDescriptorClearBeforeGenerationAdvance, + LockFreeCheckpointId.DirectoryAfterCancelLocationClearBeforeDescriptorRejection + ]; + + private static readonly byte[][] PrimaryKeys = GenerateBucketPairCollisions(count: 64, PrimarySlotCount); + private static readonly byte[][] OverflowKeys = GenerateBucketPairCollisions(count: 64, OverflowSlotCount); + + private readonly ITestOutputHelper _output; + + public LockFreeDirectoryGenerationStressTests(ITestOutputHelper output) + { + _output = output; + } + + [Fact] + public void ConfiguredProductionStressFencesEveryDirectoryMutationTransitionAcrossSlotReuse() + { + ValidateTransitionMatrix(); + if (!IsSupportedLockFreeHost()) + { + _output.WriteLine( + $"SC017 skipped: OS={RuntimeInformation.OSDescription}; architecture={RuntimeInformation.ProcessArchitecture}."); + return; + } + + int configuredRepetitions = ReadRepetitions(); + ulong seed = ReadSeed(); + var evidence = new EvidenceCounters(); + _output.WriteLine( + $"SC017 start: seed=0x{seed:X16}; configuredRepetitions={configuredRepetitions}; " + + $"transitionCount={TransitionCases.Length}; distribution=quotient-plus-remainder."); + _output.WriteLine( + "SC017 catalog mapping: covered helper-mutation checkpoints=" + + string.Join(",", Sc017MutationCheckpoints) + + "; excluded non-mutation brackets=DirectoryBeforeDescriptorPublication" + + " (no published canonical mutation),DirectoryAfterDescriptorClear (mutation already zero)," + + "DirectoryAfterInsertCompletionStateValidationBeforeLocationRead" + + " (terminal caller classification after helping)," + + "DirectoryBeforeInsertOuterLoopBudgetCheck" + + " (caller retry boundary before helper dispatch)," + + "DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation" + + " (read-only exact-reference correction window)."); + + int quotient = Math.DivRem(configuredRepetitions, TransitionCases.Length, out int remainder); + for (var transitionIndex = 0; transitionIndex < TransitionCases.Length; transitionIndex++) + { + int repetitions = quotient + (transitionIndex < remainder ? 1 : 0); + ulong transitionSeed = seed ^ (0x9E37_79B9_7F4A_7C15UL * checked((ulong)(transitionIndex + 1))); + RunTransition( + TransitionCases[transitionIndex], + transitionIndex, + repetitions, + transitionSeed, + evidence); + _output.WriteLine( + $"SC017 transition={TransitionCases[transitionIndex].Name}; " + + $"seed=0x{transitionSeed:X16}; repetitions={repetitions}; result=pass."); + } + + Assert.Equal(configuredRepetitions, evidence.ExecutedRepetitions); + Assert.Equal(0, evidence.WrongGenerationMutationCount); + Assert.Equal(0, evidence.CorruptionCount); + Assert.Equal(0, evidence.FalseMissCount); + Assert.Equal(0, evidence.LeakedCapacityCount); + _output.WriteLine( + $"SC017 complete: seed=0x{seed:X16}; executedRepetitions={evidence.ExecutedRepetitions}; " + + $"wrongGenerationMutations={evidence.WrongGenerationMutationCount}; " + + $"corruption={evidence.CorruptionCount}; falseMisses={evidence.FalseMissCount}; " + + $"leakedCapacity={evidence.LeakedCapacityCount}."); + } + + [Fact] + public void TransitionMatrixMapsEveryDirectoryCatalogEntryAndEveryMutationPhase() + { + ValidateTransitionMatrix(); + } + + private static void ValidateTransitionMatrix() + { + Assert.Equal(QualificationTransitionCount, TransitionCases.Length); + + LockFreeCheckpointId[] catalog = LockFreeCheckpointCatalog.Entries + .Where(static entry => entry.Family == LockFreeCheckpointFamily.Directory) + .Select(static entry => entry.Id) + .Order() + .ToArray(); + LockFreeCheckpointId[] classified = Sc017MutationCheckpoints + .Concat(NonMutationDirectoryBrackets) + .Distinct() + .Order() + .ToArray(); + Assert.Equal(catalog, classified); + + LockFreeCheckpointId[] covered = TransitionCases + .Select(static transition => transition.Checkpoint) + .Distinct() + .Order() + .ToArray(); + Assert.Equal(Sc017MutationCheckpoints.Order().ToArray(), covered); + + int[] operationPhases = [1, 2, 3, 5]; + foreach (LockFreeCheckpointId checkpoint in new[] + { + LockFreeCheckpointId.DirectoryAfterOperationValidation, + LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch + }) + { + foreach (DelayedOperation operation in Enum.GetValues()) + { + foreach (DirectoryTarget target in Enum.GetValues()) + { + foreach (int phase in operationPhases) + { + Assert.Contains( + TransitionCases, + transition => transition.Checkpoint == checkpoint + && transition.Operation == operation + && transition.Target == target + && transition.ExpectedPhase == phase); + } + } + } + } + + foreach (DirectoryTarget target in Enum.GetValues()) + { + foreach (int phase in new[] { 1, 2 }) + { + Assert.Contains( + TransitionCases, + transition => transition.Checkpoint == LockFreeCheckpointId.DirectoryAfterLocationValidation + && transition.Operation == DelayedOperation.Unlink + && transition.Target == target + && transition.ExpectedPhase == phase); + } + + Assert.Contains( + TransitionCases, + transition => transition.Checkpoint + == LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead + && transition.Operation == DelayedOperation.Unlink + && transition.Target == target + && transition.ExpectedPhase == 1); + Assert.Contains( + TransitionCases, + transition => transition.Checkpoint + == LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation + && transition.Operation == DelayedOperation.Insert + && transition.Target == target + && transition.ExpectedPhase == 2); + Assert.Contains( + TransitionCases, + transition => transition.Checkpoint + == LockFreeCheckpointId.DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication + && transition.Operation == DelayedOperation.Insert + && transition.Target == target + && transition.ExpectedPhase == 3); + } + + foreach (LockFreeCheckpointId spillCheckpoint in new[] + { + LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas, + LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication, + LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan, + LockFreeCheckpointId.DirectoryAfterSpillSummaryClear + }) + { + Assert.Contains( + TransitionCases, + transition => transition.Checkpoint == spillCheckpoint + && transition.Target == DirectoryTarget.Overflow); + } + } + + private static void RunTransition( + TransitionCase transition, + int transitionIndex, + int repetitions, + ulong seed, + EvidenceCounters evidence) + { + int slotCount = transition.Target == DirectoryTarget.Overflow + ? OverflowSlotCount + : PrimarySlotCount; + byte[][] keys = transition.Target == DirectoryTarget.Overflow + ? OverflowKeys + : PrimaryKeys; + string name = $"sms-sc017-{transitionIndex:D2}-{Guid.NewGuid():N}"; + using var controller = new CheckpointPauseController(); + using MemoryStore delayedStore = CreateInstrumentedStore(name, slotCount, controller); + using MemoryStore helperStore = OpenHelperStore(name, slotCount); + using MemoryStore reuseStore = OpenHelperStore(name, slotCount); + using var delayedActor = new DelayedActor(delayedStore); + LockFreeKeyDirectory directory = ReadDirectory(helperStore); + LockFreeSlotTable slots = ReadSlots(helperStore); + var random = new DeterministicRandom(seed); + var helperValue = new byte[1]; + + if (transition.Target == DirectoryTarget.Overflow) + { + for (var index = 0; index < OverflowAnchorCount; index++) + { + helperValue[0] = unchecked((byte)index); + RequireExactStatus( + helperStore.TryPublish(keys[index], helperValue, default, StoreWaitOptions.Infinite), + StoreStatus.Success, + evidence, + $"{transition.Name}: publish anchor {index}"); + } + + Require( + directory.PrimaryOccupancy == OverflowAnchorCount && directory.OverflowOccupancy == 0, + $"{transition.Name}: the collision anchors did not fill exactly the two primary buckets."); + } + + int keyPoolStart = transition.Target == DirectoryTarget.Overflow + ? OverflowAnchorCount + : 0; + int keyPoolLength = keys.Length - keyPoolStart; + + for (var repetition = 0; repetition < repetitions; repetition++) + { + int oldOffset = random.Next(keyPoolLength); + int newOffset = random.Next(keyPoolLength - 1); + if (newOffset >= oldOffset) + { + newOffset++; + } + + byte[] oldKey = keys[keyPoolStart + oldOffset]; + byte[] newKey = keys[keyPoolStart + newOffset]; + byte oldMarker = unchecked((byte)random.NextUInt64()); + byte newMarker = unchecked((byte)random.NextUInt64()); + helperValue[0] = oldMarker; + if (transition.Operation == DelayedOperation.Unlink) + { + RequireExactStatus( + helperStore.TryPublish(oldKey, helperValue, default, StoreWaitOptions.Infinite), + StoreStatus.Success, + evidence, + $"{transition.Name}/{repetition}: publish unlink target"); + } + + controller.Arm(transition.Checkpoint, transition.CheckpointOccurrence); + delayedActor.Start(transition.Operation, oldKey, oldMarker); + bool resumed = false; + try + { + Require( + controller.WaitUntilPaused(TimeSpan.FromSeconds(10)), + $"{transition.Name}/{repetition}: delayed actor did not reach " + + $"{transition.Checkpoint} occurrence {transition.CheckpointOccurrence}."); + + ulong pausedMutation = directory.ReadCanonicalMutation(CanonicalBucket); + Require(pausedMutation != 0, $"{transition.Name}/{repetition}: canonical mutation is absent."); + IndexBinding mutationBinding = IndexBinding.Decode(pausedMutation); + int targetSlotIndex = mutationBinding.SlotIndex; + SlotSnapshot paused = ReadSlotSnapshot(slots, targetSlotIndex); + IndexBinding oldBinding = ValidatePausedTransition(transition, paused, targetSlotIndex); + Require( + pausedMutation == paused.DirectoryBinding, + $"{transition.Name}/{repetition}: canonical mutation no longer names the validated generation."); + ValidateSpillCheckpointState(transition, directory, paused.DirectoryBinding); + + RequireExactStatus( + directory.HelpMutation( + CanonicalBucket, + LockFreeOperationBudget.UnboundedScan, + maxSteps: 128), + StoreStatus.Success, + evidence, + $"{transition.Name}/{repetition}: independent helper completion"); + + if (transition.Operation == DelayedOperation.Insert) + { + // Adversarial generation-fencing injection. The live + // delayed writer means this intentionally violates the + // administrative override's process-wide quiescence + // precondition. Assertions below cover only stale-helper + // isolation and mapped-state convergence, not a supported + // public result contract for the delayed call. + StoreStatus recoveryStatus = helperStore.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: true), + StoreWaitOptions.Infinite, + out ReservationRecoveryReport recovery); + RequireExactStatus( + recoveryStatus, + StoreStatus.Success, + evidence, + $"{transition.Name}/{repetition}: recover delayed reservation"); + Require( + recovery.RecoveredReservationCount >= 1, + $"{transition.Name}/{repetition}: exact delayed reservation was not recovered."); + } + + helperValue[0] = newMarker; + // Keep completion/recovery and exact-slot reuse on distinct + // participants. This deterministically models the sustained + // three-or-more-process helper churn that exposed the delayed + // location-publication source-loss race. + StoreStatus republish = reuseStore.TryPublish( + newKey, + helperValue, + default, + StoreWaitOptions.Infinite); + if (republish == StoreStatus.StoreFull) + { + evidence.LeakedCapacityCount++; + } + + RequireExactStatus( + republish, + StoreStatus.Success, + evidence, + $"{transition.Name}/{repetition}: publish later generation"); + + DirectoryGenerationSnapshot beforeResume = ReadPublishedSnapshot( + directory, + slots, + newKey, + targetSlotIndex, + evidence, + $"{transition.Name}/{repetition}: before delayed resume"); + Require( + beforeResume.Binding.SlotIndex == oldBinding.SlotIndex + && beforeResume.Binding.Generation == oldBinding.Generation + 1, + $"{transition.Name}/{repetition}: helper did not reuse the exact slot at the immediately following generation."); + + controller.Continue(); + resumed = true; + StoreStatus delayedStatus = delayedActor.WaitUntilComplete(TimeSpan.FromSeconds(10)); + ValidateDelayedStatus(transition, delayedStatus, evidence, repetition); + + DirectoryGenerationSnapshot afterResume = ReadPublishedSnapshot( + directory, + slots, + newKey, + targetSlotIndex, + evidence, + $"{transition.Name}/{repetition}: after delayed resume"); + if (beforeResume != afterResume) + { + evidence.WrongGenerationMutationCount++; + throw new XunitException( + $"{transition.Name}/{repetition}: resumed helper changed the later generation. " + + $"before={beforeResume}; after={afterResume}."); + } + + RequireMissing( + reuseStore, + oldKey, + evidence, + $"{transition.Name}/{repetition}: old generation remained visible"); + RequirePublishedValue( + reuseStore, + newKey, + newMarker, + evidence, + $"{transition.Name}/{repetition}: later generation false miss or byte mismatch"); + if (transition.Target == DirectoryTarget.Overflow) + { + int anchorIndex = random.Next(OverflowAnchorCount); + RequirePublishedValue( + helperStore, + keys[anchorIndex], + unchecked((byte)anchorIndex), + evidence, + $"{transition.Name}/{repetition}: colliding anchor false miss"); + } + + RequireExactStatus( + reuseStore.TryRemove(newKey, StoreWaitOptions.Infinite), + StoreStatus.Success, + evidence, + $"{transition.Name}/{repetition}: remove later generation"); + Require( + directory.ReadCanonicalMutation(CanonicalBucket) == 0, + $"{transition.Name}/{repetition}: canonical mutation leaked after cleanup."); + if (transition.Target == DirectoryTarget.Overflow) + { + SpillSummary summary = SpillSummary.Decode( + directory.ReadSpillSummary(CanonicalBucket)); + Require( + !summary.IsPresent && directory.OverflowOccupancy == 0, + $"{transition.Name}/{repetition}: spill state leaked after cleanup."); + } + + evidence.ExecutedRepetitions++; + } + finally + { + if (!resumed) + { + controller.Continue(); + _ = delayedActor.TryWaitUntilComplete(TimeSpan.FromSeconds(10)); + } + } + } + + VerifyAllCapacityRecoverable(helperStore, directory, keys, transition, helperValue, evidence); + } + + private static void VerifyAllCapacityRecoverable( + MemoryStore store, + LockFreeKeyDirectory directory, + byte[][] keys, + TransitionCase transition, + byte[] value, + EvidenceCounters evidence) + { + if (transition.Target == DirectoryTarget.Overflow) + { + for (var index = 0; index < OverflowAnchorCount; index++) + { + RequireExactStatus( + store.TryRemove(keys[index], StoreWaitOptions.Infinite), + StoreStatus.Success, + evidence, + $"{transition.Name}: remove capacity anchor {index}"); + } + } + + int slotCount = transition.Target == DirectoryTarget.Overflow + ? OverflowSlotCount + : PrimarySlotCount; + for (var index = 0; index < slotCount; index++) + { + value[0] = unchecked((byte)(0x80 + index)); + StoreStatus publish = store.TryPublish( + keys[index], + value, + default, + StoreWaitOptions.Infinite); + if (publish == StoreStatus.StoreFull) + { + evidence.LeakedCapacityCount++; + } + + RequireExactStatus( + publish, + StoreStatus.Success, + evidence, + $"{transition.Name}: fill-to-capacity slot {index}"); + } + + RequireExactStatus( + store.TryPublish(keys[slotCount], value, default, StoreWaitOptions.Infinite), + StoreStatus.StoreFull, + evidence, + $"{transition.Name}: full-capacity sentinel"); + for (var index = 0; index < slotCount; index++) + { + RequirePublishedValue( + store, + keys[index], + unchecked((byte)(0x80 + index)), + evidence, + $"{transition.Name}: fill-to-capacity visibility {index}"); + RequireExactStatus( + store.TryRemove(keys[index], StoreWaitOptions.Infinite), + StoreStatus.Success, + evidence, + $"{transition.Name}: drain capacity slot {index}"); + } + + Require( + directory.PrimaryOccupancy == 0 + && directory.OverflowOccupancy == 0 + && directory.ReadCanonicalMutation(CanonicalBucket) == 0, + $"{transition.Name}: directory capacity did not drain to zero."); + if (transition.Target == DirectoryTarget.Overflow) + { + Require( + !SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)).IsPresent, + $"{transition.Name}: spill summary remained Present after full drain."); + } + } + + private static IndexBinding ValidatePausedTransition( + TransitionCase transition, + SlotSnapshot snapshot, + int expectedSlotIndex) + { + Require(snapshot.DirectoryBinding != 0, $"{transition.Name}: paused slot has no exact binding."); + IndexBinding binding = IndexBinding.Decode(snapshot.DirectoryBinding); + Require(binding.SlotIndex == expectedSlotIndex, $"{transition.Name}: paused the wrong value slot."); + Require( + binding.Generation == SlotGeneration(snapshot.Control), + $"{transition.Name}: slot control and binding generations disagree."); + Require(snapshot.DirectoryOperation != 0, $"{transition.Name}: paused slot has no directory operation."); + DirectoryOperation operation = DirectoryOperation.Decode( + unchecked((ulong)snapshot.DirectoryOperation)); + int expectedIntent = transition.Operation == DelayedOperation.Insert ? 1 : 2; + Require(operation.Intent == expectedIntent, $"{transition.Name}: unexpected mutation intent."); + Require( + operation.Phase == transition.ExpectedPhase, + $"{transition.Name}: expected mutation phase {transition.ExpectedPhase}, observed {operation.Phase}."); + Require(operation.Generation == binding.Generation, $"{transition.Name}: operation generation mismatch."); + if (transition.ExpectedPhase == 1) + { + Require(operation.Kind == 0, $"{transition.Name}: Prepared unexpectedly names a target."); + } + else + { + int expectedKind = transition.Target == DirectoryTarget.Overflow ? 2 : 1; + Require(operation.Kind == expectedKind, $"{transition.Name}: mutation names the wrong target section."); + } + + return binding; + } + + private static void ValidateSpillCheckpointState( + TransitionCase transition, + LockFreeKeyDirectory directory, + ulong oldBinding) + { + if (transition.Checkpoint == LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas) + { + SpillSummary summary = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Require(!summary.IsPresent, $"{transition.Name}: summary was already Present before its CAS."); + } + else if (transition.Checkpoint == LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication) + { + SpillSummary summary = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Require( + summary.IsPresent && summary.Binding == oldBinding, + $"{transition.Name}: Present summary does not identify the validated insertion."); + } + else if (transition.Checkpoint == LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan) + { + SpillSummary summary = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Require( + summary.IsPresent && summary.Binding == oldBinding && directory.OverflowOccupancy == 0, + $"{transition.Name}: empty-scan checkpoint did not retain the captured Present token."); + } + else if (transition.Checkpoint == LockFreeCheckpointId.DirectoryAfterSpillSummaryClear) + { + SpillSummary summary = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Require( + !summary.IsPresent && summary.Binding == oldBinding && directory.OverflowOccupancy == 0, + $"{transition.Name}: versioned clear did not preserve the exact old identity."); + } + } + + private static DirectoryGenerationSnapshot ReadPublishedSnapshot( + LockFreeKeyDirectory directory, + LockFreeSlotTable slots, + byte[] key, + int expectedSlotIndex, + EvidenceCounters evidence, + string context) + { + StoreStatus lookup = directory.TryLookup( + key, + StoreKey.Hash(key), + LockFreeOperationBudget.UnboundedScan, + out ulong rawBinding, + out DirectoryLocation location); + if (lookup == StoreStatus.CorruptStore) + { + evidence.CorruptionCount++; + } + else if (lookup == StoreStatus.NotFound) + { + evidence.FalseMissCount++; + } + + RequireExactStatus(lookup, StoreStatus.Success, evidence, context); + IndexBinding binding = IndexBinding.Decode(rawBinding); + Require(binding.SlotIndex == expectedSlotIndex, $"{context}: lookup resolved a different slot."); + SlotSnapshot slot = ReadSlotSnapshot(slots, expectedSlotIndex); + Require( + slot.DirectoryBinding == rawBinding + && SlotGeneration(slot.Control) == binding.Generation + && location.Generation == binding.Generation, + $"{context}: later-generation binding/location/control are inconsistent."); + return new DirectoryGenerationSnapshot( + binding, + location.Value, + slot, + directory.ReadSpillSummary(CanonicalBucket), + directory.ReadCanonicalMutation(CanonicalBucket)); + } + + private static SlotSnapshot ReadSlotSnapshot(LockFreeSlotTable slots, int slotIndex) + { + for (var attempt = 0; attempt < 8; attempt++) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + long control1 = AtomicControlWord.LoadAcquire(ref slot.Control); + var snapshot = new SlotSnapshot( + control1, + slot.DirectoryBinding, + AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation), + AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation), + slot.KeyHash, + slot.KeyLength, + slot.DescriptorLength, + slot.ValueLength, + AtomicControlWord.LoadAcquire(ref slot.BytesAdvanced), + slot.CommitSequence, + slot.KeyOffset, + slot.DescriptorOffset, + slot.PayloadOffset); + if (control1 == AtomicControlWord.LoadAcquire(ref slot.Control)) + { + return snapshot; + } + } + + throw new XunitException($"Slot {slotIndex} did not stabilize for an exact generation snapshot."); + } + + private static void ValidateDelayedStatus( + TransitionCase transition, + StoreStatus status, + EvidenceCounters evidence, + int repetition) + { + if (status == StoreStatus.CorruptStore) + { + evidence.CorruptionCount++; + } + + bool allowed = transition.Operation == DelayedOperation.Insert + ? status != StoreStatus.CorruptStore + : status is StoreStatus.Success or StoreStatus.NotFound; + Require(allowed, $"{transition.Name}/{repetition}: delayed actor returned {status}."); + } + + private static void RequirePublishedValue( + MemoryStore store, + byte[] key, + byte expected, + EvidenceCounters evidence, + string context) + { + StoreStatus acquire = store.TryAcquire(key, StoreWaitOptions.Infinite, out ValueLease lease); + if (acquire == StoreStatus.NotFound) + { + evidence.FalseMissCount++; + } + else if (acquire == StoreStatus.CorruptStore) + { + evidence.CorruptionCount++; + } + + RequireExactStatus(acquire, StoreStatus.Success, evidence, context); + try + { + Require(lease.ValueSpan.Length == 1 && lease.ValueSpan[0] == expected, context); + } + finally + { + RequireExactStatus(lease.Release(), StoreStatus.Success, evidence, $"{context}: release"); + } + } + + private static void RequireMissing( + MemoryStore store, + byte[] key, + EvidenceCounters evidence, + string context) + { + StoreStatus status = store.TryAcquire(key, StoreWaitOptions.Infinite, out ValueLease lease); + if (status == StoreStatus.CorruptStore) + { + evidence.CorruptionCount++; + } + + if (status == StoreStatus.Success) + { + _ = lease.Release(); + } + + RequireExactStatus(status, StoreStatus.NotFound, evidence, context); + } + + private static void RequireExactStatus( + StoreStatus actual, + StoreStatus expected, + EvidenceCounters evidence, + string context) + { + if (actual == StoreStatus.CorruptStore) + { + evidence.CorruptionCount++; + } + + Require(actual == expected, $"{context}: expected {expected}, observed {actual}."); + } + + private static void Require(bool condition, string message) + { + if (!condition) + { + throw new XunitException(message); + } + } + + private static MemoryStore CreateInstrumentedStore( + string name, + int slotCount, + CheckpointPauseController controller) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, slotCount, OpenMode.CreateNew), + LockFreeCheckpointFactory.CreateInstrumented(controller.Observe), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore OpenHelperStore(string name, int slotCount) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, slotCount, OpenMode.OpenExisting), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options(string name, int slotCount, OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: sizeof(long), + leaseRecordCount: Math.Max(2, slotCount), + participantRecordCount: 4, + openMode, + enableLeaseRecovery: true); + + private static LockFreeKeyDirectory ReadDirectory(MemoryStore store) + { + object engine = ReadEngine(store); + FieldInfo field = engine.GetType().GetField( + "_directory", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new XunitException("Lock-free engine._directory is absent."); + return Assert.IsType(field.GetValue(engine)); + } + + private static LockFreeSlotTable ReadSlots(MemoryStore store) + { + object engine = ReadEngine(store); + FieldInfo field = engine.GetType().GetField( + "_slots", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new XunitException("Lock-free engine._slots is absent."); + return Assert.IsType(field.GetValue(engine)); + } + + private static object ReadEngine(MemoryStore store) + { + FieldInfo field = typeof(MemoryStore).GetField( + "_engine", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new XunitException("MemoryStore._engine is absent."); + return field.GetValue(store) ?? throw new XunitException("MemoryStore._engine is null."); + } + + private static byte[][] GenerateBucketPairCollisions(int count, int slotCount) + { + var keys = new List(count); + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + uint bucketMask = checked((uint)((primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + for (long candidate = 1; keys.Count < count; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + ulong hash = StoreKey.Hash(key); + int first = (int)(Mix(hash) & bucketMask); + int second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & bucketMask); + if (second == first) + { + second = (first + 1) & (int)bucketMask; + } + + if (first == CanonicalBucket && second == 1) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static int NextPowerOfTwo(int value) + { + var result = 1; + while (result < value) + { + result <<= 1; + } + + return result; + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static int ReadRepetitions() + { + string? configured = Environment.GetEnvironmentVariable(RepetitionsVariable); + int repetitions = string.IsNullOrWhiteSpace(configured) + ? TransitionCases.Length + : int.Parse(configured, NumberStyles.None, CultureInfo.InvariantCulture); + if (repetitions < TransitionCases.Length) + { + throw new XunitException( + $"{RepetitionsVariable} must be at least {TransitionCases.Length} so every transition executes."); + } + + return repetitions; + } + + private static ulong ReadSeed() + { + string? configured = Environment.GetEnvironmentVariable(SeedVariable); + if (string.IsNullOrWhiteSpace(configured)) + { + return DefaultSeed; + } + + return configured.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + ? ulong.Parse(configured.AsSpan(2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture) + : ulong.Parse(configured, NumberStyles.None, CultureInfo.InvariantCulture); + } + + private static long SlotGeneration(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private static TransitionCase Operation( + string name, + DelayedOperation operation, + DirectoryTarget target, + int occurrence, + int phase) => + new( + name, + operation, + target, + LockFreeCheckpointId.DirectoryAfterOperationValidation, + occurrence, + phase); + + private static TransitionCase Location( + string name, + DirectoryTarget target, + int occurrence, + int phase) => + new( + name, + DelayedOperation.Unlink, + target, + LockFreeCheckpointId.DirectoryAfterLocationValidation, + occurrence, + phase); + + private static TransitionCase OperationRevalidation( + string name, + DelayedOperation operation, + DirectoryTarget target, + int occurrence, + int phase) => + new( + name, + operation, + target, + LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch, + occurrence, + phase); + + private static TransitionCase InsertBindingChangedStateValidation( + string name, + DirectoryTarget target) => + new( + name, + DelayedOperation.Insert, + target, + LockFreeCheckpointId.DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication, + CheckpointOccurrence: 1, + ExpectedPhase: 3); + + private static TransitionCase UnlinkLocationRead( + string name, + DirectoryTarget target) => + new( + name, + DelayedOperation.Unlink, + target, + LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead, + CheckpointOccurrence: 1, + ExpectedPhase: 1); + + private static TransitionCase LocationPublication( + string name, + DirectoryTarget target, + LockFreeCheckpointId checkpoint = + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation) => + new( + name, + DelayedOperation.Insert, + target, + checkpoint, + CheckpointOccurrence: 1, + ExpectedPhase: 2); + + private static TransitionCase Spill( + string name, + DelayedOperation operation, + LockFreeCheckpointId checkpoint, + int phase) => + new(name, operation, DirectoryTarget.Overflow, checkpoint, 1, phase); + + private sealed class EvidenceCounters + { + internal long ExecutedRepetitions; + internal long WrongGenerationMutationCount; + internal long CorruptionCount; + internal long FalseMissCount; + internal long LeakedCapacityCount; + } + + private sealed class CheckpointPauseController : IDisposable + { + private readonly object _sync = new(); + private readonly ManualResetEventSlim _paused = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: true); + private LockFreeCheckpointId _target; + private int _targetOccurrence; + private int _observedOccurrences; + private bool _armed; + private bool _disposed; + + internal void Arm(LockFreeCheckpointId checkpoint, int occurrence) + { + lock (_sync) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_armed) + { + throw new InvalidOperationException("A checkpoint pause is already armed."); + } + + _target = checkpoint; + _targetOccurrence = occurrence; + _observedOccurrences = 0; + _paused.Reset(); + _resume.Reset(); + _armed = true; + } + } + + internal void Observe(LockFreeCheckpointEntry entry) + { + bool pause = false; + lock (_sync) + { + if (!_disposed && _armed && entry.Id == _target) + { + _observedOccurrences++; + if (_observedOccurrences == _targetOccurrence) + { + _armed = false; + pause = true; + } + } + } + + if (pause) + { + _paused.Set(); + _resume.Wait(); + } + } + + internal bool WaitUntilPaused(TimeSpan timeout) => _paused.Wait(timeout); + + internal void Continue() => _resume.Set(); + + public void Dispose() + { + lock (_sync) + { + if (_disposed) + { + return; + } + + _disposed = true; + _armed = false; + _resume.Set(); + _paused.Set(); + } + + _paused.Dispose(); + _resume.Dispose(); + } + } + + private sealed class DelayedActor : IDisposable + { + private readonly MemoryStore _store; + private readonly ManualResetEventSlim _start = new(initialState: false); + private readonly ManualResetEventSlim _complete = new(initialState: true); + private readonly Thread _thread; + private readonly byte[] _value = new byte[1]; + private DelayedOperation _operation; + private byte[] _key = []; + private byte _marker; + private StoreStatus _status; + private Exception? _exception; + private bool _stop; + private bool _disposed; + + internal DelayedActor(MemoryStore store) + { + _store = store; + _thread = new Thread(Run) + { + IsBackground = true, + Name = "SharedMemoryStore.SC017.DelayedActor" + }; + _thread.Start(); + } + + internal void Start(DelayedOperation operation, byte[] key, byte marker) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (!_complete.IsSet) + { + throw new InvalidOperationException("The delayed actor is already running."); + } + + _operation = operation; + _key = key; + _marker = marker; + _exception = null; + _complete.Reset(); + _start.Set(); + } + + internal StoreStatus WaitUntilComplete(TimeSpan timeout) + { + if (!_complete.Wait(timeout)) + { + throw new XunitException("The delayed SC-017 actor did not complete within the test bound."); + } + + if (_exception is not null) + { + throw new XunitException($"The delayed SC-017 actor failed: {_exception}"); + } + + return _status; + } + + internal bool TryWaitUntilComplete(TimeSpan timeout) => _complete.Wait(timeout); + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _stop = true; + _start.Set(); + if (!_thread.Join(TimeSpan.FromSeconds(10))) + { + throw new XunitException("The delayed SC-017 actor thread did not stop."); + } + + _start.Dispose(); + _complete.Dispose(); + } + + private void Run() + { + while (true) + { + _start.Wait(); + _start.Reset(); + if (_stop) + { + return; + } + + try + { + _value[0] = _marker; + _status = _operation == DelayedOperation.Insert + ? _store.TryPublish(_key, _value, default, StoreWaitOptions.Infinite) + : _store.TryRemove(_key, StoreWaitOptions.Infinite); + } + catch (Exception exception) + { + _exception = exception; + } + finally + { + _complete.Set(); + } + } + } + } + + private struct DeterministicRandom + { + private ulong _state; + + internal DeterministicRandom(ulong seed) + { + _state = seed == 0 ? 0xD1B5_4A32_D192_ED03UL : seed; + } + + internal int Next(int exclusiveMaximum) => + checked((int)(NextUInt64() % checked((uint)exclusiveMaximum))); + + internal ulong NextUInt64() + { + ulong value = _state; + value ^= value >> 12; + value ^= value << 25; + value ^= value >> 27; + _state = value; + return value * 0x2545_F491_4F6C_DD1DUL; + } + } + + private readonly record struct TransitionCase( + string Name, + DelayedOperation Operation, + DirectoryTarget Target, + LockFreeCheckpointId Checkpoint, + int CheckpointOccurrence, + int ExpectedPhase); + + private readonly record struct SlotSnapshot( + long Control, + ulong DirectoryBinding, + long DirectoryLocation, + long DirectoryOperation, + ulong KeyHash, + int KeyLength, + int DescriptorLength, + int ValueLength, + long BytesAdvanced, + long CommitSequence, + long KeyOffset, + long DescriptorOffset, + long PayloadOffset); + + private readonly record struct DirectoryGenerationSnapshot( + IndexBinding Binding, + ulong Location, + SlotSnapshot Slot, + ulong SpillSummary, + ulong CanonicalMutation); + + private enum DelayedOperation + { + Insert, + Unlink + } + + private enum DirectoryTarget + { + Primary, + Overflow + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryReferenceRevalidationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryReferenceRevalidationTests.cs new file mode 100644 index 0000000..99bf65a --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryReferenceRevalidationTests.cs @@ -0,0 +1,715 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Interop; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeDirectoryReferenceRevalidationTests +{ + private const int TargetPrimary = 1; + private const int TargetOverflow = 2; + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(10); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CachedExactLookupRemovedBeforeSlotClassificationRetriesWithoutCorruption( + bool useOverflow) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + int slotCount = useOverflow ? 20 : 2; + string name = $"sms-v2-lookup-witness-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions createOptions = Options( + name, + slotCount, + OpenMode.CreateNew); + SharedMemoryStoreOptions openOptions = Options( + name, + slotCount, + OpenMode.OpenExisting); + StoreLayoutV2 layout = StoreLayoutV2.FromOptions(createOptions); + byte[][] keys = useOverflow + ? GenerateBucketPairCollisions(count: 17, layout) + : [[0x51]]; + byte[] targetKey = keys[^1]; + + using var lookupScheduler = new ControlledLockFreeScheduler(); + using var unlinkScheduler = new ControlledLockFreeScheduler(); + using MemoryStore lookupStore = OpenInstrumented(createOptions, lookupScheduler); + using MemoryStore unlinkStore = OpenInstrumented(openOptions, unlinkScheduler); + for (var index = 0; index < keys.Length; index++) + { + Assert.Equal( + StoreStatus.Success, + lookupStore.TryPublish(keys[index], [unchecked((byte)(0x20 + index))])); + } + + StoreInternals internals = ReadInternals(lookupStore); + DirectoryEntry oldEntry = FindEntry(internals.Directory, targetKey); + Assert.Equal(useOverflow ? TargetOverflow : TargetPrimary, oldEntry.Location.Kind); + IndexBinding oldBinding = IndexBinding.Decode(oldEntry.Binding); + + lookupScheduler.PauseAt(LockFreeCheckpointId.ReserveAfterExistingLookup); + Task republish = PublishAsync(lookupStore, targetKey, 0xE1); + Assert.True( + lookupScheduler.WaitUntilPaused(TestTimeout), + "The publisher did not pause after returning an exact lookup witness."); + + unlinkScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterUnlinkDescriptorClearBeforeGenerationAdvance); + Task remove = RemoveAsync(unlinkStore, targetKey); + Assert.True( + unlinkScheduler.WaitUntilPaused(TestTimeout), + "The unlink helper did not pause after winning the descriptor clear."); + + try + { + ref ValueSlotMetadataV2 oldSlot = ref internals.Slots.Slot(oldBinding.SlotIndex); + Assert.Equal( + LockFreeSlotTable.ReclaimingState, + (int)(unchecked((ulong)AtomicControlWord.LoadAcquire(ref oldSlot.Control)) & 0x7UL)); + Assert.Equal(0, AtomicControlWord.LoadAcquire(ref oldSlot.DirectoryOperation)); + Assert.Equal(0, AtomicControlWord.LoadAcquire(ref oldSlot.DirectoryLocation)); + Assert.Equal(0UL, ReadCell(internals, oldEntry.Location)); + + lookupScheduler.Continue(); + OperationResult publishResult = await republish.WaitAsync(TestTimeout); + Assert.Equal(StoreStatus.Success, publishResult.Status); + Assert.Null(publishResult.CorruptionOrigin); + } + finally + { + lookupScheduler.Continue(); + unlinkScheduler.Continue(); + } + + OperationResult removeResult = await remove.WaitAsync(TestTimeout); + Assert.NotEqual(StoreStatus.CorruptStore, removeResult.Status); + Assert.Null(removeResult.CorruptionOrigin); + DirectoryEntry currentEntry = FindEntry(internals.Directory, targetKey); + Assert.NotEqual(oldEntry.Binding, currentEntry.Binding); + Assert.NotEqual( + oldBinding.SlotIndex, + IndexBinding.Decode(currentEntry.Binding).SlotIndex); + Assert.Equal(StoreStatus.Success, lookupStore.TryAcquire(targetKey, out ValueLease current)); + Assert.Equal(0xE1, current.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, current.Release()); + } + + [Fact] + public void StableInvalidSlotMetadataWithLiveExactSourceStillFailsClosed() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = OpenInstrumented( + Options( + $"sms-v2-live-invalid-witness-{Guid.NewGuid():N}", + slotCount: 2, + OpenMode.CreateNew), + scheduler); + byte[] key = [0x61]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x71])); + StoreInternals internals = ReadInternals(store); + DirectoryEntry entry = FindEntry(internals.Directory, key); + IndexBinding binding = IndexBinding.Decode(entry.Binding); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(binding.SlotIndex); + long validOperation = AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation); + Assert.NotEqual(0, validOperation); + + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, 0); + StoreStatus status; + try + { + status = store.TryPublish( + key, + [0x72], + descriptor: default, + StoreWaitOptions.Infinite); + } + finally + { + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, validOperation); + } + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Equal(entry.Binding, ReadCell(internals, entry.Location)); + AssertMappingLatched(internals); + Assert.Equal(StoreStatus.CorruptStore, store.TryAcquire(key, out _)); + } + + [Fact] + public async Task ChangedInvalidPrimaryReferenceIsRetriedWithoutCorruption() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var controller = new InvalidReferenceController(); + using MemoryStore store = CreateInstrumentedStore(slotCount: 4, controller); + StoreInternals internals = ReadInternals(store); + byte[][] keys = GenerateBucketPairCollisions(count: 2, internals.Layout); + + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[0], [0x11])); + DirectoryEntry anchor = FindEntry(internals.Directory, keys[0]); + Assert.Equal(TargetPrimary, anchor.Location.Kind); + ulong invalid = NextGeneration(anchor.Binding); + controller.Arm( + LockFreeCheckpointId.ReserveBeforeSlotClaim, + () => WriteCell(internals, anchor.Location, invalid)); + + Task publish = PublishAsync(store, keys[1], 0x12); + Assert.True(controller.WaitUntilPaused(TestTimeout)); + try + { + Assert.Equal(invalid, ReadCell(internals, anchor.Location)); + WriteCell(internals, anchor.Location, anchor.Binding); + } + finally + { + controller.Continue(); + } + + OperationResult result = await publish.WaitAsync(TestTimeout); + Assert.Equal(StoreStatus.Success, result.Status); + Assert.Null(result.CorruptionOrigin); + Assert.Equal(anchor.Binding, ReadCell(internals, anchor.Location)); + AssertAcquirable(store, keys[0]); + AssertAcquirable(store, keys[1]); + } + + [Fact] + public async Task ChangedInvalidOverflowReferenceIsRetriedWithoutCorruption() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + const int slotCount = 20; + using var controller = new InvalidReferenceController(); + using MemoryStore store = CreateInstrumentedStore(slotCount, controller); + StoreInternals internals = ReadInternals(store); + byte[][] keys = GenerateBucketPairCollisions(count: 18, internals.Layout); + for (var index = 0; index < 17; index++) + { + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[index], [unchecked((byte)(0x20 + index))])); + } + + DirectoryEntry anchor = FindEntry(internals.Directory, keys[16]); + Assert.Equal(TargetOverflow, anchor.Location.Kind); + ulong invalid = NextGeneration(anchor.Binding); + controller.Arm( + LockFreeCheckpointId.ReserveBeforeSlotClaim, + () => WriteCell(internals, anchor.Location, invalid)); + + Task publish = PublishAsync(store, keys[17], 0x41); + Assert.True(controller.WaitUntilPaused(TestTimeout)); + try + { + Assert.Equal(invalid, ReadCell(internals, anchor.Location)); + WriteCell(internals, anchor.Location, anchor.Binding); + } + finally + { + controller.Continue(); + } + + OperationResult result = await publish.WaitAsync(TestTimeout); + Assert.Equal(StoreStatus.Success, result.Status); + Assert.Null(result.CorruptionOrigin); + Assert.Equal(anchor.Binding, ReadCell(internals, anchor.Location)); + AssertAcquirable(store, keys[16]); + AssertAcquirable(store, keys[17]); + } + + [Fact] + public async Task ChangedInvalidSpillSummaryReferenceIsRetriedWithoutCorruption() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + const int slotCount = 20; + using var controller = new InvalidReferenceController(); + using MemoryStore store = CreateInstrumentedStore(slotCount, controller); + StoreInternals internals = ReadInternals(store); + byte[][] keys = GenerateBucketPairCollisions(count: 18, internals.Layout); + for (var index = 0; index < keys.Length; index++) + { + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[index], [unchecked((byte)(0x50 + index))])); + } + + DirectoryEntry retained = FindEntry(internals.Directory, keys[16]); + DirectoryEntry removed = FindEntry(internals.Directory, keys[17]); + Assert.Equal(TargetOverflow, retained.Location.Kind); + Assert.Equal(TargetOverflow, removed.Location.Kind); + int canonicalBucket = CanonicalBucket(StoreKey.Hash(keys[16]), internals.Layout.PrimaryBucketCount); + ulong invalidSummary = SpillSummary.EncodePresent(NextGeneration(retained.Binding)); + ulong replacementSummary = SpillSummary.EncodePresent(retained.Binding); + controller.Arm( + LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas, + () => WriteSpillSummary(internals, canonicalBucket, invalidSummary)); + + Task remove = RemoveAsync(store, keys[17]); + Assert.True(controller.WaitUntilPaused(TestTimeout)); + try + { + Assert.Equal(invalidSummary, ReadSpillSummary(internals, canonicalBucket)); + WriteSpillSummary(internals, canonicalBucket, replacementSummary); + } + finally + { + controller.Continue(); + } + + OperationResult result = await remove.WaitAsync(TestTimeout); + Assert.Equal(StoreStatus.Success, result.Status); + Assert.Null(result.CorruptionOrigin); + Assert.Equal(replacementSummary, ReadSpillSummary(internals, canonicalBucket)); + AssertAcquirable(store, keys[16]); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(keys[17], out _)); + } + + [Fact] + public async Task UnchangedInvalidReferenceStillFailsClosed() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var controller = new InvalidReferenceController(); + using MemoryStore store = CreateInstrumentedStore(slotCount: 4, controller); + StoreInternals internals = ReadInternals(store); + byte[][] keys = GenerateBucketPairCollisions(count: 2, internals.Layout); + + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[0], [0x71])); + DirectoryEntry anchor = FindEntry(internals.Directory, keys[0]); + ulong invalid = NextGeneration(anchor.Binding); + controller.Arm( + LockFreeCheckpointId.ReserveBeforeSlotClaim, + () => WriteCell(internals, anchor.Location, invalid)); + + Task publish = PublishAsync(store, keys[1], 0x72); + Assert.True(controller.WaitUntilPaused(TestTimeout)); + try + { + Assert.Equal(invalid, ReadCell(internals, anchor.Location)); + } + finally + { + controller.Continue(); + } + + OperationResult result = await publish.WaitAsync(TestTimeout); + Assert.Equal(StoreStatus.CorruptStore, result.Status); + Assert.NotNull(result.CorruptionOrigin); + Assert.Equal(invalid, ReadCell(internals, anchor.Location)); + } + + [Theory] + [InlineData(MalformedOwnerShape.OutOfRangeInitializing)] + [InlineData(MalformedOwnerShape.OutOfRangeReserved)] + [InlineData(MalformedOwnerShape.OwnedPublished)] + [InlineData(MalformedOwnerShape.OwnedRemoveRequested)] + [InlineData(MalformedOwnerShape.OwnedAborting)] + [InlineData(MalformedOwnerShape.OwnedReclaiming)] + [InlineData(MalformedOwnerShape.OwnedRetired)] + [InlineData(MalformedOwnerShape.NonterminalRetired)] + public void StableMalformedSlotOwnerShapeFailsClosedOnEveryLookupPath( + MalformedOwnerShape malformedShape) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var controller = new InvalidReferenceController(); + using MemoryStore store = CreateInstrumentedStore(slotCount: 2, controller); + byte[] key = [0x73]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x83])); + + StoreInternals internals = ReadInternals(store); + DirectoryEntry entry = FindEntry(internals.Directory, key); + IndexBinding binding = IndexBinding.Decode(entry.Binding); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(binding.SlotIndex); + long originalControl = AtomicControlWord.LoadAcquire(ref slot.Control); + long malformedControl = MalformedControl(malformedShape, binding.Generation); + + AtomicControlWord.StoreRelease(ref slot.Control, malformedControl); + try + { + Assert.Equal(StoreStatus.CorruptStore, store.TryAcquire(key, out _)); + + // Each operation receives the same exact live directory witness. + // Reinstalling it here also makes this assertion distinguish a + // fail-closed result from a classifier that silently erased the + // malformed reference as ordinary stale state. + WriteCell(internals, entry.Location, entry.Binding); + Assert.Equal(StoreStatus.CorruptStore, store.TryRemove(key)); + + WriteCell(internals, entry.Location, entry.Binding); + Assert.Equal( + StoreStatus.CorruptStore, + internals.Directory.TryLookup( + key, + StoreKey.Hash(key), + out _, + out _)); + Assert.Equal(entry.Binding, ReadCell(internals, entry.Location)); + } + finally + { + WriteCell(internals, entry.Location, entry.Binding); + AtomicControlWord.StoreRelease(ref slot.Control, originalControl); + } + + AssertMappingLatched(internals); + Assert.Equal(StoreStatus.CorruptStore, store.TryAcquire(key, out _)); + } + + private static Task PublishAsync(MemoryStore store, byte[] key, byte value) => + Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + StoreStatus status = store.TryPublish( + key, + [value], + descriptor: default, + StoreWaitOptions.Infinite); + return new OperationResult(status, LockFreeCorruptionTrace.Consume()); + }); + + private static Task RemoveAsync(MemoryStore store, byte[] key) => + Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + StoreStatus status = store.TryRemove(key, StoreWaitOptions.Infinite); + return new OperationResult(status, LockFreeCorruptionTrace.Consume()); + }); + + private static void AssertAcquirable(MemoryStore store, byte[] key) + { + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + private static unsafe void AssertMappingLatched(StoreInternals internals) => + Assert.Equal( + LayoutV2Constants.StoreCorrupt, + AtomicControlWord.LoadAcquire( + ref ((StoreHeaderV2*)internals.Region.Pointer)->Control)); + + private static DirectoryEntry FindEntry(LockFreeKeyDirectory directory, byte[] key) + { + StoreStatus status = directory.TryLookup( + key, + StoreKey.Hash(key), + out ulong binding, + out DirectoryLocation location); + Assert.Equal(StoreStatus.Success, status); + return new DirectoryEntry(binding, location); + } + + private static ulong NextGeneration(ulong binding) + { + IndexBinding decoded = IndexBinding.Decode(binding); + return IndexBinding.Encode(decoded.SlotIndex, checked(decoded.Generation + 1)); + } + + private static long MalformedControl(MalformedOwnerShape shape, long generation) + { + const int malformedParticipant = 13; + Assert.False(ParticipantToken.IsStructurallyValid(malformedParticipant, 4)); + int validParticipant = checked((int)ParticipantToken.Encode( + recordIndex: 0, + generation: 1, + participantCount: 4)); + Assert.True(ParticipantToken.IsStructurallyValid( + unchecked((ulong)validParticipant), + 4)); + + (int state, long controlGeneration, int participant) = shape switch + { + MalformedOwnerShape.OutOfRangeInitializing => + (LockFreeSlotTable.InitializingState, generation, malformedParticipant), + MalformedOwnerShape.OutOfRangeReserved => + (LockFreeSlotTable.ReservedState, generation, malformedParticipant), + MalformedOwnerShape.OwnedPublished => + (LockFreeSlotTable.PublishedState, generation, validParticipant), + MalformedOwnerShape.OwnedRemoveRequested => + (LockFreeSlotTable.RemoveRequestedState, generation, validParticipant), + MalformedOwnerShape.OwnedAborting => + (LockFreeSlotTable.AbortingState, generation, validParticipant), + MalformedOwnerShape.OwnedReclaiming => + (LockFreeSlotTable.ReclaimingState, generation, validParticipant), + MalformedOwnerShape.OwnedRetired => + ( + LockFreeSlotTable.RetiredState, + LockFreeSlotTable.TerminalGeneration, + validParticipant), + MalformedOwnerShape.NonterminalRetired => + (LockFreeSlotTable.RetiredState, generation, 0), + _ => throw new ArgumentOutOfRangeException(nameof(shape)), + }; + return unchecked((long)AtomicControlWord.EncodeSlot( + state, + controlGeneration, + participant)); + } + + private static ulong ReadCell(StoreInternals internals, DirectoryLocation location) => + unchecked((ulong)AtomicControlWord.LoadAcquire(ref Cell(internals, location))); + + private static void WriteCell(StoreInternals internals, DirectoryLocation location, ulong binding) => + AtomicControlWord.StoreRelease(ref Cell(internals, location), unchecked((long)binding)); + + private static unsafe ref long Cell(StoreInternals internals, DirectoryLocation location) + { + long offset = location.Kind switch + { + TargetPrimary => PrimaryCellOffset(internals.Layout, checked((int)location.Index)), + TargetOverflow => internals.Layout.OverflowDirectoryOffset + + (location.Index * internals.Layout.OverflowStride), + _ => throw new ArgumentOutOfRangeException(nameof(location)), + }; + return ref *(long*)(internals.Region.Pointer + offset); + } + + private static long PrimaryCellOffset(StoreLayoutV2 layout, int absoluteCellIndex) + { + int bucket = absoluteCellIndex / LayoutV2Constants.PrimaryLanesPerBucket; + int lane = absoluteCellIndex % LayoutV2Constants.PrimaryLanesPerBucket; + return layout.PrimaryDirectoryOffset + + ((long)bucket * layout.PrimaryBucketStride) + + 16 + + (lane * sizeof(long)); + } + + private static ulong ReadSpillSummary(StoreInternals internals, int canonicalBucket) => + unchecked((ulong)AtomicControlWord.LoadAcquire(ref SpillSummaryWord(internals, canonicalBucket))); + + private static void WriteSpillSummary( + StoreInternals internals, + int canonicalBucket, + ulong raw) => + AtomicControlWord.StoreRelease( + ref SpillSummaryWord(internals, canonicalBucket), + unchecked((long)raw)); + + private static unsafe ref long SpillSummaryWord(StoreInternals internals, int canonicalBucket) + { + long offset = internals.Layout.PrimaryDirectoryOffset + + ((long)canonicalBucket * internals.Layout.PrimaryBucketStride); + return ref *(long*)(internals.Region.Pointer + offset); + } + + private static MemoryStore CreateInstrumentedStore( + int slotCount, + InvalidReferenceController controller) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-invalid-reference-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: slotCount, + participantRecordCount: 4, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + controller.CreateCheckpoint(), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options( + string name, + int slotCount, + OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: slotCount, + participantRecordCount: 4, + openMode, + enableLeaseRecovery: true); + + private static MemoryStore OpenInstrumented( + SharedMemoryStoreOptions options, + ControlledLockFreeScheduler scheduler) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + scheduler.CreateInstrumentedCheckpoint(), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static StoreInternals ReadInternals(MemoryStore store) + { + object engine = ReadPrivate(store, "_engine"); + return new StoreInternals( + ReadPrivate(engine, "_directory"), + ReadPrivate(engine, "_slots"), + ReadPrivate(engine, "_region"), + ReadPrivate(engine, "_layout")); + } + + private static T ReadPrivate(object owner, string fieldName) + { + FieldInfo field = owner.GetType().GetField( + fieldName, + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Missing field {owner.GetType().FullName}.{fieldName}."); + return Assert.IsAssignableFrom(field.GetValue(owner)); + } + + private static byte[][] GenerateBucketPairCollisions(int count, StoreLayoutV2 layout) + { + var keys = new List(count); + uint bucketMask = checked((uint)(layout.PrimaryBucketCount - 1)); + for (long candidate = 1; keys.Count < count; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + ulong hash = StoreKey.Hash(key); + int first = (int)(Mix(hash) & bucketMask); + int second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & bucketMask); + if (second == first) + { + second = (first + 1) & (int)bucketMask; + } + + if (first == 0 && second == 1) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static int CanonicalBucket(ulong hash, int bucketCount) => + (int)(Mix(hash) & checked((uint)(bucketCount - 1))); + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed class InvalidReferenceController : IDisposable + { + private readonly ManualResetEventSlim _paused = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: false); + private LockFreeCheckpointId _mutationCheckpoint; + private Action? _mutation; + private int _mutationApplied; + private int _revalidationReached; + private bool _disposed; + + internal InstrumentedLockFreeCheckpoint CreateCheckpoint() => + LockFreeCheckpointFactory.CreateInstrumented(Observe); + + internal void Arm(LockFreeCheckpointId mutationCheckpoint, Action mutation) + { + ObjectDisposedException.ThrowIf(_disposed, this); + _mutationCheckpoint = mutationCheckpoint; + _mutation = mutation; + } + + internal bool WaitUntilPaused(TimeSpan timeout) => _paused.Wait(timeout); + + internal void Continue() => _resume.Set(); + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _resume.Set(); + _paused.Set(); + _resume.Dispose(); + _paused.Dispose(); + } + + private void Observe(LockFreeCheckpointEntry entry) + { + if (entry.Id == _mutationCheckpoint + && Interlocked.CompareExchange(ref _mutationApplied, 1, 0) == 0) + { + (_mutation ?? throw new InvalidOperationException("The mutation is not armed."))(); + } + + if (entry.Id + != LockFreeCheckpointId.DirectoryAfterInvalidReferenceConfirmationBeforeBindingRevalidation + || Interlocked.CompareExchange(ref _revalidationReached, 1, 0) != 0) + { + return; + } + + _paused.Set(); + if (!_resume.Wait(TestTimeout)) + { + throw new TimeoutException("Invalid-reference revalidation was not resumed."); + } + } + } + + private readonly record struct StoreInternals( + LockFreeKeyDirectory Directory, + LockFreeSlotTable Slots, + MemoryMappedStoreRegion Region, + StoreLayoutV2 Layout); + + private readonly record struct DirectoryEntry(ulong Binding, DirectoryLocation Location); + + private readonly record struct OperationResult(StoreStatus Status, string? CorruptionOrigin); + + public enum MalformedOwnerShape + { + OutOfRangeInitializing, + OutOfRangeReserved, + OwnedPublished, + OwnedRemoveRequested, + OwnedAborting, + OwnedReclaiming, + OwnedRetired, + NonterminalRetired, + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryStateTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryStateTests.cs new file mode 100644 index 0000000..df1bd53 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeDirectoryStateTests.cs @@ -0,0 +1,83 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeDirectoryStateTests +{ + [Fact] + public async Task ConcurrentSameKeyPublishCallsHaveOneCurrentWinner() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-unit-publish-history-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 4, + participantRecordCount: 4, + openMode: OpenMode.CreateNew); + var openStatus = Store.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, openStatus); + using var ownedStore = Assert.IsType(store); + + var statuses = new StoreStatus[2]; + using var start = new Barrier(participantCount: 3); + var first = Task.Run(() => + { + start.SignalAndWait(); + statuses[0] = ownedStore.TryPublish([0x41], [0x11]); + }); + var second = Task.Run(() => + { + start.SignalAndWait(); + statuses[1] = ownedStore.TryPublish([0x41], [0x22]); + }); + start.SignalAndWait(); + await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(StoreProfile.LockFree, ownedStore.Profile); + Assert.Single(statuses, static status => status == StoreStatus.Success); + Assert.Single(statuses, static status => status == StoreStatus.DuplicateKey); + } + + [Fact] + public void DirectoryImplementationExposesLookupInsertUnlinkAndHelpingWithoutOwnerLocks() + { + var directoryType = typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeKeyDirectory", + throwOnError: false, + ignoreCase: false); + Assert.True(directoryType is not null, "The layout-v2 engine requires a LockFreeKeyDirectory implementation."); + + var methodNames = directoryType! + .GetMethods(BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .Select(static method => method.Name) + .ToArray(); + Assert.Contains(methodNames, static name => name.Contains("Lookup", StringComparison.Ordinal)); + Assert.Contains(methodNames, static name => name.Contains("Insert", StringComparison.Ordinal)); + Assert.Contains(methodNames, static name => name.Contains("Unlink", StringComparison.Ordinal)); + Assert.Contains(methodNames, static name => name.Contains("Help", StringComparison.Ordinal)); + + var forbiddenTypes = new[] + { + typeof(Mutex), + typeof(Semaphore), + typeof(SemaphoreSlim), + typeof(ReaderWriterLockSlim) + }; + var fields = directoryType.GetFields( + BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain(fields, field => forbiddenTypes.Contains(field.FieldType)); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeFutureGenerationLocationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeFutureGenerationLocationTests.cs new file mode 100644 index 0000000..fba11ad --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeFutureGenerationLocationTests.cs @@ -0,0 +1,273 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed unsafe class LockFreeFutureGenerationLocationTests +{ + private const int IntentInsert = 1; + private const int IntentUnlink = 2; + private const int PhasePrepared = 1; + private const int PhaseTargetSelected = 2; + private const int TargetPrimary = 1; + private const long OperationGeneration = 17; + private const long FutureGeneration = OperationGeneration + 1; + private const ulong KeyHash = 0xd6e8_feb8_6659_fd93UL; + + [Fact] + public void PreparedUnlinkHelperRejectsStableFutureGenerationLocation() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore( + out LockFreeKeyDirectory directory, + out LockFreeSlotTable slots, + out MemoryMappedStoreRegion region, + out StoreLayoutV2 layout); + int canonicalBucket = CanonicalBucket(KeyHash, layout.PrimaryBucketCount); + int targetIndex = canonicalBucket * LayoutV2Constants.PrimaryLanesPerBucket; + ulong binding = IndexBinding.Encode(slotIndex: 0, OperationGeneration); + ulong operation = DirectoryOperation.Encode( + IntentUnlink, + PhasePrepared, + targetKind: 0, + targetIndex: 0, + OperationGeneration); + ulong futureLocation = DirectoryLocation.Encode( + TargetPrimary, + targetIndex, + FutureGeneration); + + SeedOperation( + slots, + region, + layout, + canonicalBucket, + binding, + operation, + futureLocation, + LockFreeSlotTable.ReclaimingState); + + _ = LockFreeCorruptionTrace.Consume(); + Assert.Equal( + StoreStatus.CorruptStore, + directory.HelpMutation(canonicalBucket, maxSteps: 1)); + Assert.NotNull(LockFreeCorruptionTrace.Consume()); + + AssertLocation(slots, futureLocation); + } + + [Fact] + public void TargetSelectedUnlinkHelperRejectsStableFutureGenerationLocation() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore( + out LockFreeKeyDirectory directory, + out LockFreeSlotTable slots, + out MemoryMappedStoreRegion region, + out StoreLayoutV2 layout); + int canonicalBucket = CanonicalBucket(KeyHash, layout.PrimaryBucketCount); + int targetIndex = canonicalBucket * LayoutV2Constants.PrimaryLanesPerBucket; + ulong binding = IndexBinding.Encode(slotIndex: 0, OperationGeneration); + ulong operation = DirectoryOperation.Encode( + IntentUnlink, + PhaseTargetSelected, + TargetPrimary, + targetIndex, + OperationGeneration); + ulong futureLocation = DirectoryLocation.Encode( + TargetPrimary, + targetIndex, + FutureGeneration); + + SeedOperation( + slots, + region, + layout, + canonicalBucket, + binding, + operation, + futureLocation, + LockFreeSlotTable.ReclaimingState); + SetPrimaryCell(region, layout, targetIndex, binding); + + _ = LockFreeCorruptionTrace.Consume(); + Assert.Equal( + StoreStatus.CorruptStore, + directory.HelpMutation(canonicalBucket, maxSteps: 1)); + Assert.NotNull(LockFreeCorruptionTrace.Consume()); + + AssertLocation(slots, futureLocation); + } + + [Fact] + public void TargetSelectedInsertHelperRejectsStableFutureGenerationLocation() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore( + out LockFreeKeyDirectory directory, + out LockFreeSlotTable slots, + out MemoryMappedStoreRegion region, + out StoreLayoutV2 layout); + int canonicalBucket = CanonicalBucket(KeyHash, layout.PrimaryBucketCount); + int targetIndex = canonicalBucket * LayoutV2Constants.PrimaryLanesPerBucket; + ulong binding = IndexBinding.Encode(slotIndex: 0, OperationGeneration); + ulong operation = DirectoryOperation.Encode( + IntentInsert, + PhaseTargetSelected, + TargetPrimary, + targetIndex, + OperationGeneration); + ulong futureLocation = DirectoryLocation.Encode( + TargetPrimary, + targetIndex, + FutureGeneration); + + SeedOperation( + slots, + region, + layout, + canonicalBucket, + binding, + operation, + futureLocation, + LockFreeSlotTable.InitializingState); + SetPrimaryCell(region, layout, targetIndex, binding); + + _ = LockFreeCorruptionTrace.Consume(); + Assert.Equal( + StoreStatus.CorruptStore, + directory.HelpMutation(canonicalBucket, maxSteps: 1)); + Assert.NotNull(LockFreeCorruptionTrace.Consume()); + + AssertLocation(slots, futureLocation); + } + + private static void SeedOperation( + LockFreeSlotTable slots, + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + int canonicalBucket, + ulong binding, + ulong operation, + ulong location, + int slotState) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + slot.DirectoryBinding = binding; + slot.KeyHash = KeyHash; + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, unchecked((long)location)); + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, unchecked((long)operation)); + AtomicControlWord.StoreRelease( + ref slot.Control, + unchecked((long)AtomicControlWord.EncodeSlot( + slotState, + OperationGeneration, + participantToken: 0))); + + SetBucketMutation(region, layout, canonicalBucket, binding); + } + + private static void AssertLocation(LockFreeSlotTable slots, ulong expected) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + Assert.Equal( + expected, + unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation))); + } + + private static void SetBucketMutation( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + int canonicalBucket, + ulong binding) + { + long offset = layout.PrimaryDirectoryOffset + + ((long)canonicalBucket * layout.PrimaryBucketStride) + + 8; + ref long mutation = ref *(long*)(region.Pointer + offset); + AtomicControlWord.StoreRelease(ref mutation, unchecked((long)binding)); + } + + private static void SetPrimaryCell( + MemoryMappedStoreRegion region, + StoreLayoutV2 layout, + int targetIndex, + ulong binding) + { + int bucket = targetIndex / LayoutV2Constants.PrimaryLanesPerBucket; + int lane = targetIndex % LayoutV2Constants.PrimaryLanesPerBucket; + long offset = layout.PrimaryDirectoryOffset + + ((long)bucket * layout.PrimaryBucketStride) + + 16 + + (lane * sizeof(long)); + ref long cell = ref *(long*)(region.Pointer + offset); + AtomicControlWord.StoreRelease(ref cell, unchecked((long)binding)); + } + + private static int CanonicalBucket(ulong hash, int bucketCount) => + (int)(Mix(hash) & checked((uint)(bucketCount - 1))); + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static MemoryStore CreateStore( + out LockFreeKeyDirectory directory, + out LockFreeSlotTable slots, + out MemoryMappedStoreRegion region, + out StoreLayoutV2 layout) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-future-location-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + MemoryStore result = Assert.IsType(store); + object engine = ReadPrivate(result, "_engine"); + directory = ReadPrivate(engine, "_directory"); + slots = ReadPrivate(engine, "_slots"); + region = ReadPrivate(engine, "_region"); + layout = ReadPrivate(engine, "_layout"); + return result; + } + + private static T ReadPrivate(object owner, string fieldName) + { + FieldInfo field = owner.GetType().GetField( + fieldName, + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Missing field {owner.GetType().FullName}.{fieldName}."); + return Assert.IsAssignableFrom(field.GetValue(owner)); + } + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeFutureGenerationOperationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeFutureGenerationOperationTests.cs new file mode 100644 index 0000000..cd1b3f7 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeFutureGenerationOperationTests.cs @@ -0,0 +1,144 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeFutureGenerationOperationTests +{ + private const int IntentInsert = 1; + private const int IntentUnlink = 2; + private const int PhasePrepared = 1; + + [Fact] + public void OlderInsertPreparationDoesNotClearOrReplaceFutureGenerationOperation() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore( + out LockFreeKeyDirectory directory, + out LockFreeSlotTable slots); + byte[] key = [0x41]; + ulong keyHash = StoreKey.Hash(key); + Assert.Equal( + StoreStatus.Success, + slots.TryClaimReservation( + keyHash, + keyLength: key.Length, + descriptorLength: 0, + payloadLength: 1, + out var reservation)); + key.CopyTo(slots.GetInitializingKeySpan(reservation)); + + long operationGeneration = IndexBinding.Decode(reservation.SlotBinding).Generation; + ulong futureOperation = DirectoryOperation.Encode( + IntentInsert, + PhasePrepared, + targetKind: 0, + targetIndex: 0, + generation: operationGeneration + 1); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + AtomicControlWord.StoreRelease( + ref slot.DirectoryOperation, + unchecked((long)futureOperation)); + + StoreStatus status = directory.TryInsert( + key, + keyHash, + reservation.SlotBinding, + out _); + + Assert.Equal(StoreStatus.CorruptStore, status); + AssertOperation(ref slot, futureOperation); + } + + [Fact] + public void OlderUnlinkPreparationDoesNotClearOrReplaceFutureGenerationOperation() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore( + out LockFreeKeyDirectory directory, + out LockFreeSlotTable slots); + byte[] key = [0x42]; + ulong keyHash = StoreKey.Hash(key); + Assert.Equal( + StoreStatus.Success, + slots.TryClaimReservation( + keyHash, + keyLength: key.Length, + descriptorLength: 0, + payloadLength: 1, + out var reservation)); + key.CopyTo(slots.GetInitializingKeySpan(reservation)); + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(reservation)); + + long operationGeneration = IndexBinding.Decode(reservation.SlotBinding).Generation; + ulong futureOperation = DirectoryOperation.Encode( + IntentUnlink, + PhasePrepared, + targetKind: 0, + targetIndex: 0, + generation: operationGeneration + 1); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + AtomicControlWord.StoreRelease( + ref slot.DirectoryOperation, + unchecked((long)futureOperation)); + + StoreStatus status = directory.TryUnlink(reservation.SlotBinding); + + Assert.Equal(StoreStatus.CorruptStore, status); + AssertOperation(ref slot, futureOperation); + } + + private static void AssertOperation(ref ValueSlotMetadataV2 slot, ulong expected) + { + Assert.Equal( + expected, + unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation))); + } + + private static MemoryStore CreateStore( + out LockFreeKeyDirectory directory, + out LockFreeSlotTable slots) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-future-operation-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + MemoryStore result = Assert.IsType(store); + object engine = ReadPrivate(result, "_engine"); + directory = ReadPrivate(engine, "_directory"); + slots = ReadPrivate(engine, "_slots"); + return result; + } + + private static T ReadPrivate(object owner, string fieldName) + { + FieldInfo field = owner.GetType().GetField( + fieldName, + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Missing field {owner.GetType().FullName}.{fieldName}."); + return Assert.IsAssignableFrom(field.GetValue(owner)); + } + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeInsertCancellationRaceTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeInsertCancellationRaceTests.cs new file mode 100644 index 0000000..cd5b20b --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeInsertCancellationRaceTests.cs @@ -0,0 +1,2213 @@ +using System.Buffers; +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeInsertCancellationRaceTests +{ + private const int CanonicalBucket = 0; + private const int OverflowSlotCount = 17; + private const int OverflowAnchorCount = 16; + private const int RaceSlotCount = 4; + // The finite budget is real wall-clock time. Leave enough pre-checkpoint + // setup margin for parallel full-solution runs, then wait past that same + // deadline so the controlled schedule still proves the expired-budget path. + private static readonly StoreWaitOptions FiniteRaceWait = + new(TimeSpan.FromSeconds(2)); + private static readonly TimeSpan ExpiredRaceDelay = TimeSpan.FromMilliseconds(2_250); + + [Fact] + public Task PreparedInsertCanceledAfterCurrentOperationRevalidationDoesNotReportCorruption() => + RunSingleSlotCancellationRace( + LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch, + expectedPhase: 1, + expectedSlotState: LockFreeSlotTable.InitializingState, + expectedStatus: StoreStatus.InvalidReservation, + key: [0x31]); + + [Fact] + public Task BindingChangedInsertCanceledAfterStateValidationDoesNotPublishReservedOrReportCorruption() => + RunSingleSlotCancellationRace( + LockFreeCheckpointId.DirectoryAfterInsertBindingChangedStateValidationBeforeReservedPublication, + expectedPhase: 3, + expectedSlotState: LockFreeSlotTable.InitializingState, + expectedStatus: StoreStatus.InvalidReservation, + key: [0x32]); + + [Fact] + public Task CompletedExplicitReserveCanceledBeforeLocationReadStillReturnsOrderedSuccess() => + RunSingleSlotCancellationRace( + LockFreeCheckpointId.DirectoryAfterInsertCompletionStateValidationBeforeLocationRead, + expectedPhase: 5, + expectedSlotState: LockFreeSlotTable.ReservedState, + expectedStatus: StoreStatus.Success, + key: [0x33]); + + [Fact] + public Task ExplicitReserveCanceledBeforePendingClassificationStillReturnsOrderedSuccess() => + RunSingleSlotCancellationRace( + LockFreeCheckpointId.ReserveAfterDirectoryInsertBeforePendingClassification, + expectedPhase: 5, + expectedSlotState: LockFreeSlotTable.ReservedState, + expectedStatus: StoreStatus.Success, + key: [0x34]); + + [Theory] + [InlineData(TargetAfterCancellation.Empty)] + [InlineData(TargetAfterCancellation.ValidReplacement)] + [InlineData(TargetAfterCancellation.Malformed)] + [InlineData(TargetAfterCancellation.OutOfRange)] + public async Task DelayedInsertCancellationTargetHandoffIsClassifiedExactly( + TargetAfterCancellation targetAfterCancellation) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] key = keys[0]; + string name = $"sms-v2-insert-unlink-handoff-{Guid.NewGuid():N}"; + using var insertScheduler = new ControlledLockFreeScheduler(); + using var unlinkScheduler = new ControlledLockFreeScheduler(); + using MemoryStore owner = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.CreateNew, + insertScheduler); + using MemoryStore unlinker = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.OpenExisting, + unlinkScheduler); + LockFreeKeyDirectory ownerDirectory = ReadDirectory(owner); + LockFreeKeyDirectory unlinkDirectory = ReadDirectory(unlinker); + LockFreeSlotTable slots = ReadSlots(owner); + + Assert.Equal( + StoreStatus.Success, + owner.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + ReservationHandle handle = reservation.HandleForEngine; + Assert.Equal( + StoreStatus.Success, + ownerDirectory.TryLookup( + key, + StoreKey.Hash(key), + LockFreeOperationBudget.UnboundedScan, + out ulong binding, + out DirectoryLocation location)); + Assert.Equal(handle.SlotBinding, binding); + Assert.Equal(1, location.Kind); + Assert.Equal(binding, ReadDirectoryCell(ownerDirectory, location)); + + // Recreate the reachable window after an insert helper has claimed its + // exact target cell but before it publishes DirectoryLocation. The + // successful reservation supplies fully initialized immutable slot + // metadata; only the three atomic protocol words are rewound. + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(handle)); + IndexBinding decoded = IndexBinding.Decode(binding); + int slotIndex = decoded.SlotIndex; + ulong targetSelected = DirectoryOperation.Encode( + intent: 1, + phase: 2, + location.Kind, + location.Index, + decoded.Generation); + AtomicControlWord.StoreRelease(ref slots.Slot(slotIndex).DirectoryLocation, 0); + AtomicControlWord.StoreRelease( + ref slots.Slot(slotIndex).DirectoryOperation, + unchecked((long)targetSelected)); + WriteCanonicalMutation(ownerDirectory, CanonicalBucket, binding); + + insertScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch); + Task<(StoreStatus Status, string? CorruptionOrigin)> delayedInsert = Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + InstrumentedLockFreeCheckpoint checkpoint = + insertScheduler.CreateInstrumentedCheckpoint(); + StoreStatus status = ownerDirectory.HelpMutation( + CanonicalBucket, + LockFreeOperationBudget.UnboundedScan, + ref checkpoint, + maxSteps: 1); + return (status, LockFreeCorruptionTrace.Consume()); + }); + + var insertResumed = false; + var unlinkResumed = false; + Task<(StoreStatus Status, string? CorruptionOrigin)>? delayedUnlink = null; + try + { + Assert.True(insertScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal(targetSelected, ReadDirectoryOperation(slots, slotIndex)); + + unlinkScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation); + delayedUnlink = Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + InstrumentedLockFreeCheckpoint checkpoint = + unlinkScheduler.CreateInstrumentedCheckpoint(); + StoreStatus status = unlinkDirectory.TryUnlink( + binding, + LockFreeOperationBudget.UnboundedScan, + ref checkpoint); + return (status, LockFreeCorruptionTrace.Consume()); + }); + Assert.True(unlinkScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + DirectoryOperation preparedUnlink = DirectoryOperation.Decode( + ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(2, preparedUnlink.Intent); + Assert.Equal(1, preparedUnlink.Phase); + Assert.Equal( + 0, + AtomicControlWord.LoadAcquire(ref slots.Slot(slotIndex).DirectoryLocation)); + Assert.Equal(binding, ReadDirectoryCell(ownerDirectory, location)); + + // The delayed helper still dispatches its validated Insert snapshot. + // CancelInsert exact-clears the target, but cannot replace the newer + // Unlink/Prepared descriptor that now owns the canonical mutation. + insertScheduler.Continue(); + insertResumed = true; + (StoreStatus insertStatus, string? insertCorruption) = + await delayedInsert.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.StoreBusy, insertStatus); + Assert.Null(insertCorruption); + Assert.Equal(0UL, ReadDirectoryCell(ownerDirectory, location)); + Assert.Equal(preparedUnlink.Value, ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(binding, ownerDirectory.ReadCanonicalMutation(CanonicalBucket)); + + ulong replacement = targetAfterCancellation switch + { + TargetAfterCancellation.Empty => 0, + TargetAfterCancellation.ValidReplacement => IndexBinding.Encode( + (decoded.SlotIndex + 1) % RaceSlotCount, + decoded.Generation), + TargetAfterCancellation.Malformed => ulong.MaxValue, + TargetAfterCancellation.OutOfRange => IndexBinding.Encode( + RaceSlotCount, + decoded.Generation), + _ => throw new ArgumentOutOfRangeException(nameof(targetAfterCancellation)), + }; + if (replacement != 0) + { + WriteDirectoryCell(ownerDirectory, location, replacement); + } + + unlinkScheduler.Continue(); + unlinkResumed = true; + (StoreStatus unlinkStatus, string? unlinkCorruption) = + await delayedUnlink.WaitAsync(TimeSpan.FromSeconds(5)); + + if (targetAfterCancellation is + TargetAfterCancellation.Malformed or TargetAfterCancellation.OutOfRange) + { + Assert.Equal(StoreStatus.CorruptStore, unlinkStatus); + Assert.NotNull(unlinkCorruption); + Assert.Equal(replacement, ReadDirectoryCell(ownerDirectory, location)); + return; + } + + Assert.Equal(StoreStatus.Success, unlinkStatus); + Assert.Null(unlinkCorruption); + Assert.Equal(replacement, ReadDirectoryCell(ownerDirectory, location)); + if (replacement != 0) + { + // The delayed unlink must preserve a valid winner it does not + // own. The test supplied that stand-in binding directly, so + // it also withdraws it before asserting old-slot reclamation. + WriteDirectoryCell(ownerDirectory, location, binding: 0); + } + } + finally + { + if (!insertResumed) + { + insertScheduler.Continue(); + } + + if (!unlinkResumed) + { + unlinkScheduler.Continue(); + } + } + + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.NotFound, owner.TryAcquire(key, out _)); + AssertDirectoryDrained(ownerDirectory); + + byte[] laterKey = keys[1]; + Assert.Equal( + StoreStatus.Success, + owner.TryPublish(laterKey, [0xA5], default, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, owner.TryAcquire(laterKey, out ValueLease laterLease)); + Assert.Equal(0xA5, laterLease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, laterLease.Release()); + Assert.Equal(StoreStatus.Success, owner.TryRemove(laterKey, StoreWaitOptions.Infinite)); + AssertDirectoryDrained(ownerDirectory); + AssertAllSlotCapacityReusable(owner, keys, RaceSlotCount); + } + + [Fact] + public async Task PreparedUnlinkFirstLocationPublisherWinsAndWithdrawsCompetingBinding() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] key = keys[0]; + string name = $"sms-v2-unlink-location-arbitration-{Guid.NewGuid():N}"; + using var firstScheduler = new ControlledLockFreeScheduler(); + using var competingScheduler = new ControlledLockFreeScheduler(); + using MemoryStore firstStore = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.CreateNew, + firstScheduler); + using MemoryStore competingStore = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.OpenExisting, + competingScheduler); + LockFreeKeyDirectory firstDirectory = ReadDirectory(firstStore); + LockFreeKeyDirectory competingDirectory = ReadDirectory(competingStore); + LockFreeSlotTable slots = ReadSlots(firstStore); + + Assert.Equal( + StoreStatus.Success, + firstStore.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + ReservationHandle handle = reservation.HandleForEngine; + Assert.Equal( + StoreStatus.Success, + firstDirectory.TryLookup( + key, + StoreKey.Hash(key), + LockFreeOperationBudget.UnboundedScan, + out ulong binding, + out DirectoryLocation earlierLocation)); + Assert.Equal(handle.SlotBinding, binding); + Assert.Equal(1, earlierLocation.Kind); + Assert.Equal(0, earlierLocation.Index % LayoutV2Constants.PrimaryLanesPerBucket); + + IndexBinding decoded = IndexBinding.Decode(binding); + int slotIndex = decoded.SlotIndex; + DirectoryLocation laterLocation = DirectoryLocation.Decode(DirectoryLocation.Encode( + earlierLocation.Kind, + earlierLocation.Index + 1, + decoded.Generation)); + Assert.Equal( + earlierLocation.Index / LayoutV2Constants.PrimaryLanesPerBucket, + laterLocation.Index / LayoutV2Constants.PrimaryLanesPerBucket); + + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(handle)); + WriteDirectoryCell(firstDirectory, earlierLocation, binding: 0); + WriteDirectoryCell(firstDirectory, laterLocation, binding); + ulong prepared = DirectoryOperation.Encode( + intent: 2, + phase: 1, + targetKind: 0, + targetIndex: 0, + decoded.Generation); + AtomicControlWord.StoreRelease(ref slots.Slot(slotIndex).DirectoryLocation, 0); + AtomicControlWord.StoreRelease( + ref slots.Slot(slotIndex).DirectoryOperation, + unchecked((long)prepared)); + WriteCanonicalMutation(firstDirectory, CanonicalBucket, binding); + + firstScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation); + Task<(StoreStatus Status, string? CorruptionOrigin)> firstTask = + UnlinkAsync(firstDirectory, binding, firstScheduler); + Assert.True(firstScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(binding, ReadDirectoryCell(firstDirectory, laterLocation)); + + // The second helper starts later but finds the earlier scan lane. Both + // helpers have now recovered a different exact target while the shared + // location remains empty. + WriteDirectoryCell(firstDirectory, earlierLocation, binding); + competingScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation); + Task<(StoreStatus Status, string? CorruptionOrigin)> competingTask = + UnlinkAsync(competingDirectory, binding, competingScheduler); + Assert.True(competingScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + + var firstResumed = false; + var competingResumed = false; + try + { + // Arm the descriptor-selection pause before releasing the first + // publisher, leaving no scheduler gap after its Location CAS. + firstScheduler.ContinueAndPauseAt( + LockFreeCheckpointId.DirectoryAfterLocationValidation); + firstResumed = true; + Assert.True(firstScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal(laterLocation.Value, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(prepared, ReadDirectoryOperation(slots, slotIndex)); + + competingScheduler.Continue(); + competingResumed = true; + (StoreStatus competingStatus, string? competingCorruption) = + await competingTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, competingStatus); + Assert.Null(competingCorruption); + Assert.Equal(0UL, ReadDirectoryCell(firstDirectory, earlierLocation)); + + firstScheduler.Continue(); + (StoreStatus firstStatus, string? firstCorruption) = + await firstTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, firstStatus); + Assert.Null(firstCorruption); + } + finally + { + if (!firstResumed || !firstTask.IsCompleted) + { + firstScheduler.Continue(); + } + + if (!competingResumed) + { + competingScheduler.Continue(); + } + } + + Assert.Equal(0UL, ReadDirectoryCell(firstDirectory, earlierLocation)); + Assert.Equal(0UL, ReadDirectoryCell(firstDirectory, laterLocation)); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(0UL, ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(0UL, firstDirectory.ReadCanonicalMutation(CanonicalBucket)); + AssertDirectoryDrained(firstDirectory); + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + AssertAllSlotCapacityReusable(firstStore, keys, RaceSlotCount); + } + + [Fact] + public async Task DelayedInsertCleanupAndLatePublisherConvergeAfterTargetSelection() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] key = keys[0]; + string name = $"sms-v2-unlink-target-handoff-{Guid.NewGuid():N}"; + using var insertScheduler = new ControlledLockFreeScheduler(); + using var firstScheduler = new ControlledLockFreeScheduler(); + using var lateScheduler = new ControlledLockFreeScheduler(); + using MemoryStore insertStore = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.CreateNew, + insertScheduler); + using MemoryStore firstStore = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.OpenExisting, + firstScheduler); + using MemoryStore lateStore = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.OpenExisting, + lateScheduler); + LockFreeKeyDirectory insertDirectory = ReadDirectory(insertStore); + LockFreeKeyDirectory firstDirectory = ReadDirectory(firstStore); + LockFreeKeyDirectory lateDirectory = ReadDirectory(lateStore); + LockFreeSlotTable slots = ReadSlots(insertStore); + + Assert.Equal( + StoreStatus.Success, + insertStore.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + ReservationHandle handle = reservation.HandleForEngine; + Assert.Equal( + StoreStatus.Success, + insertDirectory.TryLookup( + key, + StoreKey.Hash(key), + LockFreeOperationBudget.UnboundedScan, + out ulong binding, + out DirectoryLocation selectedLocation)); + Assert.Equal(handle.SlotBinding, binding); + Assert.Equal(1, selectedLocation.Kind); + Assert.Equal(0, selectedLocation.Index % LayoutV2Constants.PrimaryLanesPerBucket); + + IndexBinding decoded = IndexBinding.Decode(binding); + int slotIndex = decoded.SlotIndex; + DirectoryLocation lateLocation = DirectoryLocation.Decode(DirectoryLocation.Encode( + selectedLocation.Kind, + selectedLocation.Index + 1, + decoded.Generation)); + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(handle)); + ulong insertTargetSelected = DirectoryOperation.Encode( + intent: 1, + phase: 2, + selectedLocation.Kind, + selectedLocation.Index, + decoded.Generation); + AtomicControlWord.StoreRelease( + ref slots.Slot(slotIndex).DirectoryOperation, + unchecked((long)insertTargetSelected)); + AtomicControlWord.StoreRelease( + ref slots.Slot(slotIndex).DirectoryLocation, + unchecked((long)selectedLocation.Value)); + WriteDirectoryCell(insertDirectory, selectedLocation, binding); + WriteDirectoryCell(insertDirectory, lateLocation, binding: 0); + WriteCanonicalMutation(insertDirectory, CanonicalBucket, binding); + + insertScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterCurrentOperationRevalidationBeforeDispatch); + Task<(StoreStatus Status, string? CorruptionOrigin)> delayedInsert = Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + InstrumentedLockFreeCheckpoint checkpoint = + insertScheduler.CreateInstrumentedCheckpoint(); + StoreStatus status = insertDirectory.HelpMutation( + CanonicalBucket, + LockFreeOperationBudget.UnboundedScan, + ref checkpoint, + maxSteps: 1); + return (status, LockFreeCorruptionTrace.Consume()); + }); + Assert.True(insertScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + firstScheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterLocationValidation); + Task<(StoreStatus Status, string? CorruptionOrigin)> firstTask = + UnlinkAsync(firstDirectory, binding, firstScheduler); + Assert.True(firstScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + DirectoryOperation prepared = DirectoryOperation.Decode( + ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(2, prepared.Intent); + Assert.Equal(1, prepared.Phase); + Assert.Equal(selectedLocation.Value, ReadDirectoryLocation(slots, slotIndex)); + + var insertResumed = false; + var firstResumed = false; + var lateResumed = false; + Task<(StoreStatus Status, string? CorruptionOrigin)>? lateTask = null; + try + { + // The old Insert snapshot dispatches cancellation after U/Prepared + // took ownership. It exact-clears A and Location A but cannot + // replace the newer descriptor. + insertScheduler.Continue(); + insertResumed = true; + (StoreStatus insertStatus, string? insertCorruption) = + await delayedInsert.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.StoreBusy, insertStatus); + Assert.Null(insertCorruption); + Assert.Equal(0UL, ReadDirectoryCell(insertDirectory, selectedLocation)); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(prepared.Value, ReadDirectoryOperation(slots, slotIndex)); + + // A second Prepared helper recovers B and publishes it, then stays + // paused before post-CAS source validation. + WriteDirectoryCell(insertDirectory, lateLocation, binding); + lateScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation); + lateTask = UnlinkAsync(lateDirectory, binding, lateScheduler); + Assert.True(lateScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + lateScheduler.ContinueAndPauseAt( + LockFreeCheckpointId.DirectoryAfterLocationPublicationBeforeSourceRevalidation); + lateResumed = true; + Assert.True(lateScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal(lateLocation.Value, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(prepared.Value, ReadDirectoryOperation(slots, slotIndex)); + + // The first helper now selects A while B is the published location, + // then pauses as TargetSelected before reading that conflicting + // witness. This is the exact terminal-tolerance window. + firstScheduler.ContinueAndPauseAt( + LockFreeCheckpointId.DirectoryAfterUnlinkOperationValidationBeforeLocationRead); + firstResumed = true; + Assert.True(firstScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + DirectoryOperation selected = DirectoryOperation.Decode( + ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(2, selected.Intent); + Assert.Equal(2, selected.Phase); + Assert.Equal(selectedLocation.Kind, selected.Kind); + Assert.Equal(selectedLocation.Index, selected.Index); + Assert.Equal(lateLocation.Value, ReadDirectoryLocation(slots, slotIndex)); + + firstScheduler.Continue(); + (StoreStatus firstStatus, string? firstCorruption) = + await firstTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, firstStatus); + Assert.Null(firstCorruption); + + lateScheduler.Continue(); + (StoreStatus lateStatus, string? lateCorruption) = + await lateTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, lateStatus); + Assert.Null(lateCorruption); + } + finally + { + if (!insertResumed) + { + insertScheduler.Continue(); + } + + if (!firstResumed || !firstTask.IsCompleted) + { + firstScheduler.Continue(); + } + + if (lateTask is not null && (!lateResumed || !lateTask.IsCompleted)) + { + lateScheduler.Continue(); + } + } + + Assert.Equal(0UL, ReadDirectoryCell(insertDirectory, selectedLocation)); + Assert.Equal(0UL, ReadDirectoryCell(insertDirectory, lateLocation)); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(0UL, ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(0UL, insertDirectory.ReadCanonicalMutation(CanonicalBucket)); + AssertDirectoryDrained(insertDirectory); + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + AssertAllSlotCapacityReusable(insertStore, keys, RaceSlotCount); + } + + [Fact] + public async Task PreparedUnlinkPublisherDelayedBeforeLocationCasWithdrawsLatePublication() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] key = keys[0]; + string name = $"sms-v2-unlink-late-location-cas-{Guid.NewGuid():N}"; + using var publisherScheduler = new ControlledLockFreeScheduler(); + using MemoryStore publisherStore = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.CreateNew, + publisherScheduler); + using MemoryStore completingStore = OpenStore(name, RaceSlotCount); + LockFreeKeyDirectory publisherDirectory = ReadDirectory(publisherStore); + LockFreeKeyDirectory completingDirectory = ReadDirectory(completingStore); + LockFreeSlotTable slots = ReadSlots(publisherStore); + + Assert.Equal( + StoreStatus.Success, + publisherStore.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + ReservationHandle handle = reservation.HandleForEngine; + Assert.Equal( + StoreStatus.Success, + publisherDirectory.TryLookup( + key, + StoreKey.Hash(key), + LockFreeOperationBudget.UnboundedScan, + out ulong binding, + out DirectoryLocation location)); + Assert.Equal(handle.SlotBinding, binding); + + IndexBinding decoded = IndexBinding.Decode(binding); + int slotIndex = decoded.SlotIndex; + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(handle)); + ulong prepared = DirectoryOperation.Encode( + intent: 2, + phase: 1, + targetKind: 0, + targetIndex: 0, + decoded.Generation); + AtomicControlWord.StoreRelease(ref slots.Slot(slotIndex).DirectoryLocation, 0); + AtomicControlWord.StoreRelease( + ref slots.Slot(slotIndex).DirectoryOperation, + unchecked((long)prepared)); + WriteDirectoryCell(publisherDirectory, location, binding); + WriteCanonicalMutation(publisherDirectory, CanonicalBucket, binding); + + publisherScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterEmptyLocationSourceRevalidationBeforePublicationCas); + Task<(StoreStatus Status, string? CorruptionOrigin)> delayedPublisher = + UnlinkAsync(publisherDirectory, binding, publisherScheduler); + Assert.True(publisherScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + + _ = LockFreeCorruptionTrace.Consume(); + Assert.Equal( + StoreStatus.Success, + completingDirectory.TryUnlink(binding, LockFreeOperationBudget.UnboundedScan)); + Assert.Null(LockFreeCorruptionTrace.Consume()); + Assert.Equal(0UL, ReadDirectoryCell(publisherDirectory, location)); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(0UL, ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(0UL, publisherDirectory.ReadCanonicalMutation(CanonicalBucket)); + + var resumed = false; + try + { + publisherScheduler.Continue(); + resumed = true; + (StoreStatus publisherStatus, string? publisherCorruption) = + await delayedPublisher.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, publisherStatus); + Assert.Null(publisherCorruption); + } + finally + { + if (!resumed) + { + publisherScheduler.Continue(); + } + } + + // The delayed stale CAS briefly installed Location C after unlink was + // terminal; its post-CAS reconciliation must remove that exact word. + Assert.Equal(0UL, ReadDirectoryCell(publisherDirectory, location)); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(0UL, ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(0UL, publisherDirectory.ReadCanonicalMutation(CanonicalBucket)); + AssertDirectoryDrained(publisherDirectory); + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + AssertAllSlotCapacityReusable(publisherStore, keys, RaceSlotCount); + } + + [Theory] + [InlineData(CompetingUnlinkTarget.ExactBinding)] + [InlineData(CompetingUnlinkTarget.Empty)] + [InlineData(CompetingUnlinkTarget.ValidReplacement)] + [InlineData(CompetingUnlinkTarget.Malformed)] + [InlineData(CompetingUnlinkTarget.OutOfRange)] + public void TargetSelectedUnlinkClassifiesAnotherSameGenerationLocationExactly( + CompetingUnlinkTarget competingTarget) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] key = keys[0]; + using MemoryStore store = CreateStore( + $"sms-v2-unlink-location-terminal-{Guid.NewGuid():N}", + RaceSlotCount); + LockFreeKeyDirectory directory = ReadDirectory(store); + LockFreeSlotTable slots = ReadSlots(store); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + ReservationHandle handle = reservation.HandleForEngine; + Assert.Equal( + StoreStatus.Success, + directory.TryLookup( + key, + StoreKey.Hash(key), + LockFreeOperationBudget.UnboundedScan, + out ulong binding, + out DirectoryLocation selectedLocation)); + Assert.Equal(handle.SlotBinding, binding); + Assert.Equal(1, selectedLocation.Kind); + + IndexBinding decoded = IndexBinding.Decode(binding); + int slotIndex = decoded.SlotIndex; + DirectoryLocation lateLocation = DirectoryLocation.Decode(DirectoryLocation.Encode( + selectedLocation.Kind, + selectedLocation.Index + 1, + decoded.Generation)); + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(handle)); + WriteDirectoryCell(directory, selectedLocation, binding); + ulong lateTarget = competingTarget switch + { + CompetingUnlinkTarget.ExactBinding => binding, + CompetingUnlinkTarget.Empty => 0, + CompetingUnlinkTarget.ValidReplacement => IndexBinding.Encode( + (decoded.SlotIndex + 1) % RaceSlotCount, + decoded.Generation), + CompetingUnlinkTarget.Malformed => ulong.MaxValue, + CompetingUnlinkTarget.OutOfRange => IndexBinding.Encode( + RaceSlotCount, + decoded.Generation), + _ => throw new ArgumentOutOfRangeException(nameof(competingTarget)), + }; + WriteDirectoryCell(directory, lateLocation, lateTarget); + AtomicControlWord.StoreRelease( + ref slots.Slot(slotIndex).DirectoryLocation, + unchecked((long)lateLocation.Value)); + AtomicControlWord.StoreRelease( + ref slots.Slot(slotIndex).DirectoryOperation, + unchecked((long)DirectoryOperation.Encode( + intent: 2, + phase: 2, + selectedLocation.Kind, + selectedLocation.Index, + decoded.Generation))); + WriteCanonicalMutation(directory, CanonicalBucket, binding); + + _ = LockFreeCorruptionTrace.Consume(); + StoreStatus unlinkStatus = directory.TryUnlink( + binding, + LockFreeOperationBudget.UnboundedScan); + string? corruptionOrigin = LockFreeCorruptionTrace.Consume(); + if (competingTarget is + CompetingUnlinkTarget.Malformed or CompetingUnlinkTarget.OutOfRange) + { + Assert.Equal(StoreStatus.CorruptStore, unlinkStatus); + Assert.NotNull(corruptionOrigin); + Assert.Equal(lateTarget, ReadDirectoryCell(directory, lateLocation)); + return; + } + + Assert.Equal(StoreStatus.Success, unlinkStatus); + Assert.Null(corruptionOrigin); + Assert.Equal(0UL, ReadDirectoryCell(directory, selectedLocation)); + Assert.Equal(lateTarget == binding ? 0UL : lateTarget, ReadDirectoryCell(directory, lateLocation)); + Assert.Equal(0UL, ReadDirectoryLocation(slots, slotIndex)); + Assert.Equal(0UL, ReadDirectoryOperation(slots, slotIndex)); + Assert.Equal(0UL, directory.ReadCanonicalMutation(CanonicalBucket)); + if (lateTarget != 0 && lateTarget != binding) + { + // The valid replacement was injected as a stand-in only; exact + // unlink cleanup must preserve it, and the test withdraws it before + // checking capacity recovery. + WriteDirectoryCell(directory, lateLocation, binding: 0); + } + + AssertDirectoryDrained(directory); + + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + AssertAllSlotCapacityReusable(store, keys, RaceSlotCount); + } + + [Fact] + public async Task TentativeExplicitReserveCannotCauseDuplicateUnlessAHelperFirstOrdersReserved() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] ownerKey = keys[0]; + byte[] helperKey = keys[1]; + string name = $"sms-v2-tentative-explicit-{Guid.NewGuid():N}"; + using var ownerScheduler = new ControlledLockFreeScheduler(); + using var helperScheduler = new ControlledLockFreeScheduler(); + using MemoryStore owner = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.CreateNew, + ownerScheduler); + using MemoryStore helper = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.OpenExisting, + helperScheduler); + using MemoryStore contender = OpenStore(name, RaceSlotCount); + LockFreeKeyDirectory directory = ReadDirectory(owner); + LockFreeSlotTable slots = ReadSlots(owner); + + ownerScheduler.PauseAt(LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck); + Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)> ownerTask = + ReserveAsync(owner, ownerKey, FiniteRaceWait); + + var ownerResumed = false; + var helperResumed = false; + ReservationHandle ownerHandle = default; + Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)>? helperTask = null; + (StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin) ownerResult = default; + (StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin) helperResult = default; + StoreStatus contenderStatus = StoreStatus.UnknownFailure; + ValueReservation contenderReservation = default; + var ownerWasReservedAtContenderReturn = false; + try + { + Assert.True(ownerScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + ownerHandle = CaptureCurrentReservation( + directory, + slots, + expectedPhase: 1, + expectedTargetKind: 0); + AssertPublicationIntent( + slots, + ownerHandle, + SlotPublicationIntent.ExplicitReservation); + + helperScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation); + helperTask = ReserveAsync(helper, helperKey, StoreWaitOptions.Infinite); + Assert.True(helperScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal( + ownerHandle, + CaptureCurrentReservation( + directory, + slots, + expectedPhase: 2, + expectedTargetKind: 1)); + + contenderStatus = contender.TryReserve( + ownerKey, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.NoWait, + out contenderReservation); + ownerWasReservedAtContenderReturn = + HasExactSlotState(slots, ownerHandle, LockFreeSlotTable.ReservedState); + + await Task.Delay(ExpiredRaceDelay); + ownerScheduler.Continue(); + ownerResumed = true; + ownerResult = await ownerTask.WaitAsync(TimeSpan.FromSeconds(5)); + + helperScheduler.Continue(); + helperResumed = true; + helperResult = await helperTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally + { + if (!ownerResumed) + { + ownerScheduler.Continue(); + } + + if (!helperResumed) + { + helperScheduler.Continue(); + } + } + + AbortIfPending(contenderReservation); + AbortIfPending(ownerResult.Reservation); + AbortIfPending(helperResult.Reservation); + + string evidence = + $"contender={contenderStatus}; ownerReservedBeforeContenderResponse={ownerWasReservedAtContenderReturn}; " + + $"owner={ownerResult.Status}; ownerCorruption={ownerResult.CorruptionOrigin ?? "none"}; " + + $"helper={helperResult.Status}; helperCorruption={helperResult.CorruptionOrigin ?? "none"}."; + Assert.True( + contenderStatus != StoreStatus.DuplicateKey || ownerWasReservedAtContenderReturn, + "A same-key contender returned DuplicateKey from an Initializing-only binding. " + evidence); + if (ownerWasReservedAtContenderReturn) + { + Assert.Equal(StoreStatus.Success, ownerResult.Status); + Assert.Equal(ownerHandle, ownerResult.Reservation.HandleForEngine); + } + else + { + Assert.Equal(StoreStatus.StoreBusy, ownerResult.Status); + Assert.NotEqual(StoreStatus.DuplicateKey, contenderStatus); + } + + Assert.Equal(StoreStatus.Success, helperResult.Status); + Assert.Equal(StoreStatus.NotFound, owner.TryAcquire(ownerKey, out _)); + Assert.Equal(StoreStatus.NotFound, owner.TryAcquire(helperKey, out _)); + AssertDirectoryDrained(directory); + AssertAllSlotCapacityReusable(owner, keys, RaceSlotCount); + } + + [Fact] + public async Task PreReservedAbortWinsOverTentativeBindingAndCannotCauseDuplicate() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] ownerKey = keys[0]; + byte[] helperKey = keys[1]; + string name = $"sms-v2-tentative-abort-{Guid.NewGuid():N}"; + using var ownerScheduler = new ControlledLockFreeScheduler(); + using var helperScheduler = new ControlledLockFreeScheduler(); + using MemoryStore owner = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.CreateNew, + ownerScheduler); + using MemoryStore helper = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.OpenExisting, + helperScheduler); + using MemoryStore contender = OpenStore(name, RaceSlotCount); + LockFreeKeyDirectory directory = ReadDirectory(owner); + LockFreeSlotTable slots = ReadSlots(owner); + + ownerScheduler.PauseAt(LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck); + Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)> ownerTask = + ReserveAsync(owner, ownerKey, StoreWaitOptions.Infinite); + + var ownerResumed = false; + var helperResumed = false; + Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)>? helperTask = null; + (StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin) ownerResult = default; + (StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin) helperResult = default; + StoreStatus contenderStatus = StoreStatus.UnknownFailure; + ValueReservation contenderReservation = default; + try + { + Assert.True(ownerScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + ReservationHandle ownerHandle = CaptureCurrentReservation( + directory, + slots, + expectedPhase: 1, + expectedTargetKind: 0); + + helperScheduler.PauseAt( + LockFreeCheckpointId.DirectoryAfterLocationPublisherBindingValidation); + helperTask = ReserveAsync(helper, helperKey, StoreWaitOptions.Infinite); + Assert.True(helperScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal( + ownerHandle, + CaptureCurrentReservation( + directory, + slots, + expectedPhase: 2, + expectedTargetKind: 1)); + + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(ownerHandle)); + AssertSlotState(slots, ownerHandle, LockFreeSlotTable.AbortingState); + contenderStatus = contender.TryReserve( + ownerKey, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.NoWait, + out contenderReservation); + + ownerScheduler.Continue(); + ownerResumed = true; + ownerResult = await ownerTask.WaitAsync(TimeSpan.FromSeconds(5)); + + helperScheduler.Continue(); + helperResumed = true; + helperResult = await helperTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally + { + if (!ownerResumed) + { + ownerScheduler.Continue(); + } + + if (!helperResumed) + { + helperScheduler.Continue(); + } + } + + AbortIfPending(contenderReservation); + AbortIfPending(helperResult.Reservation); + Assert.Equal(StoreStatus.InvalidReservation, ownerResult.Status); + Assert.False(ownerResult.Reservation.IsValid); + Assert.True( + contenderStatus is StoreStatus.Success or StoreStatus.StoreBusy, + $"A pre-Reserved abort left only a tentative binding, but the contender returned {contenderStatus}; " + + $"ownerCorruption={ownerResult.CorruptionOrigin ?? "none"}; " + + $"helper={helperResult.Status}; helperCorruption={helperResult.CorruptionOrigin ?? "none"}."); + Assert.Equal(StoreStatus.Success, helperResult.Status); + Assert.Equal(StoreStatus.NotFound, owner.TryAcquire(ownerKey, out _)); + Assert.Equal(StoreStatus.NotFound, owner.TryAcquire(helperKey, out _)); + AssertDirectoryDrained(directory); + AssertAllSlotCapacityReusable(owner, keys, RaceSlotCount); + } + + [Fact] + public async Task HelperOrderedExplicitReserveReturnsSuccessAfterOwnerDeadline() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] ownerKey = keys[0]; + byte[] helperKey = keys[1]; + string name = $"sms-v2-reserved-before-budget-{Guid.NewGuid():N}"; + using var ownerScheduler = new ControlledLockFreeScheduler(); + using MemoryStore owner = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.CreateNew, + ownerScheduler); + using MemoryStore helper = OpenStore(name, RaceSlotCount); + LockFreeKeyDirectory directory = ReadDirectory(owner); + LockFreeSlotTable slots = ReadSlots(owner); + + ownerScheduler.PauseAt(LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck); + Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)> ownerTask = + ReserveAsync(owner, ownerKey, FiniteRaceWait); + + var ownerResumed = false; + (StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin) ownerResult = default; + StoreStatus helperStatus = StoreStatus.UnknownFailure; + ValueReservation helperReservation = default; + ReservationHandle ownerHandle = default; + try + { + Assert.True(ownerScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + ownerHandle = CaptureCurrentReservation( + directory, + slots, + expectedPhase: 1, + expectedTargetKind: 0); + + helperStatus = helper.TryReserve( + helperKey, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out helperReservation); + Assert.Equal(StoreStatus.Success, helperStatus); + AssertSlotState(slots, ownerHandle, LockFreeSlotTable.ReservedState); + + await Task.Delay(ExpiredRaceDelay); + ownerScheduler.Continue(); + ownerResumed = true; + ownerResult = await ownerTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally + { + if (!ownerResumed) + { + ownerScheduler.Continue(); + } + } + + AbortIfPending(ownerResult.Reservation); + AbortIfPending(helperReservation); + Assert.True( + ownerResult.Status == StoreStatus.Success, + $"A helper ordered Initializing->Reserved before the owner's expired budget check, " + + $"but the owner returned {ownerResult.Status}; corruptionOrigin={ownerResult.CorruptionOrigin ?? "none"}."); + Assert.Equal(ownerHandle, ownerResult.Reservation.HandleForEngine); + Assert.Equal(StoreStatus.Success, helperStatus); + Assert.Equal(StoreStatus.NotFound, owner.TryAcquire(ownerKey, out _)); + Assert.Equal(StoreStatus.NotFound, owner.TryAcquire(helperKey, out _)); + AssertDirectoryDrained(directory); + AssertAllSlotCapacityReusable(owner, keys, RaceSlotCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AtomicConvenienceReservedStateRemainsTentativeUntilPublished(bool segmented) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: RaceSlotCount + 2, RaceSlotCount); + byte[] key = keys[0]; + string name = $"sms-v2-atomic-intent-{segmented}-{Guid.NewGuid():N}"; + using var publisherScheduler = new ControlledLockFreeScheduler(); + using MemoryStore publisher = CreateInstrumentedStore( + name, + RaceSlotCount, + OpenMode.CreateNew, + publisherScheduler); + using MemoryStore contender = OpenStore(name, RaceSlotCount); + LockFreeKeyDirectory directory = ReadDirectory(publisher); + LockFreeSlotTable slots = ReadSlots(publisher); + + publisherScheduler.PauseAt( + LockFreeCheckpointId.ReserveAfterDirectoryInsertBeforePendingClassification); + Task<(StoreStatus Status, long CopiedBytes, string? CorruptionOrigin)> publisherTask = + PublishAsync(publisher, key, segmented, FiniteRaceWait); + + var publisherResumed = false; + StoreStatus contenderStatus = StoreStatus.UnknownFailure; + ValueReservation contenderReservation = default; + (StoreStatus Status, long CopiedBytes, string? CorruptionOrigin) publisherResult = default; + try + { + Assert.True(publisherScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + ReservationHandle publisherHandle = CaptureReservation( + slots, + LockFreeSlotTable.ReservedState, + expectedPhase: 5, + expectedTargetKind: 1); + AssertPublicationIntent( + slots, + publisherHandle, + SlotPublicationIntent.AtomicPublication); + + contenderStatus = contender.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.NoWait, + out contenderReservation); + + await Task.Delay(ExpiredRaceDelay); + publisherScheduler.Continue(); + publisherResumed = true; + publisherResult = await publisherTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally + { + if (!publisherResumed) + { + publisherScheduler.Continue(); + } + } + + AbortIfPending(contenderReservation); + StoreStatus acquireAfterPublisher = publisher.TryAcquire(key, out ValueLease lease); + if (acquireAfterPublisher == StoreStatus.Success) + { + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, publisher.TryRemove(key, StoreWaitOptions.Infinite)); + } + + Assert.True( + contenderStatus == StoreStatus.StoreBusy, + $"{(segmented ? "TryPublishSegments" : "TryPublish")} exposed its internal Reserved state as " + + $"public key ownership; the same-key contender returned {contenderStatus} before Published."); + Assert.True( + publisherResult.Status == StoreStatus.StoreBusy, + $"The expired atomic convenience publisher returned {publisherResult.Status}; " + + $"copiedBytes={publisherResult.CopiedBytes}; corruptionOrigin={publisherResult.CorruptionOrigin ?? "none"}."); + Assert.Equal(StoreStatus.NotFound, acquireAfterPublisher); + AssertDirectoryDrained(directory); + AssertAllSlotCapacityReusable(publisher, keys, RaceSlotCount); + } + + [Fact] + public void ExplicitReservedStateBlocksSameKeyContenders() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-explicit-intent-{Guid.NewGuid():N}"; + using MemoryStore owner = CreateStore(name, slotCount: 2); + using MemoryStore contender = OpenStore(name, slotCount: 2); + byte[] key = [0x38]; + + Assert.Equal( + StoreStatus.Success, + owner.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + Assert.Equal( + StoreStatus.DuplicateKey, + contender.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.NoWait, + out ValueReservation duplicate)); + Assert.False(duplicate.IsValid); + AssertPublicationIntent( + ReadSlots(owner), + reservation.HandleForEngine, + SlotPublicationIntent.ExplicitReservation); + + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.NotFound, owner.TryAcquire(key, out _)); + AssertDirectoryDrained(ReadDirectory(owner)); + } + + [Fact] + public void ExactCellWithoutMetadataReadyMarkerFailsClosedInsteadOfReportingDuplicate() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-missing-marker-{Guid.NewGuid():N}"; + using MemoryStore owner = CreateStore(name, slotCount: 2); + using MemoryStore contender = OpenStore(name, slotCount: 2); + byte[] key = [0x39]; + + Assert.Equal( + StoreStatus.Success, + owner.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + LockFreeSlotTable slots = ReadSlots(owner); + IndexBinding binding = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + long validOperation = AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation); + Assert.NotEqual(0, validOperation); + + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, 0); + try + { + Assert.Equal( + StoreStatus.CorruptStore, + contender.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.NoWait, + out ValueReservation duplicate)); + Assert.False(duplicate.IsValid); + } + finally + { + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, validOperation); + } + + Assert.Equal(StoreStatus.CorruptStore, reservation.Abort(StoreWaitOptions.Infinite)); + } + + [Theory] + [InlineData(LockFreeSlotTable.PublishedState, false)] + [InlineData(LockFreeSlotTable.RemoveRequestedState, false)] + [InlineData(LockFreeSlotTable.FreeState, false)] + [InlineData(LockFreeSlotTable.InitializingState, true)] + [InlineData(LockFreeSlotTable.ReservedState, true)] + [InlineData(LockFreeSlotTable.RetiredState, false)] + public void TentativeAbortFailsClosedForImpossibleSameGenerationState( + int impossibleState, + bool wrongOwner) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = CreateStore( + $"sms-v2-tentative-impossible-{impossibleState}-{Guid.NewGuid():N}", + slotCount: 1); + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + [0x3A], + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + + ReservationHandle handle = reservation.HandleForEngine; + LockFreeSlotTable slots = ReadSlots(store); + IndexBinding binding = IndexBinding.Decode(handle.SlotBinding); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + long reserved = AtomicControlWord.LoadAcquire(ref slot.Control); + int participant = 0; + if (wrongOwner) + { + participant = handle.ParticipantToken == 1 ? 2 : 1; + } + + long impossible = unchecked((long)AtomicControlWord.EncodeSlot( + impossibleState, + binding.Generation, + participant)); + AtomicControlWord.StoreRelease(ref slot.Control, impossible); + try + { + Assert.Equal( + TentativeReservationAbortResult.Corrupt, + slots.TryBeginTentativeAbort(handle)); + } + finally + { + AtomicControlWord.StoreRelease(ref slot.Control, reserved); + } + + bool structurallyMalformed = wrongOwner + || impossibleState == LockFreeSlotTable.RetiredState; + Assert.Equal( + structurallyMalformed ? StoreStatus.CorruptStore : StoreStatus.Success, + reservation.Abort(StoreWaitOptions.Infinite)); + if (!structurallyMalformed) + { + AssertDirectoryDrained(ReadDirectory(store)); + } + } + + [Fact] + public async Task RejectedNoWaitCandidateRevalidatesWinnerButCannotClaimAgain() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + string name = $"sms-v2-no-wait-rejected-retry-{Guid.NewGuid():N}"; + using var gate = new RejectedCandidateRetryGate(); + StoreOpenStatus opened = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, slotCount: 2, OpenMode.CreateNew), + LockFreeCheckpointFactory.CreateInstrumented(gate.Observe), + out MemoryStore? candidateStore); + Assert.Equal(StoreOpenStatus.Success, opened); + using MemoryStore candidate = Assert.IsType(candidateStore); + using MemoryStore winner = OpenStore(name, slotCount: 2); + byte[] key = [0x3B]; + + Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)> candidateTask = + ReserveAsync(candidate, key, StoreWaitOptions.NoWait); + Assert.True(gate.WaitBeforeDirectoryInsert(TimeSpan.FromSeconds(5))); + + Assert.Equal( + StoreStatus.Success, + winner.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation winningReservation)); + + gate.ContinueDirectoryInsert(); + Assert.True(gate.WaitAtFreshConflictResolution(TimeSpan.FromSeconds(5))); + Assert.Equal(StoreStatus.Success, winningReservation.Abort(StoreWaitOptions.Infinite)); + gate.ContinueConflictResolution(); + + (StoreStatus status, ValueReservation reservation, string? corruptionOrigin) = + await candidateTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.StoreBusy, status); + Assert.False(reservation.IsValid); + Assert.Null(corruptionOrigin); + Assert.Equal(1, gate.SlotClaimCount); + AssertDirectoryDrained(ReadDirectory(candidate)); + } + + [Fact] + public void PostInsertClassifierRejectsSameGenerationPublishedControlAsCorruption() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = CreateStore( + $"sms-v2-insert-classifier-published-{Guid.NewGuid():N}", + slotCount: 1); + Assert.Equal( + StoreStatus.Success, + store.TryReserve([0x35], 1, default, StoreWaitOptions.Infinite, out ValueReservation reservation)); + + ReservationHandle handle = reservation.HandleForEngine; + LockFreeSlotTable slots = ReadSlots(store); + IndexBinding binding = IndexBinding.Decode(handle.SlotBinding); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + long reserved = AtomicControlWord.LoadAcquire(ref slot.Control); + Assert.Equal(LockFreeSlotTable.ReservedState, SlotState(reserved)); + Assert.Equal(binding.Generation, SlotGeneration(reserved)); + long published = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.PublishedState, + binding.Generation, + participantToken: 0)); + AtomicControlWord.StoreRelease(ref slot.Control, published); + try + { + Assert.Equal( + StoreStatus.CorruptStore, + slots.ClassifyReservationAfterDirectoryInsert(handle)); + } + finally + { + Assert.Equal( + published, + AtomicControlWord.CompareExchange(ref slot.Control, reserved, published)); + } + + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + AssertDirectoryDrained(ReadDirectory(store)); + } + + [Fact] + public void PostInsertClassifierRejectsLowerGenerationControlAsCorruption() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = CreateStore( + $"sms-v2-insert-classifier-lower-generation-{Guid.NewGuid():N}", + slotCount: 1); + Assert.Equal( + StoreStatus.Success, + store.TryReserve([0x36], 1, default, StoreWaitOptions.Infinite, out ValueReservation first)); + Assert.Equal(StoreStatus.Success, first.Abort(StoreWaitOptions.Infinite)); + Assert.Equal( + StoreStatus.Success, + store.TryReserve([0x37], 1, default, StoreWaitOptions.Infinite, out ValueReservation reservation)); + + ReservationHandle handle = reservation.HandleForEngine; + LockFreeSlotTable slots = ReadSlots(store); + IndexBinding binding = IndexBinding.Decode(handle.SlotBinding); + Assert.True(binding.Generation > 1); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + long reserved = AtomicControlWord.LoadAcquire(ref slot.Control); + Assert.Equal(LockFreeSlotTable.ReservedState, SlotState(reserved)); + Assert.Equal(binding.Generation, SlotGeneration(reserved)); + long staleAborting = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.AbortingState, + binding.Generation - 1, + participantToken: 0)); + AtomicControlWord.StoreRelease(ref slot.Control, staleAborting); + try + { + Assert.Equal( + StoreStatus.CorruptStore, + slots.ClassifyReservationAfterDirectoryInsert(handle)); + } + finally + { + Assert.Equal( + staleAborting, + AtomicControlWord.CompareExchange(ref slot.Control, reserved, staleAborting)); + } + + Assert.Equal(StoreStatus.CorruptStore, reservation.Abort(StoreWaitOptions.Infinite)); + } + + [Fact] + public async Task OverflowInsertObservingExactEmptyAfterConcurrentCancellationDoesNotReportCorruption() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(count: OverflowSlotCount + 2, OverflowSlotCount); + string name = $"sms-v2-insert-cancel-overflow-{Guid.NewGuid():N}"; + using var insertScheduler = new ControlledLockFreeScheduler(); + using var abortScheduler = new ControlledLockFreeScheduler(); + using MemoryStore insertStore = CreateInstrumentedStore( + name, + OverflowSlotCount, + OpenMode.CreateNew, + insertScheduler); + using MemoryStore abortStore = CreateInstrumentedStore( + name, + OverflowSlotCount, + OpenMode.OpenExisting, + abortScheduler); + + for (var index = 0; index < OverflowAnchorCount; index++) + { + Assert.Equal( + StoreStatus.Success, + insertStore.TryPublish( + keys[index], + [unchecked((byte)index)], + default, + StoreWaitOptions.Infinite)); + } + + LockFreeKeyDirectory insertDirectory = ReadDirectory(insertStore); + Assert.Equal(OverflowAnchorCount, insertDirectory.PrimaryOccupancy); + Assert.Equal(0, insertDirectory.OverflowOccupancy); + + insertScheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication); + Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)> reserveTask = Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + StoreStatus status = insertStore.TryReserve( + keys[OverflowAnchorCount], + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation); + return (status, reservation, LockFreeCorruptionTrace.Consume()); + }); + + var insertResumed = false; + var abortResumed = false; + Task? abortTask = null; + try + { + Assert.True(insertScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + LockFreeSlotTable slots = ReadSlots(insertStore); + ReservationHandle reservationHandle = CaptureCurrentReservation( + insertDirectory, + slots, + expectedPhase: 2, + expectedTargetKind: 2); + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(reservationHandle)); + AssertSlotState(slots, reservationHandle, LockFreeSlotTable.AbortingState); + + abortScheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterSpillSummaryClear); + LockFreeKeyDirectory abortDirectory = ReadDirectory(abortStore); + abortTask = Task.Run(() => + { + InstrumentedLockFreeCheckpoint checkpoint = + abortScheduler.CreateInstrumentedCheckpoint(); + return abortDirectory.TryUnlink( + reservationHandle.SlotBinding, + LockFreeOperationBudget.UnboundedScan, + ref checkpoint); + }); + Assert.True(abortScheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + SpillSummary empty = SpillSummary.Decode( + abortDirectory.ReadSpillSummary(CanonicalBucket)); + Assert.False(empty.IsPresent); + Assert.Equal(reservationHandle.SlotBinding, empty.Binding); + Assert.Equal( + reservationHandle.SlotBinding, + abortDirectory.ReadCanonicalMutation(CanonicalBucket)); + + // Resume the already validated insert while the cancellation helper + // retains Empty(binding) and the exact canonical mutation. Empty is + // a legitimate terminal observation for this canceled lifecycle. + insertScheduler.Continue(); + insertResumed = true; + (StoreStatus insertStatus, ValueReservation reservation, string? corruptionOrigin) = + await reserveTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True( + insertStatus != StoreStatus.CorruptStore, + $"The resumed insert reported corruption at {corruptionOrigin ?? "an untraced origin"}."); + Assert.False(reservation.IsValid); + + abortScheduler.Continue(); + abortResumed = true; + StoreStatus abortStatus = await abortTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.NotEqual(StoreStatus.CorruptStore, abortStatus); + } + finally + { + if (!insertResumed) + { + insertScheduler.Continue(); + } + + if (!abortResumed) + { + abortScheduler.Continue(); + } + } + + for (var index = 0; index < OverflowAnchorCount; index++) + { + Assert.Equal( + StoreStatus.Success, + insertStore.TryRemove(keys[index], StoreWaitOptions.Infinite)); + } + + Assert.Equal(StoreStatus.NotFound, insertStore.TryAcquire(keys[OverflowAnchorCount], out _)); + AssertAllSlotCapacityReusable(insertStore, keys); + } + + private static async Task RunSingleSlotCancellationRace( + LockFreeCheckpointId checkpoint, + int expectedPhase, + int expectedSlotState, + StoreStatus expectedStatus, + byte[] key) + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-insert-cancel-{checkpoint}-{Guid.NewGuid():N}", + slotCount: 1, + OpenMode.CreateNew, + scheduler); + LockFreeKeyDirectory directory = ReadDirectory(store); + LockFreeSlotTable slots = ReadSlots(store); + scheduler.PauseAt(checkpoint); + + Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)> reserveTask = Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + StoreStatus status = store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation); + return (status, reservation, LockFreeCorruptionTrace.Consume()); + }); + + var resumed = false; + try + { + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + ReservationHandle reservationHandle = CaptureReservation( + slots, + expectedSlotState, + expectedPhase, + expectedTargetKind: expectedPhase == 1 ? 0 : 1); + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(reservationHandle)); + AssertSlotState(slots, reservationHandle, LockFreeSlotTable.AbortingState); + + scheduler.Continue(); + resumed = true; + (StoreStatus status, ValueReservation reservation, string? corruptionOrigin) = + await reserveTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True( + status == expectedStatus, + $"Expected {expectedStatus} after cancellation at {checkpoint}, observed {status}; " + + $"corruptionOrigin={corruptionOrigin ?? "none"}."); + if (expectedStatus == StoreStatus.Success) + { + Assert.Equal(reservationHandle, reservation.HandleForEngine); + } + else + { + Assert.False(reservation.IsValid); + } + } + finally + { + if (!resumed) + { + scheduler.Continue(); + } + } + + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(key, out _)); + AssertDirectoryDrained(directory); + byte[] replacementKey = [unchecked((byte)(key[0] + 0x40))]; + Assert.Equal( + StoreStatus.Success, + store.TryPublish(replacementKey, [0xA5], default, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.StoreFull, store.TryPublish([0x7F], [0x5A])); + Assert.Equal(StoreStatus.Success, store.TryRemove(replacementKey, StoreWaitOptions.Infinite)); + AssertDirectoryDrained(directory); + } + + private static ReservationHandle CaptureReservation( + LockFreeSlotTable slots, + int expectedSlotState, + int expectedPhase, + int expectedTargetKind) + { + StoreLayoutV2 layout = ReadPrivate(slots, "_layout"); + ReservationHandle found = default; + var foundCount = 0; + for (var slotIndex = 0; slotIndex < layout.SlotCount; slotIndex++) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + if (SlotState(control) != expectedSlotState) + { + continue; + } + + ulong participantToken = unchecked((ulong)control) >> 36; + if (participantToken == 0) + { + continue; + } + + ulong binding = slot.DirectoryBinding; + IndexBinding decoded = IndexBinding.Decode(binding); + DirectoryOperation operation = DirectoryOperation.Decode( + unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation))); + if (decoded.SlotIndex != slotIndex + || decoded.Generation != SlotGeneration(control) + || operation.Intent != 1 + || operation.Phase != expectedPhase + || operation.Kind != expectedTargetKind + || operation.Generation != decoded.Generation) + { + continue; + } + + found = new ReservationHandle( + ReadPrivate(slots, "_storeId"), + participantToken, + binding, + slot.ValueLength); + foundCount++; + } + + Assert.Equal(1, foundCount); + return found; + } + + private static ReservationHandle CaptureCurrentReservation( + LockFreeKeyDirectory directory, + LockFreeSlotTable slots, + int expectedPhase, + int expectedTargetKind) + { + ulong binding = FindCurrentMutation(directory); + IndexBinding decoded = IndexBinding.Decode(binding); + ref ValueSlotMetadataV2 slot = ref slots.Slot(decoded.SlotIndex); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + Assert.Equal(LockFreeSlotTable.InitializingState, SlotState(control)); + Assert.Equal(decoded.Generation, SlotGeneration(control)); + Assert.Equal(binding, slot.DirectoryBinding); + + DirectoryOperation operation = DirectoryOperation.Decode( + unchecked((ulong)AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation))); + Assert.Equal(1, operation.Intent); + Assert.Equal(expectedPhase, operation.Phase); + Assert.Equal(expectedTargetKind, operation.Kind); + Assert.Equal(decoded.Generation, operation.Generation); + + ulong participantToken = unchecked((ulong)control) >> 36; + Assert.NotEqual(0UL, participantToken); + return new ReservationHandle( + ReadPrivate(slots, "_storeId"), + participantToken, + binding, + slot.ValueLength); + } + + private static ulong FindCurrentMutation(LockFreeKeyDirectory directory) + { + StoreLayoutV2 layout = ReadPrivate(directory, "_layout"); + ulong found = 0; + for (var bucket = 0; bucket < layout.PrimaryBucketCount; bucket++) + { + ulong mutation = directory.ReadCanonicalMutation(bucket); + if (mutation == 0) + { + continue; + } + + Assert.Equal(0UL, found); + found = mutation; + } + + Assert.NotEqual(0UL, found); + return found; + } + + private static void AssertDirectoryDrained(LockFreeKeyDirectory directory) + { + StoreLayoutV2 layout = ReadPrivate(directory, "_layout"); + Assert.Equal(0, directory.PrimaryOccupancy); + Assert.Equal(0, directory.OverflowOccupancy); + for (var bucket = 0; bucket < layout.PrimaryBucketCount; bucket++) + { + Assert.Equal(0UL, directory.ReadCanonicalMutation(bucket)); + } + } + + private static void AssertSlotState( + LockFreeSlotTable slots, + in ReservationHandle reservation, + int expectedState) + { + IndexBinding binding = IndexBinding.Decode(reservation.SlotBinding); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + Assert.Equal(expectedState, SlotState(control)); + Assert.Equal(binding.Generation, SlotGeneration(control)); + } + + private static void AssertAllSlotCapacityReusable( + MemoryStore store, + byte[][] keys, + int slotCount = OverflowSlotCount) + { + for (var index = 0; index < slotCount; index++) + { + Assert.Equal( + StoreStatus.Success, + store.TryPublish( + keys[index], + [unchecked((byte)(0x80 + index))], + default, + StoreWaitOptions.Infinite)); + } + + Assert.Equal( + StoreStatus.StoreFull, + store.TryPublish( + keys[slotCount], + [0xFF], + default, + StoreWaitOptions.Infinite)); + + for (var index = 0; index < slotCount; index++) + { + Assert.Equal( + StoreStatus.Success, + store.TryRemove(keys[index], StoreWaitOptions.Infinite)); + } + } + + private static Task<(StoreStatus Status, ValueReservation Reservation, string? CorruptionOrigin)> + ReserveAsync( + MemoryStore store, + byte[] key, + StoreWaitOptions waitOptions) => + Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + StoreStatus status = store.TryReserve( + key, + payloadLength: 1, + descriptor: default, + waitOptions, + out ValueReservation reservation); + return (status, reservation, LockFreeCorruptionTrace.Consume()); + }); + + private static Task<(StoreStatus Status, string? CorruptionOrigin)> UnlinkAsync( + LockFreeKeyDirectory directory, + ulong binding, + ControlledLockFreeScheduler scheduler) => + Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + InstrumentedLockFreeCheckpoint checkpoint = scheduler.CreateInstrumentedCheckpoint(); + StoreStatus status = directory.TryUnlink( + binding, + LockFreeOperationBudget.UnboundedScan, + ref checkpoint); + return (status, LockFreeCorruptionTrace.Consume()); + }); + + private static Task<(StoreStatus Status, long CopiedBytes, string? CorruptionOrigin)> + PublishAsync( + MemoryStore store, + byte[] key, + bool segmented, + StoreWaitOptions waitOptions) => + Task.Run(() => + { + _ = LockFreeCorruptionTrace.Consume(); + long copiedBytes; + StoreStatus status; + if (segmented) + { + var payload = new ReadOnlySequence(new byte[] { 0xA5 }); + status = store.TryPublishSegments( + key, + payload, + descriptor: default, + waitOptions, + out copiedBytes); + } + else + { + status = store.TryPublish( + key, + [0xA5], + descriptor: default, + waitOptions); + copiedBytes = status == StoreStatus.Success ? 1 : 0; + } + + return (status, copiedBytes, LockFreeCorruptionTrace.Consume()); + }); + + private static bool HasExactSlotState( + LockFreeSlotTable slots, + in ReservationHandle reservation, + int expectedState) + { + IndexBinding binding = IndexBinding.Decode(reservation.SlotBinding); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + return SlotState(control) == expectedState + && SlotGeneration(control) == binding.Generation + && slot.DirectoryBinding == reservation.SlotBinding; + } + + private static void AssertPublicationIntent( + LockFreeSlotTable slots, + in ReservationHandle reservation, + SlotPublicationIntent expected) + { + IndexBinding binding = IndexBinding.Decode(reservation.SlotBinding); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + Assert.Equal((int)expected, Volatile.Read(ref slot.PublicationIntent)); + } + + private static void AbortIfPending(ValueReservation reservation) + { + if (reservation.IsValid) + { + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + } + } + + private static MemoryStore CreateInstrumentedStore( + string name, + int slotCount, + OpenMode openMode, + ControlledLockFreeScheduler scheduler) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, slotCount, openMode), + scheduler.CreateInstrumentedCheckpoint(), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore CreateStore(string name, int slotCount) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, slotCount, OpenMode.CreateNew), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore OpenStore(string name, int slotCount) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, slotCount, OpenMode.OpenExisting), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options(string name, int slotCount, OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: sizeof(long), + leaseRecordCount: Math.Max(2, slotCount), + participantRecordCount: 4, + openMode, + enableLeaseRecovery: true); + + private static LockFreeKeyDirectory ReadDirectory(MemoryStore store) => + ReadPrivate(ReadPrivate(store, "_engine"), "_directory"); + + private static LockFreeSlotTable ReadSlots(MemoryStore store) => + ReadPrivate(ReadPrivate(store, "_engine"), "_slots"); + + private static ulong ReadDirectoryOperation(LockFreeSlotTable slots, int slotIndex) => + unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slots.Slot(slotIndex).DirectoryOperation)); + + private static ulong ReadDirectoryLocation(LockFreeSlotTable slots, int slotIndex) => + unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slots.Slot(slotIndex).DirectoryLocation)); + + private static ulong ReadDirectoryCell( + LockFreeKeyDirectory directory, + DirectoryLocation location) => + unchecked((ulong)AtomicControlWord.LoadAcquire(ref DirectoryCell(directory, location))); + + private static void WriteDirectoryCell( + LockFreeKeyDirectory directory, + DirectoryLocation location, + ulong binding) => + AtomicControlWord.StoreRelease( + ref DirectoryCell(directory, location), + unchecked((long)binding)); + + private static void WriteCanonicalMutation( + LockFreeKeyDirectory directory, + int canonicalBucket, + ulong binding) => + AtomicControlWord.StoreRelease( + ref CanonicalMutation(directory, canonicalBucket), + unchecked((long)binding)); + + private static unsafe ref long DirectoryCell( + LockFreeKeyDirectory directory, + DirectoryLocation location) + { + ISharedStoreRegion region = ReadPrivate(directory, "_region"); + StoreLayoutV2 layout = ReadPrivate(directory, "_layout"); + long offset = location.Kind switch + { + 1 => PrimaryCellOffset(layout, checked((int)location.Index)), + 2 => layout.OverflowDirectoryOffset + (location.Index * layout.OverflowStride), + _ => throw new ArgumentOutOfRangeException(nameof(location)), + }; + return ref *(long*)(region.Pointer + offset); + } + + private static unsafe ref long CanonicalMutation( + LockFreeKeyDirectory directory, + int canonicalBucket) + { + ISharedStoreRegion region = ReadPrivate(directory, "_region"); + StoreLayoutV2 layout = ReadPrivate(directory, "_layout"); + long offset = layout.PrimaryDirectoryOffset + + ((long)canonicalBucket * layout.PrimaryBucketStride) + + sizeof(long); + return ref *(long*)(region.Pointer + offset); + } + + private static long PrimaryCellOffset(StoreLayoutV2 layout, int absoluteCellIndex) + { + int bucket = absoluteCellIndex / LayoutV2Constants.PrimaryLanesPerBucket; + int lane = absoluteCellIndex % LayoutV2Constants.PrimaryLanesPerBucket; + return layout.PrimaryDirectoryOffset + + ((long)bucket * layout.PrimaryBucketStride) + + (2 * sizeof(long)) + + (lane * sizeof(long)); + } + + private static T ReadPrivate(object owner, string fieldName) + { + FieldInfo field = owner.GetType().GetField( + fieldName, + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Missing field {owner.GetType().FullName}.{fieldName}."); + return Assert.IsAssignableFrom(field.GetValue(owner)); + } + + private static byte[][] GenerateBucketPairCollisions(int count, int slotCount) + { + var keys = new List(count); + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + uint bucketMask = checked((uint)((primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + for (long candidate = 1; keys.Count < count; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + ulong hash = StoreKey.Hash(key); + int first = (int)(Mix(hash) & bucketMask); + int second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & bucketMask); + if (second == first) + { + second = (first + 1) & (int)bucketMask; + } + + if (first == CanonicalBucket && second == 1) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static int NextPowerOfTwo(int value) + { + var result = 1; + while (result < value) + { + result <<= 1; + } + + return result; + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static int SlotState(long control) => (int)(unchecked((ulong)control) & 0x7UL); + + private static long SlotGeneration(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + public enum TargetAfterCancellation + { + Empty, + ValidReplacement, + Malformed, + OutOfRange, + } + + public enum CompetingUnlinkTarget + { + ExactBinding, + Empty, + ValidReplacement, + Malformed, + OutOfRange, + } + + private sealed class RejectedCandidateRetryGate : IDisposable + { + private readonly ManualResetEventSlim _beforeInsert = new(false); + private readonly ManualResetEventSlim _continueInsert = new(false); + private readonly ManualResetEventSlim _atConflict = new(false); + private readonly ManualResetEventSlim _continueConflict = new(false); + private int _slotClaimCount; + private int _beforeInsertObserved; + private int _conflictObserved; + + internal int SlotClaimCount => Volatile.Read(ref _slotClaimCount); + + internal void Observe(LockFreeCheckpointEntry entry) + { + if (entry.Id == LockFreeCheckpointId.SlotClaimAfterParticipantRecheck) + { + Interlocked.Increment(ref _slotClaimCount); + } + + if (entry.Id == LockFreeCheckpointId.DirectoryBeforeDescriptorPublication + && Interlocked.CompareExchange(ref _beforeInsertObserved, 1, 0) == 0) + { + _beforeInsert.Set(); + _continueInsert.Wait(); + return; + } + + if (entry.Id == LockFreeCheckpointId.ReserveAfterExistingLookup + && Interlocked.CompareExchange(ref _conflictObserved, 1, 0) == 0) + { + _atConflict.Set(); + _continueConflict.Wait(); + } + } + + internal bool WaitBeforeDirectoryInsert(TimeSpan timeout) => + _beforeInsert.Wait(timeout); + + internal void ContinueDirectoryInsert() => _continueInsert.Set(); + + internal bool WaitAtFreshConflictResolution(TimeSpan timeout) => + _atConflict.Wait(timeout); + + internal void ContinueConflictResolution() => _continueConflict.Set(); + + public void Dispose() + { + _continueInsert.Set(); + _continueConflict.Set(); + _beforeInsert.Dispose(); + _continueInsert.Dispose(); + _atConflict.Dispose(); + _continueConflict.Dispose(); + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeLeaseCapacityRegressionTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeLeaseCapacityRegressionTests.cs new file mode 100644 index 0000000..2fb52bc --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeLeaseCapacityRegressionTests.cs @@ -0,0 +1,590 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeLeaseCapacityRegressionTests +{ + private static readonly TimeSpan TestBound = TimeSpan.FromSeconds(10); + + [Fact] + public void StableCapacityEmitsOneExactProofAndReturnsLeaseTableFull() + { + if (!IsSupportedHost()) + { + return; + } + + var proofs = new LeaseProofRecorder(); + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(static _ => { }, proofs); + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-lease-full-proof-{Guid.NewGuid():N}", + OpenMode.CreateNew, + leaseRecordCount: 2, + checkpoint); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x11], [0x21])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0x11], out ValueLease first)); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0x11], out ValueLease second)); + + Assert.Equal( + StoreStatus.LeaseTableFull, + store.TryAcquire([0x11], StoreWaitOptions.Infinite, out ValueLease rejected)); + Assert.False(rejected.IsValid); + + LeaseProofObservation candidate = Assert.Single( + proofs.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Candidate); + LeaseProofObservation confirmed = Assert.Single( + proofs.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Confirmed); + Assert.Equal(candidate.Token, confirmed.Token); + Assert.Equal(2, candidate.LeaseRecordCount); + Assert.Equal(candidate.LeaseRecordCount, confirmed.LeaseRecordCount); + Assert.DoesNotContain( + proofs.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Rejected); + + Assert.Equal(StoreStatus.Success, first.Release()); + Assert.Equal(StoreStatus.Success, second.Release()); + } + + [Fact] + public async Task ProofGateIsPerHandleAndRespectsNoWaitFiniteAndInfinitePolicies() + { + if (!IsSupportedHost()) + { + return; + } + + string name = $"sms-v2-lease-full-gate-{Guid.NewGuid():N}"; + using var pause = new LeaseProofPause(); + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(static _ => { }, pause); + using MemoryStore firstHandle = CreateInstrumentedStore( + name, + OpenMode.CreateNew, + leaseRecordCount: 1, + checkpoint); + using MemoryStore otherHandle = OpenOrdinaryStore( + name, + leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, firstHandle.TryPublish([0x21], [0x31])); + Assert.Equal(StoreStatus.Success, firstHandle.TryAcquire([0x21], out ValueLease held)); + + Task heldProof = Task.Run(() => firstHandle.TryAcquire( + [0x21], + StoreWaitOptions.Infinite, + out _)); + Task? infiniteContender = null; + try + { + pause.WaitUntilProofPaused(); + + Assert.Equal( + StoreStatus.StoreBusy, + firstHandle.TryAcquire([0x21], StoreWaitOptions.NoWait, out _)); + + var started = Stopwatch.StartNew(); + Assert.Equal( + StoreStatus.StoreBusy, + firstHandle.TryAcquire( + [0x21], + new StoreWaitOptions(TimeSpan.FromMilliseconds(50)), + out _)); + Assert.InRange(started.Elapsed, TimeSpan.Zero, TimeSpan.FromSeconds(1)); + + // Proof state is private to one open handle. A second handle may + // independently prove the same stable physical lease exhaustion. + Assert.Equal( + StoreStatus.LeaseTableFull, + otherHandle.TryAcquire([0x21], StoreWaitOptions.Infinite, out _)); + + using var contenderEntered = new ManualResetEventSlim(initialState: false); + infiniteContender = Task.Run(() => + { + contenderEntered.Set(); + return firstHandle.TryAcquire( + [0x21], + StoreWaitOptions.Infinite, + out _); + }); + Assert.True(contenderEntered.Wait(TestBound)); + Assert.False(infiniteContender.IsCompleted); + + pause.ResumeProof(); + Assert.Equal(StoreStatus.LeaseTableFull, await heldProof.WaitAsync(TestBound)); + Assert.Equal( + StoreStatus.LeaseTableFull, + await infiniteContender.WaitAsync(TestBound)); + } + finally + { + pause.ResumeProof(); + _ = await heldProof.WaitAsync(TestBound); + if (infiniteContender is not null) + { + _ = await infiniteContender.WaitAsync(TestBound); + } + + Assert.Equal(StoreStatus.Success, held.Release()); + } + } + + [Fact] + public async Task ReleaseBetweenCollectsReturnsStoreBusyForNoWaitWithoutConfirmingFull() + { + if (!IsSupportedHost()) + { + return; + } + + using var pause = new LeaseProofPause(); + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(static _ => { }, pause); + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-lease-full-movement-{Guid.NewGuid():N}", + OpenMode.CreateNew, + leaseRecordCount: 1, + checkpoint); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x31], [0x41])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0x31], out ValueLease held)); + + Task contender = Task.Run(() => store.TryAcquire( + [0x31], + StoreWaitOptions.NoWait, + out _)); + try + { + pause.WaitUntilProofPaused(); + Assert.Equal(StoreStatus.Success, held.Release()); + pause.ResumeProof(); + + Assert.Equal(StoreStatus.StoreBusy, await contender.WaitAsync(TestBound)); + LeaseProofObservation candidate = Assert.Single( + pause.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Candidate); + LeaseProofObservation rejected = Assert.Single( + pause.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Rejected); + Assert.Equal(candidate.Token, rejected.Token); + Assert.DoesNotContain( + pause.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Confirmed); + + Assert.Equal( + StoreStatus.Success, + store.TryAcquire([0x31], StoreWaitOptions.Infinite, out ValueLease replacement)); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + finally + { + pause.ResumeProof(); + _ = await contender.WaitAsync(TestBound); + if (held.IsValid) + { + Assert.Equal(StoreStatus.Success, held.Release()); + } + } + } + + [Fact] + public async Task BindingRemovedBeforeFullCandidateReturnsNotFoundAfterExactProof() + { + if (!IsSupportedHost()) + { + return; + } + + using var pause = new AcquireBeforeClaimPause(); + var proofs = new LeaseProofRecorder(); + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(pause.Observe, proofs); + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-lease-full-binding-{Guid.NewGuid():N}", + OpenMode.CreateNew, + leaseRecordCount: 1, + checkpoint, + slotCount: 2); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x61], [0x71])); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x62], [0x72])); + + Task targetAcquire = Task.Run(() => store.TryAcquire( + [0x61], + StoreWaitOptions.Infinite, + out _)); + ValueLease blocker = default; + try + { + pause.WaitUntilPaused(); + Assert.Equal( + StoreStatus.Success, + store.TryRemove([0x61], StoreWaitOptions.Infinite)); + Assert.Equal( + StoreStatus.Success, + store.TryAcquire([0x62], StoreWaitOptions.Infinite, out blocker)); + pause.Resume(); + + Assert.Equal(StoreStatus.NotFound, await targetAcquire.WaitAsync(TestBound)); + Assert.Single( + proofs.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Confirmed); + } + finally + { + pause.Resume(); + _ = await targetAcquire.WaitAsync(TestBound); + if (blocker.IsValid) + { + Assert.Equal(StoreStatus.Success, blocker.Release()); + } + } + } + + [Theory] + [InlineData(MalformedLeaseControl.InvalidState)] + [InlineData(MalformedLeaseControl.ZeroIncarnation)] + [InlineData(MalformedLeaseControl.FreeWithOwner)] + [InlineData(MalformedLeaseControl.ClaimingWithoutOwner)] + [InlineData(MalformedLeaseControl.ReleasingWithOwner)] + [InlineData(MalformedLeaseControl.NonterminalRetired)] + [InlineData(MalformedLeaseControl.InvalidParticipantIndex)] + public void MalformedControlIntroducedAfterFirstCollectFailsClosed( + MalformedLeaseControl malformedKind) + { + if (!IsSupportedHost()) + { + return; + } + + LockFreeLeaseRegistry? registry = null; + long originalControl = 0; + var proofs = new LeaseProofRecorder(() => + { + ref LeaseRecordV2 record = ref Assert.IsType(registry).Record(0); + originalControl = AtomicControlWord.LoadAcquire(ref record.Control); + long malformed = CreateMalformedControl(malformedKind, originalControl); + AtomicControlWord.StoreRelease(ref record.Control, malformed); + }); + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(static _ => { }, proofs); + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-lease-full-malformed-{Guid.NewGuid():N}", + OpenMode.CreateNew, + leaseRecordCount: 1, + checkpoint); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x41], [0x51])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0x41], out ValueLease held)); + registry = ReadLeases(store); + + StoreStatus status; + try + { + status = store.TryAcquire( + [0x41], + StoreWaitOptions.Infinite, + out _); + } + finally + { + if (originalControl != 0) + { + AtomicControlWord.StoreRelease( + ref Assert.IsType(registry).Record(0).Control, + originalControl); + } + } + + Assert.Equal(StoreStatus.CorruptStore, status); + LeaseProofObservation candidate = Assert.Single( + proofs.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Candidate); + LeaseProofObservation rejected = Assert.Single( + proofs.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Rejected); + Assert.Equal(candidate.Token, rejected.Token); + Assert.DoesNotContain( + proofs.Snapshot(), + static item => item.Kind == LeaseProofObservationKind.Confirmed); + Assert.Equal(StoreStatus.CorruptStore, held.Release()); + } + + [Fact] + public void MalformedControlInFirstCollectFailsClosedWithoutCapacityCandidate() + { + if (!IsSupportedHost()) + { + return; + } + + var proofs = new LeaseProofRecorder(); + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(static _ => { }, proofs); + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-lease-full-first-malformed-{Guid.NewGuid():N}", + OpenMode.CreateNew, + leaseRecordCount: 1, + checkpoint); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x51], [0x61])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0x51], out ValueLease held)); + LockFreeLeaseRegistry registry = ReadLeases(store); + ref LeaseRecordV2 record = ref registry.Record(0); + long original = AtomicControlWord.LoadAcquire(ref record.Control); + AtomicControlWord.StoreRelease( + ref record.Control, + CreateMalformedControl(MalformedLeaseControl.InvalidParticipantIndex, original)); + + StoreStatus status; + try + { + status = store.TryAcquire([0x51], StoreWaitOptions.Infinite, out _); + } + finally + { + AtomicControlWord.StoreRelease(ref record.Control, original); + } + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Empty(proofs.Snapshot()); + Assert.Equal(StoreStatus.CorruptStore, held.Release()); + } + + private static long CreateMalformedControl( + MalformedLeaseControl kind, + long originalControl) + { + const int invalidParticipantToken = 13; + int owner = checked((int)((unchecked((ulong)originalControl) >> 36) & 0x0fff_ffffUL)); + Assert.NotEqual(0, owner); + return kind switch + { + MalformedLeaseControl.InvalidState => Signed(AtomicControlWord.EncodeLease( + state: 6, + generation: 1, + participantToken: 0)), + MalformedLeaseControl.ZeroIncarnation => + LockFreeLeaseRegistry.ActiveState | ((long)owner << 36), + MalformedLeaseControl.FreeWithOwner => Signed(AtomicControlWord.EncodeLease( + LockFreeLeaseRegistry.FreeState, + generation: 1, + owner)), + MalformedLeaseControl.ClaimingWithoutOwner => Signed(AtomicControlWord.EncodeLease( + LockFreeLeaseRegistry.ClaimingState, + generation: 1, + participantToken: 0)), + MalformedLeaseControl.ReleasingWithOwner => Signed(AtomicControlWord.EncodeLease( + LockFreeLeaseRegistry.ReleasingState, + generation: 1, + owner)), + MalformedLeaseControl.NonterminalRetired => Signed(AtomicControlWord.EncodeLease( + LockFreeLeaseRegistry.RetiredState, + generation: 1, + participantToken: 0)), + MalformedLeaseControl.InvalidParticipantIndex => Signed(AtomicControlWord.EncodeLease( + LockFreeLeaseRegistry.ActiveState, + generation: 1, + invalidParticipantToken)), + _ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null) + }; + } + + private static long Signed(ulong value) => unchecked((long)value); + + private static MemoryStore CreateInstrumentedStore( + string name, + OpenMode openMode, + int leaseRecordCount, + InstrumentedLockFreeCheckpoint checkpoint, + int slotCount = 1) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, openMode, leaseRecordCount, slotCount), + checkpoint, + out MemoryStore? candidate); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(candidate); + } + + private static MemoryStore OpenOrdinaryStore(string name, int leaseRecordCount) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting, leaseRecordCount, slotCount: 1), + out MemoryStore? candidate); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(candidate); + } + + private static SharedMemoryStoreOptions Options( + string name, + OpenMode openMode, + int leaseRecordCount, + int slotCount) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: 1, + leaseRecordCount, + participantRecordCount: 4, + openMode, + enableLeaseRecovery: true); + + private static LockFreeLeaseRegistry ReadLeases(MemoryStore store) + { + FieldInfo engineField = typeof(MemoryStore).GetField( + "_engine", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("MemoryStore._engine is absent."); + object engine = engineField.GetValue(store) + ?? throw new InvalidOperationException("MemoryStore._engine is null."); + FieldInfo leasesField = engine.GetType().GetField( + "_leases", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Lock-free engine._leases is absent."); + return Assert.IsType(leasesField.GetValue(engine)); + } + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + public enum MalformedLeaseControl + { + InvalidState, + ZeroIncarnation, + FreeWithOwner, + ClaimingWithoutOwner, + ReleasingWithOwner, + NonterminalRetired, + InvalidParticipantIndex + } + + private enum LeaseProofObservationKind + { + Candidate, + Confirmed, + Rejected + } + + private readonly record struct LeaseProofObservation( + LeaseProofObservationKind Kind, + long Token, + int LeaseRecordCount); + + private class LeaseProofRecorder : ILockFreeLeaseTableFullProofObserver + { + private readonly ConcurrentQueue _observations = new(); + private readonly Action? _candidateAction; + private long _nextToken; + + internal LeaseProofRecorder(Action? candidateAction = null) + { + _candidateAction = candidateAction; + } + + public virtual long BeginCandidate(int leaseRecordCount) + { + long token = Interlocked.Increment(ref _nextToken); + _observations.Enqueue(new LeaseProofObservation( + LeaseProofObservationKind.Candidate, + token, + leaseRecordCount)); + _candidateAction?.Invoke(); + return token; + } + + public void CompleteCandidate(long token, bool confirmed) + { + int leaseRecordCount = _observations + .First(item => + item.Kind == LeaseProofObservationKind.Candidate + && item.Token == token) + .LeaseRecordCount; + _observations.Enqueue(new LeaseProofObservation( + confirmed + ? LeaseProofObservationKind.Confirmed + : LeaseProofObservationKind.Rejected, + token, + leaseRecordCount)); + } + + internal LeaseProofObservation[] Snapshot() => _observations.ToArray(); + } + + private sealed class LeaseProofPause : LeaseProofRecorder, IDisposable + { + private readonly ManualResetEventSlim _proofPaused = new(initialState: false); + private readonly ManualResetEventSlim _resumeProof = new(initialState: false); + private int _proofPauseClaimed; + + public override long BeginCandidate(int leaseRecordCount) + { + long token = base.BeginCandidate(leaseRecordCount); + if (Interlocked.CompareExchange(ref _proofPauseClaimed, 1, 0) == 0) + { + _proofPaused.Set(); + if (!_resumeProof.Wait(TestBound)) + { + throw new Xunit.Sdk.XunitException( + "Paused LeaseTableFull proof was not resumed within the test bound."); + } + } + + return token; + } + + internal void WaitUntilProofPaused() => Assert.True( + _proofPaused.Wait(TestBound), + "The LeaseTableFull proof candidate was not reached."); + + internal void ResumeProof() => _resumeProof.Set(); + + public void Dispose() + { + _resumeProof.Set(); + _proofPaused.Dispose(); + _resumeProof.Dispose(); + } + } + + private sealed class AcquireBeforeClaimPause : IDisposable + { + private readonly ManualResetEventSlim _paused = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: false); + private int _claimed; + + internal void Observe(LockFreeCheckpointEntry entry) + { + if (entry.Id != LockFreeCheckpointId.AcquireBeforeLeaseClaimCas + || Interlocked.CompareExchange(ref _claimed, 1, 0) != 0) + { + return; + } + + _paused.Set(); + if (!_resume.Wait(TestBound)) + { + throw new Xunit.Sdk.XunitException( + "Acquire-before-claim pause was not resumed within the test bound."); + } + } + + internal void WaitUntilPaused() => Assert.True( + _paused.Wait(TestBound), + "Acquire did not reach the before-lease-claim checkpoint."); + + internal void Resume() => _resume.Set(); + + public void Dispose() + { + _resume.Set(); + _paused.Dispose(); + _resume.Dispose(); + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeLeaseRecoveryTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeLeaseRecoveryTests.cs new file mode 100644 index 0000000..45d723e --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeLeaseRecoveryTests.cs @@ -0,0 +1,536 @@ +using System.Diagnostics; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeLeaseRecoveryTests +{ + [Fact] + public void CurrentProcessOverrideDoesNotRevokeLiveClaimInitialization() + { + Assert.Equal( + LockFreeLeaseRegistry.RecoveryDisposition.Active, + LockFreeLeaseRegistry.RecoveryDispositionFor( + LockFreeLeaseRegistry.ClaimingState, + ParticipantClassificationKind.CurrentProcess, + LayoutV2Constants.ParticipantActive, + participantHandoffPublished: false, + recoverCurrentProcessLeases: true)); + + Assert.Equal( + LockFreeLeaseRegistry.RecoveryDisposition.Recover, + LockFreeLeaseRegistry.RecoveryDispositionFor( + LockFreeLeaseRegistry.ActiveState, + ParticipantClassificationKind.CurrentProcess, + LayoutV2Constants.ParticipantActive, + participantHandoffPublished: false, + recoverCurrentProcessLeases: true)); + + Assert.Equal( + LockFreeLeaseRegistry.RecoveryDisposition.Recover, + LockFreeLeaseRegistry.RecoveryDispositionFor( + LockFreeLeaseRegistry.ClaimingState, + ParticipantClassificationKind.CurrentProcess, + LayoutV2Constants.ParticipantClosing, + participantHandoffPublished: true, + recoverCurrentProcessLeases: false)); + + Assert.Equal( + LockFreeLeaseRegistry.RecoveryDisposition.Recover, + LockFreeLeaseRegistry.RecoveryDispositionFor( + LockFreeLeaseRegistry.ActiveState, + ParticipantClassificationKind.Live, + LayoutV2Constants.ParticipantClosing, + participantHandoffPublished: true, + recoverCurrentProcessLeases: false)); + + Assert.Equal( + LockFreeLeaseRegistry.RecoveryDisposition.Recover, + LockFreeLeaseRegistry.RecoveryDispositionFor( + LockFreeLeaseRegistry.ClaimingState, + ParticipantClassificationKind.Unsupported, + LayoutV2Constants.ParticipantRecovering, + participantHandoffPublished: true, + recoverCurrentProcessLeases: false)); + } + + [Fact] + public async Task LiveReleaseAndRecordReuseWinningBeforeRecoveryCasPreservesLaterIncarnation() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 2, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [11])); + Assert.Equal(StoreStatus.Success, store.TryPublish([2], [22])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var first)); + ulong firstLeaseToken = first.HandleForEngine.LeaseToken; + scheduler.PauseAt(LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + + LeaseRecoveryReport report = default; + var recovery = Task.Run(() => store.TryRecoverLeases( + new LeaseRecoveryOptions(true), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal(StoreStatus.Success, first.Release()); + Assert.Equal(StoreStatus.Success, store.TryAcquire([2], out var replacement)); + Assert.NotEqual(firstLeaseToken, replacement.HandleForEngine.LeaseToken); + scheduler.Continue(); + + Assert.Equal(StoreStatus.Success, await recovery.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(new LeaseRecoveryReport(1, 0, 0, 0, 0), report); + Assert.True(replacement.IsValid); + Assert.Equal(22, replacement.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task RecoveryExactCasWinningBeforeLiveReleaseFencesOldHandleAndRestoresCapacity() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + scheduler.PauseAt(LockFreeCheckpointId.ReleaseBeforeActiveReleaseCas); + + StoreStatus releaseStatus = default; + var release = Task.Run(() => releaseStatus = lease.Release()); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(true), out var report)); + Assert.Equal(new LeaseRecoveryReport(1, 1, 0, 0, 0), report); + Assert.False(lease.IsValid); + scheduler.Continue(); + await release.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.LeaseAlreadyReleased, releaseStatus); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task OrdinaryAcquireHelpsPausedUnownedReleaseAndReusesCapacityOneLeaseRecord() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var first)); + ulong firstToken = first.HandleForEngine.LeaseToken; + scheduler.PauseAt(LockFreeCheckpointId.ReleaseAfterOwnershipReleaseCas); + + StoreStatus releaseStatus = default; + var release = Task.Run(() => releaseStatus = first.Release(StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + // The lease record is unowned Releasing. Ordinary acquisition, without + // invoking explicit recovery, must finish that exact incarnation and + // claim the next one instead of leaking LeaseTableFull. + Assert.Equal( + StoreStatus.Success, + store.TryAcquire([1], StoreWaitOptions.Infinite, out var replacement)); + Assert.NotEqual(firstToken, replacement.HandleForEngine.LeaseToken); + Assert.True(replacement.IsValid); + + scheduler.Continue(); + await release.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, releaseStatus); + Assert.False(first.IsValid); + Assert.True(replacement.IsValid); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task CancellationBeforeRecoveryCasPreservesLeaseAndReturnsPartialReport() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + + LeaseRecoveryReport report = default; + var recovery = Task.Run(() => store.TryRecoverLeases( + new LeaseRecoveryOptions(true), + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + cancellation.Cancel(); + scheduler.Continue(); + + Assert.Equal(StoreStatus.OperationCanceled, await recovery.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(new LeaseRecoveryReport(1, 0, 0, 0, 0), report); + Assert.True(lease.IsValid); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public async Task DeadlineBeforeRecoveryCasPreservesLeaseAndReturnsPartialReport() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + + LeaseRecoveryReport report = default; + var recovery = Task.Run(() => store.TryRecoverLeases( + new LeaseRecoveryOptions(true), + new StoreWaitOptions(TimeSpan.FromMilliseconds(50)), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + await Task.Delay(TimeSpan.FromMilliseconds(150)); + scheduler.Continue(); + + Assert.Equal(StoreStatus.StoreBusy, await recovery.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(new LeaseRecoveryReport(1, 0, 0, 0, 0), report); + Assert.True(lease.IsValid); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public async Task CancellationAfterExactRecoveryCasFinishesHelpableTransition() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryAfterExactRecoveryCas); + + LeaseRecoveryReport report = default; + var recovery = Task.Run(() => store.TryRecoverLeases( + new LeaseRecoveryOptions(true), + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + cancellation.Cancel(); + scheduler.Continue(); + + Assert.Equal(StoreStatus.Success, await recovery.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(new LeaseRecoveryReport(1, 1, 0, 0, 0), report); + Assert.False(lease.IsValid); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task SecondRecoveryHelpsPublishedRecoveringAndDelayedWinnerCannotDamageReuse() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryAfterExactRecoveryCas); + + LeaseRecoveryReport winnerReport = default; + var winner = Task.Run(() => store.TryRecoverLeases( + new LeaseRecoveryOptions(true), + out winnerReport)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(true), out var helperReport)); + Assert.Equal(new LeaseRecoveryReport(1, 0, 0, 0, 0), helperReport); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var replacement)); + + scheduler.Continue(); + Assert.Equal(StoreStatus.Success, await winner.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(new LeaseRecoveryReport(1, 1, 0, 0, 0), winnerReport); + Assert.False(lease.IsValid); + Assert.True(replacement.IsValid); + Assert.Equal(7, replacement.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public void ReportsLiveAndRecoveredRecordsAndRestoresEveryLeaseRecord() + { + using var store = CreateStore(slotCount: 1, leaseRecordCount: 2); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var first)); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var second)); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(false), out var active)); + Assert.Equal(new LeaseRecoveryReport(2, 0, 2, 0, 0), active); + Assert.True(first.IsValid); + Assert.True(second.IsValid); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(true), out var recovered)); + Assert.Equal(new LeaseRecoveryReport(2, 2, 0, 0, 0), recovered); + Assert.False(first.IsValid); + Assert.False(second.IsValid); + + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var replacement1)); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var replacement2)); + Assert.Equal(StoreStatus.Success, replacement1.Release()); + Assert.Equal(StoreStatus.Success, replacement2.Release()); + } + + [Fact] + public void QuiescentCurrentProcessOverrideRecoversLeasesFromEveryLocalHandle() + { + const int slotCount = 1; + const int leaseRecordCount = 2; + string name = $"sms-v2-quiescent-lease-recovery-{Guid.NewGuid():N}"; + using var firstStore = CreateStore(name, slotCount, leaseRecordCount, OpenMode.CreateNew); + Assert.Equal( + StoreOpenStatus.Success, + MemoryStore.TryCreateOrOpen( + Options(name, slotCount, leaseRecordCount, OpenMode.OpenExisting), + out MemoryStore? opened)); + using MemoryStore secondStore = Assert.IsType(opened); + + Assert.Equal(StoreStatus.Success, firstStore.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, firstStore.TryAcquire([1], out var first)); + Assert.Equal(StoreStatus.Success, secondStore.TryAcquire([1], out var second)); + Assert.Equal(7, first.ValueSpan[0]); + Assert.Equal(7, second.ValueSpan[0]); + + // From this boundary until recovery returns, acquisition, projection, + // borrowed-span use, and release are quiescent on both local handles. + Assert.Equal( + StoreStatus.Success, + firstStore.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: true), + StoreWaitOptions.Infinite, + out LeaseRecoveryReport report)); + Assert.Equal(new LeaseRecoveryReport(2, 2, 0, 0, 0), report); + Assert.False(first.IsValid); + Assert.False(second.IsValid); + } + + [Fact] + public void RecoveryTriggersFreshCooperativeReclaimForRemoveRequestedBinding() + { + using var store = CreateStore(slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(StoreStatus.RemovePending, store.TryRemove([1])); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(true), out var report)); + Assert.Equal(1, report.RecoveredLeaseCount); + Assert.False(lease.IsValid); + + // If recovery merely recycled the lease, NoWait would observe the still + // published removal and return RemovePending. NotFound proves recovery's + // reclaimer completed the exact removed binding. + Assert.Equal(StoreStatus.NotFound, store.TryRemove([1], StoreWaitOptions.NoWait)); + Assert.Equal(StoreStatus.Success, store.TryPublish([2], [9])); + } + + [Fact] + public void StaleOwnerRecoveryRetiresParticipantAndRestoresParticipantCapacity() + { + if (!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + { + return; + } + + const int slotCount = 1; + const int leaseRecordCount = 1; + var name = $"sms-v2-lease-retirement-{Guid.NewGuid():N}"; + using var store = CreateStore(name, slotCount, leaseRecordCount, OpenMode.CreateNew); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7], [3])); + + var readyFile = Path.Combine(Path.GetTempPath(), $"sms-ready-{Guid.NewGuid():N}"); + var continueFile = Path.Combine(Path.GetTempPath(), $"sms-continue-{Guid.NewGuid():N}"); + using var owner = StartLeaseOwner( + name, + slotCount, + leaseRecordCount, + readyFile, + continueFile); + try + { + Assert.True(WaitForFile(readyFile, TimeSpan.FromSeconds(10))); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(false), out var live)); + Assert.Equal(new LeaseRecoveryReport(1, 0, 1, 0, 0), live); + + var existing = Options(name, slotCount, leaseRecordCount, OpenMode.OpenExisting); + Assert.Equal( + StoreOpenStatus.ParticipantTableFull, + MemoryStore.TryCreateOrOpen(existing, out var rejected)); + Assert.Null(rejected); + + owner.Kill(entireProcessTree: true); + Assert.True(owner.WaitForExit(10_000)); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverLeases(new LeaseRecoveryOptions(false), out var report)); + Assert.Equal(new LeaseRecoveryReport(1, 1, 0, 0, 0), report); + + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(existing, out var replacement)); + using var opened = Assert.IsType(replacement); + } + finally + { + TryStop(owner); + File.Delete(readyFile); + File.Delete(continueFile); + } + } + + private static MemoryStore CreateInstrumentedStore( + ControlledLockFreeScheduler scheduler, + int slotCount, + int leaseRecordCount) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options($"sms-v2-lease-recovery-{Guid.NewGuid():N}", slotCount, leaseRecordCount), + scheduler.CreateInstrumentedCheckpoint(), + out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore CreateStore(int slotCount, int leaseRecordCount) + { + return CreateStore( + $"sms-v2-lease-recovery-{Guid.NewGuid():N}", + slotCount, + leaseRecordCount, + OpenMode.CreateNew); + } + + private static MemoryStore CreateStore( + string name, + int slotCount, + int leaseRecordCount, + OpenMode openMode) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, slotCount, leaseRecordCount, openMode), + out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options( + string name, + int slotCount, + int leaseRecordCount, + OpenMode openMode = OpenMode.CreateNew) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 2, + openMode, + enableLeaseRecovery: true); + + private static Process StartLeaseOwner( + string name, + int slotCount, + int leaseRecordCount, + string readyFile, + string continueFile) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + foreach (var argument in new[] + { + "exec", + LocateAgentAssembly(), + "lease-hold", + name, + slotCount.ToString(System.Globalization.CultureInfo.InvariantCulture), + "16", + "4", + "8", + leaseRecordCount.ToString(System.Globalization.CultureInfo.InvariantCulture), + "2", + "01", + "07", + "03", + readyFile, + continueFile + }) + { + startInfo.ArgumentList.Add(argument); + } + + return Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start the lock-free lease owner agent."); + } + + private static string LocateAgentAssembly() + { + var configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null + && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + var root = directory?.FullName + ?? throw new DirectoryNotFoundException("Repository root not found."); + var path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + return File.Exists(path) + ? path + : throw new FileNotFoundException("Lock-free agent assembly was not built.", path); + } + + private static bool WaitForFile(string path, TimeSpan timeout) + { + var deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + var spin = new SpinWait(); + while (!File.Exists(path)) + { + if (Stopwatch.GetTimestamp() >= deadline) + { + return false; + } + + spin.SpinOnce(); + } + + return true; + } + + private static void TryStop(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(5_000); + } + } + catch + { + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeLeaseRegistryTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeLeaseRegistryTests.cs new file mode 100644 index 0000000..583b222 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeLeaseRegistryTests.cs @@ -0,0 +1,208 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.LockFree; +using Store = SharedMemoryStore.MemoryStore; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeLeaseRegistryTests +{ + private const long TerminalIncarnation = 0x1_ffff_ffffL; + + [Fact] + public void RegistrySurfaceSeparatesClaimActivationReleaseRecoveryAndStableScan() + { + var type = RequireLeaseRegistry(); + var methods = type.GetMethods(BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + + Assert.Contains(methods, static method => method.Name.Contains("Claim", StringComparison.Ordinal)); + Assert.Contains(methods, static method => method.Name.Contains("Activat", StringComparison.Ordinal)); + Assert.Contains(methods, static method => method.Name.Contains("Release", StringComparison.Ordinal)); + Assert.Contains(methods, static method => method.Name.Contains("Recover", StringComparison.Ordinal)); + Assert.Contains(methods, static method => method.Name.Contains("Scan", StringComparison.Ordinal)); + Assert.Contains(methods, static method => method.Name.Contains("Participant", StringComparison.Ordinal) + || method.Name.Contains("Revalid", StringComparison.Ordinal)); + + var forbidden = new[] { typeof(Mutex), typeof(Semaphore), typeof(SemaphoreSlim), typeof(ReaderWriterLockSlim) }; + Assert.DoesNotContain( + type.GetFields(BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic), + field => forbidden.Contains(field.FieldType)); + } + + [Fact] + public void FirstLeaseClaimCarriesStoreParticipantSlotAndLeaseIncarnations() + { + _ = RequireLeaseRegistry(); + using var store = CreateStore(leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7], [9])); + + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + var handle = lease.HandleForEngine; + + Assert.NotEqual(0UL, handle.StoreId); + Assert.NotEqual(0UL, handle.ParticipantToken); + Assert.NotEqual(0UL, handle.SlotBinding); + Assert.NotEqual(0UL, handle.LeaseToken); + Assert.True(lease.IsValid); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public void ExactReleaseClearsOwnershipAndReuseAdvancesLeaseIncarnation() + { + _ = RequireLeaseRegistry(); + using var store = CreateStore(leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var first)); + var staleCopy = first; + var firstHandle = first.HandleForEngine; + Assert.Equal(StoreStatus.Success, first.Release()); + Assert.False(first.IsValid); + Assert.Equal(StoreStatus.LeaseAlreadyReleased, staleCopy.Release()); + + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var second)); + var secondHandle = second.HandleForEngine; + Assert.Equal(firstHandle.StoreId, secondHandle.StoreId); + Assert.Equal(firstHandle.ParticipantToken, secondHandle.ParticipantToken); + Assert.Equal(firstHandle.SlotBinding, secondHandle.SlotBinding); + Assert.NotEqual(firstHandle.LeaseToken, secondHandle.LeaseToken); + + Assert.NotEqual(StoreStatus.Success, staleCopy.Release()); + Assert.True(second.IsValid); + Assert.Equal(StoreStatus.Success, second.Release()); + } + + [Fact] + public void CancellationBeforeActivationLeaksNoRecordAndCancellationAfterActivationCannotUndoLease() + { + _ = RequireLeaseRegistry(); + using var store = CreateStore(leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + + using (var before = new CancellationTokenSource()) + { + before.Cancel(); + var canceled = new StoreWaitOptions(TimeSpan.FromSeconds(1), before.Token); + Assert.Equal(StoreStatus.OperationCanceled, store.TryAcquire([1], canceled, out var rejected)); + Assert.False(rejected.IsValid); + } + + using var after = new CancellationTokenSource(); + var activeWait = new StoreWaitOptions(TimeSpan.FromSeconds(1), after.Token); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], activeWait, out var active)); + after.Cancel(); + Assert.True(active.IsValid); + Assert.Equal(7, active.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, active.Release()); + + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var reused)); + Assert.Equal(StoreStatus.Success, reused.Release()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CancellationOrDeadlineImmediatelyAroundActivationHasOneBoundedOutcome(bool cancellation) + { + _ = RequireLeaseRegistry(); + + var before = ActivationBoundaryOracle.Apply( + activationCompleted: false, + cancellationObserved: cancellation, + deadlineExpired: !cancellation); + var after = ActivationBoundaryOracle.Apply( + activationCompleted: true, + cancellationObserved: cancellation, + deadlineExpired: !cancellation); + + Assert.Equal(cancellation ? StoreStatus.OperationCanceled : StoreStatus.StoreBusy, before.Status); + Assert.Equal(LeaseState.Free, before.State); + Assert.False(before.HasParticipantOwnership); + Assert.Equal(StoreStatus.Success, after.Status); + Assert.Equal(LeaseState.Active, after.State); + Assert.True(after.HasParticipantOwnership); + } + + [Fact] + public void TerminalLeaseIncarnationRetiresInsteadOfWrapping() + { + var type = RequireLeaseRegistry(); + var advanceOrRetire = type.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .SingleOrDefault(method => + method.Name.Contains("Advance", StringComparison.OrdinalIgnoreCase) + && method.Name.Contains("Retire", StringComparison.OrdinalIgnoreCase) + && method.GetParameters() is [{ ParameterType: var parameterType }] + && parameterType == typeof(long) + && (method.ReturnType == typeof(long) || method.ReturnType == typeof(ulong))); + Assert.True(advanceOrRetire is not null, "Lease registry needs a pure advance-or-retire transition seam."); + + var nextFree = Convert.ToUInt64(advanceOrRetire!.Invoke(null, [TerminalIncarnation - 1])); + var retired = Convert.ToUInt64(advanceOrRetire.Invoke(null, [TerminalIncarnation])); + + Assert.Equal(AtomicControlWord.EncodeLease(state: 0, TerminalIncarnation, participantToken: 0), nextFree); + Assert.Equal(AtomicControlWord.EncodeLease(state: 5, TerminalIncarnation, participantToken: 0), retired); + } + + private static Type RequireLeaseRegistry() + { + var type = typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeLeaseRegistry", + throwOnError: false, + ignoreCase: false); + Assert.True(type is not null, "The layout-v2 engine requires LockFreeLeaseRegistry."); + return type!; + } + + private static Store CreateStore(int leaseRecordCount) + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + throw new PlatformNotSupportedException("The lock-free profile is qualified only on Windows/Linux x64."); + } + + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-lease-registry-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + var status = Store.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + var result = Assert.IsType(store); + Assert.Equal(StoreProfile.LockFree, result.Profile); + return result; + } + + private enum LeaseState + { + Free, + Active + } + + private readonly record struct ActivationBoundaryResult( + StoreStatus Status, + LeaseState State, + bool HasParticipantOwnership); + + private static class ActivationBoundaryOracle + { + public static ActivationBoundaryResult Apply( + bool activationCompleted, + bool cancellationObserved, + bool deadlineExpired) + { + Assert.True(cancellationObserved ^ deadlineExpired); + return activationCompleted + ? new ActivationBoundaryResult(StoreStatus.Success, LeaseState.Active, HasParticipantOwnership: true) + : new ActivationBoundaryResult( + cancellationObserved ? StoreStatus.OperationCanceled : StoreStatus.StoreBusy, + LeaseState.Free, + HasParticipantOwnership: false); + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeLifecycleGateTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeLifecycleGateTests.cs new file mode 100644 index 0000000..71c35ee --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeLifecycleGateTests.cs @@ -0,0 +1,139 @@ +using System.Diagnostics; +using System.Reflection; +using SharedMemoryStore.Lifecycle; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeLifecycleGateTests +{ + [Fact] + public async Task OperationEntryDoesNotWaitForTheFormerMonitor() + { + var gate = new StoreLifecycleGate(); + var formerMonitor = typeof(StoreLifecycleGate) + .GetField("_sync", BindingFlags.Instance | BindingFlags.NonPublic)? + .GetValue(gate); + + if (formerMonitor is null) + { + Assert.True(gate.TryEnter(out var operation)); + operation.Dispose(); + return; + } + + using var releaseMonitor = new ManualResetEventSlim(); + var monitorEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var monitorReleased = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var holder = new Thread(() => + { + Monitor.Enter(formerMonitor); + monitorEntered.SetResult(); + releaseMonitor.Wait(); + Monitor.Exit(formerMonitor); + monitorReleased.SetResult(); + }); + holder.Start(); + await monitorEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + StoreLifecycleGate.Operation enteredOperation = default; + var entry = Task.Run(() => gate.TryEnter(out enteredOperation)); + var completedWithoutMonitor = await CompletesWithinAsync(entry, TimeSpan.FromMilliseconds(250)); + releaseMonitor.Set(); + + await monitorReleased.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var entered = await entry.WaitAsync(TimeSpan.FromSeconds(5)); + if (entered) + { + enteredOperation.Dispose(); + } + + Assert.True(completedWithoutMonitor, "Operation entry still depends on a process-local Monitor instead of the atomic lifetime word."); + } + + [Fact] + public async Task DisposePausedBehindEnteredOperationRejectsNewEntriesThenCompletes() + { + var gate = new StoreLifecycleGate(); + Assert.True(gate.TryEnter(out var enteredOperation)); + + var disposer = Task.Run(gate.TryBeginDispose); + Assert.True( + SpinWait.SpinUntil(() => gate.IsDisposingOrDisposed, TimeSpan.FromSeconds(5)), + "Disposal did not publish its entry-closed state."); + + Assert.False(gate.TryEnter(out _)); + Assert.False(await CompletesWithinAsync(disposer, TimeSpan.FromMilliseconds(100))); + + enteredOperation.Dispose(); + Assert.True(await disposer.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.False(gate.IsDisposed); + + gate.CompleteDispose(); + Assert.True(gate.IsDisposed); + Assert.False(gate.TryEnter(out _)); + } + + [Fact] + public async Task SecondDisposePausedDuringFirstDisposeObservesCompletionAndDoesNotOwnCleanup() + { + var gate = new StoreLifecycleGate(); + Assert.True(gate.TryBeginDispose()); + + var secondDisposer = Task.Run(gate.TryBeginDispose); + Assert.False(await CompletesWithinAsync(secondDisposer, TimeSpan.FromMilliseconds(100))); + Assert.False(gate.TryEnter(out _)); + + gate.CompleteDispose(); + Assert.False(await secondDisposer.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.True(gate.IsDisposed); + } + + [Fact] + public async Task OperationExitUnblocksOnlyAfterEveryEnteredOperationLeaves() + { + var gate = new StoreLifecycleGate(); + Assert.True(gate.TryEnter(out var first)); + Assert.True(gate.TryEnter(out var second)); + + var disposer = Task.Run(gate.TryBeginDispose); + Assert.True(SpinWait.SpinUntil(() => gate.IsDisposingOrDisposed, TimeSpan.FromSeconds(5))); + + first.Dispose(); + Assert.False(await CompletesWithinAsync(disposer, TimeSpan.FromMilliseconds(100))); + + second.Dispose(); + Assert.True(await disposer.WaitAsync(TimeSpan.FromSeconds(5))); + gate.CompleteDispose(); + } + + [Fact] + public void WarmOperationEntryAndExitAllocateNoManagedMemory() + { + var gate = new StoreLifecycleGate(); + for (var index = 0; index < 1_000; index++) + { + Assert.True(gate.TryEnter(out var warmup)); + warmup.Dispose(); + } + + const int Iterations = 100_000; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var index = 0; index < Iterations; index++) + { + if (!gate.TryEnter(out var operation)) + { + throw new UnreachableException("An undisposed gate rejected an operation."); + } + + operation.Dispose(); + } + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(0, allocated); + } + + private static async Task CompletesWithinAsync(Task task, TimeSpan timeout) + { + return await Task.WhenAny(task, Task.Delay(timeout)) == task; + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeLifecycleSuccessorValidationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeLifecycleSuccessorValidationTests.cs new file mode 100644 index 0000000..27dc779 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeLifecycleSuccessorValidationTests.cs @@ -0,0 +1,263 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeLifecycleSuccessorValidationTests +{ + [Fact] + public async Task OrderedLeaseReleaseLatchesSameIncarnationReactivationButRemainsSuccessful() + { + if (!IsSupportedHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore( + scheduler, + $"sms-v2-lease-successor-{Guid.NewGuid():N}"); + byte[] key = [0x11]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x21])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + + LockFreeLeaseRegistry leases = ReadEngineField(store, "_leases"); + long active = ReadLeaseControl(leases, recordIndex: 0); + long incarnation = LeaseIncarnation(active); + scheduler.PauseAt(LockFreeCheckpointId.ReleaseAfterOwnershipReleaseCas); + + StoreStatus releaseStatus = default; + Task release = Task.Run(() => releaseStatus = lease.Release(StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + try + { + Assert.Equal( + LockFreeLeaseRegistry.ReleasingState, + LeaseState(ReadLeaseControl(leases, recordIndex: 0))); + + // This word is structurally canonical but cannot follow + // Releasing(g): Active(g) would resurrect already-ended protection. + WriteLeaseControl(leases, recordIndex: 0, active); + } + finally + { + scheduler.Continue(); + } + + await release.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, releaseStatus); + Assert.Equal(active, ReadLeaseControl(leases, recordIndex: 0)); + Assert.Equal(StoreStatus.CorruptStore, store.TryAcquire(key, out _)); + Assert.Equal(incarnation, LeaseIncarnation(active)); + } + + [Fact] + public async Task ParticipantGenerationAdvanceLatchesStableSameGenerationRegression() + { + if (!IsSupportedHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore( + scheduler, + $"sms-v2-participant-successor-{Guid.NewGuid():N}"); + LockFreeParticipantRegistry participants = + ReadEngineField(store, "_participants"); + LockFreeParticipantRegistry.Registration registration = + ReadEngineField(store, "_registration"); + long reclaiming = ParticipantControl( + LayoutV2Constants.ParticipantReclaiming, + registration.Generation, + pid: 0); + WriteParticipantControl(participants, registration.RecordIndex, reclaiming); + + scheduler.PauseAt(LockFreeCheckpointId.ParticipantBeforeReclaimGenerationAdvanceCas); + InstrumentedLockFreeCheckpoint instrumented = scheduler.CreateInstrumentedCheckpoint(); + ParticipantTransitionResult transition = default; + Task helper = Task.Run(() => + { + InstrumentedLockFreeCheckpoint checkpoint = instrumented; + transition = participants.HelpReclaiming( + registration.RecordIndex, + registration.Generation, + ref checkpoint); + }); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + try + { + Assert.Equal( + reclaiming, + ReadParticipantControl(participants, registration.RecordIndex)); + + // Active(g) is canonical in isolation, but Reclaiming(g) has no + // same-generation successor except Retired(g) at terminal rollover. + WriteParticipantControl( + participants, + registration.RecordIndex, + registration.ActiveControl); + } + finally + { + scheduler.Continue(); + } + + await helper.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(ParticipantTransitionResult.Inconsistent, transition); + Assert.Equal( + registration.ActiveControl, + ReadParticipantControl(participants, registration.RecordIndex)); + Assert.Equal(StoreStatus.CorruptStore, store.TryPublish([0x31], [0x41])); + } + + [Fact] + public async Task RemoveOwnershipCasLatchesStablePublishedRegression() + { + if (!IsSupportedHost()) + { + return; + } + + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore( + scheduler, + $"sms-v2-reclaimer-successor-{Guid.NewGuid():N}"); + byte[] key = [0x51]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x61])); + LockFreeSlotTable slots = ReadEngineField(store, "_slots"); + long published = ReadSlotControl(slots, slotIndex: 0); + long generation = SlotGeneration(published); + scheduler.PauseAt(LockFreeCheckpointId.ReclaimAfterLeaseScanBeforeOwnershipCas); + + StoreStatus removeStatus = default; + Task remove = Task.Run(() => removeStatus = store.TryRemove( + key, + StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + try + { + Assert.Equal( + LockFreeSlotTable.RemoveRequestedState, + SlotState(ReadSlotControl(slots, slotIndex: 0))); + + // Published(g) is structurally valid but cannot follow the already + // ordered RemoveRequested(g) lifecycle. + WriteSlotControl(slots, slotIndex: 0, published); + } + finally + { + scheduler.Continue(); + } + + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.CorruptStore, removeStatus); + Assert.Equal(published, ReadSlotControl(slots, slotIndex: 0)); + Assert.Equal(generation, SlotGeneration(published)); + Assert.Equal(StoreStatus.CorruptStore, store.TryAcquire(key, out _)); + } + + private static MemoryStore CreateInstrumentedStore( + ControlledLockFreeScheduler scheduler, + string name) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 8, + maxDescriptorBytes: 0, + maxKeyBytes: 4, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + scheduler.CreateInstrumentedCheckpoint(), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static T ReadEngineField(MemoryStore store, string name) + { + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + return Assert.IsType(engine.GetType() + .GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + } + + private static long ParticipantControl(int state, int generation, int pid) => + unchecked((long)AtomicControlWord.EncodeParticipant(state, generation, pid)); + + private static long ReadLeaseControl(LockFreeLeaseRegistry leases, int recordIndex) + { + ref LeaseRecordV2 record = ref leases.Record(recordIndex); + return AtomicControlWord.LoadAcquire(ref record.Control); + } + + private static void WriteLeaseControl( + LockFreeLeaseRegistry leases, + int recordIndex, + long control) + { + ref LeaseRecordV2 record = ref leases.Record(recordIndex); + AtomicControlWord.StoreRelease(ref record.Control, control); + } + + private static long ReadParticipantControl( + LockFreeParticipantRegistry participants, + int recordIndex) + { + ref ParticipantRecordV2 record = ref participants.Record(recordIndex); + return AtomicControlWord.LoadAcquire(ref record.Control); + } + + private static void WriteParticipantControl( + LockFreeParticipantRegistry participants, + int recordIndex, + long control) + { + ref ParticipantRecordV2 record = ref participants.Record(recordIndex); + AtomicControlWord.StoreRelease(ref record.Control, control); + } + + private static long ReadSlotControl(LockFreeSlotTable slots, int slotIndex) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + return AtomicControlWord.LoadAcquire(ref slot.Control); + } + + private static void WriteSlotControl( + LockFreeSlotTable slots, + int slotIndex, + long control) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + AtomicControlWord.StoreRelease(ref slot.Control, control); + } + + private static long LeaseIncarnation(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static int LeaseState(long control) => + (int)(unchecked((ulong)control) & 0x7UL); + + private static long SlotGeneration(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static int SlotState(long control) => + (int)(unchecked((ulong)control) & 0x7UL); + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeOperationBudgetTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeOperationBudgetTests.cs new file mode 100644 index 0000000..1a7cf94 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeOperationBudgetTests.cs @@ -0,0 +1,533 @@ +using System.Buffers; +using System.Diagnostics; +using System.Reflection; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using System.Runtime.InteropServices; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeOperationBudgetTests +{ + [Fact] + public void PublicNoWaitAllowsOneProbeChunkButStructuralAttemptKeepsLegacyFullScan() + { + LockFreeOperationBudget noWait = LockFreeOperationBudget.Start(StoreWaitOptions.NoWait); + + Assert.Equal(StoreStatus.Success, noWait.CheckPeriodic(0)); + Assert.Equal(StoreStatus.Success, noWait.CheckPeriodic(63)); + Assert.Equal(StoreStatus.StoreBusy, noWait.CheckPeriodic(64)); + + Assert.Equal( + StoreStatus.Success, + LockFreeOperationBudget.StructuralAttempt.CheckPeriodic(64)); + Assert.False(LockFreeOperationBudget.StructuralAttempt.TryContinueAfterContention( + attempt: 128, + out StoreStatus terminal)); + Assert.Equal(StoreStatus.StoreBusy, terminal); + } + + [Fact] + public void InfiniteBudgetStillObservesExplicitCancellation() + { + using var cancellation = new CancellationTokenSource(); + var budget = LockFreeOperationBudget.Start( + new StoreWaitOptions(Timeout.InfiniteTimeSpan, cancellation.Token)); + cancellation.Cancel(); + + Assert.Equal(StoreStatus.OperationCanceled, budget.Check()); + Assert.False(budget.TryContinueAfterContention(0, out StoreStatus terminal)); + Assert.Equal(StoreStatus.OperationCanceled, terminal); + } + + [Fact] + public void RemainingWaitRejectsExpiredFiniteButPreservesTrueNoWait() + { + long oldStart = Stopwatch.GetTimestamp() - Stopwatch.Frequency; + var expired = LockFreeOperationBudget.Start( + new StoreWaitOptions(TimeSpan.FromMilliseconds(10)), + oldStart); + Assert.Equal( + StoreStatus.StoreBusy, + expired.TryGetRemainingWaitOptions(out _)); + + var noWait = LockFreeOperationBudget.Start(StoreWaitOptions.NoWait, oldStart); + Assert.Equal( + StoreStatus.Success, + noWait.TryGetRemainingWaitOptions(out StoreWaitOptions remaining)); + Assert.Equal(StoreWaitOptions.NoWait, remaining); + } + + [Fact] + public void ByteLinearHashAndCopyUseCanonicalIdentityAndBoundNoWaitQuantum() + { + byte[] belowQuantum = Enumerable.Repeat((byte)0x37, 4_095).ToArray(); + byte[] fullQuantum = Enumerable.Repeat((byte)0x42, 4_096).ToArray(); + LockFreeOperationBudget infinite = LockFreeOperationBudget.Start(StoreWaitOptions.Infinite); + LockFreeOperationBudget noWait = LockFreeOperationBudget.Start(StoreWaitOptions.NoWait); + + Assert.Equal( + StoreStatus.Success, + LockFreeByteOperations.TryHash(belowQuantum, infinite, out ulong hash)); + Assert.Equal(SharedMemoryStore.Layout.StoreKey.Hash(belowQuantum), hash); + Assert.Equal( + StoreStatus.StoreBusy, + LockFreeByteOperations.TryHash(fullQuantum, noWait, out _)); + + byte[] destination = new byte[belowQuantum.Length]; + Assert.Equal( + StoreStatus.Success, + LockFreeByteOperations.TryCopy(belowQuantum, destination, infinite)); + Assert.Equal(belowQuantum, destination); + Assert.Equal( + StoreStatus.StoreBusy, + LockFreeByteOperations.TryCopy(fullQuantum, new byte[fullQuantum.Length], noWait)); + + byte[] overQuantum = new byte[4_160]; + Assert.Equal( + StoreStatus.StoreBusy, + LockFreeByteOperations.TryCopy( + overQuantum, + new byte[overQuantum.Length], + noWait, + out int partiallyCopied)); + Assert.Equal(4_096, partiallyCopied); + } + + [Fact] + public void LargeExactKeyNoWaitReturnsBusyInsteadOfFalseNotFound() + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + byte[] key = Enumerable.Repeat((byte)0x5A, 4_096).ToArray(); + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-key-budget-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: key.Length, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(options, out MemoryStore? store)); + using MemoryStore owned = Assert.IsType(store); + Assert.Equal( + StoreStatus.Success, + owned.TryPublish(key, [0x7B], default, StoreWaitOptions.Infinite)); + + LockFreeKeyDirectory directory = ReadDirectory(owned); + ulong hash = StoreKey.Hash(key); + LockFreeOperationBudget noWait = LockFreeOperationBudget.Start(StoreWaitOptions.NoWait); + Assert.Equal( + StoreStatus.StoreBusy, + directory.TryLookup(key, hash, noWait, out _, out _)); + Assert.Equal( + StoreStatus.Success, + directory.TryLookup( + key, + hash, + LockFreeOperationBudget.UnboundedScan, + out ulong exactBinding, + out DirectoryLocation exactLocation)); + Assert.NotEqual(0UL, exactBinding); + Assert.NotEqual(0UL, exactLocation.Value); + + Assert.Equal(StoreStatus.StoreBusy, owned.TryAcquire(key, StoreWaitOptions.NoWait, out _)); + Assert.Equal(StoreStatus.Success, owned.TryAcquire(key, StoreWaitOptions.Infinite, out ValueLease lease)); + Assert.Equal(0x7B, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release(StoreWaitOptions.Infinite)); + } + + [Fact] + public void ManyTinySegmentsNoWaitReturnsBusyAndAbortsOwnedReservation() + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + byte[][] buffers = Enumerable.Range(0, 65) + .Select(index => new[] { checked((byte)index) }) + .ToArray(); + ReadOnlySequence payload = Sequence(buffers); + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-segment-budget-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes: 65, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(options, out MemoryStore? store)); + using MemoryStore owned = Assert.IsType(store); + + Assert.Equal( + StoreStatus.StoreBusy, + owned.TryPublishSegments([1], payload, [], StoreWaitOptions.NoWait, out long copied)); + Assert.Equal(64, copied); + Assert.Equal(StoreStatus.Success, owned.TryGetDiagnostics(out var afterAbort)); + Assert.Equal(1, afterAbort.FreeSlotCount); + Assert.Equal(StoreStatus.Success, owned.TryPublish([1], [7], [], StoreWaitOptions.Infinite)); + } + + [Fact] + public void MalformedSequenceRetainsBytesCopiedBeforeFailure() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(static _ => { }); + using MemoryStore store = CreateInstrumentedStore(maxValueBytes: 64, checkpoint); + var first = new BufferSegment(new byte[8]); + BufferSegment last = first.Append(new byte[8], runningIndex: 1); + var malformed = new ReadOnlySequence(first, 0, last, last.Memory.Length); + + StoreStatus status = store.TryPublishSegments( + [1], + malformed, + [], + StoreWaitOptions.Infinite, + out long copied); + + Assert.Equal(StoreStatus.UnknownFailure, status); + Assert.Equal(8, copied); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out var afterAbort)); + Assert.Equal(1, afterAbort.FreeSlotCount); + } + + [Fact] + public void UnderEnumeratedMalformedSequenceRetainsActualCopiedBytes() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(static _ => { }); + using MemoryStore store = CreateInstrumentedStore(maxValueBytes: 128, checkpoint); + var first = new BufferSegment(new byte[8]); + BufferSegment last = first.Append(new byte[8], runningIndex: 64); + var malformed = new ReadOnlySequence(first, 0, last, last.Memory.Length); + + StoreStatus status = store.TryPublishSegments( + [1], + malformed, + [], + StoreWaitOptions.Infinite, + out long copied); + + Assert.Equal(StoreStatus.UnknownFailure, status); + Assert.Equal(16, copied); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out var afterAbort)); + Assert.Equal(1, afterAbort.FreeSlotCount); + } + + [Fact] + public void OneLargeSegmentNoWaitReportsCompletedChunksBeforeBudgetExpiry() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + const int payloadLength = 4_160; + InstrumentedLockFreeCheckpoint checkpoint = + LockFreeCheckpointFactory.CreateInstrumented(static _ => { }); + using MemoryStore store = CreateInstrumentedStore(payloadLength, checkpoint); + var payload = new ReadOnlySequence(new byte[payloadLength]); + + Assert.Equal( + StoreStatus.StoreBusy, + store.TryPublishSegments( + [1], + payload, + [], + StoreWaitOptions.NoWait, + out long copied)); + Assert.Equal(4_096, copied); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out var afterAbort)); + Assert.Equal(1, afterAbort.FreeSlotCount); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7], [], StoreWaitOptions.Infinite)); + } + + [Fact] + public void CommitCancellationAfterFullSegmentCopyRetainsFullCount() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var cancellation = new CancellationTokenSource(); + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + if (entry.Id == LockFreeCheckpointId.CommitBeforePublicationCas) + { + cancellation.Cancel(); + } + }); + using MemoryStore store = CreateInstrumentedStore(maxValueBytes: 8, checkpoint); + ReadOnlySequence payload = Sequence([[1, 2], [3, 4]]); + var wait = new StoreWaitOptions(Timeout.InfiniteTimeSpan, cancellation.Token); + + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryPublishSegments([1], payload, [], wait, out long copied)); + Assert.Equal(4, copied); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out var afterAbort)); + Assert.Equal(1, afterAbort.FreeSlotCount); + } + + [Fact] + public void AdvanceNoWaitStopsAfterOneBoundedCasProbeQuantum() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + LockFreeSlotTable? slots = null; + var slotIndex = -1; + var injectedCasWins = 0; + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + if (entry.Id == LockFreeCheckpointId.AdvanceBeforeBytesAdvancedCas) + { + IncrementBytesAdvanced(Assert.IsType(slots), slotIndex); + Interlocked.Increment(ref injectedCasWins); + } + }); + + using MemoryStore store = CreateInstrumentedStore(maxValueBytes: 128, checkpoint: checkpoint); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 128, default, out ValueReservation reservation)); + slots = ReadSlotTable(store); + slotIndex = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding).SlotIndex; + + Assert.Equal(StoreStatus.StoreBusy, reservation.Advance(1, StoreWaitOptions.NoWait)); + Assert.Equal(64, Volatile.Read(ref injectedCasWins)); + Assert.Equal(64, reservation.BytesWritten); + Assert.True(reservation.IsValid); + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + } + + [Fact] + public void AdvanceInfiniteRetriesAfterCasLossUntilItSucceeds() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + LockFreeSlotTable? slots = null; + var slotIndex = -1; + var beforeCasCount = 0; + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + if (entry.Id != LockFreeCheckpointId.AdvanceBeforeBytesAdvancedCas) + { + return; + } + + if (Interlocked.Increment(ref beforeCasCount) == 1) + { + IncrementBytesAdvanced(Assert.IsType(slots), slotIndex); + } + }); + + using MemoryStore store = CreateInstrumentedStore(maxValueBytes: 2, checkpoint: checkpoint); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 2, default, out ValueReservation reservation)); + slots = ReadSlotTable(store); + slotIndex = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding).SlotIndex; + + Assert.Equal(StoreStatus.Success, reservation.Advance(1, StoreWaitOptions.Infinite)); + Assert.Equal(2, Volatile.Read(ref beforeCasCount)); + Assert.Equal(2, reservation.BytesWritten); + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + } + + [Fact] + public void AdvanceInfiniteObservesCancellationAfterCasLossWithoutItsOwnWrite() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using var cancellation = new CancellationTokenSource(); + LockFreeSlotTable? slots = null; + var slotIndex = -1; + var beforeCasCount = 0; + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + if (entry.Id != LockFreeCheckpointId.AdvanceBeforeBytesAdvancedCas) + { + return; + } + + int occurrence = Interlocked.Increment(ref beforeCasCount); + if (occurrence == 1) + { + IncrementBytesAdvanced(Assert.IsType(slots), slotIndex); + } + else if (occurrence == 2) + { + cancellation.Cancel(); + } + }); + + using MemoryStore store = CreateInstrumentedStore(maxValueBytes: 2, checkpoint: checkpoint); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 2, default, out ValueReservation reservation)); + slots = ReadSlotTable(store); + slotIndex = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding).SlotIndex; + + var infiniteCancelable = new StoreWaitOptions(Timeout.InfiniteTimeSpan, cancellation.Token); + Assert.Equal(StoreStatus.OperationCanceled, reservation.Advance(1, infiniteCancelable)); + Assert.Equal(2, Volatile.Read(ref beforeCasCount)); + Assert.Equal(1, reservation.BytesWritten); + Assert.True(reservation.IsValid); + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + } + + [Fact] + public void InfiniteReservationClaimRepairsOlderDirectoryResidueBeforeReuse() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + using MemoryStore store = CreateInstrumentedStore( + maxValueBytes: 1, + checkpoint: LockFreeCheckpointFactory.CreateInstrumented(static _ => { })); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1], default, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, store.TryRemove([1], StoreWaitOptions.Infinite)); + + LockFreeSlotTable slots = ReadSlotTable(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + AtomicControlWord.StoreRelease( + ref slot.DirectoryLocation, + unchecked((long)DirectoryLocation.Encode(kind: 1, index: 0, generation: 1))); + AtomicControlWord.StoreRelease( + ref slot.DirectoryOperation, + unchecked((long)DirectoryOperation.Encode( + intent: 1, + phase: 1, + targetKind: 1, + targetIndex: 0, + generation: 1))); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve([2], 1, default, StoreWaitOptions.Infinite, out ValueReservation reservation)); + Assert.True(reservation.IsValid); + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + } + + private static ReadOnlySequence Sequence(byte[][] buffers) + { + BufferSegment? first = null; + BufferSegment? last = null; + foreach (byte[] buffer in buffers) + { + last = last is null + ? first = new BufferSegment(buffer) + : last.Append(buffer); + } + + return new ReadOnlySequence(first!, 0, last!, last!.Memory.Length); + } + + private static LockFreeKeyDirectory ReadDirectory(MemoryStore store) + { + FieldInfo engineField = typeof(MemoryStore).GetField( + "_engine", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new Xunit.Sdk.XunitException("MemoryStore._engine is absent."); + object engine = engineField.GetValue(store) + ?? throw new Xunit.Sdk.XunitException("MemoryStore._engine is null."); + FieldInfo directoryField = engine.GetType().GetField( + "_directory", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new Xunit.Sdk.XunitException("Lock-free engine._directory is absent."); + return Assert.IsType(directoryField.GetValue(engine)); + } + + private static LockFreeSlotTable ReadSlotTable(MemoryStore store) + { + FieldInfo engineField = typeof(MemoryStore).GetField( + "_engine", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new Xunit.Sdk.XunitException("MemoryStore._engine is absent."); + object engine = engineField.GetValue(store) + ?? throw new Xunit.Sdk.XunitException("MemoryStore._engine is null."); + FieldInfo slotsField = engine.GetType().GetField( + "_slots", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new Xunit.Sdk.XunitException("Lock-free engine._slots is absent."); + return Assert.IsType(slotsField.GetValue(engine)); + } + + private static MemoryStore CreateInstrumentedStore( + int maxValueBytes, + InstrumentedLockFreeCheckpoint checkpoint) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-advance-budget-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes, + maxDescriptorBytes: 0, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew); + Assert.Equal( + StoreOpenStatus.Success, + LockFreeInstrumentedStoreFactory.TryCreateOrOpen(options, checkpoint, out MemoryStore? store)); + return Assert.IsType(store); + } + + private static void IncrementBytesAdvanced(LockFreeSlotTable slots, int slotIndex) + { + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + long observed = AtomicControlWord.LoadAcquire(ref slot.BytesAdvanced); + Assert.Equal( + observed, + AtomicControlWord.CompareExchange(ref slot.BytesAdvanced, observed + 1, observed)); + } + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private sealed class BufferSegment : ReadOnlySequenceSegment + { + internal BufferSegment(byte[] buffer) + { + Memory = buffer; + } + + internal BufferSegment Append(byte[] buffer, long? runningIndex = null) + { + var next = new BufferSegment(buffer) + { + RunningIndex = runningIndex ?? RunningIndex + Memory.Length + }; + Next = next; + return next; + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeParticipantRegistryTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeParticipantRegistryTests.cs new file mode 100644 index 0000000..5743859 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeParticipantRegistryTests.cs @@ -0,0 +1,712 @@ +using System.Reflection; +using System.Diagnostics; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.Leasing; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeParticipantRegistryTests +{ + [Fact] + public void ParticipantStatesAndCrashClassificationsCoverEveryLifecyclePhase() + { + Assert.Equal(0, LayoutV2Constants.ParticipantFree); + Assert.Equal(1, LayoutV2Constants.ParticipantRegistering); + Assert.Equal(2, LayoutV2Constants.ParticipantActive); + Assert.Equal(3, LayoutV2Constants.ParticipantClosing); + Assert.Equal(4, LayoutV2Constants.ParticipantRecovering); + Assert.Equal(5, LayoutV2Constants.ParticipantReclaiming); + Assert.Equal(6, LayoutV2Constants.ParticipantRetired); + + Assert.Equal(ParticipantOwnerClass.Live, ClassifyIdentity(42, 1, 100, 42, 1, 100)); + Assert.Equal(ParticipantOwnerClass.Stale, ClassifyIdentity(42, 1, 100, 42, 1, 101)); + Assert.Equal(ParticipantOwnerClass.Unsupported, ClassifyIdentity(42, 0, 0, 42, 0, 0)); + Assert.Equal(ParticipantOwnerClass.Inconsistent, ClassifyIdentity(0, 1, 100, 42, 1, 100)); + } + + [Fact] + public void RegistryExposesRecordLocalRecoveryHelpingRetirementAndDiagnostics() + { + Type registry = RequireType("SharedMemoryStore.LockFree.LockFreeParticipantRegistry"); + MethodInfo[] methods = registry.GetMethods( + BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + + Assert.Contains(methods, method => method.Name.Contains("Register", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Close", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("Unregister", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Recover", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Reclaim", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Help", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Diagnostic", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("Snapshot", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("Count", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Advance", StringComparison.OrdinalIgnoreCase) + && method.Name.Contains("Retire", StringComparison.OrdinalIgnoreCase)); + + Type[] forbidden = [typeof(Mutex), typeof(Semaphore), typeof(SemaphoreSlim), typeof(ReaderWriterLockSlim)]; + Assert.DoesNotContain( + registry.GetFields(BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic), + field => forbidden.Contains(field.FieldType)); + } + + [Fact] + public void ParticipantIdentitySurfaceIncludesPidKindStartValueAndConservativeClassification() + { + Type incarnation = RequireType("SharedMemoryStore.LockFree.ParticipantIncarnation"); + var members = incarnation.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.Contains(members, member => member.Name.Contains("Pid", StringComparison.OrdinalIgnoreCase) + || member.Name.Contains("ProcessId", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(members, member => member.Name.Contains("Identity", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(members, member => member.Name.Contains("Start", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(members, member => member.Name.Contains("Namespace", StringComparison.OrdinalIgnoreCase)); + + Type classifier = RequireType("SharedMemoryStore.Leasing.LeaseOwnerClassifier"); + Assert.Contains( + classifier.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic), + method => method.Name.Contains("Classif", StringComparison.OrdinalIgnoreCase) + && method.GetParameters().Any(parameter => parameter.ParameterType == incarnation)); + } + + [Fact] + public void CompleteParticipantTokenRoundTripsEveryRecordAndChangesOnReuse() + { + const int participantCount = 7; + for (var recordIndex = 0; recordIndex < participantCount; recordIndex++) + { + ulong first = ParticipantToken.Encode(recordIndex, generation: 1, participantCount); + ParticipantToken decoded = ParticipantToken.Decode(first, participantCount); + Assert.Equal(recordIndex, decoded.RecordIndex); + Assert.Equal(1, decoded.Generation); + Assert.NotEqual(first, ParticipantToken.Encode(recordIndex, generation: 2, participantCount)); + } + } + + [Fact] + public void FirstRegisteringClaimAtomicallyCarriesPidAndIncarnation() + { + const int incarnation = 17; + int pid = Environment.ProcessId; + ulong claim = AtomicControlWord.EncodeParticipant( + LayoutV2Constants.ParticipantRegistering, + incarnation, + pid); + + Assert.Equal(LayoutV2Constants.ParticipantRegistering, (int)(claim & 0x7UL)); + Assert.Equal(incarnation, (int)((claim >> 3) & 0x0fff_ffffUL)); + Assert.Equal(pid, (int)(claim >> 31)); + Assert.Equal(0UL, claim >> 63); + } + + [Fact] + public unsafe void HeaderAndActiveParticipantPublishTheAdmittedPidNamespaceIdentity() + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture + != System.Runtime.InteropServices.Architecture.X64) + { + return; + } + + string name = $"sms-v2-participant-pidns-{Guid.NewGuid():N}"; + using MemoryStore store = Open(name, OpenMode.CreateNew, participantCount: 2); + (_, object engine) = ReadEngineComponents(store); + var region = Assert.IsAssignableFrom(engine.GetType() + .GetField("_region", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + ref StoreHeaderV2 header = ref *(StoreHeaderV2*)region.Pointer; + (LockFreeParticipantRegistry registry, LockFreeParticipantRegistry.Registration registration) = + ReadParticipantComponents(store); + ParticipantClassification active = registry.ClassifyParticipant(registration.Token); + + Assert.Equal(ParticipantClassificationKind.CurrentProcess, active.Kind); + Assert.Equal(header.PidNamespaceId, active.Incarnation.PidNamespaceId); + if (OperatingSystem.IsLinux()) + { + Assert.NotEqual(0UL, header.PidNamespaceId); + } + else + { + Assert.Equal(0UL, header.PidNamespaceId); + } + Assert.Equal( + LayoutV2Constants.PidNamespaceRecoveryEnabled, + AtomicControlWord.LoadAcquire(ref header.PidNamespaceMode)); + } + + [Fact] + public async Task LinuxPidNamespaceMismatchPublishesMixedBeforeRegisteringAndPreservesTheClaim() + { + if (!OperatingSystem.IsLinux() + || System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture + != System.Runtime.InteropServices.Architecture.X64) + { + return; + } + + string name = $"sms-v2-participant-pidns-mismatch-{Guid.NewGuid():N}"; + using MemoryStore controller = Open(name, OpenMode.CreateNew, participantCount: 2); + (_, object engine) = ReadEngineComponents(controller); + var region = Assert.IsAssignableFrom(engine.GetType() + .GetField("_region", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + ulong admittedNamespace = ReadHeaderPidNamespaceId(region); + Assert.NotEqual(0UL, admittedNamespace); + ulong otherNamespace = admittedNamespace == ulong.MaxValue + ? admittedNamespace - 1 + : admittedNamespace + 1; + long sequenceBefore = ReadHeaderSequence(region); + (LockFreeParticipantRegistry registry, _) = ReadParticipantComponents(controller); + ulong registeringToken = ParticipantToken.Encode(recordIndex: 1, generation: 1, participantCount: 2); + using var scheduler = new ControlledLockFreeScheduler(); + scheduler.PauseAt(LockFreeCheckpointId.ParticipantAfterIdentityKindWrite); + + WriteHeaderPidNamespaceId(region, otherNamespace); + try + { + StoreOpenStatus status = default; + MemoryStore? candidate = null; + Task opening = Task.Run(() => status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting, participantCount: 2), + scheduler.CreateInstrumentedCheckpoint(), + out candidate)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal(otherNamespace, ReadHeaderPidNamespaceId(region)); + Assert.Equal(sequenceBefore, ReadHeaderSequence(region)); + Assert.Equal( + LayoutV2Constants.PidNamespaceRecoveryMixed, + ReadHeaderPidNamespaceMode(region)); + ParticipantClassification partial = registry.ClassifyParticipant(registeringToken); + Assert.Equal(ParticipantClassificationKind.Unsupported, partial.Kind); + Assert.Equal(LayoutV2Constants.ParticipantRegistering, partial.Incarnation.State); + Assert.Equal( + ParticipantTransitionResult.Unsupported, + registry.TryRecoverParticipant(registeringToken)); + ParticipantClassification preserved = registry.ClassifyParticipant(registeringToken); + Assert.Equal(LayoutV2Constants.ParticipantRegistering, preserved.Incarnation.State); + Assert.Equal(partial.Incarnation.Control, preserved.Incarnation.Control); + + scheduler.Continue(); + await opening.WaitAsync(TimeSpan.FromSeconds(5)); + using MemoryStore opened = Assert.IsType(candidate); + Assert.Equal(StoreOpenStatus.Success, status); + ParticipantClassification active = registry.ClassifyParticipant(registeringToken); + Assert.Equal(ParticipantClassificationKind.CurrentProcess, active.Kind); + Assert.Equal(LayoutV2Constants.ParticipantActive, active.Incarnation.State); + Assert.Equal(StoreStatus.Success, opened.TryPublish([0x7A], [0x7B])); + } + finally + { + scheduler.Continue(); + WriteHeaderPidNamespaceId(region, admittedNamespace); + } + } + + [Fact] + public void RegisteringWithNewIdentityKindAndStalePriorStartUsesPresenceOnlyClassification() + { + Assert.True(LeaseOwnerClassifier.TryCaptureCurrentProcessIdentity( + out int identityKind, + out long currentStart, + out ulong pidNamespaceId)); + long stalePriorStart = currentStart == long.MaxValue ? currentStart - 1 : currentStart + 1; + var mixed = new ParticipantIncarnation( + RecordIndex: 0, + Generation: 2, + Token: 2, + State: LayoutV2Constants.ParticipantRegistering, + ProcessId: Environment.ProcessId, + IdentityKind: identityKind, + ProcessStartValue: stalePriorStart, + OpenSequence: 1, + PidNamespaceId: pidNamespaceId, + ReservedValue: 0, + Control: 0); + + Assert.Equal(LeaseOwnerKind.StaleProcess, LeaseOwnerClassifier.Classify(mixed).Kind); + Assert.Equal( + LeaseOwnerKind.CurrentProcess, + LockFreeParticipantRegistry.ClassifySnapshotOwner(mixed, pidNamespaceId).Kind); + } + + [Theory] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterIdentityKindWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterReservedWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterProcessStartWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterPidNamespaceWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterOpenSequenceWrite)] + public async Task RecoveryCannotRetireLiveRegisteringParticipantAtAnyOrdinaryFieldWrite( + int checkpointValue) + { + string name = $"sms-v2-registering-live-{checkpointValue}-{Guid.NewGuid():N}"; + using MemoryStore controller = Open(name, OpenMode.CreateNew, participantCount: 2); + using var scheduler = new ControlledLockFreeScheduler(); + scheduler.PauseAt((LockFreeCheckpointId)checkpointValue); + + StoreOpenStatus openStatus = default; + MemoryStore? second = null; + var opening = Task.Run(() => openStatus = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting, participantCount: 2), + scheduler.CreateInstrumentedCheckpoint(), + out second)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal(StoreStatus.Success, controller.TryGetDiagnostics(out DiagnosticsSnapshot paused)); + Assert.Equal(1, paused.ActiveParticipantCount); + Assert.Equal(1, paused.RegisteringParticipantCount); + Assert.Equal( + StoreStatus.Success, + controller.TryRecoverReservations( + new ReservationRecoveryOptions(false), + out ReservationRecoveryReport report)); + Assert.Equal(default(ReservationRecoveryReport), report); + Assert.Equal(StoreStatus.Success, controller.TryGetDiagnostics(out DiagnosticsSnapshot preserved)); + Assert.Equal(1, preserved.ActiveParticipantCount); + Assert.Equal(1, preserved.RegisteringParticipantCount); + + scheduler.Continue(); + await opening.WaitAsync(TimeSpan.FromSeconds(5)); + using MemoryStore opened = Assert.IsType(second); + Assert.Equal(StoreOpenStatus.Success, openStatus); + Assert.Equal(StoreStatus.Success, controller.TryGetDiagnostics(out DiagnosticsSnapshot active)); + Assert.Equal(2, active.ActiveParticipantCount); + Assert.Equal(0, active.RegisteringParticipantCount); + } + + [Theory] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterIdentityKindWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterReservedWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterProcessStartWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterPidNamespaceWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterOpenSequenceWrite)] + [InlineData((int)LockFreeCheckpointId.ParticipantAfterActivePublication)] + public void RegistrationObserverExceptionRetiresTheUnescapedExactClaim( + int checkpointValue) + { + string name = $"sms-v2-registering-exception-{checkpointValue}-{Guid.NewGuid():N}"; + using MemoryStore controller = Open(name, OpenMode.CreateNew, participantCount: 2); + var target = (LockFreeCheckpointId)checkpointValue; + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + if (entry.Id == target) + { + throw new InjectedRegistrationException(); + } + }); + + Assert.Equal( + StoreOpenStatus.MappingFailed, + LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting, participantCount: 2), + checkpoint, + out MemoryStore? failed)); + Assert.Null(failed); + + using MemoryStore replacement = Open(name, OpenMode.OpenExisting, participantCount: 2); + Assert.Equal(StoreStatus.Success, replacement.TryPublish([0x41], [0x42])); + Assert.Equal(StoreStatus.Success, replacement.TryRemove([0x41])); + } + + [Fact] + public void PostRegistrationConstructionExceptionRetiresTheExactActiveIncarnation() + { + string name = $"sms-v2-construction-exception-{Guid.NewGuid():N}"; + using MemoryStore controller = Open(name, OpenMode.CreateNew, participantCount: 2); + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented(entry => + { + if (entry.Id == LockFreeCheckpointId.ParticipantAfterRegistrationBeforeEngineConstruction) + { + throw new InjectedRegistrationException(); + } + }); + + Assert.Equal( + StoreOpenStatus.MappingFailed, + LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting, participantCount: 2), + checkpoint, + out MemoryStore? failed)); + Assert.Null(failed); + + using MemoryStore replacement = Open(name, OpenMode.OpenExisting, participantCount: 2); + Assert.Equal(StoreStatus.Success, replacement.TryPublish([0x51], [0x52])); + Assert.Equal(StoreStatus.Success, replacement.TryRemove([0x51])); + } + + [Fact] + public void FullParticipantTableRejectsOnlyNewHandleAndReusesAdvancedIncarnation() + { + string name = $"sms-v2-participant-table-{Guid.NewGuid():N}"; + using var first = Open(name, OpenMode.CreateNew, participantCount: 2); + var second = Open(name, OpenMode.OpenExisting, participantCount: 2); + + Assert.Equal(StoreStatus.Success, second.TryReserve([2], 1, default, out var secondReservation)); + ulong secondToken = secondReservation.HandleForEngine.ParticipantToken; + Assert.Equal(StoreStatus.Success, secondReservation.Abort()); + + Assert.Equal( + StoreOpenStatus.ParticipantTableFull, + MemoryStore.TryCreateOrOpen(Options(name, OpenMode.OpenExisting, participantCount: 2), out var rejected)); + Assert.Null(rejected); + Assert.Equal(StoreStatus.Success, first.TryPublish([1], [1])); + + second.Dispose(); + using var reused = Open(name, OpenMode.OpenExisting, participantCount: 2); + Assert.Equal(StoreStatus.Success, reused.TryReserve([3], 1, default, out var replacement)); + ulong replacementToken = replacement.HandleForEngine.ParticipantToken; + Assert.NotEqual(secondToken, replacementToken); + Assert.Equal( + ParticipantToken.Decode(secondToken, 2).RecordIndex, + ParticipantToken.Decode(replacementToken, 2).RecordIndex); + Assert.Equal( + ParticipantToken.Decode(secondToken, 2).Generation + 1, + ParticipantToken.Decode(replacementToken, 2).Generation); + Assert.Equal(StoreStatus.Success, replacement.Abort()); + } + + [Theory] + [InlineData(LayoutV2Constants.ParticipantClosing)] + [InlineData(LayoutV2Constants.ParticipantRecovering)] + public void ClaimClosedLiveParticipantIsRetiredWithoutOwnerLivenessClassification( + int handedOffState) + { + string name = $"sms-v2-participant-handoff-{handedOffState}-{Guid.NewGuid():N}"; + using MemoryStore controller = Open(name, OpenMode.CreateNew, participantCount: 2); + using MemoryStore handedOff = Open(name, OpenMode.OpenExisting, participantCount: 2); + (LockFreeParticipantRegistry registry, LockFreeParticipantRegistry.Registration registration) = + ReadParticipantComponents(handedOff); + + ParticipantTransitionResult transition; + if (handedOffState == LayoutV2Constants.ParticipantClosing) + { + transition = registry.TryBeginClose(registration); + } + else + { + ParticipantClassification active = registry.ClassifyParticipant(registration.Token); + Assert.Equal(ParticipantClassificationKind.CurrentProcess, active.Kind); + Assert.Equal(LayoutV2Constants.ParticipantActive, active.Incarnation.State); + transition = registry.TryBeginRecovery(active.Incarnation); + } + + Assert.Equal(ParticipantTransitionResult.Succeeded, transition); + Assert.Equal( + StoreStatus.Success, + controller.TryRecoverReservations( + new ReservationRecoveryOptions(false), + StoreWaitOptions.Infinite, + out ReservationRecoveryReport report)); + Assert.Equal(default, report); + Assert.Equal(StoreStatus.Success, controller.TryGetDiagnostics(out DiagnosticsSnapshot snapshot)); + Assert.Equal(1, snapshot.ActiveParticipantCount); + Assert.Equal(0, snapshot.ClosingParticipantCount); + Assert.Equal(0, snapshot.RecoveringParticipantCount); + Assert.Equal(1, snapshot.FreeParticipantCount); + + using MemoryStore replacement = Open(name, OpenMode.OpenExisting, participantCount: 2); + Assert.Equal(StoreStatus.Success, replacement.TryPublish([0x61], [0x62])); + Assert.Equal(StoreStatus.Success, replacement.TryRemove([0x61])); + } + + [Fact] + public void UnescapedPostRegistrationOwnerCanCloseAndRetireWithoutAReferenceScan() + { + string name = $"sms-v2-participant-construction-cleanup-{Guid.NewGuid():N}"; + using MemoryStore controller = Open(name, OpenMode.CreateNew, participantCount: 2); + using MemoryStore unescaped = Open(name, OpenMode.OpenExisting, participantCount: 2); + (LockFreeParticipantRegistry registry, LockFreeParticipantRegistry.Registration registration) = + ReadParticipantComponents(unescaped); + + registry.RetireUnreferencedRegistration(registration); + + Assert.Equal(StoreStatus.Success, controller.TryGetDiagnostics(out DiagnosticsSnapshot cleaned)); + Assert.Equal(1, cleaned.ActiveParticipantCount); + Assert.Equal(1, cleaned.FreeParticipantCount); + using MemoryStore replacement = Open(name, OpenMode.OpenExisting, participantCount: 2); + Assert.Equal(StoreStatus.Success, replacement.TryPublish([0x71], [0x72])); + Assert.Equal(StoreStatus.Success, replacement.TryRemove([0x71])); + } + + [Fact] + public void InstrumentedOpenBracketsFirstParticipantClaimAndActivePublication() + { + using var scheduler = new ControlledLockFreeScheduler(); + var status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options($"sms-v2-participant-checkpoints-{Guid.NewGuid():N}", OpenMode.CreateNew, participantCount: 1), + scheduler.CreateInstrumentedCheckpoint(), + out var store); + using var opened = Assert.IsType(store); + Assert.Equal(StoreOpenStatus.Success, status); + + LockFreeCheckpointId[] observed = scheduler.Snapshot() + .Select(observation => observation.Entry.Id) + .ToArray(); + Assert.Contains(LockFreeCheckpointId.ParticipantBeforeRegisteringCas, observed); + Assert.Contains(LockFreeCheckpointId.ParticipantAfterActivePublication, observed); + Assert.True( + Array.IndexOf(observed, LockFreeCheckpointId.ParticipantBeforeRegisteringCas) + < Array.IndexOf(observed, LockFreeCheckpointId.ParticipantAfterActivePublication)); + } + + [Fact] + public async Task FreshReferenceScanAfterRecoveryFencePreventsParticipantReuse() + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture + != System.Runtime.InteropServices.Architecture.X64) + { + return; + } + + const int slotCount = 4; + const int participantCount = 2; + string name = $"sms-v2-participant-fence-{Guid.NewGuid():N}"; + using var scheduler = new ControlledLockFreeScheduler(); + StoreOpenStatus opened = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, OpenMode.CreateNew, participantCount), + scheduler.CreateInstrumentedCheckpoint(), + out MemoryStore? candidate); + Assert.Equal(StoreOpenStatus.Success, opened); + using MemoryStore store = Assert.IsType(candidate); + + await CreateOrphanParticipant(name, slotCount, participantCount); + const int orphanRecordIndex = 1; + const int orphanGeneration = 1; + ulong orphanToken = ParticipantToken.Encode( + orphanRecordIndex, + orphanGeneration, + participantCount); + scheduler.PauseAt(LockFreeCheckpointId.ParticipantAfterRecoveryFenceBeforeReferenceScan); + + ReservationRecoveryReport report = default; + Task recovery = Task.Run(() => store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + StoreWaitOptions.Infinite, + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot fenced)); + Assert.Equal(1, fenced.RecoveringParticipantCount); + + PublishSyntheticPreFenceReference(store, orphanToken); + scheduler.Continue(); + Assert.Equal(StoreStatus.Success, await recovery.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(default, report); + + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot retained)); + Assert.Equal(1, retained.RecoveringParticipantCount); + Assert.Equal(0, retained.FreeParticipantCount); + + ClearSyntheticReference(store, orphanToken); + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + StoreWaitOptions.Infinite, + out _)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot reclaimed)); + Assert.Equal(0, reclaimed.RecoveringParticipantCount); + Assert.Equal(1, reclaimed.FreeParticipantCount); + } + + private static async Task CreateOrphanParticipant( + string name, + int slotCount, + int participantCount) + { + var startInfo = new ProcessStartInfo("dotnet") + { + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + foreach (string argument in new[] + { + "exec", + LocateAgentAssembly(), + "participant-orphan", + name, + slotCount.ToString(System.Globalization.CultureInfo.InvariantCulture), + "16", + "4", + "8", + slotCount.ToString(System.Globalization.CultureInfo.InvariantCulture), + participantCount.ToString(System.Globalization.CultureInfo.InvariantCulture) + }) + { + startInfo.ArgumentList.Add(argument); + } + + using Process process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start the orphan participant agent."); + await process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10)); + string output = await process.StandardOutput.ReadToEndAsync(); + string error = await process.StandardError.ReadToEndAsync(); + Assert.True( + process.ExitCode == 0, + $"Orphan participant failed with exit {process.ExitCode}. stdout={output} stderr={error}"); + } + + private static unsafe void PublishSyntheticPreFenceReference(MemoryStore store, ulong participantToken) + { + (LockFreeSlotTable slots, _) = ReadEngineComponents(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + long free = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.FreeState, + generation: 1, + participantToken: 0)); + long initializing = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.InitializingState, + generation: 1, + checked((int)participantToken))); + Assert.Equal( + free, + AtomicControlWord.CompareExchange(ref slot.Control, initializing, free)); + } + + private static unsafe void ClearSyntheticReference(MemoryStore store, ulong participantToken) + { + (LockFreeSlotTable slots, _) = ReadEngineComponents(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + long initializing = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.InitializingState, + generation: 1, + checked((int)participantToken))); + long free = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.FreeState, + generation: 1, + participantToken: 0)); + Assert.Equal( + initializing, + AtomicControlWord.CompareExchange(ref slot.Control, free, initializing)); + } + + private static (LockFreeSlotTable Slots, object Engine) ReadEngineComponents(MemoryStore store) + { + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + var slots = Assert.IsType(engine.GetType() + .GetField("_slots", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + return (slots, engine); + } + + private static unsafe ulong ReadHeaderPidNamespaceId( + SharedMemoryStore.Interop.MemoryMappedStoreRegion region) => + ((StoreHeaderV2*)region.Pointer)->PidNamespaceId; + + private static unsafe long ReadHeaderPidNamespaceMode( + SharedMemoryStore.Interop.MemoryMappedStoreRegion region) => + AtomicControlWord.LoadAcquire(ref ((StoreHeaderV2*)region.Pointer)->PidNamespaceMode); + + private static unsafe long ReadHeaderSequence( + SharedMemoryStore.Interop.MemoryMappedStoreRegion region) => + ((StoreHeaderV2*)region.Pointer)->Sequence; + + private static unsafe void WriteHeaderPidNamespaceId( + SharedMemoryStore.Interop.MemoryMappedStoreRegion region, + ulong pidNamespaceId) => + ((StoreHeaderV2*)region.Pointer)->PidNamespaceId = pidNamespaceId; + + private static ( + LockFreeParticipantRegistry Registry, + LockFreeParticipantRegistry.Registration Registration) ReadParticipantComponents( + MemoryStore store) + { + (_, object engine) = ReadEngineComponents(store); + var registry = Assert.IsType(engine.GetType() + .GetField("_participants", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + var registration = Assert.IsType(engine.GetType() + .GetField("_registration", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + return (registry, registration); + } + + private static string LocateAgentAssembly() + { + string configuration = new DirectoryInfo(AppContext.BaseDirectory).Parent?.Name ?? "Release"; + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null + && !File.Exists(Path.Combine(directory.FullName, "SharedMemoryStore.slnx"))) + { + directory = directory.Parent; + } + + string root = directory?.FullName + ?? throw new DirectoryNotFoundException("Repository root not found."); + string path = Path.Combine( + root, + "tests", + "SharedMemoryStore.LockFreeAgent", + "bin", + configuration, + "net10.0", + "SharedMemoryStore.LockFreeAgent.dll"); + return File.Exists(path) + ? path + : throw new FileNotFoundException("Lock-free agent assembly was not built.", path); + } + + private static Type RequireType(string name) + { + Type? type = typeof(MemoryStore).Assembly.GetType(name, throwOnError: false, ignoreCase: false); + Assert.True(type is not null, $"Required participant recovery surface {name} is missing."); + return type!; + } + + private static ParticipantOwnerClass ClassifyIdentity( + int storedPid, + int storedKind, + long storedStart, + int observedPid, + int observedKind, + long observedStart) + { + if (storedPid <= 0 || storedKind is < 0 or > 2 || storedStart < 0) + { + return ParticipantOwnerClass.Inconsistent; + } + + if (storedKind == LayoutV2Constants.IdentityUnknown || observedKind == LayoutV2Constants.IdentityUnknown) + { + return ParticipantOwnerClass.Unsupported; + } + + return storedPid == observedPid && storedKind == observedKind && storedStart == observedStart + ? ParticipantOwnerClass.Live + : ParticipantOwnerClass.Stale; + } + + private static MemoryStore Open(string name, OpenMode mode, int participantCount) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(Options(name, mode, participantCount), out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode mode, int participantCount) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 4, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 4, + participantRecordCount: participantCount, + openMode: mode, + enableLeaseRecovery: true); + + private enum ParticipantOwnerClass + { + Live, + Stale, + Unsupported, + Inconsistent + } + + private sealed class InjectedRegistrationException : Exception; +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreePublishAllocationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreePublishAllocationTests.cs new file mode 100644 index 0000000..cb917fc --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreePublishAllocationTests.cs @@ -0,0 +1,214 @@ +using System.Reflection; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreePublishAllocationTests +{ + [Fact] + public void RetainedMemoryManagersAreSparseAndSpanOnlyHandlesCreateNone() + { + using var store = CreateLockFreeStore(slotCount: 257); + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + object reservationMemory = engine.GetType() + .GetField("_reservationMemory", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)!; + FieldInfo pagesField = reservationMemory.GetType() + .GetField("_managerPages", BindingFlags.Instance | BindingFlags.NonPublic)!; + + Assert.Null(pagesField.GetValue(reservationMemory)); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, [], out var spanOnly)); + spanOnly.GetSpan()[0] = 1; + Assert.Null(pagesField.GetValue(reservationMemory)); + Assert.Equal(StoreStatus.Success, spanOnly.Abort()); + + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, [], out var retained)); + Assert.False(retained.DangerousGetMemory(1).IsEmpty); + var pages = Assert.IsAssignableFrom(pagesField.GetValue(reservationMemory)); + Assert.Equal(2, pages.Length); + Assert.Single(pages.Cast(), static page => page is not null); + var populatedPage = Assert.IsAssignableFrom(pages.Cast().Single(static page => page is not null)); + Assert.Single(populatedPage.Cast(), static manager => manager is not null); + Assert.Equal(StoreStatus.Success, retained.Abort()); + } + + [Fact] + public void WarmReservationAbortAndReuseAllocateZeroBytes() + { + using var store = CreateLockFreeStore(slotCount: 1); + var key = new byte[] { 1 }; + + // Cross the tiered-PGO promotion threshold before measuring; otherwise + // the runtime's one-time 24-byte call-counting transition is charged to + // the library hot path even though subsequent operations allocate zero. + for (var index = 0; index < 10_000; index++) + { + ReserveThenAbort(store, key); + } + + CollectGarbage(); + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var index = 0; index < 10_000; index++) + { + ReserveThenAbort(store, key); + } + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(0, allocated); + } + + [Fact] + public void WarmDuplicateReservationAndPublicationFailuresAllocateZeroBytes() + { + using var store = CreateLockFreeStore(slotCount: 2); + var key = new byte[] { 1 }; + var value = new byte[] { 2 }; + Assert.Equal(StoreStatus.Success, store.TryReserve(key, 1, default, out var owner)); + + for (var index = 0; index < 1_000; index++) + { + RequireStatus(StoreStatus.DuplicateKey, store.TryReserve(key, 1, default, out _)); + RequireStatus(StoreStatus.DuplicateKey, store.TryPublish(key, value)); + } + + CollectGarbage(); + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var index = 0; index < 10_000; index++) + { + RequireStatus(StoreStatus.DuplicateKey, store.TryReserve(key, 1, default, out _)); + RequireStatus(StoreStatus.DuplicateKey, store.TryPublish(key, value)); + } + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(0, allocated); + Assert.Equal(StoreStatus.Success, owner.Abort()); + } + + [Fact] + public void WarmInvalidIncompleteAndStaleReservationFailuresAllocateZeroBytes() + { + using var store = CreateLockFreeStore(slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 2, default, out var pending)); + + for (var index = 0; index < 1_000; index++) + { + RequireStatus(StoreStatus.ReservationWriteOutOfRange, pending.Advance(3)); + RequireStatus(StoreStatus.ReservationIncomplete, pending.Commit()); + } + + CollectGarbage(); + var beforePending = GC.GetAllocatedBytesForCurrentThread(); + for (var index = 0; index < 10_000; index++) + { + RequireStatus(StoreStatus.ReservationWriteOutOfRange, pending.Advance(3)); + RequireStatus(StoreStatus.ReservationIncomplete, pending.Commit()); + } + var pendingAllocated = GC.GetAllocatedBytesForCurrentThread() - beforePending; + Assert.Equal(0, pendingAllocated); + + Assert.Equal(StoreStatus.Success, pending.Abort()); + for (var index = 0; index < 1_000; index++) + { + RequireStatus(StoreStatus.InvalidReservation, pending.Advance(1)); + RequireStatus(StoreStatus.InvalidReservation, pending.Commit()); + RequireStatus(StoreStatus.InvalidReservation, pending.Abort()); + } + + CollectGarbage(); + var beforeStale = GC.GetAllocatedBytesForCurrentThread(); + for (var index = 0; index < 10_000; index++) + { + RequireStatus(StoreStatus.InvalidReservation, pending.Advance(1)); + RequireStatus(StoreStatus.InvalidReservation, pending.Commit()); + RequireStatus(StoreStatus.InvalidReservation, pending.Abort()); + } + var staleAllocated = GC.GetAllocatedBytesForCurrentThread() - beforeStale; + + Assert.Equal(0, staleAllocated); + } + + [Fact] + public void SuccessfulDirectCommitsAllocateZeroBytesThroughConfiguredCapacity() + { + const int SlotCount = 256; + + // Warm every direct-ingest method in a disposable store. Span-only + // direct ingest does not require retained-memory manager creation. + using (var warmup = CreateLockFreeStore(slotCount: 1)) + { + CommitOne(warmup, 1); + } + + using var store = CreateLockFreeStore(slotCount: SlotCount); + CollectGarbage(); + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var index = 0; index < SlotCount; index++) + { + CommitOne(store, index); + } + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(0, allocated); + + Span overflowKey = stackalloc byte[sizeof(int)]; + BitConverter.TryWriteBytes(overflowKey, SlotCount); + _ = store.TryReserve(overflowKey, 1, default, out _); // warm the capacity failure + CollectGarbage(); + var beforeFull = GC.GetAllocatedBytesForCurrentThread(); + var fullStatus = store.TryReserve(overflowKey, 1, default, out _); + var fullAllocated = GC.GetAllocatedBytesForCurrentThread() - beforeFull; + Assert.Equal(StoreStatus.StoreFull, fullStatus); + Assert.Equal(0, fullAllocated); + } + + private static void ReserveThenAbort(MemoryStore store, byte[] key) + { + RequireStatus(StoreStatus.Success, store.TryReserve(key, 1, default, out var reservation)); + RequireStatus(StoreStatus.Success, reservation.Abort()); + } + + private static void CommitOne(MemoryStore store, int id) + { + Span key = stackalloc byte[sizeof(int)]; + BitConverter.TryWriteBytes(key, id); + RequireStatus(StoreStatus.Success, store.TryReserve(key, 1, default, out var reservation)); + reservation.GetSpan()[0] = unchecked((byte)id); + RequireStatus(StoreStatus.Success, reservation.Advance(1)); + RequireStatus(StoreStatus.Success, reservation.Commit()); + } + + private static MemoryStore CreateLockFreeStore(int slotCount) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-publish-allocation-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 8, + maxDescriptorBytes: 2, + maxKeyBytes: 8, + leaseRecordCount: 8, + participantRecordCount: 1, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + var status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + var result = Assert.IsType(store); + Assert.Equal(StoreProfile.LockFree, result.Profile); + return result; + } + + private static void RequireStatus(StoreStatus expected, StoreStatus actual) + { + if (actual != expected) + { + throw new InvalidOperationException($"Expected {expected}, received {actual}."); + } + } + + private static void CollectGarbage() + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeReclamationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeReclamationTests.cs new file mode 100644 index 0000000..f581500 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeReclamationTests.cs @@ -0,0 +1,794 @@ +using System.Reflection; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeReclamationTests +{ + [Theory] + [InlineData(StoreStatus.Success, StoreStatus.Success)] + [InlineData(StoreStatus.RemovePending, StoreStatus.RemovePending)] + [InlineData(StoreStatus.StoreBusy, StoreStatus.RemovePending)] + [InlineData(StoreStatus.NotFound, StoreStatus.RemovePending)] + [InlineData(StoreStatus.InvalidReservation, StoreStatus.RemovePending)] + [InlineData(StoreStatus.CorruptStore, StoreStatus.CorruptStore)] + public void PostLogicalRemovePreservesDurableOutcome( + StoreStatus reclaimStatus, + StoreStatus expected) + { + Assert.Equal( + expected, + LockFreeStoreEngine.NormalizePostLogicalRemoveOutcome(reclaimStatus)); + } + + [Theory] + [InlineData(StoreStatus.Success, StoreStatus.Success)] + [InlineData(StoreStatus.RemovePending, StoreStatus.DuplicateKey)] + [InlineData(StoreStatus.NotFound, StoreStatus.DuplicateKey)] + [InlineData(StoreStatus.StoreBusy, StoreStatus.StoreBusy)] + [InlineData(StoreStatus.OperationCanceled, StoreStatus.OperationCanceled)] + [InlineData(StoreStatus.CorruptStore, StoreStatus.CorruptStore)] + public void ExistingGenerationNormalizationNeverMasksOperationalOrStructuralFailure( + StoreStatus reclaimStatus, + StoreStatus expected) + { + Assert.Equal( + expected, + LockFreeStoreEngine.NormalizeExistingGenerationReclaimOutcome(reclaimStatus)); + } + + [Theory] + [InlineData(StoreStatus.Success, StoreStatus.Success)] + [InlineData(StoreStatus.NotFound, StoreStatus.Success)] + [InlineData(StoreStatus.StoreBusy, StoreStatus.StoreBusy)] + [InlineData(StoreStatus.OperationCanceled, StoreStatus.OperationCanceled)] + [InlineData(StoreStatus.CorruptStore, StoreStatus.CorruptStore)] + public void AbortingUnlinkScanNormalizesOnlyExactAbsence( + StoreStatus unlinkStatus, + StoreStatus expected) + { + Assert.Equal( + expected, + LockFreeReclaimer.NormalizeAbortingUnlinkOutcome(unlinkStatus)); + } + + [Theory] + [InlineData(StoreStatus.Success, StoreStatus.Success)] + [InlineData(StoreStatus.NotFound, StoreStatus.Success)] + [InlineData(StoreStatus.RemovePending, StoreStatus.Success)] + [InlineData(StoreStatus.StoreBusy, StoreStatus.StoreBusy)] + [InlineData(StoreStatus.OperationCanceled, StoreStatus.OperationCanceled)] + [InlineData(StoreStatus.CorruptStore, StoreStatus.CorruptStore)] + public void ReclaimScanNormalizesOnlyConservativeLifecycleRaces( + StoreStatus reclaimStatus, + StoreStatus expected) + { + Assert.Equal( + expected, + LockFreeReclaimer.NormalizeObservedReclaimOutcome(reclaimStatus)); + } + + [Fact] + public void FinalReleaseReclaimsExactRemovedGenerationAndAllowsRepublish() + { + using var store = CreateStore(slotCount: 1, leaseRecordCount: 2); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1, 2, 3])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(StoreStatus.RemovePending, store.TryRemove([1])); + Assert.Equal(StoreStatus.DuplicateKey, store.TryPublish([1], [4])); + + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [4])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var replacement)); + Assert.Equal(4, replacement.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task ExistingLookupThatDisappearsDuringFinalReleaseContinuesRepublishInsteadOfReturningNotFound() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + byte[] key = [0x71]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [1])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out var lease)); + Assert.Equal(StoreStatus.RemovePending, store.TryRemove(key)); + scheduler.PauseAt(LockFreeCheckpointId.ReserveAfterExistingLookup); + + StoreStatus republishStatus = default; + var republish = Task.Run(() => republishStatus = store.TryPublish( + key, + [2], + default, + StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + // Final release removes the exact old directory generation while the + // publisher is between its existing-key lookup and revalidation. + Assert.Equal(StoreStatus.Success, lease.Release(StoreWaitOptions.Infinite)); + scheduler.Continue(); + await republish.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, republishStatus); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out var current)); + Assert.Equal(2, current.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, current.Release()); + } + + [Fact] + public async Task SecondHandlePublishesAtCapacityOneWhileAbortOwnerIsPausedAfterOwnershipRelease() + { + string name = $"sms-v2-reclamation-abort-help-{Guid.NewGuid():N}"; + var createOptions = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + var openOptions = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 2, + openMode: OpenMode.OpenExisting, + enableLeaseRecovery: true); + using var scheduler = new ControlledLockFreeScheduler(); + Assert.Equal( + StoreOpenStatus.Success, + LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + createOptions, + scheduler.CreateInstrumentedCheckpoint(), + out var firstHandle)); + using var first = Assert.IsType(firstHandle); + Assert.Equal(StoreOpenStatus.Success, MemoryStore.TryCreateOrOpen(openOptions, out var secondHandle)); + using var second = Assert.IsType(secondHandle); + + Assert.Equal(StoreStatus.Success, first.TryReserve([0x81], 1, default, out var reservation)); + reservation.GetSpan(1)[0] = 1; + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + scheduler.PauseAt(LockFreeCheckpointId.AbortAfterOwnershipReleaseCas); + + StoreStatus abortStatus = default; + var abort = Task.Run(() => abortStatus = reservation.Abort(StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + // The other process/handle sees an unowned Aborting slot. Allocation + // pressure must derive the exact binding from control, help it to the + // next generation, and retry its claim rather than return StoreFull. + Assert.Equal( + StoreStatus.Success, + second.TryPublish([0x82], [2], default, StoreWaitOptions.Infinite)); + + scheduler.Continue(); + await abort.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, abortStatus); + Assert.Equal(StoreStatus.Success, second.TryAcquire([0x82], out var current)); + Assert.Equal(2, current.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, current.Release()); + } + + [Fact] + public void StaleCopiedLeaseCannotReleaseReusedLeaseRecordOrSlotGeneration() + { + using var store = CreateStore(slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var first)); + var stale = first; + Assert.Equal(StoreStatus.RemovePending, store.TryRemove([1])); + Assert.Equal(StoreStatus.Success, first.Release()); + + Assert.Equal(StoreStatus.Success, store.TryPublish([2], [2])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([2], out var current)); + Assert.Contains(stale.Release(), new[] { StoreStatus.InvalidLease, StoreStatus.LeaseAlreadyReleased }); + Assert.True(current.IsValid); + Assert.Equal(2, current.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, current.Release()); + } + + [Fact] + public void ExactUnlinkOfRemovedKeyPreservesUnrelatedPublishedBinding() + { + using var store = CreateStore(slotCount: 2, leaseRecordCount: 2); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [11])); + Assert.Equal(StoreStatus.Success, store.TryPublish([2], [22])); + + Assert.Equal(StoreStatus.Success, store.TryRemove([1])); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, store.TryAcquire([2], out var unrelated)); + Assert.Equal(22, unrelated.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, unrelated.Release()); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [33])); + } + + [Fact] + public void RetryingRemovalBeforeRepublishCannotAffectLaterGeneration() + { + using var store = CreateStore(slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + Assert.Equal(StoreStatus.Success, store.TryRemove([1])); + Assert.Equal(StoreStatus.NotFound, store.TryRemove([1])); + + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [2])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var current)); + Assert.Equal(2, current.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, current.Release()); + } + + [Fact] + public async Task ConcurrentFinalReleaseAndRetryingRemoveReclaimExactlyOnce() + { + using var store = CreateStore(slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(StoreStatus.RemovePending, store.TryRemove([1])); + + StoreStatus releaseStatus = default; + StoreStatus retryStatus = default; + using var start = new Barrier(3); + var release = Task.Run(() => + { + start.SignalAndWait(); + releaseStatus = lease.Release(); + }); + var retry = Task.Run(() => + { + start.SignalAndWait(); + retryStatus = store.TryRemove([1]); + }); + start.SignalAndWait(); + await Task.WhenAll(release, retry).WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, releaseStatus); + Assert.Contains(retryStatus, new[] + { + StoreStatus.Success, + StoreStatus.RemovePending, + StoreStatus.NotFound + }); + Assert.Equal(StoreStatus.Success, store.TryPublish([2], [2])); + } + + [Fact] + public async Task CancellationAfterLogicalRemovalHandsPendingReclaimToFinalRelease() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.RemoveAfterLeaseClassification); + + StoreStatus removeStatus = default; + var remove = Task.Run(() => removeStatus = store.TryRemove( + [1], + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token))); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + cancellation.Cancel(); + scheduler.Continue(); + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "Remove never published a helpable pending-removal state."); + Assert.Equal(StoreStatus.RemovePending, removeStatus); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [2])); + } + + [Fact] + public async Task PausedStaleRemoveCannotRemoveRepublishedSlotGeneration() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [1])); + scheduler.PauseAt(LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas); + + StoreStatus staleStatus = default; + var staleRemove = Task.Run(() => staleStatus = store.TryRemove([1])); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal(StoreStatus.Success, store.TryRemove([1])); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [2])); + scheduler.Continue(); + await staleRemove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.NotFound, staleStatus); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var current)); + Assert.Equal(2, current.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, current.Release()); + } + + [Fact] + public async Task DelayedTargetSelectedInsertHelperCannotOrphanProgressedBinding() + { + byte[][] keys = GenerateCanonicalBucketCollisions(count: 5, slotCount: 2); + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 2, leaseRecordCount: 2); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterOperationValidation, occurrence: 2); + + StoreStatus firstPublishStatus = default; + var firstPublish = Task.Run(() => firstPublishStatus = store.TryPublish(keys[0], [0xA1])); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + StoreStatus oneStepStatus = default; + SlotSnapshot progressed = default; + try + { + LockFreeKeyDirectory directory = ReadDirectory(store); + oneStepStatus = directory.HelpMutation(canonicalBucketIndex: 0, maxSteps: 1); + progressed = RequireOperationSnapshot(store, slotCount: 2, intent: 1, phase: 3); + } + finally + { + scheduler.Continue(); + } + + await firstPublish.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "Insert helper never paused after observing TargetSelected."); + Assert.Equal(StoreStatus.StoreBusy, oneStepStatus); + Assert.Equal(LockFreeSlotTable.InitializingState, SlotState(progressed.Control)); + AssertGenerationConsistent(progressed); + Assert.Equal(StoreStatus.Success, firstPublishStatus); + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[1], [0xB2])); + AssertPublishedValue(store, keys[0], 0xA1); + AssertPublishedValue(store, keys[1], 0xB2); + + Assert.Equal(StoreStatus.Success, store.TryRemove(keys[0])); + Assert.Equal(StoreStatus.Success, store.TryRemove(keys[1])); + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[2], [0xC3])); + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[3], [0xD4])); + Assert.Equal(StoreStatus.StoreFull, store.TryPublish(keys[4], [0xE5])); + AssertPublishedValue(store, keys[2], 0xC3); + AssertPublishedValue(store, keys[3], 0xD4); + } + + [Fact] + public async Task DelayedTargetSelectedUnlinkHelperCannotOrphanProgressedBinding() + { + byte[][] keys = GenerateCanonicalBucketCollisions(count: 5, slotCount: 2); + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 2, leaseRecordCount: 2); + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[0], [0x11])); + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[1], [0x22])); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterOperationValidation, occurrence: 2); + + StoreStatus removeStatus = default; + var remove = Task.Run(() => removeStatus = store.TryRemove(keys[0])); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + StoreStatus oneStepStatus = default; + SlotSnapshot progressed = default; + try + { + LockFreeKeyDirectory directory = ReadDirectory(store); + oneStepStatus = directory.HelpMutation(canonicalBucketIndex: 0, maxSteps: 1); + progressed = RequireOperationSnapshot(store, slotCount: 2, intent: 2, phase: 3); + } + finally + { + scheduler.Continue(); + } + + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "Unlink helper never paused after observing TargetSelected."); + Assert.Equal(StoreStatus.StoreBusy, oneStepStatus); + Assert.Equal(LockFreeSlotTable.ReclaimingState, SlotState(progressed.Control)); + Assert.Equal(StoreStatus.Success, removeStatus); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(keys[0], out _)); + AssertPublishedValue(store, keys[1], 0x22); + + Assert.Equal(StoreStatus.Success, store.TryRemove(keys[1])); + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[2], [0x33])); + Assert.Equal(StoreStatus.Success, store.TryPublish(keys[3], [0x44])); + Assert.Equal(StoreStatus.StoreFull, store.TryPublish(keys[4], [0x55])); + AssertPublishedValue(store, keys[2], 0x33); + AssertPublishedValue(store, keys[3], 0x44); + } + + [Theory] + [InlineData("DirectoryAfterOperationValidation")] + [InlineData("DirectoryAfterLocationValidation")] + public async Task DelayedValidatedUnlinkHelperCannotDamageReusedSlotGeneration(string checkpointName) + { + LockFreeCheckpointId checkpoint = RequireCheckpoint(checkpointName); + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([0xA1], [1, 2], [3])); + SlotSnapshot oldGeneration = ReadSlotSnapshot(store, slotIndex: 0); + scheduler.PauseAt(checkpoint); + + StoreStatus delayedStatus = default; + var delayed = Task.Run(() => delayedStatus = store.TryRemove([0xA1])); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + StoreStatus helperStatus = default; + StoreStatus republishStatus = default; + SlotSnapshot beforeDelayedResume = default; + DirectoryOccupancy occupancyBeforeDelayedResume = default; + try + { + helperStatus = store.TryRemove([0xA1]); + republishStatus = store.TryPublish([0xB2], [0x21, 0x22, 0x23], [0x31, 0x32]); + beforeDelayedResume = ReadSlotSnapshot(store, slotIndex: 0); + occupancyBeforeDelayedResume = ReadDirectoryOccupancy(store); + } + finally + { + scheduler.Continue(); + } + + await delayed.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, $"The helper never reached {checkpointName}."); + Assert.DoesNotContain(StoreStatus.CorruptStore, new[] { helperStatus, republishStatus, delayedStatus }); + Assert.Contains(helperStatus, new[] { StoreStatus.Success, StoreStatus.NotFound }); + Assert.Contains(delayedStatus, new[] { StoreStatus.Success, StoreStatus.NotFound }); + Assert.Equal(StoreStatus.Success, republishStatus); + Assert.Equal(SlotGeneration(oldGeneration.Control) + 1, SlotGeneration(beforeDelayedResume.Control)); + Assert.NotEqual(oldGeneration.DirectoryBinding, beforeDelayedResume.DirectoryBinding); + AssertGenerationConsistent(beforeDelayedResume); + Assert.Equal(beforeDelayedResume, ReadSlotSnapshot(store, slotIndex: 0)); + Assert.Equal(new DirectoryOccupancy(Primary: 1, Overflow: 0), occupancyBeforeDelayedResume); + Assert.Equal(occupancyBeforeDelayedResume, ReadDirectoryOccupancy(store)); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([0xA1], out _)); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0xB2], out var current)); + Assert.Equal(new byte[] { 0x21, 0x22, 0x23 }, current.ValueSpan.ToArray()); + Assert.Equal(new byte[] { 0x31, 0x32 }, current.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, current.Release()); + Assert.Equal(StoreStatus.DuplicateKey, store.TryPublish([0xB2], [9])); + + Assert.Equal(StoreStatus.Success, store.TryRemove([0xB2])); + Assert.Equal(StoreStatus.Success, store.TryPublish([0xC3], [0x41], [0x42])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0xC3], out var final)); + Assert.Equal(0x41, final.ValueSpan[0]); + Assert.Equal(0x42, final.DescriptorSpan[0]); + Assert.Equal(StoreStatus.Success, final.Release()); + } + + [Fact] + public async Task DelayedReclaimLoserCannotZeroOrdinaryMetadataAfterFreeToInitializingReuse() + { + LockFreeCheckpointId checkpoint = RequireCheckpoint("ReclaimAfterMetadataValidation"); + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1, leaseRecordCount: 1); + Assert.Equal(StoreStatus.Success, store.TryPublish([0xD1], [1, 2, 3], [4])); + SlotSnapshot oldGeneration = ReadSlotSnapshot(store, slotIndex: 0); + scheduler.PauseAt(checkpoint); + + StoreStatus delayedStatus = default; + var delayed = Task.Run(() => delayedStatus = store.TryRemove([0xD1])); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + StoreStatus republishStatus = default; + SlotSnapshot beforeDelayedResume = default; + DirectoryOccupancy occupancyBeforeDelayedResume = default; + try + { + // SlotCount=1 forces allocation pressure to run a second reclaim + // helper, advance the generation, and claim Initializing for D2. + republishStatus = store.TryPublish([0xD2], [0x51, 0x52, 0x53, 0x54], [0x61, 0x62]); + beforeDelayedResume = ReadSlotSnapshot(store, slotIndex: 0); + occupancyBeforeDelayedResume = ReadDirectoryOccupancy(store); + } + finally + { + scheduler.Continue(); + } + + await delayed.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "The losing reclaimer never reached its post-validation window."); + Assert.DoesNotContain(StoreStatus.CorruptStore, new[] { republishStatus, delayedStatus }); + Assert.Equal(StoreStatus.Success, republishStatus); + Assert.Contains(delayedStatus, new[] { StoreStatus.Success, StoreStatus.NotFound }); + Assert.Equal(SlotGeneration(oldGeneration.Control) + 1, SlotGeneration(beforeDelayedResume.Control)); + Assert.NotEqual(oldGeneration.DirectoryBinding, beforeDelayedResume.DirectoryBinding); + AssertGenerationConsistent(beforeDelayedResume); + Assert.Equal(beforeDelayedResume, ReadSlotSnapshot(store, slotIndex: 0)); + Assert.Equal(new DirectoryOccupancy(Primary: 1, Overflow: 0), occupancyBeforeDelayedResume); + Assert.Equal(occupancyBeforeDelayedResume, ReadDirectoryOccupancy(store)); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([0xD1], out _)); + Assert.Equal(StoreStatus.Success, store.TryAcquire([0xD2], out var current)); + Assert.Equal(new byte[] { 0x51, 0x52, 0x53, 0x54 }, current.ValueSpan.ToArray()); + Assert.Equal(new byte[] { 0x61, 0x62 }, current.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, current.Release()); + + Assert.Equal(StoreStatus.Success, store.TryRemove([0xD2])); + Assert.Equal(StoreStatus.Success, store.TryPublish([0xD3], [0x71], [0x72])); + } + + [Fact] + public void ReclaimCleanupWritesAreFencedByExactOldReclaimingControl() + { + var delayed = new ReclaimValidation( + Generation: 70, + ExpectedReclaimingControl: TaggedControl(generation: 70, state: 6)); + var reused = new OrdinarySlotState( + Generation: 71, + Control: TaggedControl(generation: 71, state: 1), + DirectoryBinding: ((ulong)71 << 31) | 1, + DirectoryOperation: ((ulong)71 << 16) | 1, + DirectoryLocation: ((ulong)71 << 16) | 2, + KeyHash: 0xABCD, + KeyLength: 3, + DescriptorLength: 2, + ValueLength: 8, + BytesAdvanced: 0, + CommitSequence: 99); + + OrdinarySlotState afterDelayedResume = ResumeReclaimCleanup(reused, delayed); + + Assert.Equal(reused, afterDelayedResume); + } + + [Fact] + public void ProductionHasRecordLocalReclaimerWithLogicalRemoveAndHelpingEntryPoints() + { + var type = typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeReclaimer", + throwOnError: false, + ignoreCase: false); + Assert.True(type is not null, "LockFreeReclaimer is required for remove/release cooperation."); + + MethodInfo[] methods = type!.GetMethods( + BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + Assert.Contains(methods, method => method.Name.Contains("Logical", StringComparison.OrdinalIgnoreCase) + && method.Name.Contains("Remove", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Reclaim", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Help", StringComparison.OrdinalIgnoreCase)); + } + + private static LockFreeCheckpointId RequireCheckpoint(string name) + { + bool found = Enum.TryParse(name, ignoreCase: false, out LockFreeCheckpointId checkpoint) + && Enum.IsDefined(checkpoint); + Assert.True(found, $"The canonical checkpoint catalog is missing {name}."); + return checkpoint; + } + + private static OrdinarySlotState ResumeReclaimCleanup( + OrdinarySlotState current, + ReclaimValidation delayed) + { + if (current.Generation != delayed.Generation + || current.Control != delayed.ExpectedReclaimingControl) + { + return current; + } + + return current with + { + DirectoryBinding = 0, + DirectoryOperation = 0, + DirectoryLocation = 0, + KeyHash = 0, + KeyLength = 0, + DescriptorLength = 0, + ValueLength = 0, + BytesAdvanced = 0, + CommitSequence = 0 + }; + } + + private static long TaggedControl(long generation, int state) => + unchecked((long)(((ulong)generation << 3) | (uint)state)); + + private readonly record struct ReclaimValidation(long Generation, long ExpectedReclaimingControl); + + private readonly record struct OrdinarySlotState( + long Generation, + long Control, + ulong DirectoryBinding, + ulong DirectoryOperation, + ulong DirectoryLocation, + ulong KeyHash, + int KeyLength, + int DescriptorLength, + int ValueLength, + long BytesAdvanced, + long CommitSequence); + + private readonly record struct SlotSnapshot( + long Control, + ulong DirectoryBinding, + long DirectoryLocation, + long DirectoryOperation, + ulong KeyHash, + int KeyLength, + int DescriptorLength, + int ValueLength, + long BytesAdvanced, + long CommitSequence, + long KeyOffset, + long DescriptorOffset, + long PayloadOffset); + + private readonly record struct DirectoryOccupancy(int Primary, int Overflow); + + private static SlotSnapshot ReadSlotSnapshot(MemoryStore store, int slotIndex) + { + object engine = ReadPrivateField(store, "_engine"); + LockFreeSlotTable slots = ReadPrivateField(engine, "_slots"); + ref ValueSlotMetadataV2 slot = ref slots.Slot(slotIndex); + long control = AtomicControlWord.LoadAcquire(ref slot.Control); + var snapshot = new SlotSnapshot( + control, + slot.DirectoryBinding, + AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation), + AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation), + slot.KeyHash, + slot.KeyLength, + slot.DescriptorLength, + slot.ValueLength, + slot.BytesAdvanced, + slot.CommitSequence, + slot.KeyOffset, + slot.DescriptorOffset, + slot.PayloadOffset); + Assert.Equal(control, AtomicControlWord.LoadAcquire(ref slot.Control)); + return snapshot; + } + + private static DirectoryOccupancy ReadDirectoryOccupancy(MemoryStore store) + { + LockFreeKeyDirectory directory = ReadDirectory(store); + return new DirectoryOccupancy(directory.PrimaryOccupancy, directory.OverflowOccupancy); + } + + private static LockFreeKeyDirectory ReadDirectory(MemoryStore store) + { + object engine = ReadPrivateField(store, "_engine"); + return ReadPrivateField(engine, "_directory"); + } + + private static SlotSnapshot RequireOperationSnapshot( + MemoryStore store, + int slotCount, + int intent, + int phase) + { + for (int slotIndex = 0; slotIndex < slotCount; slotIndex++) + { + SlotSnapshot snapshot = ReadSlotSnapshot(store, slotIndex); + if (snapshot.DirectoryOperation == 0) + { + continue; + } + + DirectoryOperation operation = DirectoryOperation.Decode( + unchecked((ulong)snapshot.DirectoryOperation)); + if (operation.Intent == intent && operation.Phase == phase) + { + return snapshot; + } + } + + Assert.Fail($"No intent={intent}, phase={phase} directory operation was present."); + return default; + } + + private static void AssertPublishedValue(MemoryStore store, byte[] key, byte expected) + { + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out var lease)); + Assert.Equal(expected, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + private static byte[][] GenerateCanonicalBucketCollisions(int count, int slotCount) + { + var keys = new List(count); + int primaryLanes = NextPowerOfTwo(Math.Max(32, checked(slotCount * 4))); + uint bucketMask = checked((uint)((primaryLanes / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + for (long candidate = 0; keys.Count < count; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + if ((Mix(StoreKey.Hash(key)) & bucketMask) == 0) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static int NextPowerOfTwo(int value) + { + var result = 1; + while (result < value) + { + result <<= 1; + } + + return result; + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static void AssertGenerationConsistent(SlotSnapshot snapshot) + { + long generation = SlotGeneration(snapshot.Control); + Assert.Equal(generation, IndexBinding.Decode(snapshot.DirectoryBinding).Generation); + if (snapshot.DirectoryLocation != 0) + { + Assert.Equal( + generation, + DirectoryLocation.Decode(unchecked((ulong)snapshot.DirectoryLocation)).Generation); + } + + if (snapshot.DirectoryOperation != 0) + { + Assert.Equal( + generation, + DirectoryOperation.Decode(unchecked((ulong)snapshot.DirectoryOperation)).Generation); + } + } + + private static long SlotGeneration(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static int SlotState(long control) => + (int)(unchecked((ulong)control) & 0x7UL); + + private static T ReadPrivateField(object owner, string name) + where T : class + { + FieldInfo? field = owner.GetType().GetField( + name, + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsAssignableFrom(field!.GetValue(owner)); + } + + private static MemoryStore CreateStore(int slotCount, int leaseRecordCount) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-reclamation-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + var status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore CreateInstrumentedStore( + ControlledLockFreeScheduler scheduler, + int slotCount, + int leaseRecordCount) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-reclamation-instrumented-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + var status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + scheduler.CreateInstrumentedCheckpoint(), + out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeRecoveryCorruptionPropagationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeRecoveryCorruptionPropagationTests.cs new file mode 100644 index 0000000..bc6aade --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeRecoveryCorruptionPropagationTests.cs @@ -0,0 +1,136 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed unsafe class LockFreeRecoveryCorruptionPropagationTests +{ + [Fact] + public void ReservationRecoveryReturnsCorruptAfterAnEarlierSuccessfulRecovery() + { + if (!IsSupportedHost()) + { + return; + } + + string name = $"sms-v2-reservation-owner-corruption-{Guid.NewGuid():N}"; + using MemoryStore controller = Open(Options(name, OpenMode.CreateNew)); + using MemoryStore malformedOwner = Open(Options(name, OpenMode.OpenExisting)); + Assert.Equal( + StoreStatus.Success, + controller.TryReserve([0x31], 1, default, StoreWaitOptions.Infinite, out ValueReservation first)); + Assert.Equal( + StoreStatus.Success, + malformedOwner.TryReserve([0x32], 1, default, StoreWaitOptions.Infinite, out ValueReservation target)); + Assert.True( + IndexBinding.Decode(first.HandleForEngine.SlotBinding).SlotIndex + < IndexBinding.Decode(target.HandleForEngine.SlotBinding).SlotIndex); + + EngineInternals targetInternals = ReadInternals(malformedOwner); + ref ParticipantRecordV2 participant = ref ParticipantRecord(targetInternals); + Volatile.Write(ref participant.Reserved, 1); + + StoreStatus status = controller.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: true), + StoreWaitOptions.Infinite, + out ReservationRecoveryReport report); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.True(report.RecoveredReservationCount >= 1); + Assert.Equal(1, Volatile.Read(ref participant.Reserved)); + AssertCorrupt(targetInternals.Region); + } + + [Fact] + public void LeaseRecoveryReturnsCorruptAfterAnEarlierSuccessfulRecovery() + { + if (!IsSupportedHost()) + { + return; + } + + string name = $"sms-v2-lease-owner-corruption-{Guid.NewGuid():N}"; + using MemoryStore controller = Open(Options(name, OpenMode.CreateNew)); + using MemoryStore malformedOwner = Open(Options(name, OpenMode.OpenExisting)); + byte[] key = [0x41]; + Assert.Equal(StoreStatus.Success, controller.TryPublish(key, [0x51])); + Assert.Equal(StoreStatus.Success, controller.TryAcquire(key, out ValueLease first)); + Assert.Equal(StoreStatus.Success, malformedOwner.TryAcquire(key, out ValueLease target)); + Assert.True( + IndexBinding.Decode(first.HandleForEngine.LeaseToken).SlotIndex + < IndexBinding.Decode(target.HandleForEngine.LeaseToken).SlotIndex); + + EngineInternals targetInternals = ReadInternals(malformedOwner); + ref ParticipantRecordV2 participant = ref ParticipantRecord(targetInternals); + Volatile.Write(ref participant.Reserved, 1); + + StoreStatus status = controller.TryRecoverLeases( + new LeaseRecoveryOptions(RecoverCurrentProcessLeases: true), + StoreWaitOptions.Infinite, + out LeaseRecoveryReport report); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.True(report.RecoveredLeaseCount >= 1); + Assert.Equal(1, Volatile.Read(ref participant.Reserved)); + AssertCorrupt(targetInternals.Region); + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode mode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 4, + maxValueBytes: 8, + maxDescriptorBytes: 4, + maxKeyBytes: 4, + leaseRecordCount: 4, + participantRecordCount: 4, + openMode: mode, + enableLeaseRecovery: true); + + private static MemoryStore Open(SharedMemoryStoreOptions options) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static EngineInternals ReadInternals(MemoryStore store) + { + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + return new EngineInternals( + ReadField(engine, "_registration"), + ReadField(engine, "_region"), + ReadField(engine, "_layout")); + } + + private static T ReadField(object owner, string name) => + Assert.IsType(owner.GetType() + .GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(owner)); + + private static ref ParticipantRecordV2 ParticipantRecord(in EngineInternals internals) => + ref *(ParticipantRecordV2*)( + internals.Region.Pointer + + internals.Layout.ParticipantOffset + + ((long)internals.Registration.RecordIndex * internals.Layout.ParticipantStride)); + + private static void AssertCorrupt(MemoryMappedStoreRegion region) => + Assert.Equal( + LayoutV2Constants.StoreCorrupt, + AtomicControlWord.LoadAcquire(ref ((StoreHeaderV2*)region.Pointer)->Control)); + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private readonly record struct EngineInternals( + LockFreeParticipantRegistry.Registration Registration, + MemoryMappedStoreRegion Region, + StoreLayoutV2 Layout); +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeRemoveReuseAllocationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeRemoveReuseAllocationTests.cs new file mode 100644 index 0000000..637deea --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeRemoveReuseAllocationTests.cs @@ -0,0 +1,227 @@ +using System.Diagnostics; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeRemoveReuseAllocationTests +{ + private const int MeasuredLifecycleCycles = 1_000_000; + private const int TieredPgoWarmupCycles = 50_000; + + [Fact] + public void OneMillionWarmedCompleteLifecycleCyclesAllocateZeroBytesAndRestoreExactCapacity() + { + using var store = CreateLockFreeStore(); + var key = new byte[] { 1, 0, 2 }; + var otherKey = new byte[] { 3, 0, 4 }; + var publishedValue = new byte[] { 5, 0, 6, 7 }; + var publishedDescriptor = new byte[] { 8, 0, 9 }; + var reservedValue = new byte[] { 10, 0, 11, 12 }; + var reservedDescriptor = new byte[] { 13, 0, 14 }; + + // Warm every successful facade, directory, slot, lease, projection, + // reclaim, and reuse path far enough for tiered PGO promotion. + RunCompleteLifecycleCycles( + store, + key, + publishedValue, + publishedDescriptor, + reservedValue, + reservedDescriptor, + TieredPgoWarmupCycles); + VerifyExactCapacityRestoration(store, key, otherKey, publishedValue, publishedDescriptor); + CollectGarbage(); + + long started = Stopwatch.GetTimestamp(); + long before = GC.GetAllocatedBytesForCurrentThread(); + RunCompleteLifecycleCycles( + store, + key, + publishedValue, + publishedDescriptor, + reservedValue, + reservedDescriptor, + MeasuredLifecycleCycles); + long allocated = GC.GetAllocatedBytesForCurrentThread() - before; + TimeSpan elapsed = Stopwatch.GetElapsedTime(started); + + Assert.Equal(0, allocated); + Assert.True(elapsed > TimeSpan.Zero); + VerifyExactCapacityRestoration(store, key, otherKey, publishedValue, publishedDescriptor); + } + + private static void RunCompleteLifecycleCycles( + MemoryStore store, + byte[] key, + byte[] publishedValue, + byte[] publishedDescriptor, + byte[] reservedValue, + byte[] reservedDescriptor, + int cycles) + { + for (var index = 0; index < cycles; index++) + { + RequireStatus( + StoreStatus.Success, + store.TryPublish(key, publishedValue, publishedDescriptor), + "publish", + index); + AcquireProjectRelease( + store, + key, + expectedValueSum: 18, + expectedDescriptorSum: 17, + index); + RequireStatus(StoreStatus.Success, store.TryRemove(key), "remove published", index); + + RequireStatus( + StoreStatus.Success, + store.TryReserve( + key, + reservedValue.Length, + reservedDescriptor, + out var reservation), + "reserve", + index); + Span destination = reservation.GetSpan(reservedValue.Length); + reservedValue.AsSpan().CopyTo(destination); + RequireStatus( + StoreStatus.Success, + reservation.Advance(reservedValue.Length), + "advance", + index); + RequireStatus(StoreStatus.Success, reservation.Commit(), "commit", index); + AcquireProjectRelease( + store, + key, + expectedValueSum: 33, + expectedDescriptorSum: 27, + index); + RequireStatus(StoreStatus.Success, store.TryRemove(key), "remove reserved", index); + } + } + + private static void AcquireProjectRelease( + MemoryStore store, + byte[] key, + int expectedValueSum, + int expectedDescriptorSum, + int cycle) + { + StoreStatus acquire = store.TryAcquire(key, out var lease); + if (acquire != StoreStatus.Success + || !lease.IsValid + || lease.ValueLength != 4 + || lease.DescriptorLength != 3) + { + throw new InvalidOperationException( + $"Acquire/projection metadata failed during cycle {cycle}: {acquire}."); + } + + ReadOnlySpan value = lease.ValueSpan; + ReadOnlySpan descriptor = lease.DescriptorSpan; + if (value.Length != 4 + || value[0] + value[1] + value[2] + value[3] != expectedValueSum + || descriptor.Length != 3 + || descriptor[0] + descriptor[1] + descriptor[2] != expectedDescriptorSum) + { + throw new InvalidOperationException($"Projected bytes failed during cycle {cycle}."); + } + + StoreStatus release = lease.Release(); + if (release != StoreStatus.Success || lease.IsValid) + { + throw new InvalidOperationException( + $"Lease release failed during cycle {cycle}: {release}."); + } + } + + private static void VerifyExactCapacityRestoration( + MemoryStore store, + byte[] firstKey, + byte[] secondKey, + byte[] value, + byte[] descriptor) + { + // The store has exactly one slot. Filling it must reject a second key; + // aborting or removing the owner must make exactly that slot reusable. + RequireStatus( + StoreStatus.Success, + store.TryReserve(firstKey, value.Length, descriptor, out var reservation), + "capacity reserve", + cycle: -1); + RequireStatus( + StoreStatus.StoreFull, + store.TryPublish(secondKey, value, descriptor), + "capacity full after reserve", + cycle: -1); + RequireStatus(StoreStatus.Success, reservation.Abort(), "capacity abort", cycle: -1); + + RequireStatus( + StoreStatus.Success, + store.TryPublish(secondKey, value, descriptor), + "capacity publish after abort", + cycle: -1); + RequireStatus( + StoreStatus.StoreFull, + store.TryReserve(firstKey, value.Length, descriptor, out _), + "capacity full after publish", + cycle: -1); + AcquireProjectRelease( + store, + secondKey, + expectedValueSum: 18, + expectedDescriptorSum: 17, + cycle: -1); + RequireStatus( + StoreStatus.Success, + store.TryRemove(secondKey), + "capacity remove", + cycle: -1); + + RequireStatus( + StoreStatus.Success, + store.TryReserve(firstKey, value.Length, descriptor, out var restored), + "capacity reserve after remove", + cycle: -1); + RequireStatus(StoreStatus.Success, restored.Abort(), "capacity final abort", cycle: -1); + } + + private static MemoryStore CreateLockFreeStore() + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-remove-reuse-allocation-{Guid.NewGuid():N}", + slotCount: 1, + maxValueBytes: 8, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 1, + participantRecordCount: 1, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out var store); + Assert.Equal(StoreOpenStatus.Success, status); + var result = Assert.IsType(store); + Assert.Equal(StoreProfile.LockFree, result.Profile); + return result; + } + + private static void RequireStatus( + StoreStatus expected, + StoreStatus actual, + string operation, + int cycle) + { + if (actual != expected) + { + throw new InvalidOperationException( + $"Expected {expected} from {operation} during cycle {cycle}, received {actual}."); + } + } + + private static void CollectGarbage() + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeRemoveStateTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeRemoveStateTests.cs new file mode 100644 index 0000000..6ffa82b --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeRemoveStateTests.cs @@ -0,0 +1,357 @@ +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeRemoveStateTests +{ + [Fact] + public async Task AcquireOrderedBeforeLogicalRemoveReturnsLeaseAndRemovePending() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7, 8])); + scheduler.PauseAt(LockFreeCheckpointId.AcquireAfterPublishedRevalidation); + + ValueLease lease = default; + StoreStatus acquireStatus = default; + var acquire = Task.Run(() => acquireStatus = store.TryAcquire([1], out lease)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + StoreStatus removeStatus = store.TryRemove([1], StoreWaitOptions.NoWait); + scheduler.Continue(); + await acquire.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, acquireStatus); + Assert.Equal(StoreStatus.RemovePending, removeStatus); + Assert.Equal(new byte[] { 7, 8 }, lease.ValueSpan.ToArray()); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public async Task LogicalRemoveOrderedBeforeLeaseClaimMakesAcquireReturnNotFound() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7])); + scheduler.PauseAt(LockFreeCheckpointId.AcquireBeforeLeaseClaimCas); + + ValueLease lease = default; + StoreStatus acquireStatus = default; + var acquire = Task.Run(() => acquireStatus = store.TryAcquire([1], out lease)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + StoreStatus removeStatus = store.TryRemove([1]); + scheduler.Continue(); + await acquire.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(removeStatus, new[] { StoreStatus.Success, StoreStatus.RemovePending }); + Assert.Equal(StoreStatus.NotFound, acquireStatus); + Assert.False(lease.IsValid); + } + + [Fact] + public async Task AcquireAfterNoLeaseClassificationCannotEscapeBeforeReclaimOwnership() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [7, 8])); + scheduler.PauseAt(LockFreeCheckpointId.ReclaimAfterLeaseScanBeforeOwnershipCas); + + StoreStatus removeStatus = default; + var remove = Task.Run(() => removeStatus = store.TryRemove([1])); + Assert.True( + scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)), + "Remove never reached its sole lease scan before reclaim ownership."); + + StoreStatus acquireStatus = store.TryAcquire([1], out ValueLease lease); + + Assert.Equal(StoreStatus.NotFound, acquireStatus); + Assert.False(lease.IsValid); + Assert.Equal(0, lease.ValueSpan.Length); + + scheduler.Continue(); + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(removeStatus, new[] { StoreStatus.Success, StoreStatus.RemovePending }); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [9])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out ValueLease replacement)); + Assert.Equal(9, replacement.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task ExpiredDeadlineAfterLeaseScanDoesNotClaimReclaimOwnership() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + byte[] key = [0x21]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x31])); + scheduler.PauseAt(LockFreeCheckpointId.ReclaimAfterLeaseScanBeforeOwnershipCas); + + StoreStatus removeStatus = default; + var remove = Task.Run(() => removeStatus = store.TryRemove( + key, + new StoreWaitOptions(TimeSpan.FromSeconds(2)))); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + await Task.Delay(TimeSpan.FromMilliseconds(2_250)); + scheduler.Continue(); + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "Remove never reached the post-scan pre-ownership checkpoint."); + Assert.Equal(StoreStatus.RemovePending, removeStatus); + DiagnosticsSnapshot pending = store.GetDiagnostics(); + Assert.Equal(0, pending.ReclaimingSlotCount); + Assert.Equal(1, pending.PendingRemovalCount); + Assert.Equal(1, pending.FreeSlotCount); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(key, out _)); + + Assert.Equal( + StoreStatus.Success, + store.TryPublish(key, [0x41], default, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease replacement)); + Assert.Equal(0x41, replacement.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, replacement.Release()); + Assert.Equal(StoreStatus.Success, store.TryRemove(key, StoreWaitOptions.Infinite)); + + DiagnosticsSnapshot reclaimed = store.GetDiagnostics(); + Assert.Equal(0, reclaimed.ReclaimingSlotCount); + Assert.Equal(0, reclaimed.PendingRemovalCount); + Assert.Equal(2, reclaimed.FreeSlotCount); + } + + [Fact] + public void CancellationBeforeLogicalRemovalPreservesPublishedGeneration() + { + using var store = CreateStore(); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [5])); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + var status = store.TryRemove( + [1], + new StoreWaitOptions(TimeSpan.FromSeconds(1), cancellation.Token)); + + Assert.Equal(StoreStatus.OperationCanceled, status); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(5, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public async Task CancellationAtPreRemovalCheckpointDoesNotCrossLogicalRemovalPoint() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [5])); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas); + + StoreStatus removeStatus = default; + var remove = Task.Run(() => removeStatus = store.TryRemove( + [1], + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token))); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + cancellation.Cancel(); + scheduler.Continue(); + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "Remove never reached its pre-ordering checkpoint."); + Assert.Equal(StoreStatus.OperationCanceled, removeStatus); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var preserved)); + Assert.Equal(5, preserved.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, preserved.Release()); + } + + [Fact] + public async Task DeadlineAtPreRemovalCheckpointDoesNotCrossLogicalRemovalPoint() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [5])); + scheduler.PauseAt(LockFreeCheckpointId.RemoveBeforeLogicalRemovalCas); + + StoreStatus removeStatus = default; + var remove = Task.Run(() => removeStatus = store.TryRemove( + [1], + new StoreWaitOptions(TimeSpan.FromMilliseconds(50)))); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + await Task.Delay(TimeSpan.FromMilliseconds(150)); + scheduler.Continue(); + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "Remove never reached its pre-ordering checkpoint."); + Assert.Equal(StoreStatus.StoreBusy, removeStatus); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var preserved)); + Assert.Equal(StoreStatus.Success, preserved.Release()); + } + + [Fact] + public async Task CancellationAfterLogicalRemovalAndClassificationDoesNotUndoOutcome() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [5])); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.RemoveAfterLeaseClassification); + + StoreStatus removeStatus = default; + var remove = Task.Run(() => removeStatus = store.TryRemove( + [1], + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token))); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromMilliseconds(500)); + cancellation.Cancel(); + scheduler.Continue(); + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "Remove never reached its post-ordering classification checkpoint."); + Assert.Contains(removeStatus, new[] { StoreStatus.Success, StoreStatus.RemovePending }); + Assert.NotEqual(StoreStatus.OperationCanceled, removeStatus); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + } + + [Fact] + public async Task DeadlineAfterLogicalRemovalAndClassificationDoesNotUndoOutcome() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [5])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + scheduler.PauseAt(LockFreeCheckpointId.RemoveAfterLeaseClassification); + + StoreStatus removeStatus = default; + var remove = Task.Run(() => removeStatus = store.TryRemove( + [1], + new StoreWaitOptions(TimeSpan.FromMilliseconds(50)))); + bool paused = scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5)); + await Task.Delay(TimeSpan.FromMilliseconds(150)); + scheduler.Continue(); + await remove.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(paused, "Remove never reached its post-ordering classification checkpoint."); + Assert.Equal(StoreStatus.RemovePending, removeStatus); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public void PostRemovalNoWaitScanExpiryReturnsConservativePending() + { + using var store = CreateStore(leaseRecordCount: 8_192); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [5])); + + StoreStatus removeStatus = store.TryRemove([1], StoreWaitOptions.NoWait); + + Assert.Equal(StoreStatus.RemovePending, removeStatus); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [6])); + } + + [Fact] + public void NoWaitRemovalWithActiveLeaseReturnsConservativePendingAndLogicalAbsence() + { + using var store = CreateStore(); + Assert.Equal(StoreStatus.Success, store.TryPublish([1], [3])); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + + var status = store.TryRemove([1], StoreWaitOptions.NoWait); + + Assert.Equal(StoreStatus.RemovePending, status); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.True(lease.IsValid); + Assert.Equal(3, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public async Task LeaseProjectionRetriesAcrossLogicalRemoveWithoutReturningTransientEmptyData() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + byte[] key = [0x31]; + byte[] value = [0x41, 0x42, 0x43]; + byte[] descriptor = [0x51, 0x52]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, value, descriptor)); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + scheduler.PauseAt(LockFreeCheckpointId.ProjectAfterMetadataReadBeforeControlRevalidation); + + byte[]? projectedValue = null; + byte[]? projectedDescriptor = null; + var projection = Task.Run(() => + { + projectedValue = lease.ValueSpan.ToArray(); + projectedDescriptor = lease.DescriptorSpan.ToArray(); + }); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal(StoreStatus.RemovePending, store.TryRemove(key, StoreWaitOptions.NoWait)); + scheduler.Continue(); + await projection.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(value, projectedValue); + Assert.Equal(descriptor, projectedDescriptor); + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x61])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease replacement)); + Assert.Equal(0x61, replacement.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + [Fact] + public async Task CopiedLeaseProjectionRacingReleaseReclaimAndReuseExpiresWithoutPoisoningStore() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler); + byte[] key = [0x32]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x71, 0x72], [0x73])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + ValueLease copiedLease = lease; + Assert.Equal(StoreStatus.RemovePending, store.TryRemove(key, StoreWaitOptions.NoWait)); + scheduler.PauseAt(LockFreeCheckpointId.ProjectAfterMetadataReadBeforeControlRevalidation); + + byte[]? projectedValue = null; + var projection = Task.Run(() => projectedValue = copiedLease.ValueSpan.ToArray()); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x81, 0x82], [0x83])); + scheduler.Continue(); + await projection.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Empty(Assert.IsType(projectedValue)); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease replacement)); + Assert.Equal(new byte[] { 0x81, 0x82 }, replacement.ValueSpan.ToArray()); + Assert.Equal(new byte[] { 0x83 }, replacement.DescriptorSpan.ToArray()); + Assert.Equal(StoreStatus.Success, replacement.Release()); + } + + private static MemoryStore CreateInstrumentedStore(ControlledLockFreeScheduler scheduler) + { + var options = Options(); + var status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + options, + scheduler.CreateInstrumentedCheckpoint(), + out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore CreateStore(int leaseRecordCount = 4) + { + var status = MemoryStore.TryCreateOrOpen(Options(leaseRecordCount), out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options(int leaseRecordCount = 4) => SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-remove-state-{Guid.NewGuid():N}", + slotCount: 2, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeReservationRecoveryTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeReservationRecoveryTests.cs new file mode 100644 index 0000000..caab2d2 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeReservationRecoveryTests.cs @@ -0,0 +1,792 @@ +using System.Reflection; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeReservationRecoveryTests +{ + [Fact] + public void RecoverySurfaceSeparatesClassificationExactCasHelpingAndParticipantRetirement() + { + Type? recovery = typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeRecovery", + throwOnError: false, + ignoreCase: false); + Assert.True(recovery is not null, "LockFreeRecovery is required for owner-classified record-local recovery."); + + MethodInfo[] methods = recovery!.GetMethods( + BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + Assert.Contains(methods, method => method.Name.Contains("Reservation", StringComparison.OrdinalIgnoreCase) + && method.Name.Contains("Recover", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Classif", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Help", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(methods, method => method.Name.Contains("Participant", StringComparison.OrdinalIgnoreCase) + && (method.Name.Contains("Retire", StringComparison.OrdinalIgnoreCase) + || method.Name.Contains("Reclaim", StringComparison.OrdinalIgnoreCase))); + } + + [Fact] + public async Task CommitWinningBeforeRecoveryCasPreservesPublishedValueWithoutFailedCount() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + reservation.GetSpan()[0] = 7; + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + + StoreStatus recoveryStatus = default; + ReservationRecoveryReport report = default; + var recovery = Task.Run(() => recoveryStatus = store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal(StoreStatus.Success, reservation.Commit()); + scheduler.Continue(); + await recovery.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, recoveryStatus); + Assert.Equal(1, report.ScannedReservationCount); + Assert.Equal(0, report.RecoveredReservationCount); + Assert.Equal(0, report.ActiveReservationCount); + Assert.Equal(0, report.UnsupportedReservationCount); + Assert.Equal(0, report.FailedRecoveryCount); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(7, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public async Task AbortWinningBeforeRecoveryCasIsACompletedRaceNotAFailedRecovery() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + + ReservationRecoveryReport report = default; + var recovery = Task.Run(() => store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal(StoreStatus.Success, reservation.Abort()); + scheduler.Continue(); + Assert.Equal(StoreStatus.Success, await recovery.WaitAsync(TimeSpan.FromSeconds(5))); + + Assert.Equal(1, report.ScannedReservationCount); + Assert.Equal(0, report.RecoveredReservationCount); + Assert.Equal(0, report.ActiveReservationCount); + Assert.Equal(0, report.UnsupportedReservationCount); + Assert.Equal(0, report.FailedRecoveryCount); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Abort()); + } + + [Fact] + public async Task RecoveryWinningExactCasFencesConcurrentCommit() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + reservation.GetSpan()[0] = 9; + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + scheduler.PauseAt(LockFreeCheckpointId.CommitBeforePublicationCas); + + StoreStatus commitStatus = default; + var commit = Task.Run(() => commitStatus = reservation.Commit()); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations(new ReservationRecoveryOptions(true), out var report)); + Assert.Equal(1, report.RecoveredReservationCount); + scheduler.Continue(); + await commit.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.InvalidReservation, commitStatus); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Abort()); + } + + [Fact] + public async Task RecoveryWinningExactCasFencesConcurrentAbortAndRemainsHelpable() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + scheduler.PauseAt(LockFreeCheckpointId.AbortBeforeAbortCas); + + StoreStatus abortStatus = default; + var abort = Task.Run(() => abortStatus = reservation.Abort()); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations(new ReservationRecoveryOptions(true), out var report)); + Assert.Equal(1, report.RecoveredReservationCount); + scheduler.Continue(); + await abort.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.InvalidReservation, abortStatus); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Abort()); + } + + [Fact] + public async Task CurrentProcessInitializingWriterPausedBeforeMetadataCannotBeRecoveredOrDamageReuse() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + scheduler.PauseAt(LockFreeCheckpointId.SlotClaimAfterParticipantRecheck); + + StoreStatus reserveStatus = default; + ValueReservation reservation = default; + var reserve = Task.Run(() => reserveStatus = store.TryReserve([1], 1, default, out reservation)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations(new ReservationRecoveryOptions(true), out var report)); + Assert.Equal(new ReservationRecoveryReport(1, 0, 1, 0, 0), report); + scheduler.Continue(); + await reserve.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, reserveStatus); + Assert.True(reservation.IsValid); + reservation.GetSpan()[0] = 0x5a; + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + Assert.Equal(StoreStatus.Success, reservation.Commit()); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(0x5a, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Theory] + [InlineData((int)SlotPublicationIntent.None)] + [InlineData(3)] + public async Task DiscoverableInitializingUnknownPublicationIntentFailsClosed(int invalidIntent) + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + LockFreeSlotTable slots = ReadSlots(store); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck); + + Task<(StoreStatus Status, ValueReservation Reservation)> reserve = Task.Run(() => + { + StoreStatus status = store.TryReserve([1], 1, default, out ValueReservation reservation); + return (status, reservation); + }); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + ulong operationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + DirectoryOperation operation = DirectoryOperation.Decode(operationRaw); + Assert.Equal(LockFreeSlotTable.InitializingState, SlotState(slot.Control)); + Assert.Equal(1, operation.Intent); + Assert.Equal(SlotGeneration(slot.Control), operation.Generation); + int validIntent = Volatile.Read(ref slot.PublicationIntent); + Assert.Equal((int)SlotPublicationIntent.ExplicitReservation, validIntent); + + Volatile.Write(ref slot.PublicationIntent, invalidIntent); + try + { + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + out ReservationRecoveryReport report)); + Assert.Equal(new ReservationRecoveryReport(1, 0, 0, 0, 1), report); + Assert.Equal(LockFreeSlotTable.InitializingState, SlotState(slot.Control)); + } + finally + { + Volatile.Write(ref slot.PublicationIntent, validIntent); + scheduler.Continue(); + } + + (StoreStatus status, ValueReservation reservation) = + await reserve.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, status); + Assert.Equal(StoreStatus.CorruptStore, reservation.Abort()); + } + + [Theory] + [InlineData((int)SlotPublicationIntent.None)] + [InlineData(3)] + public void ReservedUnknownPublicationIntentFailsClosed(int invalidIntent) + { + using var store = CreateStore(slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + LockFreeSlotTable slots = ReadSlots(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + Assert.Equal(LockFreeSlotTable.ReservedState, SlotState(slot.Control)); + + int validIntent = Volatile.Read(ref slot.PublicationIntent); + Assert.Equal((int)SlotPublicationIntent.ExplicitReservation, validIntent); + Volatile.Write(ref slot.PublicationIntent, invalidIntent); + try + { + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + out ReservationRecoveryReport report)); + Assert.Equal(new ReservationRecoveryReport(1, 0, 0, 0, 1), report); + Assert.False(reservation.IsValid); + } + finally + { + Volatile.Write(ref slot.PublicationIntent, validIntent); + } + + Assert.Equal(StoreStatus.CorruptStore, reservation.Abort()); + } + + [Fact] + public async Task PreMetadataInitializingIgnoresStaleUnknownPublicationIntent() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + LockFreeSlotTable slots = ReadSlots(store); + scheduler.PauseAt(LockFreeCheckpointId.SlotClaimAfterParticipantRecheck); + + Task<(StoreStatus Status, ValueReservation Reservation)> reserve = Task.Run(() => + { + StoreStatus status = store.TryReserve([1], 1, default, out ValueReservation reservation); + return (status, reservation); + }); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + Assert.Equal(LockFreeSlotTable.InitializingState, SlotState(slot.Control)); + Assert.Equal(0, AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)); + Volatile.Write(ref slot.PublicationIntent, 3); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + out ReservationRecoveryReport report)); + Assert.Equal(new ReservationRecoveryReport(1, 0, 1, 0, 0), report); + Assert.Equal(LockFreeSlotTable.InitializingState, SlotState(slot.Control)); + + scheduler.Continue(); + (StoreStatus status, ValueReservation reservation) = + await reserve.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, status); + Assert.Equal(StoreStatus.Success, reservation.Abort()); + } + + [Theory] + [InlineData(LockFreeSlotTable.AbortingState, (int)SlotPublicationIntent.None)] + [InlineData(LockFreeSlotTable.AbortingState, 3)] + [InlineData(LockFreeSlotTable.ReclaimingState, (int)SlotPublicationIntent.None)] + [InlineData(LockFreeSlotTable.ReclaimingState, 3)] + public void DiscoverableUnownedUnknownPublicationIntentFailsClosed( + int unownedState, + int invalidIntent) + { + using var store = CreateStore(slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + LockFreeSlotTable slots = ReadSlots(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(reservation.HandleForEngine)); + + if (unownedState == LockFreeSlotTable.ReclaimingState) + { + long aborting = AtomicControlWord.LoadAcquire(ref slot.Control); + long reclaiming = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.ReclaimingState, + SlotGeneration(aborting), + participantToken: 0)); + Assert.Equal( + aborting, + AtomicControlWord.CompareExchange(ref slot.Control, reclaiming, aborting)); + } + + Assert.Equal(unownedState, SlotState(slot.Control)); + int validIntent = Volatile.Read(ref slot.PublicationIntent); + Volatile.Write(ref slot.PublicationIntent, invalidIntent); + try + { + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + out ReservationRecoveryReport report)); + Assert.Equal(new ReservationRecoveryReport(0, 0, 0, 0, 1), report); + Assert.Equal(unownedState, SlotState(slot.Control)); + } + finally + { + Volatile.Write(ref slot.PublicationIntent, validIntent); + } + + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations(new ReservationRecoveryOptions(false), out _)); + Assert.False(reservation.IsValid); + } + + [Fact] + public void UnreferencedOperationZeroAbortIgnoresStaleIntentAndReclaimsDirectly() + { + using var store = CreateStore(slotCount: 1); + LockFreeSlotTable slots = ReadSlots(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + long free = AtomicControlWord.LoadAcquire(ref slot.Control); + long generation = SlotGeneration(free); + Assert.Equal(LayoutV2Constants.SlotFree, SlotState(free)); + Assert.Equal(0, AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)); + Assert.Equal(0, AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + + long aborting = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.AbortingState, + generation, + participantToken: 0)); + Assert.Equal(free, AtomicControlWord.CompareExchange(ref slot.Control, aborting, free)); + Volatile.Write(ref slot.PublicationIntent, 3); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + out ReservationRecoveryReport report)); + Assert.Equal(default, report); + Assert.Equal(LayoutV2Constants.SlotFree, SlotState(slot.Control)); + Assert.Equal(generation + 1, SlotGeneration(slot.Control)); + Assert.Equal(0, AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation)); + Assert.Equal(0, AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation)); + } + + [Fact] + public async Task CurrentDirectoryReferenceWithoutMetadataReadyMarkerFailsClosed() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + LockFreeSlotTable slots = ReadSlots(store); + LockFreeKeyDirectory directory = ReadDirectory(store); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryBeforeInsertOuterLoopBudgetCheck); + + Task<(StoreStatus Status, ValueReservation Reservation)> reserve = Task.Run(() => + { + StoreStatus status = store.TryReserve([1], 1, default, out ValueReservation reservation); + return (status, reservation); + }); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + ulong binding = slot.DirectoryBinding; + ulong operationRaw = unchecked((ulong)AtomicControlWord.LoadAcquire( + ref slot.DirectoryOperation)); + Assert.NotEqual(0UL, operationRaw); + Assert.True(HasCanonicalMutation(directory, binding)); + + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, 0); + try + { + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + out ReservationRecoveryReport report)); + Assert.Equal(new ReservationRecoveryReport(1, 0, 0, 0, 1), report); + Assert.Equal(LockFreeSlotTable.InitializingState, SlotState(slot.Control)); + } + finally + { + AtomicControlWord.StoreRelease( + ref slot.DirectoryOperation, + unchecked((long)operationRaw)); + scheduler.Continue(); + } + + (StoreStatus status, ValueReservation reservation) = + await reserve.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, status); + Assert.Equal(StoreStatus.CorruptStore, reservation.Abort()); + } + + [Fact] + public void CurrentDirectoryCellWithoutMetadataReadyMarkerFailsClosed() + { + using var store = CreateStore(slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + LockFreeSlotTable slots = ReadSlots(store); + LockFreeKeyDirectory directory = ReadDirectory(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + ulong binding = slot.DirectoryBinding; + Assert.False(HasCanonicalMutation(directory, binding)); + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(reservation.HandleForEngine)); + + long operation = AtomicControlWord.LoadAcquire(ref slot.DirectoryOperation); + long location = AtomicControlWord.LoadAcquire(ref slot.DirectoryLocation); + Assert.NotEqual(0, operation); + Assert.NotEqual(0, location); + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, 0); + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, 0); + try + { + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations( + new ReservationRecoveryOptions(false), + out ReservationRecoveryReport report)); + Assert.Equal(new ReservationRecoveryReport(0, 0, 0, 0, 1), report); + Assert.Equal(LockFreeSlotTable.AbortingState, SlotState(slot.Control)); + } + finally + { + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, location); + AtomicControlWord.StoreRelease(ref slot.DirectoryOperation, operation); + } + + Assert.Equal( + StoreStatus.CorruptStore, + store.TryRecoverReservations(new ReservationRecoveryOptions(false), out _)); + Assert.False(reservation.IsValid); + } + + [Fact] + public void InitializingRecoveryRequiresStaleOwnerOrPublishedQuiescentHandoff() + { + ParticipantClassification currentActive = Classification( + ParticipantClassificationKind.CurrentProcess, + LayoutV2Constants.ParticipantActive); + ParticipantClassification currentClosing = Classification( + ParticipantClassificationKind.CurrentProcess, + LayoutV2Constants.ParticipantClosing); + ParticipantClassification liveRecovering = Classification( + ParticipantClassificationKind.Live, + LayoutV2Constants.ParticipantRecovering); + ParticipantClassification stale = Classification( + ParticipantClassificationKind.Stale, + LayoutV2Constants.ParticipantActive); + ParticipantClassification inconsistentClosing = Classification( + ParticipantClassificationKind.Inconsistent, + LayoutV2Constants.ParticipantClosing); + + Assert.False(LockFreeRecovery.CanRecoverReservation( + LockFreeSlotTable.InitializingState, + currentActive, + recoverCurrentProcessReservations: true)); + Assert.True(LockFreeRecovery.CanRecoverReservation( + LockFreeSlotTable.ReservedState, + currentActive, + recoverCurrentProcessReservations: true)); + Assert.True(LockFreeRecovery.CanRecoverReservation( + LockFreeSlotTable.InitializingState, + currentClosing, + recoverCurrentProcessReservations: false)); + Assert.True(LockFreeRecovery.CanRecoverReservation( + LockFreeSlotTable.InitializingState, + liveRecovering, + recoverCurrentProcessReservations: false)); + Assert.True(LockFreeRecovery.CanRecoverReservation( + LockFreeSlotTable.InitializingState, + stale, + recoverCurrentProcessReservations: false)); + Assert.False(LockFreeRecovery.CanRecoverReservation( + LockFreeSlotTable.InitializingState, + inconsistentClosing, + recoverCurrentProcessReservations: true)); + } + + [Fact] + public async Task CancellationBeforeRecoveryCasPreservesReservationAndReturnsPartialCounts() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + + StoreStatus recoveryStatus = default; + ReservationRecoveryReport report = default; + var recovery = Task.Run(() => recoveryStatus = store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + cancellation.Cancel(); + scheduler.Continue(); + await recovery.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.OperationCanceled, recoveryStatus); + Assert.Equal(1, report.ScannedReservationCount); + Assert.Equal(0, report.RecoveredReservationCount); + Assert.True(reservation.IsValid); + Assert.Equal(StoreStatus.Success, reservation.Abort()); + } + + [Fact] + public async Task DeadlineBeforeRecoveryCasPreservesReservationAndReturnsPartialCounts() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryBeforeOwnerClassification); + + StoreStatus recoveryStatus = default; + ReservationRecoveryReport report = default; + var recovery = Task.Run(() => recoveryStatus = store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + new StoreWaitOptions(TimeSpan.FromMilliseconds(50)), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + await Task.Delay(TimeSpan.FromMilliseconds(150)); + scheduler.Continue(); + await recovery.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.StoreBusy, recoveryStatus); + Assert.Equal(1, report.ScannedReservationCount); + Assert.Equal(0, report.RecoveredReservationCount); + Assert.True(reservation.IsValid); + Assert.Equal(StoreStatus.Success, reservation.Abort()); + } + + [Fact] + public async Task CancellationAfterRecoveryCasDoesNotUndoRecoveredOutcome() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryAfterExactRecoveryCas); + + StoreStatus recoveryStatus = default; + ReservationRecoveryReport report = default; + var recovery = Task.Run(() => recoveryStatus = store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + cancellation.Cancel(); + scheduler.Continue(); + await recovery.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, recoveryStatus); + Assert.Equal(1, report.RecoveredReservationCount); + Assert.False(reservation.IsValid); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Abort()); + } + + [Fact] + public async Task CancellationAfterAbortOwnershipReleaseReturnsSuccessAndLeavesHelpableCleanup() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + using var cancellation = new CancellationTokenSource(); + scheduler.PauseAt(LockFreeCheckpointId.AbortAfterOwnershipReleaseCas); + + StoreStatus abortStatus = default; + var abort = Task.Run(() => abortStatus = reservation.Abort( + new StoreWaitOptions(TimeSpan.FromSeconds(10), cancellation.Token))); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + cancellation.Cancel(); + scheduler.Continue(); + await abort.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, abortStatus); + Assert.False(reservation.IsValid); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Abort()); + } + + [Fact] + public async Task DeadlineAfterRecoveryCasDoesNotUndoRecoveredOutcome() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + scheduler.PauseAt(LockFreeCheckpointId.RecoveryAfterExactRecoveryCas); + + StoreStatus recoveryStatus = default; + ReservationRecoveryReport report = default; + var recovery = Task.Run(() => recoveryStatus = store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + new StoreWaitOptions(TimeSpan.FromMilliseconds(50)), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + await Task.Delay(TimeSpan.FromMilliseconds(150)); + scheduler.Continue(); + await recovery.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(StoreStatus.Success, recoveryStatus); + Assert.Equal(1, report.RecoveredReservationCount); + Assert.False(reservation.IsValid); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Abort()); + } + + [Fact] + public async Task DeadlineDuringLaterPublishedHelperPreservesEarlierRecoveredOutcome() + { + using var scheduler = new ControlledLockFreeScheduler(); + using var store = CreateInstrumentedStore(scheduler, slotCount: 2); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var first)); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var second)); + + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + var slots = Assert.IsType(engine.GetType() + .GetField("_slots", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + Assert.Equal(StoreStatus.Success, slots.TryBeginAbort(second.HandleForEngine)); + + // Slot zero is recovered and ordered first. Slot one is already + // Aborting; pause its second unlink validation until the public budget + // expires to verify the earlier durable recovery is not rewritten. + scheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterLocationValidation, occurrence: 2); + ReservationRecoveryReport report = default; + Task recovery = Task.Run(() => store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + new StoreWaitOptions(TimeSpan.FromMilliseconds(50)), + out report)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + await Task.Delay(TimeSpan.FromMilliseconds(150)); + scheduler.Continue(); + + Assert.Equal(StoreStatus.Success, await recovery.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(1, report.RecoveredReservationCount); + Assert.False(first.IsValid); + Assert.False(second.IsValid); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations( + new ReservationRecoveryOptions(true), + StoreWaitOptions.Infinite, + out _)); + Assert.Equal(StoreStatus.Success, store.TryReserve([3], 1, default, out var replacement1)); + Assert.Equal(StoreStatus.Success, store.TryReserve([4], 1, default, out var replacement2)); + Assert.Equal(StoreStatus.Success, replacement1.Abort()); + Assert.Equal(StoreStatus.Success, replacement2.Abort()); + } + + [Fact] + public void ReportCountsDistinguishLiveCurrentRecoveryAndRestoreAllCapacity() + { + using var store = CreateStore(slotCount: 2); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var first)); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var second)); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations(new ReservationRecoveryOptions(false), out var active)); + Assert.Equal(new ReservationRecoveryReport(2, 0, 2, 0, 0), active); + Assert.True(first.IsValid); + Assert.True(second.IsValid); + + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations(new ReservationRecoveryOptions(true), out var recovered)); + Assert.Equal(new ReservationRecoveryReport(2, 2, 0, 0, 0), recovered); + Assert.False(first.IsValid); + Assert.False(second.IsValid); + + Assert.Equal(StoreStatus.Success, store.TryReserve([3], 1, default, out var replacement1)); + Assert.Equal(StoreStatus.Success, store.TryReserve([4], 1, default, out var replacement2)); + Assert.Equal(StoreStatus.Success, replacement1.Abort()); + Assert.Equal(StoreStatus.Success, replacement2.Abort()); + } + + private static MemoryStore CreateInstrumentedStore(ControlledLockFreeScheduler scheduler, int slotCount) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options($"sms-v2-reservation-recovery-{Guid.NewGuid():N}", slotCount), + scheduler.CreateInstrumentedCheckpoint(), + out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore CreateStore(int slotCount) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options($"sms-v2-reservation-recovery-{Guid.NewGuid():N}", slotCount), + out var store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static LockFreeSlotTable ReadSlots(MemoryStore store) + { + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + return Assert.IsType(engine.GetType() + .GetField("_slots", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + } + + private static LockFreeKeyDirectory ReadDirectory(MemoryStore store) + { + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + return Assert.IsType(engine.GetType() + .GetField("_directory", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + } + + private static bool HasCanonicalMutation(LockFreeKeyDirectory directory, ulong binding) + { + StoreLayoutV2 layout = (StoreLayoutV2)typeof(LockFreeKeyDirectory) + .GetField("_layout", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(directory)!; + for (var bucketIndex = 0; bucketIndex < layout.PrimaryBucketCount; bucketIndex++) + { + if (directory.ReadCanonicalMutation(bucketIndex) == binding) + { + return true; + } + } + + return false; + } + + private static int SlotState(long control) => (int)(unchecked((ulong)control) & 0x7UL); + + private static long SlotGeneration(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static SharedMemoryStoreOptions Options(string name, int slotCount) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 4, + participantRecordCount: 2, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + + private static ParticipantClassification Classification( + ParticipantClassificationKind kind, + int participantState) => + new( + kind, + new ParticipantIncarnation( + RecordIndex: 0, + Generation: 1, + Token: 1, + State: participantState, + ProcessId: Environment.ProcessId, + IdentityKind: LayoutV2Constants.IdentityUnknown, + ProcessStartValue: 0, + OpenSequence: 1, + PidNamespaceId: 0, + ReservedValue: 0, + Control: 0)); +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeReservationStateTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeReservationStateTests.cs new file mode 100644 index 0000000..620bbcc --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeReservationStateTests.cs @@ -0,0 +1,248 @@ +using System.Reflection; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeReservationStateTests +{ + [Fact] + public void PublicationIntentFeatureAndWireAssignmentsAreStable() + { + Assert.Equal(1UL, LayoutV2Constants.SpillSummaryVersionedEmptyRequiredFeature); + Assert.Equal(2UL, LayoutV2Constants.PublicationIntentRequiredFeature); + Assert.Equal(4UL, LayoutV2Constants.PidNamespaceIdentityRequiredFeature); + Assert.Equal(7UL, LayoutV2Constants.RequiredFeatures); + Assert.Equal(0, (int)SlotPublicationIntent.None); + Assert.Equal(1, (int)SlotPublicationIntent.ExplicitReservation); + Assert.Equal(2, (int)SlotPublicationIntent.AtomicPublication); + + using MemoryStore store = CreateLockFreeStore(); + Assert.Equal(7UL, store.ProtocolInfo.RequiredFeatures); + } + + [Fact] + public void FirstClaimCarriesParticipantAndStoreIdentityAndReuseAdvancesGeneration() + { + using var store = CreateLockFreeStore(slotCount: 1); + + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var first)); + var firstHandle = first.HandleForEngine; + Assert.NotEqual(0UL, firstHandle.StoreId); + Assert.NotEqual(0UL, firstHandle.ParticipantToken); + Assert.NotEqual(0UL, firstHandle.SlotBinding); + Assert.Equal(0, ParticipantToken.Decode(firstHandle.ParticipantToken, participantCount: 1).RecordIndex); + Assert.Equal(0, IndexBinding.Decode(firstHandle.SlotBinding).SlotIndex); + + Assert.Equal(StoreStatus.Success, first.Abort()); + Assert.Equal(StoreStatus.Success, store.TryReserve([2], 1, default, out var second)); + var secondHandle = second.HandleForEngine; + + Assert.Equal(firstHandle.StoreId, secondHandle.StoreId); + Assert.Equal(firstHandle.ParticipantToken, secondHandle.ParticipantToken); + Assert.NotEqual(firstHandle.SlotBinding, secondHandle.SlotBinding); + Assert.Equal( + IndexBinding.Decode(firstHandle.SlotBinding).Generation + 1, + IndexBinding.Decode(secondHandle.SlotBinding).Generation); + + Assert.Equal(StoreStatus.InvalidReservation, first.Advance(1)); + Assert.Equal(StoreStatus.InvalidReservation, first.Commit()); + Assert.Equal(StoreStatus.InvalidReservation, first.Abort()); + Assert.Equal(StoreStatus.Success, second.Abort()); + } + + [Fact] + public void CopiedReservationHasOneExclusiveCursorAndRequiresExactAdvance() + { + using var store = CreateLockFreeStore(); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 4, [9], out var reservation)); + var copy = reservation; + + new byte[] { 1, 2 }.CopyTo(reservation.GetSpan(2)); + Assert.Equal(StoreStatus.Success, reservation.Advance(2)); + Assert.Equal(2, copy.BytesWritten); + Assert.Equal(2, copy.RemainingBytes); + Assert.Equal(StoreStatus.ReservationIncomplete, copy.Commit()); + + new byte[] { 3, 4 }.CopyTo(copy.GetSpan(2)); + Assert.Equal(StoreStatus.Success, copy.Advance(2)); + Assert.Equal(4, reservation.BytesWritten); + Assert.Equal(StoreStatus.ReservationWriteOutOfRange, reservation.Advance(1)); + Assert.Equal(StoreStatus.Success, reservation.Commit()); + Assert.Equal(StoreStatus.ReservationAlreadyCompleted, copy.Commit()); + } + + [Fact] + public void CommitAbortAndRecoveryEachFenceTheExactReservationGeneration() + { + using var store = CreateLockFreeStore(slotCount: 1, enableRecovery: true); + + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var committed)); + committed.GetSpan(1)[0] = 7; + Assert.Equal(StoreStatus.Success, committed.Advance(1)); + Assert.Equal(StoreStatus.Success, committed.Commit()); + Assert.Equal(StoreStatus.ReservationAlreadyCompleted, committed.Commit()); + Assert.Equal(StoreStatus.ReservationAlreadyCompleted, committed.Abort()); + + // A committed value still owns the only slot, so use independent stores for + // abort and recovery terminal paths. + using var abortStore = CreateLockFreeStore(slotCount: 1, enableRecovery: true); + Assert.Equal(StoreStatus.Success, abortStore.TryReserve([2], 1, default, out var aborted)); + Assert.Equal(StoreStatus.Success, aborted.Abort()); + Assert.Equal(StoreStatus.InvalidReservation, aborted.Commit()); + + Assert.Equal(StoreStatus.Success, abortStore.TryReserve([3], 1, default, out var recovered)); + Assert.Equal( + StoreStatus.Success, + abortStore.TryRecoverReservations(new ReservationRecoveryOptions(true), out var report)); + Assert.Equal(1, report.RecoveredReservationCount); + Assert.False(recovered.IsValid); + Assert.Equal(StoreStatus.InvalidReservation, recovered.Advance(1)); + Assert.Equal(StoreStatus.InvalidReservation, recovered.Commit()); + Assert.Equal(StoreStatus.InvalidReservation, recovered.Abort()); + + Assert.Equal(StoreStatus.Success, abortStore.TryReserve([4], 1, default, out var reused)); + Assert.NotEqual(recovered.HandleForEngine.SlotBinding, reused.HandleForEngine.SlotBinding); + Assert.Equal(StoreStatus.Success, reused.Abort()); + } + + [Fact] + public void TerminalGenerationPublishesRetiredInsteadOfWrappingToFree() + { + const long terminalGeneration = 0x1_ffff_ffffL; + var slotTable = typeof(MemoryStore).Assembly.GetType( + "SharedMemoryStore.LockFree.LockFreeSlotTable", + throwOnError: false, + ignoreCase: false); + Assert.True(slotTable is not null, "The layout-v2 slot state machine is missing."); + + var advanceOrRetire = slotTable!.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + .SingleOrDefault(method => + method.Name.Contains("Advance", StringComparison.OrdinalIgnoreCase) + && method.Name.Contains("Retire", StringComparison.OrdinalIgnoreCase) + && method.GetParameters() is [{ ParameterType: var parameterType }] + && parameterType == typeof(long) + && (method.ReturnType == typeof(long) || method.ReturnType == typeof(ulong))); + Assert.True( + advanceOrRetire is not null, + "LockFreeSlotTable needs a pure advance-or-retire transition seam for terminal rollover tests."); + + var nextFree = Convert.ToUInt64(advanceOrRetire!.Invoke(null, [terminalGeneration - 1])); + var retired = Convert.ToUInt64(advanceOrRetire.Invoke(null, [terminalGeneration])); + + Assert.Equal(AtomicControlWord.EncodeSlot(state: 0, terminalGeneration, participantToken: 0), nextFree); + Assert.Equal(AtomicControlWord.EncodeSlot(state: 7, terminalGeneration, participantToken: 0), retired); + } + + [Fact] + public void WritableProjectionEndsAfterCommitAbortRecoveryAndStoreDispose() + { + using var commitStore = CreateLockFreeStore(); + Assert.Equal(StoreStatus.Success, commitStore.TryReserve([1], 1, default, out var committed)); + committed.GetSpan()[0] = 1; + Assert.Equal(StoreStatus.Success, committed.Advance(1)); + Assert.Equal(StoreStatus.Success, committed.Commit()); + Assert.True(committed.GetSpan().IsEmpty); + Assert.True(committed.DangerousGetMemory().IsEmpty); + + using var abortStore = CreateLockFreeStore(); + Assert.Equal(StoreStatus.Success, abortStore.TryReserve([2], 1, default, out var aborted)); + Assert.Equal(StoreStatus.Success, aborted.Abort()); + Assert.True(aborted.GetSpan().IsEmpty); + Assert.True(aborted.DangerousGetMemory().IsEmpty); + + using var recoveryStore = CreateLockFreeStore(enableRecovery: true); + Assert.Equal(StoreStatus.Success, recoveryStore.TryReserve([3], 1, default, out var recovered)); + Assert.Equal( + StoreStatus.Success, + recoveryStore.TryRecoverReservations(new ReservationRecoveryOptions(true), out _)); + Assert.True(recovered.GetSpan().IsEmpty); + Assert.True(recovered.DangerousGetMemory().IsEmpty); + + var disposedStore = CreateLockFreeStore(); + Assert.Equal(StoreStatus.Success, disposedStore.TryReserve([4], 1, default, out var disposed)); + disposedStore.Dispose(); + Assert.True(disposed.GetSpan().IsEmpty); + Assert.True(disposed.DangerousGetMemory().IsEmpty); + Assert.Equal(StoreStatus.StoreDisposed, disposed.Advance(1)); + } + + [Fact] + public void CancellationBeforeBindingLeavesNoKeyOrSlotOwnership() + { + using var store = CreateLockFreeStore(slotCount: 1); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var canceledWait = new StoreWaitOptions(TimeSpan.FromSeconds(1), cancellation.Token); + + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryReserve([1], 1, default, canceledWait, out var canceled)); + Assert.False(canceled.IsValid); + + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var replacement)); + Assert.Equal(StoreStatus.Success, replacement.Abort()); + } + + [Fact] + public void CancellationAfterBindingAndBeforeCommitPreservesThePendingLifecycle() + { + using var store = CreateLockFreeStore(slotCount: 1); + Assert.Equal(StoreStatus.Success, store.TryReserve([1], 1, default, out var reservation)); + reservation.GetSpan()[0] = 9; + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var canceledWait = new StoreWaitOptions(TimeSpan.FromSeconds(1), cancellation.Token); + + Assert.Equal(StoreStatus.OperationCanceled, reservation.Advance(1, canceledWait)); + Assert.True(reservation.IsValid); + Assert.Equal(0, reservation.BytesWritten); + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + Assert.Equal(StoreStatus.OperationCanceled, reservation.Commit(canceledWait)); + Assert.True(reservation.IsValid); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire([1], out _)); + Assert.Equal(StoreStatus.Success, reservation.Commit()); + + cancellation.Cancel(); + Assert.Equal(StoreStatus.Success, store.TryAcquire([1], out var lease)); + Assert.Equal(9, lease.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, lease.Release()); + } + + [Fact] + public void NoWaitMayOrderBindingAndCommitWhenThereIsNoContention() + { + using var store = CreateLockFreeStore(); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve([1], 1, default, StoreWaitOptions.NoWait, out var reservation)); + reservation.GetSpan()[0] = 3; + Assert.Equal(StoreStatus.Success, reservation.Advance(1, StoreWaitOptions.NoWait)); + Assert.Equal(StoreStatus.Success, reservation.Commit(StoreWaitOptions.NoWait)); + } + + private static MemoryStore CreateLockFreeStore( + int slotCount = 4, + bool enableRecovery = true) + { + var options = SharedMemoryStoreOptions.CreateLockFree( + $"sms-v2-reservation-state-{Guid.NewGuid():N}", + slotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: 8, + participantRecordCount: 1, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: enableRecovery); + var status = MemoryStore.TryCreateOrOpen(options, out var store); + + Assert.Equal(StoreOpenStatus.Success, status); + var result = Assert.IsType(store); + Assert.Equal(StoreProfile.LockFree, result.Profile); + Assert.Equal(2, result.ProtocolInfo.LayoutMajorVersion); + return result; + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeSpillSummaryTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeSpillSummaryTests.cs new file mode 100644 index 0000000..c76ba3e --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeSpillSummaryTests.cs @@ -0,0 +1,613 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Layout; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; +using SharedMemoryStore.UnitTests.TestSupport; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeSpillSummaryTests +{ + private const int SlotCount = 20; + private const int CanonicalBucket = 0; + private const int SecondaryBucket = 1; + + [Fact] + public async Task PresentCandidateIsPublishedBeforeOverflowCellCas() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(17); + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore(scheduler); + PublishPrimaryPair(store, keys); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication); + + StoreStatus publishStatus = default; + Task publish = Task.Run(() => publishStatus = store.TryPublish( + keys[16], + [0xA1], + default, + StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + try + { + LockFreeKeyDirectory directory = ReadDirectory(store); + SpillSummary summary = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + + Assert.True(summary.IsPresent); + Assert.NotEqual(0UL, summary.Binding); + Assert.Equal(summary.Binding, directory.ReadCanonicalMutation(CanonicalBucket)); + Assert.Equal(0, directory.OverflowOccupancy); + } + finally + { + scheduler.Continue(); + } + + await publish.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, publishStatus); + Assert.Equal(1, ReadDirectory(store).OverflowOccupancy); + } + + [Fact] + public async Task EmptyFullScanAndVersionedClearBothPrecedeMutationRelease() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(18); + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore(scheduler); + PublishPrimaryPair(store, keys); + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[16], [0xB1], default, StoreWaitOptions.Infinite)); + + scheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan); + StoreStatus firstRemoveStatus = default; + Task firstRemove = Task.Run(() => firstRemoveStatus = store.TryRemove( + keys[16], + StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + try + { + LockFreeKeyDirectory directory = ReadDirectory(store); + SpillSummary summary = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + + Assert.True(summary.IsPresent); + Assert.Equal(0, directory.OverflowOccupancy); + Assert.Equal(summary.Binding, directory.ReadCanonicalMutation(CanonicalBucket)); + } + finally + { + scheduler.Continue(); + } + + await firstRemove.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, firstRemoveStatus); + SpillSummary firstEmpty = SpillSummary.Decode( + ReadDirectory(store).ReadSpillSummary(CanonicalBucket)); + Assert.False(firstEmpty.IsPresent); + Assert.False(firstEmpty.IsInitial); + + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[17], [0xB2], default, StoreWaitOptions.Infinite)); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterSpillSummaryClear); + StoreStatus secondRemoveStatus = default; + Task secondRemove = Task.Run(() => secondRemoveStatus = store.TryRemove( + keys[17], + StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + try + { + LockFreeKeyDirectory directory = ReadDirectory(store); + SpillSummary summary = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + + Assert.False(summary.IsPresent); + Assert.False(summary.IsInitial); + Assert.Equal(summary.Binding, directory.ReadCanonicalMutation(CanonicalBucket)); + Assert.Equal(0, directory.OverflowOccupancy); + } + finally + { + scheduler.Continue(); + } + + await secondRemove.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(StoreStatus.Success, secondRemoveStatus); + Assert.Equal(0UL, ReadDirectory(store).ReadCanonicalMutation(CanonicalBucket)); + } + + [Fact] + public async Task PausedOldClearerCannotClearACompletedLaterSpill() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(18); + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore(scheduler); + PublishPrimaryPair(store, keys); + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[16], [0xC1], default, StoreWaitOptions.Infinite)); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterEmptySpillSummaryScan); + + StoreStatus delayedRemoveStatus = default; + Task delayedRemove = Task.Run(() => delayedRemoveStatus = store.TryRemove( + keys[16], + StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + LockFreeKeyDirectory directory = ReadDirectory(store); + SpillSummary oldPresent = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + try + { + LockFreeOperationBudget maintenance = LockFreeOperationBudget.UnboundedScan; + Assert.Equal( + StoreStatus.Success, + directory.HelpMutation(CanonicalBucket, maintenance, maxSteps: 128)); + SpillSummary oldEmpty = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Assert.False(oldEmpty.IsPresent); + Assert.Equal(oldPresent.Binding, oldEmpty.Binding); + + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[17], [0xC2], default, StoreWaitOptions.Infinite)); + SpillSummary laterPresent = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Assert.True(laterPresent.IsPresent); + Assert.NotEqual(oldPresent.Binding, laterPresent.Binding); + } + finally + { + scheduler.Continue(); + } + + await delayedRemove.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Contains(delayedRemoveStatus, new[] { StoreStatus.Success, StoreStatus.NotFound }); + SpillSummary afterResume = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Assert.True(afterResume.IsPresent); + Assert.Equal(StoreStatus.Success, store.TryAcquire(keys[17], out ValueLease current)); + Assert.Equal(0xC2, current.ValueSpan[0]); + Assert.Equal(StoreStatus.Success, current.Release()); + } + + [Fact] + public async Task PausedOldSetterCannotAbaThroughVersionedEmpty() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(17); + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore(scheduler); + PublishPrimaryPair(store, keys); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryBeforeSpillSummaryPublicationCas); + + Task delayedPublish = Task.Run(() => store.TryPublish( + keys[16], + [0xD1], + default, + StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + LockFreeKeyDirectory directory = ReadDirectory(store); + try + { + Assert.Equal(0UL, directory.ReadSpillSummary(CanonicalBucket)); + LockFreeOperationBudget maintenance = LockFreeOperationBudget.UnboundedScan; + Assert.Equal( + StoreStatus.Success, + directory.HelpMutation(CanonicalBucket, maintenance, maxSteps: 128)); + Assert.True(SpillSummary.Decode( + directory.ReadSpillSummary(CanonicalBucket)).IsPresent); + + // Adversarial fencing injection: this deliberately violates the + // administrative override's process-wide quiescence precondition + // so a delayed writer can resume after reuse. Only mapped-state + // generation safety is asserted; the paused call's public result + // is outside the supported recovery contract. + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: true), + StoreWaitOptions.Infinite, + out ReservationRecoveryReport recovery)); + Assert.True(recovery.RecoveredReservationCount > 0); + + SpillSummary empty = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Assert.False(empty.IsPresent); + Assert.False(empty.IsInitial); + Assert.Equal(0, directory.OverflowOccupancy); + } + finally + { + scheduler.Continue(); + } + + _ = await delayedPublish.WaitAsync(TimeSpan.FromSeconds(5)); + SpillSummary afterResume = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Assert.False(afterResume.IsPresent); + Assert.False(afterResume.IsInitial); + Assert.Equal(0, directory.OverflowOccupancy); + } + + [Fact] + public async Task SetterPostCasValidationFailureStillConvergesAndReleasesMutation() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(18); + using var scheduler = new ControlledLockFreeScheduler(); + using MemoryStore store = CreateInstrumentedStore(scheduler); + PublishPrimaryPair(store, keys); + scheduler.PauseAt(LockFreeCheckpointId.DirectoryAfterSpillSummaryPublication); + + Task delayedPublish = Task.Run(() => store.TryPublish( + keys[16], + [0xD2], + default, + StoreWaitOptions.Infinite)); + Assert.True(scheduler.WaitUntilPaused(TimeSpan.FromSeconds(5))); + LockFreeKeyDirectory directory = ReadDirectory(store); + try + { + Assert.True(SpillSummary.Decode( + directory.ReadSpillSummary(CanonicalBucket)).IsPresent); + LockFreeOperationBudget maintenance = LockFreeOperationBudget.UnboundedScan; + Assert.Equal( + StoreStatus.Success, + directory.HelpMutation(CanonicalBucket, maintenance, maxSteps: 128)); + // Adversarial fencing injection; see the quiescence note in + // PausedOldSetterCannotAbaThroughVersionedEmpty. The supported + // contract does not define the paused live call's return value. + Assert.Equal( + StoreStatus.Success, + store.TryRecoverReservations( + new ReservationRecoveryOptions(RecoverCurrentProcessReservations: true), + StoreWaitOptions.Infinite, + out ReservationRecoveryReport recovery)); + Assert.True(recovery.RecoveredReservationCount > 0); + Assert.Equal(0UL, directory.ReadCanonicalMutation(CanonicalBucket)); + Assert.False(SpillSummary.Decode( + directory.ReadSpillSummary(CanonicalBucket)).IsPresent); + } + finally + { + scheduler.Continue(); + } + + _ = await delayedPublish.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0UL, directory.ReadCanonicalMutation(CanonicalBucket)); + Assert.Equal(0, directory.OverflowOccupancy); + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[17], [0xD3], default, StoreWaitOptions.Infinite)); + Assert.True(SpillSummary.Decode( + directory.ReadSpillSummary(CanonicalBucket)).IsPresent); + } + + [Fact] + public void InfiniteChurnClearsSummaryAndLaterMissingLookupsSkipOverflow() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(18); + using MemoryStore store = CreateStore(); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot before)); + + PublishPrimaryPair(store, keys); + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[16], [0xE1], default, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot during)); + Assert.True(during.SpilledBucketCount > 0); + + for (var index = 0; index < 17; index++) + { + Assert.Equal(StoreStatus.Success, store.TryRemove(keys[index], StoreWaitOptions.Infinite)); + } + + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterCleanup)); + Assert.Equal(0, afterCleanup.SpilledBucketCount); + Assert.Equal(0, afterCleanup.OverflowDirectoryOccupancy); + Assert.True(afterCleanup.OverflowScanCount > before.OverflowScanCount); + Assert.True(afterCleanup.MaxObservedOverflowScanLength >= SlotCount); + + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(keys[17], out _)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterFirstMiss)); + Assert.Equal(StoreStatus.NotFound, store.TryAcquire(keys[17], out _)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterSecondMiss)); + Assert.Equal(afterCleanup.OverflowScanCount, afterFirstMiss.OverflowScanCount); + Assert.Equal(afterFirstMiss.OverflowScanCount, afterSecondMiss.OverflowScanCount); + } + + [Fact] + public void CurrentExactSummaryWitnessAvoidsRepeatedMutationReleaseScans() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(17); + using MemoryStore store = CreateStore(); + PublishPrimaryPair(store, keys); + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[16], [0xE2], default, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterSpill)); + + for (var index = 0; index < 16; index++) + { + Assert.Equal( + StoreStatus.Success, + store.TryRemove(keys[index], StoreWaitOptions.Infinite)); + } + + Assert.Equal( + StoreStatus.Success, + store.TryGetDiagnostics(out DiagnosticsSnapshot afterPrimaryRemovals)); + Assert.Equal(afterSpill.OverflowScanCount, afterPrimaryRemovals.OverflowScanCount); + Assert.Equal(1, afterPrimaryRemovals.SpilledBucketCount); + Assert.Equal(1, afterPrimaryRemovals.OverflowDirectoryOccupancy); + + Assert.Equal(StoreStatus.Success, store.TryRemove(keys[16], StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterLastRemoval)); + Assert.True(afterLastRemoval.OverflowScanCount > afterPrimaryRemovals.OverflowScanCount); + Assert.Equal(0, afterLastRemoval.SpilledBucketCount); + Assert.Equal(0, afterLastRemoval.OverflowDirectoryOccupancy); + } + + [Fact] + public void RemovingNewestOfTwoSpillsRepointsSummaryToOlderExactWitness() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(18); + using MemoryStore store = CreateStore(); + PublishPrimaryPair(store, keys); + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[16], [0xE3], default, StoreWaitOptions.Infinite)); + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[17], [0xE4], default, StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterTwoSpills)); + + Assert.Equal(StoreStatus.Success, store.TryRemove(keys[17], StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterNewestRemoval)); + Assert.True(afterNewestRemoval.OverflowScanCount > afterTwoSpills.OverflowScanCount); + Assert.Equal(1, afterNewestRemoval.SpilledBucketCount); + Assert.Equal(1, afterNewestRemoval.OverflowDirectoryOccupancy); + + for (var index = 0; index < 16; index++) + { + Assert.Equal( + StoreStatus.Success, + store.TryRemove(keys[index], StoreWaitOptions.Infinite)); + } + + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterPrimaryRemovals)); + Assert.Equal(afterNewestRemoval.OverflowScanCount, afterPrimaryRemovals.OverflowScanCount); + Assert.Equal(1, afterPrimaryRemovals.SpilledBucketCount); + Assert.Equal(1, afterPrimaryRemovals.OverflowDirectoryOccupancy); + + Assert.Equal(StoreStatus.Success, store.TryRemove(keys[16], StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.Success, store.TryGetDiagnostics(out DiagnosticsSnapshot afterFinalRemoval)); + Assert.True(afterFinalRemoval.OverflowScanCount > afterPrimaryRemovals.OverflowScanCount); + Assert.Equal(0, afterFinalRemoval.SpilledBucketCount); + Assert.Equal(0, afterFinalRemoval.OverflowDirectoryOccupancy); + } + + [Fact] + public void OverflowReservationAbortConvergesToVersionedEmpty() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(17); + using MemoryStore store = CreateStore(); + PublishPrimaryPair(store, keys); + + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + keys[16], + 1, + default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + Assert.Equal(StoreStatus.Success, reservation.Abort(StoreWaitOptions.Infinite)); + + LockFreeKeyDirectory directory = ReadDirectory(store); + SpillSummary summary = SpillSummary.Decode(directory.ReadSpillSummary(CanonicalBucket)); + Assert.False(summary.IsPresent); + Assert.False(summary.IsInitial); + Assert.Equal(0, directory.OverflowOccupancy); + Assert.Equal(0UL, directory.ReadCanonicalMutation(CanonicalBucket)); + } + + [Fact] + public void MappingOutOfRangeEmptySummaryFailsClosedInsteadOfSuppressingLiveOverflow() + { + if (!IsSupportedLockFreeHost()) + { + return; + } + + byte[][] keys = GenerateBucketPairCollisions(17); + using MemoryStore store = CreateStore(); + PublishPrimaryPair(store, keys); + Assert.Equal( + StoreStatus.Success, + store.TryPublish(keys[16], [0xF1], default, StoreWaitOptions.Infinite)); + Assert.Equal(1, ReadDirectory(store).OverflowOccupancy); + + ulong invalidForMapping = SpillSummary.EncodeEmpty( + IndexBinding.Encode(slotIndex: SlotCount, generation: 1)); + WriteSpillSummary(store, CanonicalBucket, invalidForMapping); + + Assert.Equal(StoreStatus.CorruptStore, store.TryAcquire(keys[16], out _)); + Assert.Equal( + StoreStatus.CorruptStore, + store.TryGetDiagnostics(StoreWaitOptions.Infinite, out _)); + } + + private static void PublishPrimaryPair(MemoryStore store, byte[][] keys) + { + for (var index = 0; index < 16; index++) + { + Assert.Equal( + StoreStatus.Success, + store.TryPublish( + keys[index], + [unchecked((byte)index)], + default, + StoreWaitOptions.Infinite)); + } + + Assert.Equal(0, ReadDirectory(store).OverflowOccupancy); + } + + private static byte[][] GenerateBucketPairCollisions(int count) + { + var keys = new List(count); + int primaryLaneCount = NextPowerOfTwo(Math.Max(32, checked(SlotCount * 4))); + uint bucketMask = checked((uint)((primaryLaneCount / LayoutV2Constants.PrimaryLanesPerBucket) - 1)); + for (long candidate = 1; keys.Count < count; candidate++) + { + byte[] key = BitConverter.GetBytes(candidate); + ulong hash = StoreKey.Hash(key); + int first = (int)(Mix(hash) & bucketMask); + int second = (int)(Mix(hash ^ 0x9e37_79b9_7f4a_7c15UL) & bucketMask); + if (second == first) + { + second = (first + 1) & (int)bucketMask; + } + + if (first == CanonicalBucket && second == SecondaryBucket) + { + keys.Add(key); + } + } + + return keys.ToArray(); + } + + private static int NextPowerOfTwo(int value) + { + var result = 1; + while (result < value) + { + result <<= 1; + } + + return result; + } + + private static ulong Mix(ulong value) + { + value ^= value >> 30; + value *= 0xbf58_476d_1ce4_e5b9UL; + value ^= value >> 27; + value *= 0x94d0_49bb_1331_11ebUL; + return value ^ (value >> 31); + } + + private static LockFreeKeyDirectory ReadDirectory(MemoryStore store) + { + FieldInfo engineField = typeof(MemoryStore).GetField( + "_engine", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new Xunit.Sdk.XunitException("MemoryStore._engine is absent."); + object engine = engineField.GetValue(store) + ?? throw new Xunit.Sdk.XunitException("MemoryStore._engine is null."); + FieldInfo directoryField = engine.GetType().GetField( + "_directory", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new Xunit.Sdk.XunitException("Lock-free engine._directory is absent."); + return Assert.IsType(directoryField.GetValue(engine)); + } + + private static unsafe void WriteSpillSummary( + MemoryStore store, + int canonicalBucketIndex, + ulong raw) + { + FieldInfo engineField = typeof(MemoryStore).GetField( + "_engine", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new Xunit.Sdk.XunitException("MemoryStore._engine is absent."); + object engine = engineField.GetValue(store) + ?? throw new Xunit.Sdk.XunitException("MemoryStore._engine is null."); + var region = Assert.IsType( + engine.GetType().GetField( + "_region", + BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(engine)); + var layout = Assert.IsType( + engine.GetType().GetField( + "_layout", + BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(engine)); + long offset = layout.PrimaryDirectoryOffset + + ((long)canonicalBucketIndex * layout.PrimaryBucketStride); + ref long summary = ref *(long*)(region.Pointer + offset); + AtomicControlWord.StoreRelease(ref summary, unchecked((long)raw)); + } + + private static MemoryStore CreateInstrumentedStore(ControlledLockFreeScheduler scheduler) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options($"sms-v2-spill-summary-instrumented-{Guid.NewGuid():N}"), + scheduler.CreateInstrumentedCheckpoint(), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static MemoryStore CreateStore() + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options($"sms-v2-spill-summary-{Guid.NewGuid():N}"), + out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static SharedMemoryStoreOptions Options(string name) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: SlotCount, + maxValueBytes: 16, + maxDescriptorBytes: 4, + maxKeyBytes: 8, + leaseRecordCount: SlotCount, + participantRecordCount: 4, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + + private static bool IsSupportedLockFreeHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeStoreCorruptionLatchTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeStoreCorruptionLatchTests.cs new file mode 100644 index 0000000..091f90a --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeStoreCorruptionLatchTests.cs @@ -0,0 +1,327 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Interop; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed unsafe class LockFreeStoreCorruptionLatchTests +{ + [Fact] + public void StableMappedCorruptionPoisonsEveryHandleProjectionOperationAndFutureOpen() + { + if (!IsSupportedHost()) + { + return; + } + + string name = $"sms-v2-global-corruption-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions createOptions = Options(name, OpenMode.CreateNew); + SharedMemoryStoreOptions openOptions = Options(name, OpenMode.OpenExisting); + using MemoryStore detector = Open(createOptions); + using MemoryStore attached = Open(openOptions); + + byte[] publishedKey = [0x21]; + byte[] reservedKey = [0x22]; + Assert.Equal(StoreStatus.Success, detector.TryPublish(publishedKey, [0x31, 0x32], [0x41])); + Assert.Equal(StoreStatus.Success, attached.TryAcquire(publishedKey, out ValueLease lease)); + Assert.Equal( + StoreStatus.Success, + attached.TryReserve(reservedKey, payloadLength: 2, descriptor: [0x42], out ValueReservation reservation)); + Assert.Equal(2, lease.ValueLength); + Assert.Equal(1, lease.DescriptorLength); + Assert.Equal(2, reservation.GetSpan().Length); + Memory retainedReservationMemory = reservation.DangerousGetMemory(); + Assert.Equal(2, retainedReservationMemory.Length); + + EngineInternals detectorInternals = ReadInternals(detector); + object attachedEngine = ReadEngine(attached); + LockFreeParticipantRegistry.Registration attachedRegistration = + ReadField(attachedEngine, "_registration"); + LeaseHandle leaseHandle = lease.HandleForEngine; + ReservationHandle reservationHandle = reservation.HandleForEngine; + IndexBinding publishedBinding = IndexBinding.Decode(leaseHandle.SlotBinding); + IndexBinding reservedBinding = IndexBinding.Decode(reservationHandle.SlotBinding); + IndexBinding leaseBinding = IndexBinding.Decode(leaseHandle.LeaseToken); + ref ValueSlotMetadataV2 publishedSlot = + ref detectorInternals.Slots.Slot(publishedBinding.SlotIndex); + long originalOperation = AtomicControlWord.LoadAcquire(ref publishedSlot.DirectoryOperation); + Assert.NotEqual(0, originalOperation); + + int freeSlotIndex = FindFreeSlot(detectorInternals.Slots, excluded: reservedBinding.SlotIndex); + long freeControlBefore = AtomicControlWord.LoadAcquire( + ref detectorInternals.Slots.Slot(freeSlotIndex).Control); + + AtomicControlWord.StoreRelease(ref publishedSlot.DirectoryOperation, 0); + Assert.Equal(StoreStatus.CorruptStore, detector.TryPublish(publishedKey, [0x55])); + Assert.Equal(LayoutV2Constants.StoreCorrupt, ReadHeaderControl(detectorInternals.Region)); + + Assert.False(lease.IsValid); + Assert.Equal(0, lease.ValueLength); + Assert.Equal(0, lease.DescriptorLength); + Assert.True(lease.ValueSpan.IsEmpty); + Assert.True(lease.DescriptorSpan.IsEmpty); + Assert.Equal(StoreStatus.CorruptStore, lease.Release()); + + Assert.False(reservation.IsValid); + Assert.Equal(0, reservation.PayloadLength); + Assert.Equal(0, reservation.BytesWritten); + Assert.Equal(0, reservation.RemainingBytes); + Assert.True(reservation.GetSpan().IsEmpty); + Assert.True(reservation.DangerousGetMemory().IsEmpty); + Assert.Throws(() => retainedReservationMemory.Pin()); + Assert.Equal(StoreStatus.CorruptStore, reservation.Advance(1)); + Assert.Equal(StoreStatus.CorruptStore, reservation.Commit()); + Assert.Equal(StoreStatus.CorruptStore, reservation.Abort()); + + Assert.Equal(StoreStatus.CorruptStore, attached.TryPublish([0x23], [0x33])); + Assert.Equal( + freeControlBefore, + AtomicControlWord.LoadAcquire(ref detectorInternals.Slots.Slot(freeSlotIndex).Control)); + + ref ParticipantRecordV2 participantRecord = ref *(ParticipantRecordV2*)( + detectorInternals.Region.Pointer + + detectorInternals.Layout.ParticipantOffset + + ((long)attachedRegistration.RecordIndex * detectorInternals.Layout.ParticipantStride)); + ref LeaseRecordV2 leaseRecord = + ref detectorInternals.Leases.Record(leaseBinding.SlotIndex); + ref ValueSlotMetadataV2 reservedSlot = + ref detectorInternals.Slots.Slot(reservedBinding.SlotIndex); + long participantBeforeDispose = AtomicControlWord.LoadAcquire(ref participantRecord.Control); + long leaseBeforeDispose = AtomicControlWord.LoadAcquire(ref leaseRecord.Control); + long reservationBeforeDispose = AtomicControlWord.LoadAcquire(ref reservedSlot.Control); + + attached.Dispose(); + + Assert.Equal(participantBeforeDispose, AtomicControlWord.LoadAcquire(ref participantRecord.Control)); + Assert.Equal(leaseBeforeDispose, AtomicControlWord.LoadAcquire(ref leaseRecord.Control)); + Assert.Equal(reservationBeforeDispose, AtomicControlWord.LoadAcquire(ref reservedSlot.Control)); + + StoreOpenStatus reopenStatus = MemoryStore.TryCreateOrOpen(openOptions, out MemoryStore? rejected); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, reopenStatus); + Assert.Null(rejected); + } + + [Fact] + public void SuppressedReleaseReclaimCorruptionStillPoisonsTheMapping() + { + if (!IsSupportedHost()) + { + return; + } + + string name = $"sms-v2-suppressed-corruption-{Guid.NewGuid():N}"; + SharedMemoryStoreOptions createOptions = Options(name, OpenMode.CreateNew); + SharedMemoryStoreOptions openOptions = Options(name, OpenMode.OpenExisting); + using MemoryStore remover = Open(createOptions); + using MemoryStore reader = Open(openOptions); + byte[] key = [0x61]; + Assert.Equal(StoreStatus.Success, remover.TryPublish(key, [0x71])); + Assert.Equal(StoreStatus.Success, reader.TryAcquire(key, out ValueLease lease)); + Assert.Equal(StoreStatus.RemovePending, remover.TryRemove(key, StoreWaitOptions.Infinite)); + + EngineInternals internals = ReadInternals(remover); + IndexBinding binding = IndexBinding.Decode(lease.HandleForEngine.SlotBinding); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(binding.SlotIndex); + AtomicControlWord.StoreRelease(ref slot.DirectoryLocation, long.MaxValue); + + // Lease release is already ordered before optional physical reclaim. + // The suppressed reclaim result still has to publish the global latch. + Assert.Equal(StoreStatus.Success, lease.Release()); + Assert.Equal(LayoutV2Constants.StoreCorrupt, ReadHeaderControl(internals.Region)); + Assert.Equal(StoreStatus.CorruptStore, remover.TryAcquire(key, out _)); + + StoreOpenStatus reopenStatus = MemoryStore.TryCreateOrOpen(openOptions, out MemoryStore? rejected); + Assert.Equal(StoreOpenStatus.IncompatibleLayout, reopenStatus); + Assert.Null(rejected); + } + + [Fact] + public void LeaseProjectionRejectsStableOutOfRangeMappedMetadataBeforeFormingSpan() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open(Options( + $"sms-v2-lease-projection-corruption-{Guid.NewGuid():N}", + OpenMode.CreateNew)); + byte[] key = [0x41]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x51])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + EngineInternals internals = ReadInternals(store); + IndexBinding binding = IndexBinding.Decode(lease.HandleForEngine.SlotBinding); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(binding.SlotIndex); + slot.PayloadOffset = long.MaxValue; + + Assert.True(lease.ValueSpan.IsEmpty); + Assert.Equal(LayoutV2Constants.StoreCorrupt, ReadHeaderControl(internals.Region)); + Assert.Equal(0, lease.ValueLength); + Assert.Equal(StoreStatus.CorruptStore, lease.Release()); + } + + [Fact] + public void ActiveLeaseWithStableNonprojectableSlotLifecyclePoisonsStore() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open(Options( + $"sms-v2-lease-lifecycle-corruption-{Guid.NewGuid():N}", + OpenMode.CreateNew)); + byte[] key = [0x44]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x54])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + EngineInternals internals = ReadInternals(store); + IndexBinding binding = IndexBinding.Decode(lease.HandleForEngine.SlotBinding); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(binding.SlotIndex); + long impossible = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.ReclaimingState, + binding.Generation, + participantToken: 0)); + AtomicControlWord.StoreRelease(ref slot.Control, impossible); + + Assert.False(lease.IsValid); + Assert.Equal(LayoutV2Constants.StoreCorrupt, ReadHeaderControl(internals.Region)); + Assert.True(lease.ValueSpan.IsEmpty); + Assert.Equal(StoreStatus.CorruptStore, lease.Release()); + } + + [Fact] + public void ReservationProjectionRejectsStableOutOfRangeMappedMetadataBeforeFormingSpan() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open(Options( + $"sms-v2-reservation-projection-corruption-{Guid.NewGuid():N}", + OpenMode.CreateNew)); + Assert.Equal( + StoreStatus.Success, + store.TryReserve([0x42], payloadLength: 2, descriptor: default, out ValueReservation reservation)); + EngineInternals internals = ReadInternals(store); + IndexBinding binding = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(binding.SlotIndex); + slot.PayloadOffset = long.MaxValue; + + Assert.True(reservation.GetSpan().IsEmpty); + Assert.Equal(LayoutV2Constants.StoreCorrupt, ReadHeaderControl(internals.Region)); + Assert.Equal(StoreStatus.CorruptStore, reservation.Advance(1)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReservationMutationRejectsStableMappedLengthCorruptionBeforeAdvanceOrPublish( + bool commit) + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = Open(Options( + $"sms-v2-reservation-mutation-corruption-{commit}-{Guid.NewGuid():N}", + OpenMode.CreateNew)); + Assert.Equal( + StoreStatus.Success, + store.TryReserve([0x43], payloadLength: 1, descriptor: default, out ValueReservation reservation)); + reservation.GetSpan()[0] = 0x53; + if (commit) + { + Assert.Equal(StoreStatus.Success, reservation.Advance(1)); + } + + EngineInternals internals = ReadInternals(store); + IndexBinding binding = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding); + ref ValueSlotMetadataV2 slot = ref internals.Slots.Slot(binding.SlotIndex); + long controlBefore = AtomicControlWord.LoadAcquire(ref slot.Control); + long advancedBefore = AtomicControlWord.LoadAcquire(ref slot.BytesAdvanced); + slot.ValueLength = 9; + + StoreStatus status = commit ? reservation.Commit() : reservation.Advance(1); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Equal(LayoutV2Constants.StoreCorrupt, ReadHeaderControl(internals.Region)); + Assert.Equal(controlBefore, AtomicControlWord.LoadAcquire(ref slot.Control)); + Assert.Equal(advancedBefore, AtomicControlWord.LoadAcquire(ref slot.BytesAdvanced)); + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode mode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 4, + maxValueBytes: 8, + maxDescriptorBytes: 4, + maxKeyBytes: 4, + leaseRecordCount: 4, + participantRecordCount: 4, + openMode: mode, + enableLeaseRecovery: true); + + private static MemoryStore Open(SharedMemoryStoreOptions options) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static EngineInternals ReadInternals(MemoryStore store) + { + object engine = ReadEngine(store); + return new EngineInternals( + ReadField(engine, "_slots"), + ReadField(engine, "_leases"), + ReadField(engine, "_region"), + ReadField(engine, "_layout")); + } + + private static object ReadEngine(MemoryStore store) => + typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + + private static T ReadField(object owner, string name) => + Assert.IsType(owner.GetType() + .GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(owner)); + + private static int FindFreeSlot(LockFreeSlotTable slots, int excluded) + { + for (var index = 0; index < 4; index++) + { + if (index == excluded) + { + continue; + } + + long control = AtomicControlWord.LoadAcquire(ref slots.Slot(index).Control); + if ((unchecked((ulong)control) & 0x7UL) == LockFreeSlotTable.FreeState) + { + return index; + } + } + + throw new InvalidOperationException("No free slot was available for the mutation guard."); + } + + private static long ReadHeaderControl(MemoryMappedStoreRegion region) => + AtomicControlWord.LoadAcquire(ref ((StoreHeaderV2*)region.Pointer)->Control); + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private readonly record struct EngineInternals( + LockFreeSlotTable Slots, + LockFreeLeaseRegistry Leases, + MemoryMappedStoreRegion Region, + StoreLayoutV2 Layout); +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeStoreFullRegressionTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeStoreFullRegressionTests.cs new file mode 100644 index 0000000..8e232ec --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeStoreFullRegressionTests.cs @@ -0,0 +1,499 @@ +using System.Collections.Concurrent; +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeStoreFullRegressionTests +{ + private static readonly TimeSpan TestBound = TimeSpan.FromSeconds(10); + + [Fact] + public void CancellationAtClaimAdvancesGenerationInsteadOfRestoringSameFreeControl() + { + if (!IsSupportedHost()) + { + return; + } + + using var cancellation = new CancellationTokenSource(); + var resources = new ConcurrentQueue(); + var canceledClaim = 0; + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented( + static _ => { }, + resource => + { + resources.Enqueue(resource); + if (resource.Kind == LockFreeSlotResourceEventKind.Claim + && Interlocked.CompareExchange(ref canceledClaim, 1, 0) == 0) + { + cancellation.Cancel(); + } + }); + + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-claim-cancel-{Guid.NewGuid():N}", + OpenMode.CreateNew, + checkpoint); + LockFreeSlotTable slots = ReadSlots(store); + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + long initialFree = AtomicControlWord.LoadAcquire(ref slot.Control); + long initialGeneration = SlotGeneration(initialFree); + + StoreStatus status = store.TryPublish( + [0x11], + [0x21], + default, + new StoreWaitOptions(Timeout.InfiniteTimeSpan, cancellation.Token)); + + Assert.Equal(StoreStatus.OperationCanceled, status); + Assert.Equal(LockFreeSlotTable.FreeState, SlotState(initialFree)); + LockFreeSlotResourceEvent claim = Assert.Single( + resources, + static item => item.Kind == LockFreeSlotResourceEventKind.Claim); + LockFreeSlotResourceEvent release = Assert.Single( + resources, + static item => item.Kind is LockFreeSlotResourceEventKind.Free + or LockFreeSlotResourceEventKind.Retire); + Assert.Equal(0, claim.SlotIndex); + Assert.Equal(initialGeneration, claim.Generation); + Assert.Equal(claim.SlotIndex, release.SlotIndex); + Assert.Equal(claim.Generation, release.Generation); + + long advancedFree = AtomicControlWord.LoadAcquire(ref slot.Control); + Assert.NotEqual(initialFree, advancedFree); + Assert.Equal(LockFreeSlotTable.FreeState, SlotState(advancedFree)); + Assert.Equal(initialGeneration + 1, SlotGeneration(advancedFree)); + Assert.Equal((int)SlotPublicationIntent.None, Volatile.Read(ref slot.PublicationIntent)); + + Assert.Equal( + StoreStatus.Success, + store.TryPublish([0x12], [0x22], default, StoreWaitOptions.Infinite)); + LockFreeSlotResourceEvent replacementClaim = resources + .Where(static item => item.Kind == LockFreeSlotResourceEventKind.Claim) + .Last(); + Assert.Equal(initialGeneration + 1, replacementClaim.Generation); + } + + [Fact] + public async Task StoreFullProofGateIsPerHandleAndRespectsNoWaitAndInfinitePolicies() + { + if (!IsSupportedHost()) + { + return; + } + + string name = $"sms-v2-full-proof-gate-{Guid.NewGuid():N}"; + using var pause = new ProofPause(); + var resources = new ConcurrentQueue(); + var proofs = new ProofRecorder(); + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented( + pause.Observe, + resources.Enqueue, + proofs); + using MemoryStore firstHandle = CreateInstrumentedStore(name, OpenMode.CreateNew, checkpoint); + using MemoryStore otherHandle = OpenOrdinaryStore(name); + Assert.Equal(StoreStatus.Success, firstHandle.TryPublish([0x21], [0x31])); + resources.Clear(); + + Task heldProof = Task.Run(() => firstHandle.TryPublish( + [0x22], + [0x32], + default, + StoreWaitOptions.Infinite)); + + Task? infiniteContender = null; + try + { + pause.WaitUntilProofPaused(); + + Assert.Equal( + StoreStatus.StoreBusy, + firstHandle.TryPublish([0x23], [0x33], default, StoreWaitOptions.NoWait)); + + // The proof gate is process-local and per open handle. Holding it on + // firstHandle must not prevent another handle from proving the same + // stable physical capacity result. + Assert.Equal( + StoreStatus.StoreFull, + otherHandle.TryPublish([0x24], [0x34], default, StoreWaitOptions.Infinite)); + + infiniteContender = Task.Run(() => + { + pause.WatchInfiniteContenderThread(); + return firstHandle.TryPublish( + [0x25], + [0x35], + default, + StoreWaitOptions.Infinite); + }); + pause.WaitUntilInfiniteContenderEntered(); + Assert.False(infiniteContender.IsCompleted); + + pause.ResumeProof(); + Assert.Equal(StoreStatus.StoreFull, await heldProof.WaitAsync(TestBound)); + Assert.Equal(StoreStatus.StoreFull, await infiniteContender.WaitAsync(TestBound)); + + ProofObservation[] proofEvents = proofs.Snapshot(); + Assert.Equal(2, proofEvents.Count(static item => + item.Kind == ProofObservationKind.Candidate)); + Assert.Equal(2, proofEvents.Count(static item => + item.Kind == ProofObservationKind.Confirmed)); + Assert.DoesNotContain( + proofEvents, + static item => item.Kind == ProofObservationKind.Rejected); + Assert.Equal( + proofEvents + .Where(static item => item.Kind == ProofObservationKind.Candidate) + .Select(static item => item.Token) + .Order(), + proofEvents + .Where(static item => item.Kind == ProofObservationKind.Confirmed) + .Select(static item => item.Token) + .Order()); + Assert.All(proofEvents, static item => Assert.Equal(1, item.SlotCount)); + } + finally + { + pause.ResumeProof(); + _ = await heldProof.WaitAsync(TestBound); + if (infiniteContender is not null) + { + _ = await infiniteContender.WaitAsync(TestBound); + } + } + } + + [Fact] + public async Task MovementBetweenCollectsReturnsStoreBusyForNoWaitWithoutConfirmingFull() + { + if (!IsSupportedHost()) + { + return; + } + + string name = $"sms-v2-full-proof-movement-{Guid.NewGuid():N}"; + using var pause = new ProofPause(); + var resources = new ConcurrentQueue(); + var proofs = new ProofRecorder(); + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented( + pause.Observe, + resources.Enqueue, + proofs); + using MemoryStore store = CreateInstrumentedStore(name, OpenMode.CreateNew, checkpoint); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x31], [0x41])); + resources.Clear(); + + Task contender = Task.Run(() => store.TryPublish( + [0x32], + [0x42], + default, + StoreWaitOptions.NoWait)); + try + { + pause.WaitUntilProofPaused(); + Assert.Equal(StoreStatus.Success, store.TryRemove([0x31], StoreWaitOptions.Infinite)); + pause.ResumeProof(); + + Assert.Equal(StoreStatus.StoreBusy, await contender.WaitAsync(TestBound)); + ProofObservation candidate = Assert.Single( + proofs.Snapshot(), + static item => item.Kind == ProofObservationKind.Candidate); + ProofObservation rejected = Assert.Single( + proofs.Snapshot(), + static item => item.Kind == ProofObservationKind.Rejected); + Assert.Equal(candidate.Token, rejected.Token); + Assert.DoesNotContain( + proofs.Snapshot(), + static item => item.Kind == ProofObservationKind.Confirmed); + + Assert.Equal( + StoreStatus.Success, + store.TryPublish([0x32], [0x42], default, StoreWaitOptions.Infinite)); + } + finally + { + pause.ResumeProof(); + _ = await contender.WaitAsync(TestBound); + } + } + + [Fact] + public void MalformedControlIntroducedAfterFirstCollectFailsClosedOnSecondCollect() + { + if (!IsSupportedHost()) + { + return; + } + + LockFreeSlotTable? slots = null; + long originalControl = 0; + var injected = 0; + var resources = new ConcurrentQueue(); + var proofs = new ProofRecorder(); + InstrumentedLockFreeCheckpoint checkpoint = LockFreeCheckpointFactory.CreateInstrumented( + entry => + { + if (entry.Id != LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification + || Interlocked.CompareExchange(ref injected, 1, 0) != 0) + { + return; + } + + LockFreeSlotTable table = Assert.IsType(slots); + ref ValueSlotMetadataV2 slot = ref table.Slot(0); + originalControl = AtomicControlWord.LoadAcquire(ref slot.Control); + long malformed = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.PublishedState, + SlotGeneration(originalControl), + participantToken: 1)); + AtomicControlWord.StoreRelease(ref slot.Control, malformed); + }, + resources.Enqueue, + proofs); + + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-full-proof-malformed-{Guid.NewGuid():N}", + OpenMode.CreateNew, + checkpoint); + Assert.Equal(StoreStatus.Success, store.TryPublish([0x41], [0x51])); + slots = ReadSlots(store); + resources.Clear(); + + StoreStatus status; + try + { + status = store.TryPublish([0x42], [0x52], default, StoreWaitOptions.Infinite); + } + finally + { + if (Volatile.Read(ref injected) != 0) + { + ref ValueSlotMetadataV2 slot = ref Assert.IsType(slots).Slot(0); + AtomicControlWord.StoreRelease(ref slot.Control, originalControl); + } + } + + Assert.Equal(1, Volatile.Read(ref injected)); + Assert.Equal(StoreStatus.CorruptStore, status); + ProofObservation candidate = Assert.Single( + proofs.Snapshot(), + static item => item.Kind == ProofObservationKind.Candidate); + ProofObservation rejected = Assert.Single( + proofs.Snapshot(), + static item => item.Kind == ProofObservationKind.Rejected); + Assert.Equal(candidate.Token, rejected.Token); + Assert.DoesNotContain( + proofs.Snapshot(), + static item => item.Kind == ProofObservationKind.Confirmed); + + Assert.Equal(StoreStatus.CorruptStore, store.TryAcquire([0x41], out _)); + } + + [Fact] + public void StoreFullProofRejectsOwnedControlWithOutOfRangeParticipantIndex() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateInstrumentedStore( + $"sms-v2-full-proof-participant-{Guid.NewGuid():N}", + OpenMode.CreateNew, + LockFreeCheckpointFactory.CreateInstrumented(static _ => { })); + Assert.Equal( + StoreStatus.Success, + store.TryReserve( + [0x61], + payloadLength: 1, + descriptor: default, + StoreWaitOptions.Infinite, + out ValueReservation reservation)); + + LockFreeSlotTable slots = ReadSlots(store); + IndexBinding binding = IndexBinding.Decode(reservation.HandleForEngine.SlotBinding); + ref ValueSlotMetadataV2 slot = ref slots.Slot(binding.SlotIndex); + long originalControl = AtomicControlWord.LoadAcquire(ref slot.Control); + const int malformedParticipant = 13; // generation 1, index-plus-one 5 for count 4 + Assert.False(ParticipantToken.IsStructurallyValid(malformedParticipant, 4)); + long malformedControl = unchecked((long)AtomicControlWord.EncodeSlot( + LockFreeSlotTable.ReservedState, + binding.Generation, + malformedParticipant)); + AtomicControlWord.StoreRelease(ref slot.Control, malformedControl); + + try + { + NoOpLockFreeCheckpoint checkpoint = default; + StoreStatus status = slots.TryProveStoreFull( + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint, + out bool provenFull); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.False(provenFull); + } + finally + { + AtomicControlWord.StoreRelease(ref slot.Control, originalControl); + } + + Assert.Equal(StoreStatus.CorruptStore, reservation.Abort(StoreWaitOptions.Infinite)); + Assert.Equal(StoreStatus.CorruptStore, store.TryPublish([0x62], [0x72])); + } + + private static MemoryStore CreateInstrumentedStore( + string name, + OpenMode openMode, + InstrumentedLockFreeCheckpoint checkpoint) + { + StoreOpenStatus status = LockFreeInstrumentedStoreFactory.TryCreateOrOpen( + Options(name, openMode), + checkpoint, + out MemoryStore? candidate); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(candidate); + } + + private static MemoryStore OpenOrdinaryStore(string name) + { + StoreOpenStatus status = MemoryStore.TryCreateOrOpen( + Options(name, OpenMode.OpenExisting), + out MemoryStore? candidate); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(candidate); + } + + private static SharedMemoryStoreOptions Options(string name, OpenMode openMode) => + SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: 1, + leaseRecordCount: 2, + participantRecordCount: 4, + openMode, + enableLeaseRecovery: true); + + private static LockFreeSlotTable ReadSlots(MemoryStore store) + { + FieldInfo engineField = typeof(MemoryStore).GetField( + "_engine", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("MemoryStore._engine is absent."); + object engine = engineField.GetValue(store) + ?? throw new InvalidOperationException("MemoryStore._engine is null."); + FieldInfo slotsField = engine.GetType().GetField( + "_slots", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Lock-free engine._slots is absent."); + return Assert.IsType(slotsField.GetValue(engine)); + } + + private static int SlotState(long control) => (int)(unchecked((ulong)control) & 0x7UL); + + private static long SlotGeneration(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + private enum ProofObservationKind + { + Candidate, + Confirmed, + Rejected + } + + private readonly record struct ProofObservation( + ProofObservationKind Kind, + long Token, + int SlotCount); + + private sealed class ProofRecorder : ILockFreeStoreFullProofObserver + { + private readonly ConcurrentQueue _observations = new(); + private long _nextToken; + + public long BeginCandidate(int slotCount) + { + long token = Interlocked.Increment(ref _nextToken); + _observations.Enqueue(new ProofObservation( + ProofObservationKind.Candidate, + token, + slotCount)); + return token; + } + + public void CompleteCandidate(long token, bool confirmed) + { + int slotCount = _observations + .First(item => item.Kind == ProofObservationKind.Candidate && item.Token == token) + .SlotCount; + _observations.Enqueue(new ProofObservation( + confirmed ? ProofObservationKind.Confirmed : ProofObservationKind.Rejected, + token, + slotCount)); + } + + internal ProofObservation[] Snapshot() => _observations.ToArray(); + } + + private sealed class ProofPause : IDisposable + { + private readonly ManualResetEventSlim _proofPaused = new(initialState: false); + private readonly ManualResetEventSlim _resumeProof = new(initialState: false); + private readonly ManualResetEventSlim _infiniteContenderEntered = new(initialState: false); + private int _proofPauseClaimed; + private int _infiniteContenderThreadId; + + internal void Observe(LockFreeCheckpointEntry entry) + { + if (entry.Id == LockFreeCheckpointId.PublishBeforeSlotClaim + && Environment.CurrentManagedThreadId + == Volatile.Read(ref _infiniteContenderThreadId)) + { + _infiniteContenderEntered.Set(); + } + + if (entry.Id != LockFreeCheckpointId.StoreFullAfterFirstCollectBeforeVerification + || Interlocked.CompareExchange(ref _proofPauseClaimed, 1, 0) != 0) + { + return; + } + + _proofPaused.Set(); + if (!_resumeProof.Wait(TestBound)) + { + throw new Xunit.Sdk.XunitException( + "Paused StoreFull proof was not resumed within the test bound."); + } + } + + internal void WatchInfiniteContenderThread() => + Volatile.Write(ref _infiniteContenderThreadId, Environment.CurrentManagedThreadId); + + internal void WaitUntilProofPaused() => Assert.True( + _proofPaused.Wait(TestBound), + "The StoreFull first-collect checkpoint was not reached."); + + internal void WaitUntilInfiniteContenderEntered() => Assert.True( + _infiniteContenderEntered.Wait(TestBound), + "The infinite contender did not enter the publish operation."); + + internal void ResumeProof() => _resumeProof.Set(); + + public void Dispose() + { + _resumeProof.Set(); + _proofPaused.Dispose(); + _resumeProof.Dispose(); + _infiniteContenderEntered.Dispose(); + } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/LockFreeStructuralControlValidationTests.cs b/tests/SharedMemoryStore.UnitTests/LockFreeStructuralControlValidationTests.cs new file mode 100644 index 0000000..1c78271 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/LockFreeStructuralControlValidationTests.cs @@ -0,0 +1,335 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using SharedMemoryStore.Engines; +using SharedMemoryStore.LayoutV2; +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests; + +public sealed class LockFreeStructuralControlValidationTests +{ + [Theory] + [InlineData(LeaseMutationPath.Activate)] + [InlineData(LeaseMutationPath.CancelClaim)] + [InlineData(LeaseMutationPath.Release)] + public void ExactLeaseMutationRejectsMalformedControlWithoutRecycling( + LeaseMutationPath path) + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore($"sms-v2-lease-structure-{path}-{Guid.NewGuid():N}"); + LockFreeLeaseRegistry leases = ReadEngineField(store, "_leases"); + ulong slotBinding = IndexBinding.Encode(slotIndex: 0, generation: 1); + Assert.Equal(StoreStatus.Success, leases.TryClaim(slotBinding, acquireSequence: 1, out LeaseHandle lease)); + if (path == LeaseMutationPath.Release) + { + Assert.Equal(StoreStatus.Success, leases.TryActivate(lease)); + } + + ref LeaseRecordV2 record = ref leases.Record(0); + long original = AtomicControlWord.LoadAcquire(ref record.Control); + long incarnation = LeaseIncarnation(original); + long malformed = path switch + { + LeaseMutationPath.Activate => LeaseControl( + LockFreeLeaseRegistry.ActiveState, + incarnation, + participantToken: 0), + LeaseMutationPath.CancelClaim => LeaseControl( + LockFreeLeaseRegistry.RecoveringState, + incarnation, + checked((int)lease.ParticipantToken)), + LeaseMutationPath.Release => LeaseControl( + LockFreeLeaseRegistry.ReleasingState, + incarnation, + checked((int)lease.ParticipantToken)), + _ => throw new ArgumentOutOfRangeException(nameof(path), path, null) + }; + + try + { + AtomicControlWord.StoreRelease(ref record.Control, malformed); + + StoreStatus status = path switch + { + LeaseMutationPath.Activate => leases.TryActivate(lease), + LeaseMutationPath.CancelClaim => leases.TryCancelClaim(lease), + LeaseMutationPath.Release => leases.TryRelease(lease), + _ => throw new ArgumentOutOfRangeException(nameof(path), path, null) + }; + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref record.Control)); + } + finally + { + AtomicControlWord.StoreRelease(ref record.Control, original); + Assert.Equal( + StoreStatus.Success, + path == LeaseMutationPath.Release + ? leases.TryRelease(lease) + : leases.TryCancelClaim(lease)); + } + } + + [Fact] + public void ActiveLeaseScanRejectsMalformedNonActiveControlInsteadOfSkippingIt() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore($"sms-v2-lease-scan-structure-{Guid.NewGuid():N}"); + LockFreeLeaseRegistry leases = ReadEngineField(store, "_leases"); + ulong slotBinding = IndexBinding.Encode(slotIndex: 0, generation: 1); + Assert.Equal(StoreStatus.Success, leases.TryClaim(slotBinding, acquireSequence: 1, out LeaseHandle lease)); + Assert.Equal(StoreStatus.Success, leases.TryActivate(lease)); + ref LeaseRecordV2 record = ref leases.Record(0); + long original = AtomicControlWord.LoadAcquire(ref record.Control); + long malformed = LeaseControl( + LockFreeLeaseRegistry.FreeState, + LeaseIncarnation(original), + checked((int)lease.ParticipantToken)); + + try + { + AtomicControlWord.StoreRelease(ref record.Control, malformed); + + StoreStatus status = leases.ScanHasActiveLease( + slotBinding, + LockFreeOperationBudget.StructuralAttempt, + out bool hasActiveLease); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.False(hasActiveLease); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref record.Control)); + } + finally + { + AtomicControlWord.StoreRelease(ref record.Control, original); + Assert.Equal(StoreStatus.Success, leases.TryRelease(lease)); + } + } + + [Fact] + public void RemovePropagatesMalformedLeaseScanAsCorruptStore() + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore($"sms-v2-remove-lease-structure-{Guid.NewGuid():N}"); + byte[] key = [0x31]; + Assert.Equal(StoreStatus.Success, store.TryPublish(key, [0x41])); + Assert.Equal(StoreStatus.Success, store.TryAcquire(key, out ValueLease lease)); + LockFreeLeaseRegistry leases = ReadEngineField(store, "_leases"); + ref LeaseRecordV2 record = ref leases.Record(0); + long original = AtomicControlWord.LoadAcquire(ref record.Control); + long malformed = LeaseControl( + LockFreeLeaseRegistry.FreeState, + LeaseIncarnation(original), + checked((int)LeaseParticipant(original))); + + try + { + AtomicControlWord.StoreRelease(ref record.Control, malformed); + + StoreStatus status = store.TryRemove(key, StoreWaitOptions.Infinite); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref record.Control)); + } + finally + { + AtomicControlWord.StoreRelease(ref record.Control, original); + Assert.Equal(StoreStatus.CorruptStore, lease.Release()); + } + } + + [Theory] + [InlineData(ParticipantCleanupCorruption.ZeroIncarnation)] + [InlineData(ParticipantCleanupCorruption.InvalidActiveBinding)] + public void ParticipantDisposalCleanupRejectsMalformedLeaseWithoutMutationOrThrow( + ParticipantCleanupCorruption corruption) + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore($"sms-v2-dispose-lease-structure-{corruption}-{Guid.NewGuid():N}"); + LockFreeLeaseRegistry leases = ReadEngineField(store, "_leases"); + LockFreeReclaimer reclaimer = ReadEngineField(store, "_reclaimer"); + ulong slotBinding = IndexBinding.Encode(slotIndex: 0, generation: 1); + Assert.Equal(StoreStatus.Success, leases.TryClaim(slotBinding, acquireSequence: 1, out LeaseHandle lease)); + Assert.Equal(StoreStatus.Success, leases.TryActivate(lease)); + ref LeaseRecordV2 record = ref leases.Record(0); + long originalControl = AtomicControlWord.LoadAcquire(ref record.Control); + ulong originalBinding = record.SlotBinding; + long malformedControl = corruption == ParticipantCleanupCorruption.ZeroIncarnation + ? LockFreeLeaseRegistry.ActiveState + | (checked((long)lease.ParticipantToken) << 36) + : originalControl; + + try + { + AtomicControlWord.StoreRelease(ref record.Control, malformedControl); + if (corruption == ParticipantCleanupCorruption.InvalidActiveBinding) + { + record.SlotBinding = 0; + } + + NoOpLockFreeCheckpoint checkpoint = default; + StoreStatus status = leases.ReleaseParticipantLeases( + lease.ParticipantToken, + reclaimer, + LockFreeOperationBudget.StructuralAttempt, + ref checkpoint, + out int released); + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Equal(0, released); + Assert.Equal(malformedControl, AtomicControlWord.LoadAcquire(ref record.Control)); + Assert.Equal( + corruption == ParticipantCleanupCorruption.InvalidActiveBinding ? 0UL : originalBinding, + record.SlotBinding); + } + finally + { + record.SlotBinding = originalBinding; + AtomicControlWord.StoreRelease(ref record.Control, originalControl); + Assert.Equal(StoreStatus.Success, leases.TryRelease(lease)); + } + } + + [Theory] + [InlineData(ReservationMutationPath.MarkReserved)] + [InlineData(ReservationMutationPath.Advance)] + [InlineData(ReservationMutationPath.Commit)] + [InlineData(ReservationMutationPath.Abort)] + public void ExactReservationMutationRejectsMalformedControlInsteadOfCompletedStatus( + ReservationMutationPath path) + { + if (!IsSupportedHost()) + { + return; + } + + using MemoryStore store = CreateStore($"sms-v2-reservation-structure-{path}-{Guid.NewGuid():N}"); + LockFreeSlotTable slots = ReadEngineField(store, "_slots"); + Assert.Equal( + StoreStatus.Success, + slots.TryClaimReservation( + keyHash: 1, + keyLength: 1, + descriptorLength: 0, + payloadLength: 1, + out ReservationHandle reservation)); + if (path != ReservationMutationPath.MarkReserved) + { + Assert.Equal(StoreStatus.Success, slots.TryMarkReserved(reservation)); + } + + ref ValueSlotMetadataV2 slot = ref slots.Slot(0); + long original = AtomicControlWord.LoadAcquire(ref slot.Control); + long generation = SlotGeneration(original); + long malformed = SlotControl( + LockFreeSlotTable.PublishedState, + generation, + checked((int)reservation.ParticipantToken)); + + try + { + AtomicControlWord.StoreRelease(ref slot.Control, malformed); + + StoreStatus status = path switch + { + ReservationMutationPath.MarkReserved => slots.TryMarkReserved(reservation), + ReservationMutationPath.Advance => slots.AdvanceReservation(reservation, byteCount: 1), + ReservationMutationPath.Commit => slots.CommitReservation(reservation, commitSequence: 1), + ReservationMutationPath.Abort => slots.TryBeginAbort(reservation), + _ => throw new ArgumentOutOfRangeException(nameof(path), path, null) + }; + + Assert.Equal(StoreStatus.CorruptStore, status); + Assert.Equal(malformed, AtomicControlWord.LoadAcquire(ref slot.Control)); + } + finally + { + AtomicControlWord.StoreRelease(ref slot.Control, original); + Assert.Equal(StoreStatus.Success, slots.AbortUnboundReservation(reservation)); + } + } + + private static MemoryStore CreateStore(string name) + { + SharedMemoryStoreOptions options = SharedMemoryStoreOptions.CreateLockFree( + name, + slotCount: 1, + maxValueBytes: 4, + maxDescriptorBytes: 0, + maxKeyBytes: 4, + leaseRecordCount: 1, + participantRecordCount: 4, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + StoreOpenStatus status = MemoryStore.TryCreateOrOpen(options, out MemoryStore? store); + Assert.Equal(StoreOpenStatus.Success, status); + return Assert.IsType(store); + } + + private static T ReadEngineField(MemoryStore store, string name) + { + object engine = typeof(MemoryStore) + .GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(store)!; + return Assert.IsType(engine.GetType() + .GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(engine)); + } + + private static long LeaseControl(int state, long incarnation, int participantToken) => + unchecked((long)AtomicControlWord.EncodeLease(state, incarnation, participantToken)); + + private static long SlotControl(int state, long generation, int participantToken) => + unchecked((long)AtomicControlWord.EncodeSlot(state, generation, participantToken)); + + private static long LeaseIncarnation(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static ulong LeaseParticipant(long control) => + (unchecked((ulong)control) >> 36) & 0x0fff_ffffUL; + + private static long SlotGeneration(long control) => + (long)((unchecked((ulong)control) >> 3) & 0x1_ffff_ffffUL); + + private static bool IsSupportedHost() => + (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + public enum LeaseMutationPath + { + Activate, + CancelClaim, + Release + } + + public enum ReservationMutationPath + { + MarkReserved, + Advance, + Commit, + Abort + } + + public enum ParticipantCleanupCorruption + { + ZeroIncarnation, + InvalidActiveBinding + } +} diff --git a/tests/SharedMemoryStore.UnitTests/MemoryStoreFacadeTests.cs b/tests/SharedMemoryStore.UnitTests/MemoryStoreFacadeTests.cs new file mode 100644 index 0000000..295f8e4 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/MemoryStoreFacadeTests.cs @@ -0,0 +1,276 @@ +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; + +namespace SharedMemoryStore.UnitTests +{ + + public sealed class MemoryStoreFacadeTests + { + private static readonly Assembly StoreAssembly = typeof(MemoryStore).Assembly; + + [Fact] + public void FacadeRoutesCoreStatusOperationsThroughInjectedEngine() + { + var engineType = RequireInternalType("SharedMemoryStore.Engines.IStoreEngine"); + Assert.True(engineType.IsInterface, "IStoreEngine must be an interface so the facade has one profile-neutral dependency."); + + FakeEngineCallLog.Reset(StoreStatus.UnsupportedPlatform); + using var store = CreateFacadeWithFakeEngine(engineType); + + Assert.Equal(StoreStatus.UnsupportedPlatform, store.TryPublish([1], [2])); + Assert.Equal(StoreStatus.UnsupportedPlatform, store.TryReserve([2], 1, default, out _)); + Assert.Equal(StoreStatus.UnsupportedPlatform, store.TryAcquire([3], out _)); + Assert.Equal(StoreStatus.UnsupportedPlatform, store.TryRemove([4])); + + Assert.Equal(1, FakeEngineCallLog.Count("TryPublish")); + Assert.Equal(1, FakeEngineCallLog.Count("TryReserve")); + Assert.Equal(1, FakeEngineCallLog.Count("TryAcquire")); + Assert.Equal(1, FakeEngineCallLog.Count("TryRemove")); + } + + [Fact] + public void PublicTokensCarryOnlyFacadeAndOpaqueEngineHandle() + { + var reservationHandle = RequireOpaqueHandle( + "SharedMemoryStore.Engines.ReservationHandle", + typeof(ulong), typeof(ulong), typeof(ulong), typeof(int)); + var leaseHandle = RequireOpaqueHandle( + "SharedMemoryStore.Engines.LeaseHandle", + typeof(ulong), typeof(ulong), typeof(ulong), typeof(ulong)); + + AssertTokenFields(reservationHandle); + AssertTokenFields(leaseHandle); + } + + [Fact] + public void ReservationHandleFencesStoreParticipantAndSlotIncarnations() + { + var handleType = RequireOpaqueHandle( + "SharedMemoryStore.Engines.ReservationHandle", + typeof(ulong), typeof(ulong), typeof(ulong), typeof(int)); + + var baseline = CreateHandle(handleType, 11UL, 21UL, 31UL, 41); + Assert.NotEqual(baseline, CreateHandle(handleType, 12UL, 21UL, 31UL, 41)); + Assert.NotEqual(baseline, CreateHandle(handleType, 11UL, 22UL, 31UL, 41)); + Assert.NotEqual(baseline, CreateHandle(handleType, 11UL, 21UL, 32UL, 41)); + Assert.NotEqual(baseline, CreateHandle(handleType, 11UL, 21UL, 31UL, 42)); + } + + [Fact] + public void LeaseHandleFencesStoreParticipantSlotAndLeaseIncarnations() + { + var handleType = RequireOpaqueHandle( + "SharedMemoryStore.Engines.LeaseHandle", + typeof(ulong), typeof(ulong), typeof(ulong), typeof(ulong)); + + var baseline = CreateHandle(handleType, 11UL, 21UL, 31UL, 41UL); + Assert.NotEqual(baseline, CreateHandle(handleType, 12UL, 21UL, 31UL, 41UL)); + Assert.NotEqual(baseline, CreateHandle(handleType, 11UL, 22UL, 31UL, 41UL)); + Assert.NotEqual(baseline, CreateHandle(handleType, 11UL, 21UL, 32UL, 41UL)); + Assert.NotEqual(baseline, CreateHandle(handleType, 11UL, 21UL, 31UL, 42UL)); + } + + private static Type RequireInternalType(string fullName) + { + var type = StoreAssembly.GetType(fullName, throwOnError: false, ignoreCase: false); + Assert.True(type is not null, $"Required engine-neutral type '{fullName}' is missing."); + Assert.False(type!.IsPublic, $"'{fullName}' is an implementation seam and must not expand the public API."); + return type; + } + + private static Type RequireOpaqueHandle(string fullName, params Type[] constructorParameterTypes) + { + var type = RequireInternalType(fullName); + Assert.True(type.IsValueType, $"'{fullName}' must be an allocation-free value handle."); + Assert.NotNull(type.GetCustomAttribute()); + Assert.NotNull(type.GetConstructor( + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, + binder: null, + constructorParameterTypes, + modifiers: null)); + Assert.Empty(type.GetFields(BindingFlags.Instance | BindingFlags.Public)); + return type; + } + + private static object CreateHandle(Type handleType, params object[] arguments) + { + var instance = Activator.CreateInstance( + handleType, + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, + binder: null, + args: arguments, + culture: null); + Assert.NotNull(instance); + return instance!; + } + + private static void AssertTokenFields(Type expectedHandleType) + { + var fields = typeof(TToken).GetFields(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); + Assert.Contains(fields, field => field.FieldType == typeof(MemoryStore)); + Assert.Single(fields, field => field.FieldType == expectedHandleType); + Assert.DoesNotContain(fields, field => field.FieldType.FullName == "SharedMemoryStore.Layout.SlotLifecycleId"); + Assert.DoesNotContain(fields, field => field.FieldType == typeof(int)); + } + + private static MemoryStore CreateFacadeWithFakeEngine(Type engineType) + { + var constructor = typeof(MemoryStore) + .GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .SingleOrDefault(candidate => + { + var parameters = candidate.GetParameters(); + return parameters.Length == 1 && parameters[0].ParameterType == engineType; + }); + Assert.True(constructor is not null, "MemoryStore must expose an internal one-engine constructor for routing tests."); + + var fake = FakeEngineEmitter.Create(engineType); + return (MemoryStore)constructor!.Invoke([fake]); + } + + private static class FakeEngineEmitter + { + public static object Create(Type interfaceType) + { + var assemblyName = new AssemblyName($"SharedMemoryStore.UnitTests.Fakes.{Guid.NewGuid():N}"); + var assembly = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var ignoresAccessChecks = typeof(IgnoresAccessChecksToAttribute).GetConstructor([typeof(string)]); + Assert.NotNull(ignoresAccessChecks); + assembly.SetCustomAttribute(new CustomAttributeBuilder(ignoresAccessChecks!, ["SharedMemoryStore"])); + + var module = assembly.DefineDynamicModule(assemblyName.Name!); + var type = module.DefineType( + $"FakeStoreEngine_{Guid.NewGuid():N}", + TypeAttributes.Public | TypeAttributes.Sealed | TypeAttributes.Class); + type.AddInterfaceImplementation(interfaceType); + type.DefineDefaultConstructor(MethodAttributes.Public); + + var methods = interfaceType + .GetInterfaces() + .Append(interfaceType) + .SelectMany(type => type.GetMethods()) + .DistinctBy(method => + $"{method.Name}|{method.ReturnType.AssemblyQualifiedName}|{string.Join("|", method.GetParameters().Select(parameter => parameter.ParameterType.AssemblyQualifiedName))}") + .ToArray(); + Assert.DoesNotContain(methods, method => method.IsGenericMethodDefinition); + foreach (var method in methods) + { + EmitMethod(type, method); + } + + var generated = type.CreateType(); + return Activator.CreateInstance(generated!)!; + } + + private static void EmitMethod(TypeBuilder type, MethodInfo interfaceMethod) + { + var parameters = interfaceMethod.GetParameters(); + var implementation = type.DefineMethod( + interfaceMethod.Name, + MethodAttributes.Public | MethodAttributes.Virtual | MethodAttributes.Final | + MethodAttributes.HideBySig | MethodAttributes.NewSlot | + (interfaceMethod.IsSpecialName ? MethodAttributes.SpecialName : 0), + interfaceMethod.ReturnType, + parameters.Select(parameter => parameter.ParameterType).ToArray()); + + for (var index = 0; index < parameters.Length; index++) + { + implementation.DefineParameter(index + 1, parameters[index].Attributes, parameters[index].Name); + } + + var il = implementation.GetILGenerator(); + il.Emit(OpCodes.Ldstr, interfaceMethod.Name); + il.Emit(OpCodes.Call, typeof(FakeEngineCallLog).GetMethod(nameof(FakeEngineCallLog.Record))!); + + for (var index = 0; index < parameters.Length; index++) + { + var parameterType = parameters[index].ParameterType; + if (!parameterType.IsByRef || parameters[index].IsIn) + { + continue; + } + + il.Emit(OpCodes.Ldarg, index + 1); + il.Emit(OpCodes.Initobj, parameterType.GetElementType()!); + } + + EmitReturn(il, interfaceMethod.ReturnType); + type.DefineMethodOverride(implementation, interfaceMethod); + } + + private static void EmitReturn(ILGenerator il, Type returnType) + { + if (returnType == typeof(void)) + { + il.Emit(OpCodes.Ret); + return; + } + + if (returnType == typeof(StoreStatus)) + { + il.Emit(OpCodes.Call, typeof(FakeEngineCallLog).GetMethod(nameof(FakeEngineCallLog.GetStatus))!); + il.Emit(OpCodes.Ret); + return; + } + + if (!returnType.IsValueType) + { + il.Emit(OpCodes.Ldnull); + il.Emit(OpCodes.Ret); + return; + } + + var value = il.DeclareLocal(returnType); + il.Emit(OpCodes.Ldloca, value); + il.Emit(OpCodes.Initobj, returnType); + il.Emit(OpCodes.Ldloc, value); + il.Emit(OpCodes.Ret); + } + } + } + + public static class FakeEngineCallLog + { + private static readonly object Sync = new(); + private static readonly List Calls = []; + private static StoreStatus _status; + + public static void Reset(StoreStatus status) + { + lock (Sync) + { + Calls.Clear(); + _status = status; + } + } + + public static void Record(string methodName) + { + lock (Sync) + { + Calls.Add(methodName); + } + } + + public static int Count(string methodName) + { + lock (Sync) + { + return Calls.Count(call => call == methodName); + } + } + + public static StoreStatus GetStatus() => _status; + } + +} + +namespace System.Runtime.CompilerServices +{ + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)] + public sealed class IgnoresAccessChecksToAttribute(string assemblyName) : Attribute + { + public string AssemblyName { get; } = assemblyName; + } +} diff --git a/tests/SharedMemoryStore.UnitTests/RepositoryEnvironmentProbeTests.cs b/tests/SharedMemoryStore.UnitTests/RepositoryEnvironmentProbeTests.cs new file mode 100644 index 0000000..0d1b8b8 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/RepositoryEnvironmentProbeTests.cs @@ -0,0 +1,229 @@ +using System.Diagnostics; + +namespace SharedMemoryStore.UnitTests; + +public sealed class RepositoryEnvironmentProbeTests +{ + private const string Sha1 = "0123456789abcdef0123456789abcdef01234567"; + private const string Sha256 = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + [Theory] + [InlineData(Sha1, "clean")] + [InlineData(Sha256, "dirty")] + [InlineData("ABCDEF0123456789ABCDEF0123456789ABCDEF01", "clean")] + public void InjectedPairBypassesGit(string commit, string state) + { + var processCalls = 0; + + RepositoryProvenanceSnapshot result = RepositoryEnvironmentProbe.Capture( + ["--repository-commit", commit, "--repository-working-tree-state", state], + Path.GetTempPath(), + (_, _) => + { + processCalls++; + throw new InvalidOperationException("Git must not run for injected provenance."); + }); + + Assert.Equal(commit, result.Commit); + Assert.Equal(state, result.WorkingTreeState); + Assert.Equal(0, processCalls); + } + + [Theory] + [MemberData(nameof(InvalidArguments))] + public void InvalidPartialDuplicateOrMissingInjectionThrows(string[] args) + { + Assert.Throws(() => RepositoryEnvironmentProbe.Capture( + args, + Path.GetTempPath(), + static (_, _) => throw new InvalidOperationException("Git must not run."))); + } + + public static TheoryData InvalidArguments => new() + { + { ["--repository-commit", Sha1] }, + { ["--repository-working-tree-state", "clean"] }, + { ["--repository-commit"] }, + { ["--repository-commit", "--repository-working-tree-state", "clean"] }, + { ["--repository-working-tree-state"] }, + { ["--repository-commit", Sha1, "--repository-commit", Sha1, + "--repository-working-tree-state", "clean"] }, + { ["--repository-commit", Sha1, "--repository-working-tree-state", "clean", + "--repository-working-tree-state", "clean"] }, + { ["--repository-commit", "abc", "--repository-working-tree-state", "clean"] }, + { ["--repository-commit", new string('g', 40), + "--repository-working-tree-state", "clean"] }, + { ["--repository-commit", Sha1, "--repository-working-tree-state", "CLEAN"] }, + { ["--repository-commit", Sha1, "--repository-working-tree-state", "unknown"] } + }; + + [Fact] + public void GitFallbackUsesExplicitSafeConfigurationAndReportsClean() + { + var invocations = new List(); + + RepositoryProvenanceSnapshot result = RepositoryEnvironmentProbe.Capture( + [], + AppContext.BaseDirectory, + (startInfo, timeout) => + { + invocations.Add(startInfo.ArgumentList.ToArray()); + Assert.Equal(TimeSpan.FromSeconds(30), timeout); + Assert.Equal("0", startInfo.Environment["GIT_OPTIONAL_LOCKS"]); + return invocations.Count == 1 + ? Successful(Sha1 + Environment.NewLine) + : Successful(string.Empty); + }); + + Assert.Equal(Sha1, result.Commit); + Assert.Equal("clean", result.WorkingTreeState); + Assert.Equal(2, invocations.Count); + AssertGitPrefix(invocations[0]); + Assert.Equal(["rev-parse", "HEAD"], invocations[0][6..]); + AssertGitPrefix(invocations[1]); + Assert.Equal( + ["status", "--porcelain=v2", "--untracked-files=normal"], + invocations[1][6..]); + } + + [Fact] + public void GitFallbackReportsDirtyForPorcelainV2Output() + { + var invocation = 0; + RepositoryProvenanceSnapshot result = RepositoryEnvironmentProbe.Capture( + [], + AppContext.BaseDirectory, + (_, _) => ++invocation == 1 + ? Successful(Sha256) + : Successful("? untracked.txt")); + + Assert.Equal(Sha256, result.Commit); + Assert.Equal("dirty", result.WorkingTreeState); + } + + [Fact] + public void GitFailureMakesThePairUnknown() + { + RepositoryProvenanceSnapshot result = RepositoryEnvironmentProbe.Capture( + [], + AppContext.BaseDirectory, + static (_, _) => new BoundedProcessResult( + Started: true, + TimedOut: false, + ExitCode: 128, + StandardOutput: string.Empty, + StandardError: "not a repository")); + + Assert.Equal("unknown", result.Commit); + Assert.Equal("unknown", result.WorkingTreeState); + } + + [Fact] + public void BoundedRunnerDrainsBothStreamsAndReturnsExitCode() + { + ProcessStartInfo startInfo = CreateShellStartInfo( + OperatingSystem.IsWindows() + ? "echo standard-output & echo standard-error 1>&2 & exit /b 7" + : "printf standard-output; printf standard-error >&2; exit 7"); + + BoundedProcessResult result = BoundedProcessRunner.Run(startInfo, TimeSpan.FromSeconds(5)); + + Assert.True(result.Started); + Assert.False(result.TimedOut); + Assert.Equal(7, result.ExitCode); + Assert.Contains("standard-output", result.StandardOutput, StringComparison.Ordinal); + Assert.Contains("standard-error", result.StandardError, StringComparison.Ordinal); + Assert.False(result.Succeeded); + } + + [Fact] + public void BoundedRunnerKillsAndReapsTimedOutProcessTree() + { + ProcessStartInfo startInfo = CreateShellStartInfo( + OperatingSystem.IsWindows() + ? "ping 127.0.0.1 -n 30 > nul" + : "sleep 30"); + var stopwatch = Stopwatch.StartNew(); + + BoundedProcessResult result = BoundedProcessRunner.Run( + startInfo, + TimeSpan.FromMilliseconds(200)); + + stopwatch.Stop(); + Assert.True(result.Started); + Assert.True(result.TimedOut); + Assert.NotNull(result.ExitCode); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(10), stopwatch.Elapsed.ToString()); + Assert.False(result.Succeeded); + } + + [Fact] + public void BoundedRunnerDoesNotHangWhenExitedParentLeavesInheritedPipesOpen() + { + ProcessStartInfo startInfo = CreateExitedParentWithPipeHoldingChildStartInfo(); + var stopwatch = Stopwatch.StartNew(); + + BoundedProcessResult result = BoundedProcessRunner.Run( + startInfo, + TimeSpan.FromSeconds(2)); + + stopwatch.Stop(); + Assert.True(result.Started); + Assert.True(result.TimedOut); + Assert.Equal(0, result.ExitCode); + Assert.False(result.Succeeded); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(6), stopwatch.Elapsed.ToString()); + } + + private static BoundedProcessResult Successful(string output) => new( + Started: true, + TimedOut: false, + ExitCode: 0, + StandardOutput: output, + StandardError: string.Empty); + + private static ProcessStartInfo CreateShellStartInfo(string command) + { + var startInfo = OperatingSystem.IsWindows() + ? new ProcessStartInfo("cmd.exe") + : new ProcessStartInfo("/bin/sh"); + startInfo.UseShellExecute = false; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + startInfo.CreateNoWindow = true; + if (OperatingSystem.IsWindows()) + { + startInfo.ArgumentList.Add("/d"); + startInfo.ArgumentList.Add("/c"); + } + else + { + startInfo.ArgumentList.Add("-c"); + } + + startInfo.ArgumentList.Add(command); + return startInfo; + } + + private static ProcessStartInfo CreateExitedParentWithPipeHoldingChildStartInfo() + { + if (!OperatingSystem.IsWindows()) + { + return CreateShellStartInfo("sleep 10 &"); + } + + return CreateShellStartInfo( + "start /b ping 127.0.0.1 -n 12 & exit /b 0"); + } + + private static void AssertGitPrefix(string[] arguments) + { + Assert.Equal("-c", arguments[0]); + Assert.Equal("core.autocrlf=true", arguments[1]); + Assert.Equal("-c", arguments[2]); + Assert.Equal("core.safecrlf=false", arguments[3]); + Assert.Equal("-C", arguments[4]); + Assert.Equal(RepositoryEnvironmentProbe.FindRepositoryRoot(AppContext.BaseDirectory), arguments[5]); + } +} diff --git a/tests/SharedMemoryStore.UnitTests/SharedMemoryStore.UnitTests.csproj b/tests/SharedMemoryStore.UnitTests/SharedMemoryStore.UnitTests.csproj index a98f5a7..25a70a5 100644 --- a/tests/SharedMemoryStore.UnitTests/SharedMemoryStore.UnitTests.csproj +++ b/tests/SharedMemoryStore.UnitTests/SharedMemoryStore.UnitTests.csproj @@ -4,6 +4,7 @@ net10.0 enable enable + true false @@ -20,6 +21,9 @@ + + - \ No newline at end of file + diff --git a/tests/SharedMemoryStore.UnitTests/StoreEngineFactoryOwnershipTests.cs b/tests/SharedMemoryStore.UnitTests/StoreEngineFactoryOwnershipTests.cs new file mode 100644 index 0000000..a78e1a6 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/StoreEngineFactoryOwnershipTests.cs @@ -0,0 +1,55 @@ +using System.Buffers; +using SharedMemoryStore.Diagnostics; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Ingest; + +namespace SharedMemoryStore.UnitTests; + +public sealed class StoreEngineFactoryOwnershipTests +{ + [Fact] + public void FacadeConstructionFailureDisposesTransferredEngineExactlyOnce() + { + var engine = new ThrowingProfileEngine(); + + Assert.Throws( + () => StoreEngineFactory.WrapOwnedEngine(engine)); + + Assert.Equal(1, engine.DisposeCount); + } + + private sealed class InjectedFacadeConstructionException : Exception; + + private sealed class ThrowingProfileEngine : IStoreEngine + { + internal int DisposeCount { get; private set; } + + public StoreProfile Profile => throw new InjectedFacadeConstructionException(); + public StoreProtocolInfo ProtocolInfo => default; + public StoreStatus RecordFacadeStatus(StoreStatus status) => status; + public DiagnosticsSnapshot CreateDisposedDiagnosticsSnapshot() => default; + public StoreStatus TryPublish(ReadOnlySpan key, ReadOnlySpan value, ReadOnlySpan descriptor, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public StoreStatus TryReserve(ReadOnlySpan key, int payloadLength, ReadOnlySpan descriptor, StoreWaitOptions waitOptions, out ReservationHandle reservation) { reservation = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryPublishSegments(ReadOnlySpan key, ReadOnlySequence payload, ReadOnlySpan descriptor, StoreWaitOptions waitOptions, out long copiedBytes) { copiedBytes = 0; return StoreStatus.UnknownFailure; } + public StoreStatus TryAcquire(ReadOnlySpan key, StoreWaitOptions waitOptions, out LeaseHandle lease) { lease = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryRemove(ReadOnlySpan key, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public StoreStatus TryRecoverLeases(LeaseRecoveryOptions options, StoreWaitOptions waitOptions, out LeaseRecoveryReport report) { report = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryRecoverReservations(ReservationRecoveryOptions options, StoreWaitOptions waitOptions, out ReservationRecoveryReport report) { report = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryGetMetrics(StoreWaitOptions waitOptions, out EngineMetrics metrics) { metrics = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryGetDiagnostics(StoreWaitOptions waitOptions, out DiagnosticsSnapshot snapshot) { snapshot = default; return StoreStatus.UnknownFailure; } + public bool IsReservationPending(ReservationHandle reservation) => false; + public int GetReservationBytesWritten(ReservationHandle reservation) => 0; + public Span GetReservationSpan(ReservationHandle reservation, int sizeHint) => []; + public Memory DangerousGetReservationMemory(ReservationHandle reservation, int sizeHint) => Memory.Empty; + public StoreStatus AdvanceReservation(ReservationHandle reservation, int byteCount, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public StoreStatus CommitReservation(ReservationHandle reservation, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public StoreStatus AbortReservation(ReservationHandle reservation, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public bool IsLeaseActive(LeaseHandle lease) => false; + public int GetValueLength(LeaseHandle lease) => 0; + public int GetDescriptorLength(LeaseHandle lease) => 0; + public ReadOnlySpan GetValueSpan(LeaseHandle lease) => []; + public ReadOnlySpan GetDescriptorSpan(LeaseHandle lease) => []; + public StoreStatus ReleaseLease(LeaseHandle lease, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public void Dispose() => DisposeCount++; + } +} diff --git a/tests/SharedMemoryStore.UnitTests/StoreLifecycleGateBudgetTests.cs b/tests/SharedMemoryStore.UnitTests/StoreLifecycleGateBudgetTests.cs new file mode 100644 index 0000000..02ed553 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/StoreLifecycleGateBudgetTests.cs @@ -0,0 +1,161 @@ +using System.Buffers; +using System.Diagnostics; +using SharedMemoryStore.Diagnostics; +using SharedMemoryStore.Engines; +using SharedMemoryStore.Lifecycle; + +namespace SharedMemoryStore.UnitTests; + +public sealed class StoreLifecycleGateBudgetTests +{ + [Fact] + public void BoundedEntryPreservesExpiredCanceledAndTrueNoWaitSemantics() + { + var gate = new StoreLifecycleGate(); + long oldStart = Stopwatch.GetTimestamp() - Stopwatch.Frequency; + + Assert.Equal( + StoreStatus.StoreBusy, + gate.TryEnter( + new StoreWaitOptions(TimeSpan.FromMilliseconds(10)), + oldStart, + out _)); + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + Assert.Equal( + StoreStatus.OperationCanceled, + gate.TryEnter( + new StoreWaitOptions(TimeSpan.FromSeconds(1), cancellation.Token), + Stopwatch.GetTimestamp(), + out _)); + + Assert.Equal( + StoreStatus.Success, + gate.TryEnter(StoreWaitOptions.NoWait, Stopwatch.GetTimestamp(), out var operation)); + operation.Dispose(); + } + + [Fact] + public async Task HighContentionEntryAndDisposalConvergeWithoutUnboundedEntrants() + { + var gate = new StoreLifecycleGate(); + using var start = new ManualResetEventSlim(initialState: false); + int workerCount = Math.Max(8, Environment.ProcessorCount * 2); + long successes = 0; + Task[] workers = Enumerable.Range(0, workerCount) + .Select(_ => Task.Run(() => + { + start.Wait(); + while (true) + { + StoreStatus status = gate.TryEnter( + StoreWaitOptions.NoWait, + Stopwatch.GetTimestamp(), + out StoreLifecycleGate.Operation operation); + if (status == StoreStatus.StoreDisposed) + { + return; + } + + if (status == StoreStatus.StoreBusy) + { + continue; + } + + Assert.Equal(StoreStatus.Success, status); + Interlocked.Increment(ref successes); + Thread.SpinWait(16); + operation.Dispose(); + } + })) + .ToArray(); + + start.Set(); + Assert.True(SpinWait.SpinUntil(() => Volatile.Read(ref successes) >= 1_000, TimeSpan.FromSeconds(5))); + Task disposer = Task.Run(() => + { + Assert.True(gate.TryBeginDispose()); + gate.CompleteDispose(); + }); + + await Task.WhenAll(workers.Append(disposer)).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.True(successes > 0); + Assert.True(gate.IsDisposed); + } + + [Fact] + public void FacadePassesRemainingFiniteTimeAndDoesNotEnterEngineWhenCanceled() + { + var engine = new RecordingEngine(); + using var store = new MemoryStore(engine); + TimeSpan requested = TimeSpan.FromSeconds(1); + + Assert.Equal( + StoreStatus.Success, + store.TryPublish([1], [2], [], new StoreWaitOptions(requested))); + Assert.Equal(1, engine.PublishCalls); + Assert.InRange(engine.LastWait.Timeout, TimeSpan.FromTicks(1), requested - TimeSpan.FromTicks(1)); + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + Assert.Equal( + StoreStatus.OperationCanceled, + store.TryPublish( + [1], + [2], + [], + new StoreWaitOptions(TimeSpan.FromSeconds(1), cancellation.Token))); + Assert.Equal(1, engine.PublishCalls); + Assert.Equal(1, engine.FacadeStatusCalls); + Assert.Equal(StoreStatus.OperationCanceled, engine.LastFacadeStatus); + } + + private sealed class RecordingEngine : IStoreEngine + { + internal int PublishCalls { get; private set; } + internal int FacadeStatusCalls { get; private set; } + internal StoreStatus LastFacadeStatus { get; private set; } + internal StoreWaitOptions LastWait { get; private set; } + public StoreProfile Profile => StoreProfile.LockFree; + public StoreProtocolInfo ProtocolInfo => default; + public StoreStatus RecordFacadeStatus(StoreStatus status) + { + FacadeStatusCalls++; + LastFacadeStatus = status; + return status; + } + + public DiagnosticsSnapshot CreateDisposedDiagnosticsSnapshot() => default; + + public StoreStatus TryPublish(ReadOnlySpan key, ReadOnlySpan value, ReadOnlySpan descriptor, StoreWaitOptions waitOptions) + { + PublishCalls++; + LastWait = waitOptions; + return StoreStatus.Success; + } + + public StoreStatus TryReserve(ReadOnlySpan key, int payloadLength, ReadOnlySpan descriptor, StoreWaitOptions waitOptions, out ReservationHandle reservation) { reservation = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryPublishSegments(ReadOnlySpan key, ReadOnlySequence payload, ReadOnlySpan descriptor, StoreWaitOptions waitOptions, out long copiedBytes) { copiedBytes = 0; return StoreStatus.UnknownFailure; } + public StoreStatus TryAcquire(ReadOnlySpan key, StoreWaitOptions waitOptions, out LeaseHandle lease) { lease = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryRemove(ReadOnlySpan key, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public StoreStatus TryRecoverLeases(LeaseRecoveryOptions options, StoreWaitOptions waitOptions, out LeaseRecoveryReport report) { report = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryRecoverReservations(ReservationRecoveryOptions options, StoreWaitOptions waitOptions, out ReservationRecoveryReport report) { report = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryGetMetrics(StoreWaitOptions waitOptions, out EngineMetrics metrics) { metrics = default; return StoreStatus.UnknownFailure; } + public StoreStatus TryGetDiagnostics(StoreWaitOptions waitOptions, out DiagnosticsSnapshot snapshot) { snapshot = default; return StoreStatus.UnknownFailure; } + public bool IsReservationPending(ReservationHandle reservation) => false; + public int GetReservationBytesWritten(ReservationHandle reservation) => 0; + public Span GetReservationSpan(ReservationHandle reservation, int sizeHint) => []; + public Memory DangerousGetReservationMemory(ReservationHandle reservation, int sizeHint) => Memory.Empty; + public StoreStatus AdvanceReservation(ReservationHandle reservation, int byteCount, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public StoreStatus CommitReservation(ReservationHandle reservation, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public StoreStatus AbortReservation(ReservationHandle reservation, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public bool IsLeaseActive(LeaseHandle lease) => false; + public int GetValueLength(LeaseHandle lease) => 0; + public int GetDescriptorLength(LeaseHandle lease) => 0; + public ReadOnlySpan GetValueSpan(LeaseHandle lease) => []; + public ReadOnlySpan GetDescriptorSpan(LeaseHandle lease) => []; + public StoreStatus ReleaseLease(LeaseHandle lease, StoreWaitOptions waitOptions) => StoreStatus.UnknownFailure; + public void Dispose() { } + } +} diff --git a/tests/SharedMemoryStore.UnitTests/StoreOptionsValidationTests.cs b/tests/SharedMemoryStore.UnitTests/StoreOptionsValidationTests.cs index c1fcc90..ad65ad0 100644 --- a/tests/SharedMemoryStore.UnitTests/StoreOptionsValidationTests.cs +++ b/tests/SharedMemoryStore.UnitTests/StoreOptionsValidationTests.cs @@ -103,4 +103,54 @@ public void ValidateRejectsDimensionsThatCannotBeRepresentedByTheLayout() Assert.Equal(StoreOpenStatus.InvalidOptions, result.Status); Assert.Contains(result.Failures, failure => failure.MemberName == nameof(SharedMemoryStoreOptions.TotalBytes)); } + + [Fact] + public void ValidateAppliesTheSlotCountCeilingOnlyToTheLockFreeProfile() + { + const int firstRejectedLockFreeSlotCount = 1_048_576; + var lockFree = new SharedMemoryStoreOptions + { + Profile = StoreProfile.LockFree, + Name = StoreTestNames.Create(), + OpenMode = OpenMode.CreateNew, + SlotCount = firstRejectedLockFreeSlotCount, + MaxValueBytes = 1, + MaxDescriptorBytes = 0, + MaxKeyBytes = 1, + LeaseRecordCount = 1, + ParticipantRecordCount = 1, + TotalBytes = long.MaxValue + }; + + var lockFreeResult = lockFree.Validate(); + + Assert.False(lockFreeResult.IsValid); + Assert.Equal(StoreOpenStatus.InvalidOptions, lockFreeResult.Status); + var failure = Assert.Single( + lockFreeResult.Failures, + static candidate => candidate.MemberName == nameof(SharedMemoryStoreOptions.SlotCount)); + Assert.Contains("1,048,575", failure.Message, StringComparison.Ordinal); + + long legacyBytes = SharedMemoryStoreOptions.CalculateRequiredBytes( + firstRejectedLockFreeSlotCount, + maxValueBytes: 1, + maxDescriptorBytes: 0, + maxKeyBytes: 1, + leaseRecordCount: 1); + var legacy = new SharedMemoryStoreOptions + { + Profile = StoreProfile.Legacy, + Name = StoreTestNames.Create(), + OpenMode = OpenMode.CreateNew, + SlotCount = firstRejectedLockFreeSlotCount, + MaxValueBytes = 1, + MaxDescriptorBytes = 0, + MaxKeyBytes = 1, + LeaseRecordCount = 1, + ParticipantRecordCount = 0, + TotalBytes = legacyBytes + }; + + Assert.Equal(StoreOpenStatus.Success, legacy.Validate().Status); + } } diff --git a/tests/SharedMemoryStore.UnitTests/SyncProbeCompletionTargetPolicyTests.cs b/tests/SharedMemoryStore.UnitTests/SyncProbeCompletionTargetPolicyTests.cs new file mode 100644 index 0000000..6be1c27 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/SyncProbeCompletionTargetPolicyTests.cs @@ -0,0 +1,77 @@ +using SharedMemoryStore; + +namespace SharedMemoryStore.UnitTests; + +public sealed class SyncProbeCompletionTargetPolicyTests +{ + private const long MixedTarget = 100_000_000; + private const long FrameTarget = 100_000; + + [Theory] + [InlineData(StoreProfile.Legacy, (int)ProbeScenarioKind.MixedChurn, 0, 0)] + [InlineData(StoreProfile.Legacy, (int)ProbeScenarioKind.LargeIngest, 0, 0)] + [InlineData(StoreProfile.LockFree, (int)ProbeScenarioKind.MixedChurn, MixedTarget, 0)] + [InlineData(StoreProfile.LockFree, (int)ProbeScenarioKind.LargeIngest, 0, FrameTarget)] + [InlineData(StoreProfile.LockFree, (int)ProbeScenarioKind.Autonomous, 0, 0)] + [InlineData(StoreProfile.LockFree, (int)ProbeScenarioKind.BrokerDirected, 0, 0)] + public void LockFreeOnlyPolicyResolvesExactlyOneApplicableTarget( + StoreProfile profile, + int scenario, + long expectedOperations, + long expectedFrames) + { + ProbeRunTargets targets = ProbeCompletionTargetPolicy.Resolve( + profile, + (ProbeScenarioKind)scenario, + [StoreProfile.LockFree], + MixedTarget, + FrameTarget); + + Assert.Equal(expectedOperations, targets.OperationTarget); + Assert.Equal(expectedFrames, targets.FrameTarget); + Assert.False(targets.OperationTarget > 0 && targets.FrameTarget > 0); + Assert.Equal(expectedOperations > 0 || expectedFrames > 0, targets.HasCountTarget); + } + + [Theory] + [InlineData(StoreProfile.Legacy)] + [InlineData(StoreProfile.LockFree)] + public void DefaultBothPolicyPreservesPreviousCountBoundBehavior(StoreProfile profile) + { + StoreProfile[] both = [StoreProfile.Legacy, StoreProfile.LockFree]; + + Assert.Equal( + new ProbeRunTargets(MixedTarget, 0), + ProbeCompletionTargetPolicy.Resolve( + profile, + ProbeScenarioKind.MixedChurn, + both, + MixedTarget, + FrameTarget)); + Assert.Equal( + new ProbeRunTargets(0, FrameTarget), + ProbeCompletionTargetPolicy.Resolve( + profile, + ProbeScenarioKind.LargeIngest, + both, + MixedTarget, + FrameTarget)); + } + + [Fact] + public void NegativeTargetsAreRejectedBeforeProfileResolution() + { + Assert.Throws(() => ProbeCompletionTargetPolicy.Resolve( + StoreProfile.Legacy, + ProbeScenarioKind.Autonomous, + [], + -1, + FrameTarget)); + Assert.Throws(() => ProbeCompletionTargetPolicy.Resolve( + StoreProfile.Legacy, + ProbeScenarioKind.Autonomous, + [], + MixedTarget, + -1)); + } +} diff --git a/tests/SharedMemoryStore.UnitTests/SyncProbeTrialWatchdogTests.cs b/tests/SharedMemoryStore.UnitTests/SyncProbeTrialWatchdogTests.cs new file mode 100644 index 0000000..4cf6a13 --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/SyncProbeTrialWatchdogTests.cs @@ -0,0 +1,104 @@ +namespace SharedMemoryStore.UnitTests; + +public sealed class SyncProbeTrialWatchdogTests +{ + [Fact] + public async Task CompletionBeforeDeadlineCancelsTimeoutAction() + { + var invocations = 0; + var watchdog = new ProbeTrialWatchdog( + TimeSpan.FromSeconds(1), + () => Interlocked.Increment(ref invocations)); + + await watchdog.CompleteAsync(); + await Task.Delay(100); + + Assert.Equal(0, Volatile.Read(ref invocations)); + } + + [Fact] + public async Task AbsoluteDeadlineRejectsLateCompletionWhenTimerDispatchIsDelayed() + { + var invocations = 0; + var watchdog = new ProbeTrialWatchdog( + TimeSpan.FromMilliseconds(25), + () => Interlocked.Increment(ref invocations), + timerDueTime: TimeSpan.FromSeconds(5)); + await Task.Delay(100); + + TimeoutException timeout = await Assert.ThrowsAsync( + () => watchdog.CompleteAsync().AsTask()); + + Assert.Contains("timeout action returned", timeout.Message, StringComparison.Ordinal); + Assert.Equal(1, Volatile.Read(ref invocations)); + } + + [Fact] + public async Task PostLatchDeadlineCheckRejectsCompletionThatCrossesBoundary() + { + var invocations = 0; + var watchdog = new ProbeTrialWatchdog( + TimeSpan.FromMilliseconds(250), + () => Interlocked.Increment(ref invocations), + timerDueTime: TimeSpan.FromSeconds(5), + afterCompletionLatch: () => Thread.Sleep(400)); + + await Assert.ThrowsAsync(() => watchdog.CompleteAsync().AsTask()); + + Assert.Equal(1, Volatile.Read(ref invocations)); + } + + [Fact] + public async Task TimerCanTimeOutCompletionWhileCompletingLatchIsBlocked() + { + var completionLatchEntered = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var timeoutInvoked = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + using var releaseCompletionLatch = new ManualResetEventSlim(); + var watchdog = new ProbeTrialWatchdog( + TimeSpan.FromMilliseconds(100), + () => timeoutInvoked.TrySetResult(), + afterCompletionLatch: () => + { + completionLatchEntered.TrySetResult(); + releaseCompletionLatch.Wait(); + }); + + Task completion = Task.Factory.StartNew( + () => watchdog.CompleteAsync().AsTask(), + CancellationToken.None, + TaskCreationOptions.LongRunning, + TaskScheduler.Default).Unwrap(); + try + { + await completionLatchEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await timeoutInvoked.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.False(completion.IsCompleted); + } + finally + { + releaseCompletionLatch.Set(); + } + + await Assert.ThrowsAsync(() => completion); + } + + [Fact] + public async Task TimerDispatchInvokesTimeoutAction() + { + var invoked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var invocations = 0; + await using var watchdog = new ProbeTrialWatchdog( + TimeSpan.FromMilliseconds(25), + () => + { + Interlocked.Increment(ref invocations); + invoked.TrySetResult(); + }); + + await invoked.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, Volatile.Read(ref invocations)); + } +} diff --git a/tests/SharedMemoryStore.UnitTests/TestSupport/ControlledLockFreeScheduler.cs b/tests/SharedMemoryStore.UnitTests/TestSupport/ControlledLockFreeScheduler.cs new file mode 100644 index 0000000..92ab97f --- /dev/null +++ b/tests/SharedMemoryStore.UnitTests/TestSupport/ControlledLockFreeScheduler.cs @@ -0,0 +1,188 @@ +using SharedMemoryStore.LockFree; + +namespace SharedMemoryStore.UnitTests.TestSupport; + +/// +/// Deterministically pauses an instrumented lock-free protocol invocation at a +/// selected canonical checkpoint. The scheduler never participates in ordinary +/// production construction. +/// +internal sealed class ControlledLockFreeScheduler : IDisposable +{ + private readonly object _sync = new(); + private readonly List _observations = []; + private readonly ManualResetEventSlim _paused = new(initialState: false); + private readonly ManualResetEventSlim _resume = new(initialState: true); + private LockFreeCheckpointId? _target; + private int _targetOccurrence; + private int _observedTargetOccurrences; + private long _targetVersion; + private long _sequence; + private bool _disposed; + + internal InstrumentedLockFreeCheckpoint CreateInstrumentedCheckpoint() + { + ThrowIfDisposed(); + return LockFreeCheckpointFactory.CreateInstrumented(Observe); + } + + internal void PauseAt(LockFreeCheckpointId checkpoint, int occurrence = 1) + { + ArgumentOutOfRangeException.ThrowIfLessThan(occurrence, 1); + _ = LockFreeCheckpointCatalog.Get(checkpoint); + + lock (_sync) + { + ThrowIfDisposed(); + if (_target.HasValue) + { + throw new InvalidOperationException("A checkpoint pause is already armed."); + } + + _target = checkpoint; + _targetOccurrence = occurrence; + _observedTargetOccurrences = 0; + _targetVersion++; + _paused.Reset(); + _resume.Reset(); + } + } + + internal bool WaitUntilPaused(TimeSpan timeout) + { + ThrowIfDisposed(); + return _paused.Wait(timeout); + } + + internal bool WaitUntilPaused(TimeSpan timeout, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _paused.Wait(timeout, cancellationToken); + } + + internal void Continue() + { + ThrowIfDisposed(); + _resume.Set(); + } + + /// + /// Arms the next checkpoint before releasing the invocation currently + /// paused by this scheduler. This removes the scheduling gap between a + /// Continue call and a subsequent PauseAt call. + /// + internal void ContinueAndPauseAt( + LockFreeCheckpointId checkpoint, + int occurrence = 1) + { + ArgumentOutOfRangeException.ThrowIfLessThan(occurrence, 1); + _ = LockFreeCheckpointCatalog.Get(checkpoint); + + lock (_sync) + { + ThrowIfDisposed(); + if (!_target.HasValue || !_paused.IsSet) + { + throw new InvalidOperationException( + "No checkpoint invocation is currently paused."); + } + + _target = checkpoint; + _targetOccurrence = occurrence; + _observedTargetOccurrences = 0; + _targetVersion++; + _paused.Reset(); + _resume.Set(); + } + } + + internal IReadOnlyList Snapshot() + { + lock (_sync) + { + return _observations.ToArray(); + } + } + + internal void ClearObservations() + { + lock (_sync) + { + ThrowIfDisposed(); + _observations.Clear(); + } + } + + public void Dispose() + { + lock (_sync) + { + if (_disposed) + { + return; + } + + _disposed = true; + _target = null; + _resume.Set(); + _paused.Set(); + } + + _paused.Dispose(); + _resume.Dispose(); + } + + private void Observe(LockFreeCheckpointEntry entry) + { + bool pause; + long pauseVersion; + long sequence = Interlocked.Increment(ref _sequence); + lock (_sync) + { + if (_disposed) + { + return; + } + + _observations.Add(new Observation(sequence, Environment.CurrentManagedThreadId, entry)); + pause = _target == entry.Id + && ++_observedTargetOccurrences == _targetOccurrence; + pauseVersion = pause ? _targetVersion : 0; + } + + if (!pause) + { + return; + } + + _paused.Set(); + _resume.Wait(); + + lock (_sync) + { + if (_targetVersion == pauseVersion) + { + _target = null; + _targetOccurrence = 0; + _observedTargetOccurrences = 0; + } + else + { + // ContinueAndPauseAt deliberately left a new target armed. The + // resumed invocation cannot reach it until this callback + // returns, so reset the shared gate here without a race. + _resume.Reset(); + } + } + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(_disposed, this); + } + + internal readonly record struct Observation( + long Sequence, + int ManagedThreadId, + LockFreeCheckpointEntry Entry); +}