From 6b09939d2805a1b5782324f0ff8b1b758ad79c41 Mon Sep 17 00:00:00 2001 From: Ran Trifon Date: Fri, 10 Jul 2026 15:04:57 +0300 Subject: [PATCH 1/3] Add native C++ and Python implementations --- .dockerignore | 13 + .github/workflows/ci.yml | 75 ++ .gitignore | 13 + .specify/feature.json | 2 +- AGENTS.md | 2 +- CHANGELOG.md | 41 + CMakeLists.txt | 82 ++ README.md | 117 ++- SECURITY.md | 34 +- SUPPORT.md | 27 +- SharedMemoryStore.slnx | 2 + cmake/SharedMemoryStoreConfig.cmake.in | 13 + docs/architecture.md | 84 +- docs/concepts.md | 11 +- docs/getting-started.md | 88 +- docs/index.md | 2 +- docs/maintainers.md | 52 +- docs/packaging.md | 109 +- docs/portability.md | 92 +- docs/releases.md | 104 +- docs/samples.md | 46 + protocol/README.md | 63 ++ protocol/compatibility.json | 61 ++ protocol/fixtures/v1.2/empty.bin | Bin 0 -> 1016 bytes protocol/fixtures/v1.2/empty.snapshot.json | 90 ++ protocol/fixtures/v1.2/generate-fixtures.ps1 | 402 +++++++ protocol/fixtures/v1.2/manifest.json | 678 ++++++++++++ protocol/fixtures/v1.2/pending-removal.bin | Bin 0 -> 1016 bytes .../v1.2/pending-removal.snapshot.json | 112 ++ .../fixtures/v1.2/pending-reservation.bin | Bin 0 -> 1016 bytes .../v1.2/pending-reservation.snapshot.json | 101 ++ protocol/fixtures/v1.2/published.bin | Bin 0 -> 1016 bytes .../fixtures/v1.2/published.snapshot.json | 101 ++ protocol/fixtures/v1.2/reused-slot.bin | Bin 0 -> 1016 bytes .../fixtures/v1.2/reused-slot.snapshot.json | 111 ++ protocol/layout-v1.2.md | 191 ++++ protocol/resource-naming-v1.md | 117 +++ pyproject.toml | 48 + samples/CppBasicUsage/CMakeLists.txt | 26 + samples/CppBasicUsage/README.md | 6 + samples/CppBasicUsage/main.cpp | 38 + samples/PythonBasicUsage/README.md | 12 + samples/PythonBasicUsage/main.py | 45 + scripts/validate-docker-shared-memory.ps1 | 18 +- scripts/validate-interoperability.ps1 | 138 +++ scripts/validate-native.ps1 | 81 ++ scripts/validate-python.ps1 | 248 +++++ .../contracts/ingest-layout.md | 6 +- .../checklists/requirements.md | 36 + .../contracts/cpp-api.md | 27 + .../contracts/interoperability.md | 44 + .../contracts/native-c-api.md | 62 ++ .../contracts/packaging.md | 31 + .../contracts/python-api.md | 38 + .../data-model.md | 114 ++ specs/008-cpp-python-implementations/plan.md | 208 ++++ .../quickstart.md | 57 + .../research.md | 125 +++ specs/008-cpp-python-implementations/spec.md | 240 +++++ specs/008-cpp-python-implementations/tasks.md | 273 +++++ .../Interop/LinuxSharedMemoryRegion.cs | 12 +- .../Interop/SharedStorePlatform.cs | 5 + .../SharedMemoryStore.csproj | 2 +- src/cpp/CMakeLists.txt | 141 +++ src/cpp/include/shared_memory_store/c_api.h | 332 ++++++ src/cpp/include/shared_memory_store/store.hpp | 347 ++++++ src/cpp/src/c_api.cpp | 443 ++++++++ src/cpp/src/internal.hpp | 421 ++++++++ src/cpp/src/platform_linux.cpp | 412 ++++++++ src/cpp/src/platform_windows.cpp | 154 +++ src/cpp/src/protocol.cpp | 394 +++++++ src/cpp/src/store.cpp | 910 ++++++++++++++++ src/python/shared_memory_store/__init__.py | 45 + src/python/shared_memory_store/_native.py | 416 ++++++++ src/python/shared_memory_store/enums.py | 61 ++ src/python/shared_memory_store/store.py | 890 ++++++++++++++++ .../MultiStoreLifecycleIntegrationTests.cs | 36 + .../AgentHost.cs | 53 + .../AgentMessages.cs | 52 + .../AgentProtocol.cs | 146 +++ .../AgentSession.cs | 412 ++++++++ .../SharedMemoryStore.InteropAgent/Program.cs | 3 + .../SharedMemoryStore.InteropAgent.csproj | 15 + .../AgentLifecycleTests.cs | 98 ++ .../AgentProtocolTests.cs | 131 +++ .../CoreExchangeMatrixTests.cs | 128 +++ .../DiagnosticsInteropTests.cs | 104 ++ .../SharedMemoryStore.InteropTests/Dockerfile | 34 + .../MixedLifecycleTests.cs | 215 ++++ .../RecoveryAndOwnershipTests.cs | 243 +++++ .../SharedMemoryStore.InteropTests.csproj | 25 + .../StressInteropTests.cs | 365 +++++++ .../TestSupport/AgentProcess.cs | 214 ++++ .../TestSupport/ForeignStoreLock.cs | 180 ++++ .../TestSupport/InteropAssertions.cs | 47 + tests/cpp/CMakeLists.txt | 78 ++ tests/cpp/c_api_tests.cpp | 64 ++ tests/cpp/diagnostics_tests.cpp | 25 + tests/cpp/interop_agent.cpp | 987 ++++++++++++++++++ tests/cpp/lifecycle_tests.cpp | 49 + tests/cpp/package_consumer/CMakeLists.txt | 20 + tests/cpp/package_consumer/main.cpp | 35 + tests/cpp/protocol_tests.cpp | 51 + tests/cpp/store_tests.cpp | 38 + tests/cpp/test_support.hpp | 52 + tests/python/_support.py | 38 + tests/python/interop_agent.py | 358 +++++++ tests/python/test_diagnostics.py | 52 + tests/python/test_installed_package.py | 45 + tests/python/test_interop_agent.py | 123 +++ tests/python/test_lifecycle.py | 135 +++ tests/python/test_native_abi.py | 99 ++ tests/python/test_protocol_manifest.py | 628 +++++++++++ tests/python/test_store.py | 180 ++++ 114 files changed, 15353 insertions(+), 147 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 cmake/SharedMemoryStoreConfig.cmake.in create mode 100644 protocol/README.md create mode 100644 protocol/compatibility.json create mode 100644 protocol/fixtures/v1.2/empty.bin create mode 100644 protocol/fixtures/v1.2/empty.snapshot.json create mode 100644 protocol/fixtures/v1.2/generate-fixtures.ps1 create mode 100644 protocol/fixtures/v1.2/manifest.json create mode 100644 protocol/fixtures/v1.2/pending-removal.bin create mode 100644 protocol/fixtures/v1.2/pending-removal.snapshot.json create mode 100644 protocol/fixtures/v1.2/pending-reservation.bin create mode 100644 protocol/fixtures/v1.2/pending-reservation.snapshot.json create mode 100644 protocol/fixtures/v1.2/published.bin create mode 100644 protocol/fixtures/v1.2/published.snapshot.json create mode 100644 protocol/fixtures/v1.2/reused-slot.bin create mode 100644 protocol/fixtures/v1.2/reused-slot.snapshot.json create mode 100644 protocol/layout-v1.2.md create mode 100644 protocol/resource-naming-v1.md create mode 100644 pyproject.toml create mode 100644 samples/CppBasicUsage/CMakeLists.txt create mode 100644 samples/CppBasicUsage/README.md create mode 100644 samples/CppBasicUsage/main.cpp create mode 100644 samples/PythonBasicUsage/README.md create mode 100644 samples/PythonBasicUsage/main.py create mode 100644 scripts/validate-interoperability.ps1 create mode 100644 scripts/validate-native.ps1 create mode 100644 scripts/validate-python.ps1 create mode 100644 specs/008-cpp-python-implementations/checklists/requirements.md create mode 100644 specs/008-cpp-python-implementations/contracts/cpp-api.md create mode 100644 specs/008-cpp-python-implementations/contracts/interoperability.md create mode 100644 specs/008-cpp-python-implementations/contracts/native-c-api.md create mode 100644 specs/008-cpp-python-implementations/contracts/packaging.md create mode 100644 specs/008-cpp-python-implementations/contracts/python-api.md create mode 100644 specs/008-cpp-python-implementations/data-model.md create mode 100644 specs/008-cpp-python-implementations/plan.md create mode 100644 specs/008-cpp-python-implementations/quickstart.md create mode 100644 specs/008-cpp-python-implementations/research.md create mode 100644 specs/008-cpp-python-implementations/spec.md create mode 100644 specs/008-cpp-python-implementations/tasks.md create mode 100644 src/cpp/CMakeLists.txt create mode 100644 src/cpp/include/shared_memory_store/c_api.h create mode 100644 src/cpp/include/shared_memory_store/store.hpp create mode 100644 src/cpp/src/c_api.cpp create mode 100644 src/cpp/src/internal.hpp create mode 100644 src/cpp/src/platform_linux.cpp create mode 100644 src/cpp/src/platform_windows.cpp create mode 100644 src/cpp/src/protocol.cpp create mode 100644 src/cpp/src/store.cpp create mode 100644 src/python/shared_memory_store/__init__.py create mode 100644 src/python/shared_memory_store/_native.py create mode 100644 src/python/shared_memory_store/enums.py create mode 100644 src/python/shared_memory_store/store.py create mode 100644 tests/SharedMemoryStore.InteropAgent/AgentHost.cs create mode 100644 tests/SharedMemoryStore.InteropAgent/AgentMessages.cs create mode 100644 tests/SharedMemoryStore.InteropAgent/AgentProtocol.cs create mode 100644 tests/SharedMemoryStore.InteropAgent/AgentSession.cs create mode 100644 tests/SharedMemoryStore.InteropAgent/Program.cs create mode 100644 tests/SharedMemoryStore.InteropAgent/SharedMemoryStore.InteropAgent.csproj create mode 100644 tests/SharedMemoryStore.InteropTests/AgentLifecycleTests.cs create mode 100644 tests/SharedMemoryStore.InteropTests/AgentProtocolTests.cs create mode 100644 tests/SharedMemoryStore.InteropTests/CoreExchangeMatrixTests.cs create mode 100644 tests/SharedMemoryStore.InteropTests/DiagnosticsInteropTests.cs create mode 100644 tests/SharedMemoryStore.InteropTests/Dockerfile create mode 100644 tests/SharedMemoryStore.InteropTests/MixedLifecycleTests.cs create mode 100644 tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs create mode 100644 tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj create mode 100644 tests/SharedMemoryStore.InteropTests/StressInteropTests.cs create mode 100644 tests/SharedMemoryStore.InteropTests/TestSupport/AgentProcess.cs create mode 100644 tests/SharedMemoryStore.InteropTests/TestSupport/ForeignStoreLock.cs create mode 100644 tests/SharedMemoryStore.InteropTests/TestSupport/InteropAssertions.cs create mode 100644 tests/cpp/CMakeLists.txt create mode 100644 tests/cpp/c_api_tests.cpp create mode 100644 tests/cpp/diagnostics_tests.cpp create mode 100644 tests/cpp/interop_agent.cpp create mode 100644 tests/cpp/lifecycle_tests.cpp create mode 100644 tests/cpp/package_consumer/CMakeLists.txt create mode 100644 tests/cpp/package_consumer/main.cpp create mode 100644 tests/cpp/protocol_tests.cpp create mode 100644 tests/cpp/store_tests.cpp create mode 100644 tests/cpp/test_support.hpp create mode 100644 tests/python/_support.py create mode 100644 tests/python/interop_agent.py create mode 100644 tests/python/test_diagnostics.py create mode 100644 tests/python/test_installed_package.py create mode 100644 tests/python/test_interop_agent.py create mode 100644 tests/python/test_lifecycle.py create mode 100644 tests/python/test_native_abi.py create mode 100644 tests/python/test_protocol_manifest.py create mode 100644 tests/python/test_store.py diff --git a/.dockerignore b/.dockerignore index 216bc69..fbf2d90 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,9 +6,22 @@ artifacts/ BenchmarkDotNet.Artifacts/ bin/ obj/ +build/ +out/ TestResults/ coverage/ node_modules/ +__pycache__/ +*.py[cod] +.venv/ +venv/ +dist/ +*.egg-info/ +*.o +*.so +*.a +*.dll +*.exe *.log *.log.* *.trx diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b95c6a5..106c856 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,3 +72,78 @@ jobs: - name: Validate Docker sharing shell: pwsh run: ./scripts/validate-docker-shared-memory.ps1 -Configuration Release + + native: + name: Native (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + + steps: + - name: Check out source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Validate CMake build, tests, install, and clean consumer + shell: pwsh + run: ./scripts/validate-native.ps1 -Configuration Release + + python: + name: Python (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + + steps: + - name: Check out source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Install Python build frontend + run: python -m pip install --upgrade build + + - name: Validate source tests, wheel contents, and clean installation + shell: pwsh + run: ./scripts/validate-python.ps1 -Configuration Release + + interoperability: + name: Interoperability (${{ matrix.os }}) + needs: [native, python] + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + + steps: + - name: Check out source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up .NET 10 + uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 + with: + dotnet-version: 10.0.x + + - name: Set up Python 3.12 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Validate all ordered runtime pairs and stress lifecycles + shell: pwsh + run: >- + ./scripts/validate-interoperability.ps1 + -Configuration Release + -Stress + -StressValueCount 1000 + -StressLifecycleCycleCount 10000 diff --git a/.gitignore b/.gitignore index 4823426..095d452 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,21 @@ bin/ obj/ +build/ +out/ artifacts/ TestResults/ BenchmarkDotNet.Artifacts/ +__pycache__/ +*.py[cod] +.venv/ +venv/ +dist/ +*.egg-info/ +*.o +*.so +*.a +*.dll +*.exe *.log *.trx *.tmp diff --git a/.specify/feature.json b/.specify/feature.json index 052c863..df1504a 100644 --- a/.specify/feature.json +++ b/.specify/feature.json @@ -1,3 +1,3 @@ { - "feature_directory": "specs/007-linux-windows-support" + "feature_directory": "specs/008-cpp-python-implementations" } diff --git a/AGENTS.md b/AGENTS.md index e66241b..9f9c3fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,5 @@ For additional context about technologies to be used, project structure, shell commands, and other important information, read the current plan -at specs/007-linux-windows-support/plan.md +at specs/008-cpp-python-implementations/plan.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 12dfd34..af7c78d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,47 @@ All notable package and documentation changes are recorded in reverse chronological order. +## Unreleased + +### Added + +- Added an independently versioned CMake `SharedMemoryStore` `0.1.0` + distribution with a C++20 protocol core, Windows and Linux adapters, + fixed-width C ABI `1.0`, move-only C++ RAII wrappers, dependency-free native + tests, install/export rules, clean-consumer project, and basic sample. +- Added the Python `shared-memory-store` `0.1.0` source and wheel configuration + for Python 3.10 or newer. Its standard-library `ctypes` API packages and + validates the native library beside the Python modules and exposes + context-managed stores, leases, reservations, recovery, waits, and + diagnostics. +- Added the canonical language-neutral protocol boundary for layout `1.2`, + resource naming `1`, exact conformance fixtures, compatibility metadata, and + test-only .NET, C++, and Python JSON-lines participants. +- Added an ordered 3x3 core exchange harness plus native and Python contract, + lifecycle, ABI, diagnostics, and package tests. + +### Security + +- Preserved the trusted same-host participant boundary across all three + distributions. Native and Python callers receive lifecycle and layout + validation, but no implementation claims protection from a malicious writer + that already has legitimate access to the shared resources. +- Restricted Python native loading to the library packaged beside its modules; + it does not search the current directory, `PATH`, or system library paths. + +### Compatibility + +- NuGet `SharedMemoryStore` remains `1.0.1`; its public managed API, status + numbers, BCL-only runtime dependency surface, and layout `1.2` behavior are + unchanged. The native and Python artifacts are sibling distributions and are + not included in the NuGet package. +- NuGet, CMake, and Python versions advance independently. Cross-runtime + compatibility is declared through layout `1.2`, resource naming `1`, and C + ABI `1.0` where applicable. +- The native and Python `0.1.0` lines are alpha. Registry publication and full + Windows/Linux ordered-pair release evidence remain explicit publication + gates rather than claims made by this entry. + ## 1.0.1 - 2026-07-10 ### Packaging diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..ad65961 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,82 @@ +cmake_minimum_required(VERSION 3.20) + +project( + SharedMemoryStore + VERSION 0.1.0 + DESCRIPTION "Cross-process shared memory value store" + LANGUAGES CXX) + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) +find_package(Threads REQUIRED) + +option(SMS_BUILD_TESTS "Build the dependency-free native test suite" OFF) +option(SMS_BUILD_SAMPLES "Build native usage samples" OFF) +option(SMS_BUILD_STATIC "Build the optional static native library" OFF) + +set(_sms_install_default ON) +if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + set(_sms_install_default OFF) +endif() +option(SMS_INSTALL "Generate native install and CMake package rules" + "${_sms_install_default}") +unset(_sms_install_default) +set( + SMS_PYTHON_INSTALL_DIR + "" + CACHE STRING + "Optional wheel-relative directory for the Python native library") + +add_subdirectory(src/cpp) + +if(SMS_INSTALL AND NOT SMS_PYTHON_INSTALL_DIR STREQUAL "") + install( + FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/python/shared_memory_store/__init__.py" + "${CMAKE_CURRENT_SOURCE_DIR}/src/python/shared_memory_store/_native.py" + "${CMAKE_CURRENT_SOURCE_DIR}/src/python/shared_memory_store/enums.py" + "${CMAKE_CURRENT_SOURCE_DIR}/src/python/shared_memory_store/store.py" + DESTINATION "${SMS_PYTHON_INSTALL_DIR}" + COMPONENT Python) +endif() + +if(SMS_BUILD_TESTS) + include(CTest) + enable_testing() + add_subdirectory(tests/cpp) +endif() + +if(SMS_BUILD_SAMPLES) + add_subdirectory(samples/CppBasicUsage) +endif() + +if(SMS_INSTALL) + set( + SMS_INSTALL_CMAKEDIR + "${CMAKE_INSTALL_LIBDIR}/cmake/SharedMemoryStore" + CACHE STRING "Directory used to install the CMake package files") + + configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/SharedMemoryStoreConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/SharedMemoryStoreConfig.cmake" + INSTALL_DESTINATION "${SMS_INSTALL_CMAKEDIR}") + + write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/SharedMemoryStoreConfigVersion.cmake" + VERSION "${PROJECT_VERSION}" + COMPATIBILITY SameMajorVersion) + + install( + EXPORT SharedMemoryStoreTargets + FILE SharedMemoryStoreTargets.cmake + NAMESPACE SharedMemoryStore:: + DESTINATION "${SMS_INSTALL_CMAKEDIR}" + COMPONENT Development) + + install( + FILES + "${CMAKE_CURRENT_BINARY_DIR}/SharedMemoryStoreConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/SharedMemoryStoreConfigVersion.cmake" + DESTINATION "${SMS_INSTALL_CMAKEDIR}" + COMPONENT Development) +endif() diff --git a/README.md b/README.md index 1f6a949..4e3ac47 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,66 @@ # SharedMemoryStore -SharedMemoryStore is a `net10.0` library package for bounded named -shared-memory key-value storage. It stores opaque byte keys, optional descriptor -bytes, and immutable payload bytes in a memory-mapped region so producers and -readers can exchange data without copying payloads through a broker process. - -Package identity: - -- PackageId: `SharedMemoryStore` -- Version: `1.0.1` -- Target framework: `net10.0` -- License: MIT, see the [license file](LICENSE) -- Runtime dependencies: .NET BCL only - -The `1.0.0` package establishes the production public API contract. Linux and -Windows are supported runtime and development targets. Same-host Linux Docker -containers are supported when they share the required IPC, owner-liveness, -permission, and shared-memory capacity capabilities. C++ and Python are future -portability audiences, not current bindings. +SharedMemoryStore is a monorepo for bounded named shared-memory key-value +storage. Its .NET, native C++, and Python distributions store opaque byte keys, +optional descriptor bytes, and immutable payload bytes in one versioned +memory-mapped protocol so same-host processes can exchange data without copying +payloads through a broker process. + +Distribution identities: + +- NuGet: `SharedMemoryStore` `1.0.1`, targeting `net10.0` with .NET BCL + runtime dependencies only. +- CMake: `SharedMemoryStore` `0.1.0`, exposing a C++20 RAII API and fixed-width + C ABI `1.0` over the native shared library. +- Python: `shared-memory-store` `0.1.0`, requiring Python 3.10 or newer and + using standard-library `ctypes` with the packaged native library. +- Shared protocol: mapped layout `1.2`, resource naming `1`, little-endian + 64-bit Windows and Linux targets. +- License: MIT, see the [license file](LICENSE). + +The managed `1.0.0` line establishes the production .NET API contract. The +native and Python `0.1.0` lines are independently versioned initial +distributions; they do not change or ship inside the NuGet package. Linux and +Windows are implementation targets. Same-host Linux Docker containers require +shared IPC, owner-liveness, permission, and shared-memory capacity capabilities. +See [Portability](docs/portability.md) and +[Compatibility metadata](protocol/compatibility.json) before combining +independently released distributions. ## What It Provides -The initial public contract supports: +The shared lifecycle implemented by the three public APIs supports: - create or open a named store with explicit capacity limits. - publish immutable value bytes and optional descriptor bytes under an opaque byte key. -- acquire a `ValueLease`, read descriptor and value spans, and release or +- acquire an owning lease, read descriptor and value views, and release or dispose the lease exactly once. - remove values and reuse slots after active readers release their leases. - reserve store-owned payload memory for direct length-delimited frame ingest, advance exact write progress, and commit atomically. -- publish segmented buffered payloads through `ReadOnlySequence` without a - temporary full-payload array. +- publish segmented buffered payloads, including .NET + `ReadOnlySequence`, without a temporary full-payload array. - abort or explicitly recover incomplete reservations without exposing partial bytes to readers. - run owner-controlled stale lease recovery when enabled. - inspect caller-formatted diagnostics snapshots without library console output, including lease recovery results and key-index tombstone health. +The native core implements the mapped protocol and Windows/Linux resource +mechanisms once. The C++ API wraps the C ABI with move-only RAII stores, leases, +and reservations. The Python package uses `ctypes` and context-managed objects +over that same ABI; it does not maintain a second protocol state machine. + The store does not parse frame headers, own application schemas, provide a cross-host cache, persist data beyond process and mapping lifetime, or turn Docker into distributed storage. ## First Use -For package consumers, start with [Getting started](docs/getting-started.md) and -the [Usage guide](docs/usage.md). A local package source workflow is documented -when validating a local build before consuming the published package. +Start with [Getting started](docs/getting-started.md). It separates the NuGet, +CMake, and wheel workflows and explains how processes select the same named +store. The managed package workflow remains: ```powershell dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/package @@ -102,11 +115,31 @@ Current-process lease recovery skips other live owner processes, disposal races return documented statuses or empty token views, and slot lifecycle identity is safe across generation rollover. +Native build and test entry point: + +```powershell +pwsh ./scripts/validate-native.ps1 -Configuration Release +``` + +Python wheel build and installed-sample entry point: + +```powershell +python -m pip install build +python -m build --wheel +python -m venv artifacts/python-consumer +artifacts/python-consumer/Scripts/python -m pip install (Get-ChildItem dist/*.whl | Select-Object -First 1) +artifacts/python-consumer/Scripts/python samples/PythonBasicUsage/main.py +``` + +On Linux, use `artifacts/python-consumer/bin/python`. The Python sample must run +from an installed wheel because the native loader deliberately searches only +the package directory, never the current directory or system library path. + ## Documentation - [Documentation index](docs/index.md): complete table of contents by audience. -- [Getting started](docs/getting-started.md): install, local package source, - minimal workflow, and expected statuses. +- [Getting started](docs/getting-started.md): NuGet, CMake, wheel, minimal + workflow, and interoperability setup. - [Concepts](docs/concepts.md): store, name, key, descriptor, payload, slot, lease, reservation, wait policy, status, diagnostics, recovery, capacity, lifecycle, portability, and package contract vocabulary. @@ -127,18 +160,18 @@ safe across generation rollover. narrow-interface boundaries outside the core package. - [Performance scope](docs/performance.md): measured scope and unmeasured claims. -- [Portability](docs/portability.md): .NET 10 baseline, Linux, Windows, and - same-host Docker support, layout compatibility, and future C++/Python - constraints. +- [Portability](docs/portability.md): distribution versions, Linux, Windows, + same-host Docker, layout compatibility, and cross-runtime constraints. - [Samples](docs/samples.md): ordered runnable sample ladder from minimal usage through frame values, zero-copy ingest, optional hosted integration, and same-host Docker validation. -- [Architecture](docs/architecture.md): maintainer internals, source areas, - invariants, storage, lifecycle, synchronization, recovery, and diagnostics. +- [Architecture](docs/architecture.md): managed and native dependency + direction, source areas, storage, lifecycle, synchronization, recovery, and + diagnostics. - [Maintainers](docs/maintainers.md): documentation update rules, validation commands, contract boundaries, performance evidence, and release impact. -- [Packaging](docs/packaging.md): package metadata, package README, release - notes, and clean consumer validation. +- [Packaging](docs/packaging.md): NuGet, CMake, wheel, compatibility metadata, + release notes, and clean consumer validation. - [Release preparation](docs/releases.md): maintainer checks before publication. Detailed behavior sources: @@ -156,6 +189,12 @@ Detailed behavior sources: - [Contention configuration contract](specs/005-api-production-readiness/contracts/contention-configuration-contract.md) - [Diagnostics integration contract](specs/005-api-production-readiness/contracts/diagnostics-integration-contract.md) - [Reservation memory contract](specs/005-api-production-readiness/contracts/reservation-memory-contract.md) +- [Language-neutral protocol](protocol/README.md) +- [Native C ABI contract](specs/008-cpp-python-implementations/contracts/native-c-api.md) +- [C++ API contract](specs/008-cpp-python-implementations/contracts/cpp-api.md) +- [Python API contract](specs/008-cpp-python-implementations/contracts/python-api.md) +- [Interoperability contract](specs/008-cpp-python-implementations/contracts/interoperability.md) +- [Distribution packaging contract](specs/008-cpp-python-implementations/contracts/packaging.md) Runnable samples: @@ -164,6 +203,8 @@ Runnable samples: - [Zero-copy ingest sample](samples/ZeroCopyIngest/README.md) - [Hosted service integration sample](samples/HostedServiceIntegration/README.md) - [Docker shared-memory sample](samples/DockerSharedMemory/README.md) +- [C++ basic usage sample](samples/CppBasicUsage/README.md) +- [Python basic usage sample](samples/PythonBasicUsage/README.md) ## Project Policies @@ -200,8 +241,12 @@ dotnet test SharedMemoryStore.slnx -c Release dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/package pwsh ./scripts/validate-cross-platform.ps1 -SkipDocker pwsh ./scripts/validate-docker-shared-memory.ps1 +pwsh ./scripts/validate-native.ps1 -Configuration Release +pwsh ./scripts/validate-python.ps1 -Configuration Release +pwsh ./scripts/validate-interoperability.ps1 -Configuration Release -Stress ``` Documentation changes must keep package metadata, README content, release notes, -support policy, security policy, and contract links aligned with the current -`1.0.1` package behavior. +support policy, security policy, compatibility metadata, and contract links +aligned across managed `1.0.1`, native `0.1.0`, Python `0.1.0`, ABI `1.0`, and +layout `1.2`. diff --git a/SECURITY.md b/SECURITY.md index 5890d3f..55dffd0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,14 +1,18 @@ # Security Policy -SharedMemoryStore `1.0.x` receives best-effort community security fixes. This -policy does not promise response or remediation service levels. +Managed SharedMemoryStore `1.0.x` receives best-effort community security +fixes. The native CMake and Python `0.1.x` distributions are alpha and receive +best-effort prerelease security fixes. This policy does not promise response or +remediation service levels. ## Supported Versions | Version | Support | |---------|---------| -| `1.0.x` | Supported with best-effort community security fixes | -| `< 1.0` | Unsupported | +| Managed NuGet `1.0.x` | Supported with best-effort community security fixes | +| Native CMake `0.1.x` | Alpha; best-effort prerelease fixes | +| Python `0.1.x` | Alpha; best-effort prerelease fixes | +| Older lines | Unsupported | Older unpublished or local builds are not independently supported. Users should reproduce reports against the latest package version available to them. @@ -27,8 +31,9 @@ include vulnerability details. Include: -- affected package version. -- OS and .NET runtime. +- affected distribution, package version, C ABI version when applicable, and + layout/resource-naming versions. +- OS, architecture, and .NET runtime, C++ compiler/runtime, or Python version. - store options and operation sequence. - observed status or failure mode. - minimal reproduction that does not expose secrets. @@ -36,3 +41,20 @@ Include: Maintainers will review, request more information if needed, coordinate a fix when appropriate, and decide when public disclosure is safe. + +## Trusted Participant Boundary + +All distributions assume cooperating same-host processes that are authorized +to access the same mapping, lock, and lifecycle resources. They validate +options, record shapes, lifecycle identity, ownership, and visibility, but do +not protect against a malicious participant that already has legitimate write +access and intentionally corrupts mapped bytes, forges metadata, or bypasses +the public API. Cross-host security and hostile multi-tenant sharing are outside +the supported model. + +Reports about native ABI memory safety, unsafe handle or borrowed-view lifetime, +resource permissions, owner cleanup, library substitution, or Python wheel +contents are in scope. The Python loader intentionally accepts only its +package-adjacent native library and validates ABI and protocol metadata; a path +that loads an arbitrary current-directory, `PATH`, or system library should be +reported privately. diff --git a/SUPPORT.md b/SUPPORT.md index 3283d10..61cf93e 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,8 +1,9 @@ # Support -SharedMemoryStore `1.0.x` is a stable community package. Support is best effort and does not -include response-time service levels, production incident response, or paid -support commitments. +Managed SharedMemoryStore `1.0.x` is a stable community package. The native +CMake and Python `0.1.x` distributions are alpha. Support is best effort and +does not include response-time service levels, production incident response, or +paid support commitments. ## Where to Ask @@ -18,9 +19,10 @@ support commitments. Useful reports include: -- package version. +- distribution and package version. - OS and architecture. -- .NET SDK and runtime version. +- .NET SDK/runtime, CMake and C++ compiler/runtime, or Python version. +- C ABI, layout, and resource-naming versions when applicable. - store options. - operation being attempted. - observed `StoreOpenStatus` or `StoreStatus`. @@ -30,16 +32,27 @@ Useful reports include: ## Unsupported Scenarios -The current package does not claim support for: +The current distributions do not claim support for: -- C++ or Python bindings. - macOS runtime support. +- 32-bit processes, big-endian hosts, or architectures without recorded + conformance evidence. - Windows containers, default-isolated Docker containers, or cross-host container sharing. - network-distributed storage. - persistence beyond memory-mapped region lifetime. - application-specific frame parsing by the core store. - response-time service levels. +- protection from a malicious same-host writer that already has legitimate + access to the shared resources. +- availability from PyPI or a native package registry until those publication + channels are explicitly announced. See [Portability](docs/portability.md), [Lifecycle](docs/lifecycle.md), and [Packaging](docs/packaging.md) for current package scope. + +For cross-runtime reports, include both participant distributions, which +process created the store, the ordered producer/consumer direction, exact +binary inputs, and whether every process used the same public name, capacities, +OS identity, IPC namespace, and native library. A skipped interoperability test +or absent agent artifact is not evidence of a supported runtime pair. diff --git a/SharedMemoryStore.slnx b/SharedMemoryStore.slnx index 9014303..4e6cac9 100644 --- a/SharedMemoryStore.slnx +++ b/SharedMemoryStore.slnx @@ -20,6 +20,8 @@ + + diff --git a/cmake/SharedMemoryStoreConfig.cmake.in b/cmake/SharedMemoryStoreConfig.cmake.in new file mode 100644 index 0000000..19b4825 --- /dev/null +++ b/cmake/SharedMemoryStoreConfig.cmake.in @@ -0,0 +1,13 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(Threads) + +include("${CMAKE_CURRENT_LIST_DIR}/SharedMemoryStoreTargets.cmake") + +set(SharedMemoryStore_VERSION "@PROJECT_VERSION@") +set(SharedMemoryStore_ABI_VERSION "1.0") +set(SharedMemoryStore_LAYOUT_VERSION "1.2") +set(SharedMemoryStore_RESOURCE_NAMING_VERSION "1") + +check_required_components(SharedMemoryStore) diff --git a/docs/architecture.md b/docs/architecture.md index 6faf7a1..0e74444 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,13 +14,19 @@ Primary contracts: - [Owner recovery contract](../specs/004-store-reliability-hardening/contracts/owner-recovery-contract.md) - [Disposal and rollover contract](../specs/004-store-reliability-hardening/contracts/disposal-rollover-contract.md) - [Index health contract](../specs/004-store-reliability-hardening/contracts/index-health-contract.md) +- [Language-neutral protocol](../protocol/README.md) +- [Native C ABI contract](../specs/008-cpp-python-implementations/contracts/native-c-api.md) +- [C++ API contract](../specs/008-cpp-python-implementations/contracts/cpp-api.md) +- [Python API contract](../specs/008-cpp-python-implementations/contracts/python-api.md) +- [Interoperability contract](../specs/008-cpp-python-implementations/contracts/interoperability.md) ## Responsibility Boundary -The package is a reusable `net10.0` library. Its public responsibility is to -provide a bounded named shared-memory store for opaque byte keys, optional -descriptor bytes, immutable payload bytes, leases, direct reservations, -segmented publish, explicit recovery, and diagnostics. +The repository delivers independently consumable .NET, native C++, and Python +libraries. Their common responsibility is to provide a bounded named +shared-memory store for opaque byte keys, optional descriptor bytes, immutable +payload bytes, leases, direct reservations, segmented publish, explicit +recovery, and diagnostics through one layout and platform-resource protocol. The package does not own application schemas, frame parsing, persistence, network distribution, hosting, logging, health checks, dependency injection, or @@ -39,14 +45,37 @@ background cleanup. Those belong to consumers or optional adapters. | [`src/SharedMemoryStore/Lifecycle/`](../src/SharedMemoryStore/Lifecycle/) | Store operation gate for disposal-safe public boundaries | Public post-disposal outcomes are contracts | | [`src/SharedMemoryStore/Interop/`](../src/SharedMemoryStore/Interop/) | Platform resource names, memory-mapped region adapters, and shared synchronization adapters for Linux and Windows | Platform behavior is a documented compatibility contract | | [`src/SharedMemoryStore/Options/`](../src/SharedMemoryStore/Options/) | Option validation and detailed validation results | Public option names and validation status are contracts | +| [`protocol/`](../protocol/) | Canonical layout `1.2`, resource naming `1`, compatibility metadata, and conformance fixtures | Language-neutral bytes, names, states, hashes, and version identities are contracts | +| [`src/cpp/include/shared_memory_store/c_api.h`](../src/cpp/include/shared_memory_store/c_api.h) | Fixed-width C ABI `1.0`, versioned structures, statuses, and opaque handles | Exported names, widths, status numbers, ownership, and lifetime rules are ABI contracts | +| [`src/cpp/include/shared_memory_store/store.hpp`](../src/cpp/include/shared_memory_store/store.hpp) | Move-only C++20 RAII stores, leases, reservations, spans, reports, and diagnostics | Public C++ surface and status behavior follow the C++ distribution version | +| [`src/cpp/src/`](../src/cpp/src/) | Native protocol algorithms plus Windows and Linux mapping, lock, ownership, and cleanup adapters | Algorithms may change only while mapped and platform-resource contracts remain compatible | +| [`src/python/shared_memory_store/`](../src/python/shared_memory_store/) | Python enums and context-managed wrappers over the packaged C ABI through `ctypes` | Python public names, result shapes, view ownership, and loader policy follow the Python distribution version | +| [`tests/SharedMemoryStore.InteropTests/`](../tests/SharedMemoryStore.InteropTests/) | JSON-lines agents and ordered runtime-pair orchestration | Test protocol is test-only; observed cross-runtime behavior is release evidence | + +## Dependency Direction + +```text +Python API -> ctypes declarations -> C ABI -> C++ protocol core -> OS adapter +C++ RAII API ------------------------^ | +.NET implementation ------------------------------+-> protocol fixtures +interop agents -> public APIs only +``` + +The C ABI does not depend on Python and never exposes exceptions, C++ standard +library types, platform-sized lengths, or allocator ownership. Python loads the +native library bundled beside its modules and validates ABI, layout, record +sizes, and resource-naming identities before use. The .NET implementation +remains independent of the native ABI; both implementations depend on the +canonical protocol. ## Storage Model The mapped region contains a header, key index, lease registry, slot metadata, -descriptor storage, and payload storage. Capacity is fixed by -`SharedMemoryStoreOptions` at create/open time. `CalculateRequiredBytes` -derives the minimum region length from slot count, value length, descriptor -length, key length, and lease-record count. +descriptor storage, and payload storage. Capacity is fixed at create/open time. +Each API exposes the canonical capacity calculation from slot count, value +length, descriptor length, key length, and lease-record count. Exact records, +offsets, state numbers, and arithmetic are pinned in +[`protocol/layout-v1.2.md`](../protocol/layout-v1.2.md) and its fixtures. Keys, descriptors, and payloads are byte sequences. The layout does not encode application schemas. The consumer may place frame metadata in descriptor bytes @@ -79,9 +108,11 @@ itself. ## Lease Model -`ValueLease` is a struct token that references a slot lifecycle identity and an -active lease record. Leases protect readers from slot reuse. Release is -explicit and status-returning; dispose is best-effort. +A lease token references a slot lifecycle identity and an active lease record. +The managed `ValueLease`, C++ `value_lease`, and Python `ValueLease` protect +readers from slot reuse. Release is explicit and status-returning; disposal, +destruction, and finalization are best-effort fallbacks appropriate to each +runtime. Maintainers must preserve these invariants: @@ -94,11 +125,11 @@ Maintainers must preserve these invariants: ## Reservation Model -`ValueReservation` is a struct token for pending direct ingest. A reservation -announces payload length and descriptor bytes before payload writes. The -producer writes to `GetSpan()` or trusted direct-I/O memory from -`DangerousGetMemory()`, records exact progress with `Advance()`, and publishes -only through `Commit()` after exact completion. +A reservation token represents pending direct ingest. The managed +`ValueReservation`, C++ `value_reservation`, and Python `ValueReservation` +announce payload length and descriptor bytes before payload writes, expose +runtime-appropriate writable views, record exact progress, and publish only +after an exact commit. Pending reservations are invisible to readers but occupy capacity and block duplicate keys. Abort, dispose, and recovery remove the pending index entry @@ -114,6 +145,11 @@ deterministic shared lock resource in the runtime shared-memory location. synchronization. Busy and canceled waits return `StoreBusy` or `OperationCanceled`. +The native adapters reproduce the same Windows mapping/mutex and Linux region, +byte-range lock, owner-sidecar, lifecycle-lock, permission, and cleanup rules. +Matching mapped bytes without matching resource participation is not +interoperability. + Do not add hidden worker threads or implicit global state to avoid contention. Callers choose retry, cancellation, health check, and backoff policy. @@ -151,11 +187,13 @@ benchmark commands or measured validation notes. See ## Portability Model -Current validation is `.NET 10` on Linux, Windows, and the supported same-host -Docker profile. The layout is written so future C++ and Python implementations -can conform, but no current bindings are delivered. Do not use architecture -docs to imply cross-host, macOS, Windows-container, persistence, or distributed -cache support beyond [Portability](portability.md). +The repository now contains managed, C++20, and Python 3.10+ implementations +targeting 64-bit little-endian Linux and Windows. Distribution presence is not +the same as release validation: native tests, wheel installation, clean CMake +consumption, Windows/Linux checks, and the required ordered runtime-pair matrix +must be recorded for each release. Do not imply cross-host, macOS, +Windows-container, persistence, or distributed-cache support beyond +[Portability](portability.md). ## Review Invariants @@ -169,5 +207,5 @@ recovery, public APIs, or package metadata, answer: - Which contract, unit, integration, package, and documentation validations must pass? - Does the wording accidentally promise persistence, distributed-cache - semantics, hidden background work, unsupported platforms, or delivered future - language bindings? + semantics, hidden background work, unsupported platforms, registry + publication, or interoperability evidence that was not actually run? diff --git a/docs/concepts.md b/docs/concepts.md index e4654d3..24122c1 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -170,11 +170,12 @@ rules and abnormal termination behavior. ## Portability -The current package is C# on `.NET 10` with Linux and Windows host support and -same-host Docker support for configured Linux containers. C++ and Python are -future portability audiences, not current bindings. Future implementations must -follow the documented layout and lifecycle contracts rather than redefining -behavior per language. See [Portability](portability.md). +The repository provides independently versioned .NET, CMake/C++, and Python +distributions for 64-bit little-endian Linux and Windows hosts. They interoperate +through mapped layout `1.2`, resource naming `1`, and the same lifecycle rules; +the Python package loads the native C ABI `1.0`. Same-host Docker sharing is +supported for configured Linux containers. See [Portability](portability.md) +and [Packaging](packaging.md). ## Package Contract diff --git a/docs/getting-started.md b/docs/getting-started.md index 5e7c600..30807ca 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,20 +1,31 @@ # Getting Started -This guide gets a clean .NET project from package installation to a complete -create/open, publish, acquire, release, remove, and dispose workflow. +This guide selects one of the .NET, native C++, or Python distributions and +gets a clean consumer to a complete create/open, publish, acquire, release, +remove, and close workflow. All three use layout `1.2` and resource naming `1`. ## Prerequisites -- .NET SDK compatible with `net10.0`. +- .NET SDK compatible with `net10.0` for the managed package and cross-runtime + orchestration. +- CMake 3.20 or newer and a C++20 compiler for the native distribution. +- Python 3.10 or newer plus a PEP 517 build frontend for a source wheel build. - PowerShell 7 (`pwsh`) for repository validation scripts. - Linux or Windows for ordinary runtime and development workflows. - Docker Engine or Docker Desktop only when validating same-host container sharing. -The package version is `1.0.1`. If it has not been published to a package -feed, build a local package source from the repository. +The managed package version is `1.0.1`; the native and Python distributions are +independently versioned `0.1.0`. If an artifact has not been published to its +ecosystem feed, build it locally from the repository. -## Create a Local Package Source +| Consumer | Artifact | Public entry point | +|----------|----------|--------------------| +| .NET | NuGet `SharedMemoryStore` `1.0.1` | `MemoryStore` | +| C++ | CMake `SharedMemoryStore` `0.1.0` | `shared_memory_store::memory_store` | +| Python | wheel `shared-memory-store` `0.1.0` | `shared_memory_store.MemoryStore` | + +## .NET: Create a Local Package Source ```powershell dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/package @@ -27,7 +38,7 @@ dotnet new console -f net10.0 -n SharedMemoryStore.Tryout -o artifacts/tryout dotnet add artifacts/tryout/SharedMemoryStore.Tryout.csproj package SharedMemoryStore --source artifacts/package ``` -## Minimal Workflow +## .NET Minimal Workflow Replace `artifacts/tryout/Program.cs` with this program: @@ -86,6 +97,62 @@ Success Success ``` +## C++: Build and Consume the CMake Package + +Build the shared library, dependency-free native tests, and basic sample: + +```powershell +cmake -S . -B artifacts/native-build -DSMS_BUILD_TESTS=ON -DSMS_BUILD_SAMPLES=ON +cmake --build artifacts/native-build --config Release +ctest --test-dir artifacts/native-build -C Release --output-on-failure +cmake --install artifacts/native-build --config Release --prefix artifacts/native-install +``` + +The installed package exports `SharedMemoryStore::SharedMemoryStore` for +`find_package(SharedMemoryStore CONFIG REQUIRED)`. The C++ sample uses +`store_options::create`, `memory_store::try_create_or_open`, `try_publish`, and +a move-only `value_lease`: + +```powershell +cmake -S samples/CppBasicUsage -B artifacts/cpp-sample -DCMAKE_PREFIX_PATH=artifacts/native-install +cmake --build artifacts/cpp-sample --config Release +``` + +[`scripts/validate-native.ps1`](../scripts/validate-native.ps1) combines the +native build, tests, installation, and clean CMake consumer check. + +## Python: Build and Install a Wheel + +The Python distribution packages the platform native library beside its +modules. Build a wheel, install it into a clean environment, and run the sample: + +```powershell +python -m pip install build +python -m build --wheel +python -m venv artifacts/python-consumer +artifacts/python-consumer/Scripts/python -m pip install (Get-ChildItem dist/*.whl | Select-Object -First 1) +artifacts/python-consumer/Scripts/python samples/PythonBasicUsage/main.py +``` + +On Linux, replace `Scripts/python` with `bin/python`. Installation from a built +wheel does not require a compiler. Do not run the sample by adding +`src/python` to `PYTHONPATH`: the package intentionally loads only its adjacent, +version-checked native library. + +## Open One Store from Multiple Runtimes + +Processes interoperate only when they run on the same supported host and use +the same public store name, capacities, total mapped bytes, layout version, and +resource-naming rules. Keep the creator alive while a second runtime opens with +the equivalent `OpenExisting` mode. Keys, descriptors, and payloads are opaque +bytes and must not be converted through text. Use the compatibility metadata in +[`protocol/compatibility.json`](../protocol/compatibility.json) when combining +independently released versions. + +The test-only JSON-lines agents and ordered-pair suite live under +[`tests/SharedMemoryStore.InteropTests/`](../tests/SharedMemoryStore.InteropTests/). +Their presence does not replace per-platform release evidence. + ## Next Steps - Use [Concepts](concepts.md) for the package vocabulary before advanced @@ -101,9 +168,16 @@ Success - Use [Lifecycle](lifecycle.md) to understand ownership, lease release, removal, stale recovery, and disposal. - Use [Samples](samples.md) for the complete runnable sample ladder. +- Use [Packaging](packaging.md) for native installation, wheel contents, and + independent versioning. +- Use [Portability](portability.md) before mixing runtimes or containers. - Use [samples/BasicUsage/README.md](../samples/BasicUsage/README.md) for a runnable repository sample. - Use [samples/ZeroCopyIngest/README.md](../samples/ZeroCopyIngest/README.md) for direct reservation and segmented publish workflows. - Use [samples/DockerSharedMemory/README.md](../samples/DockerSharedMemory/README.md) when validating same-host Docker container participation. +- Use [samples/CppBasicUsage/README.md](../samples/CppBasicUsage/README.md) for + the native RAII sample. +- Use [samples/PythonBasicUsage/README.md](../samples/PythonBasicUsage/README.md) + for the installed-wheel sample. diff --git a/docs/index.md b/docs/index.md index 8958dd4..4b732ce 100644 --- a/docs/index.md +++ b/docs/index.md @@ -43,7 +43,7 @@ by goal first, then provides the full file inventory for validation and review. | Run samples | [Samples](samples.md) -> [samples/BasicUsage/README.md](../samples/BasicUsage/README.md) -> [samples/FrameValue/README.md](../samples/FrameValue/README.md) -> [samples/ZeroCopyIngest/README.md](../samples/ZeroCopyIngest/README.md) -> [samples/HostedServiceIntegration/README.md](../samples/HostedServiceIntegration/README.md) -> [samples/DockerSharedMemory/README.md](../samples/DockerSharedMemory/README.md) | | Review internals | [Architecture](architecture.md) -> [Maintainers](maintainers.md) -> contract sources | | Prepare a contribution | [CONTRIBUTING.md](../CONTRIBUTING.md) -> [Maintainers](maintainers.md) -> [Release preparation](releases.md) | -| Review future portability | [Concepts](concepts.md) -> [Portability](portability.md) -> [shared-memory-layout.md](../specs/001-frame-memory-store/contracts/shared-memory-layout.md) | +| Review portability and interoperability | [Concepts](concepts.md) -> [Portability](portability.md) -> [Protocol](../protocol/README.md) -> [Packaging](packaging.md) | ## Guides diff --git a/docs/maintainers.md b/docs/maintainers.md index f1986f9..18634a6 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -3,8 +3,10 @@ This guide defines the maintenance process for documentation, samples, package metadata, release notes, public contracts, and evidence-backed claims. Use it with [Architecture](architecture.md), [Release preparation](releases.md), and -the feature validation quickstart at -[specs/006-improve-docs-samples/quickstart.md](../specs/006-improve-docs-samples/quickstart.md). +the managed documentation quickstart at +[specs/006-improve-docs-samples/quickstart.md](../specs/006-improve-docs-samples/quickstart.md) +and the native/Python validation quickstart at +[specs/008-cpp-python-implementations/quickstart.md](../specs/008-cpp-python-implementations/quickstart.md). ## Contract Boundaries @@ -31,6 +33,17 @@ Stable public contracts include: [owner-recovery-contract.md](../specs/004-store-reliability-hardening/contracts/owner-recovery-contract.md). - production public API readiness documented by [public-api-contract.md](../specs/005-api-production-readiness/contracts/public-api-contract.md). +- layout, resource naming, versions, and conformance fixtures under + [`protocol/`](../protocol/). +- fixed-width native C ABI `1.0` documented by + [native-c-api.md](../specs/008-cpp-python-implementations/contracts/native-c-api.md). +- the public C++ and Python surfaces documented by + [cpp-api.md](../specs/008-cpp-python-implementations/contracts/cpp-api.md) + and [python-api.md](../specs/008-cpp-python-implementations/contracts/python-api.md). +- distribution compatibility and packaging documented by + [interoperability.md](../specs/008-cpp-python-implementations/contracts/interoperability.md), + [packaging.md](../specs/008-cpp-python-implementations/contracts/packaging.md), + and [`protocol/compatibility.json`](../protocol/compatibility.json). Current implementation details include private type organization, search cursor choices, compaction thresholds, and helper method structure. They may be @@ -53,6 +66,11 @@ diagnostics, or release status changes. | Performance claim | `docs/performance.md`, benchmark command/result notes, `docs/releases.md`, `CHANGELOG.md` if release-affecting | | Platform or portability scope | `docs/portability.md`, `SUPPORT.md`, sample READMEs, Docker sample docs, release notes | | Package metadata or release notes | `src/SharedMemoryStore/SharedMemoryStore.csproj`, `README.md`, `docs/packaging.md`, `docs/releases.md`, `CHANGELOG.md` | +| C ABI symbol, structure, width, or ownership rule | `c_api.h`, native ABI contract, protocol fixtures, C ABI tests, Python `ctypes` declarations, compatibility metadata, changelog | +| C++ public wrapper | `store.hpp`, C++ API contract, native tests, C++ sample, packaging and getting-started guides | +| Python public API, loader, or view lifetime | Python modules, Python API contract, wheel tests, Python sample, packaging and getting-started guides | +| Layout, resource naming, or cross-runtime behavior | `protocol/`, all three implementations, static conformance tests, ordered-pair tests, portability, security, compatibility metadata | +| Native or Python distribution version | root `CMakeLists.txt` or `pyproject.toml`, compatibility metadata, packaging, README, changelog, release preparation, support policy | | Sample command or output | sample source, sample README, `docs/samples.md`, `specs/006-improve-docs-samples/sample-validation.md` | | Documentation-only clarification | affected doc, `docs/index.md` if navigation changes, `scripts/validate-docs.ps1`, release-impact review | @@ -73,6 +91,15 @@ dotnet test SharedMemoryStore.slnx -c Release dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/package pwsh ./scripts/validate-cross-platform.ps1 -SkipDocker pwsh ./scripts/validate-docker-shared-memory.ps1 +pwsh ./scripts/validate-native.ps1 -Configuration Release +python -m pip install build +python -m build --wheel +python -m venv artifacts/python-consumer +artifacts/python-consumer/Scripts/python -m pip install (Get-ChildItem dist/*.whl | Select-Object -First 1) +$env:SMS_TEST_INSTALLED_PACKAGE = '1' +artifacts/python-consumer/Scripts/python -m unittest discover -s tests/python -v +artifacts/python-consumer/Scripts/python samples/PythonBasicUsage/main.py +dotnet test tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj -c Release ``` Use @@ -84,6 +111,15 @@ for clean package-source consumption. Use [`SharedMemoryStore.slnx`](../SharedMemoryStore.slnx) for build and test coverage. +The native wrapper validates CTest, installation, and a clean external CMake +consumer. Python validation must use a built wheel installed into a clean +environment so repository imports cannot hide a missing native artifact. The +interoperability test run must record which agent executables were available; +the existence of nine theory rows is not evidence that all nine ran on both +platforms. + +Use `bin/python` instead of `Scripts/python` on Linux. + ## Review Questions For every change, answer: @@ -97,7 +133,8 @@ For every change, answer: validated environment? - Does the change preserve the rules against hidden background work, broad core service abstractions, persistence promises, distributed-cache claims, - unsupported platforms, and delivered future bindings? + unsupported platforms, arbitrary native-library loading, and unverified + interoperability claims? ## Documentation-Only Review @@ -135,6 +172,8 @@ Before publishing: - verify package README content in [README.md](../README.md). - align [CHANGELOG.md](../CHANGELOG.md), [Release preparation](releases.md), and [Packaging](packaging.md). +- align NuGet, CMake, Python, C ABI, layout, and resource-naming versions with + [`protocol/compatibility.json`](../protocol/compatibility.json). - review [SUPPORT.md](../SUPPORT.md) and [SECURITY.md](../SECURITY.md). - update sample validation notes in [sample-validation.md](../specs/006-improve-docs-samples/sample-validation.md) @@ -145,6 +184,8 @@ Before publishing: - capture Linux, Windows, Docker, unsupported-profile, and compatibility validation evidence in [Release preparation](releases.md) before publishing a platform-support release. +- capture native CTest and clean CMake consumption, installed-wheel tests, and + every ordered runtime pair claimed for the release. ## Boundaries To Preserve @@ -158,4 +199,7 @@ Do not introduce or imply: - protection from malicious same-host writers that already have mapping access. - macOS, Windows-container, default-isolated Docker, or cross-host support beyond validated scope. -- delivered C++ or Python bindings before a feature explicitly adds them. +- native or Python registry publication before release automation and artifacts + are actually available. +- Windows/Linux or ordered-pair validation based only on target metadata or + skipped tests. diff --git a/docs/packaging.md b/docs/packaging.md index 3c0d9b4..2b704cc 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -1,10 +1,12 @@ # Packaging -The runtime package is built from +SharedMemoryStore ships independently versioned NuGet, CMake, and Python +artifacts. They share layout `1.2` and resource naming `1`; matching package +versions are not required. The managed runtime package is built from [`src/SharedMemoryStore/SharedMemoryStore.csproj`](../src/SharedMemoryStore/SharedMemoryStore.csproj) and targets `net10.0`. Runtime dependencies are limited to the .NET BCL. -## Current Package Metadata +## Managed NuGet Metadata | Field | Value | |-------|-------| @@ -16,7 +18,7 @@ and targets `net10.0`. Runtime dependencies are limited to the .NET BCL. | `PackageLicenseExpression` | `MIT` | | `PackageProjectUrl` | `https://github.com/rantri/SharedMemoryStore` | | `PackageReadmeFile` | `README.md` | -| `PackageReleaseNotes` | `Linux, Windows, and same-host Docker support hardening: fixes bounded waits, crash-safe ownership and index maintenance, private Linux resource permissions, layout overflow validation, and cleanup reliability while preserving the 1.0.0 public API and layout.` | +| `PackageReleaseNotes` | `Linux, Windows, and same-host Docker support hardening remains the managed 1.0.1 package scope. The repository also adds independently versioned native C++ and Python sibling distributions over layout 1.2 and C ABI 1.0; they are not included in the NuGet package, whose public API and runtime dependency surface remain unchanged.` | | `RepositoryType` | `git` | | `RepositoryUrl` | `https://github.com/rantri/SharedMemoryStore` | | `SymbolPackageFormat` | `snupkg` | @@ -26,7 +28,83 @@ root so NuGet consumers see the same package purpose, status, first-use workflow, support path, security path, and contract links as repository visitors. -## Build and Pack +## Native CMake Distribution + +The root CMake project is version `0.1.0`, requires CMake 3.20 or newer and a +C++20 compiler, and builds `shared_memory_store` as a shared library. Optional +tests, samples, and static library targets are controlled by +`SMS_BUILD_TESTS`, `SMS_BUILD_SAMPLES`, and `SMS_BUILD_STATIC`. + +Installation exports `SharedMemoryStore::SharedMemoryStore` and these package +identities: + +- native package `0.1.0`. +- C ABI `1.0`. +- mapped layout `1.2`. +- resource naming `1`. + +The installed development artifact includes the C header +`shared_memory_store/c_api.h` and C++ header +`shared_memory_store/store.hpp`. The C ABI uses fixed-width integers, +versioned structures, caller-owned byte ranges, and opaque store, lease, and +reservation handles. The C++ header adds move-only RAII wrappers. + +Build and validate the install plus clean `find_package` consumer with: + +```powershell +pwsh ./scripts/validate-native.ps1 -Configuration Release +``` + +## Python Wheel Distribution + +The root [`pyproject.toml`](../pyproject.toml) defines +`shared-memory-store` `0.1.0` for Python 3.10 or newer. `scikit-build-core` is a +build dependency only. A platform wheel contains the Python modules and the +native shared library directly beside them; installing a completed wheel does +not require a compiler or third-party Python runtime package. + +The loader uses standard-library `ctypes`, loads only +`shared_memory_store.dll` or `libshared_memory_store.so` from the package, and +validates C ABI major version, layout `1.2`, resource naming `1`, and canonical +record sizes. It deliberately does not search the current directory, `PATH`, +or a system library path. + +Build a wheel and inspect it through a clean environment: + +```powershell +python -m pip install build +python -m build --wheel +python -m venv artifacts/python-consumer +artifacts/python-consumer/Scripts/python -m pip install (Get-ChildItem dist/*.whl | Select-Object -First 1) +artifacts/python-consumer/Scripts/python samples/PythonBasicUsage/main.py +``` + +Use `bin/python` on Linux. Source distributions include the root CMake project, +native sources and headers, Python sources, compatibility metadata, README, and +license so their wheel build has the complete native input. + +The reproducible repository gate builds and inspects both the wheel and source +distribution, rebuilds a wheel from the source archive, and runs the installed +sample from an unrelated directory: + +```powershell +pwsh ./scripts/validate-python.ps1 -Configuration Release +``` + +## Compatibility Identities + +| Distribution | Version | ABI requirement | Creates/reads | Resource naming | +|--------------|---------|-----------------|---------------|-----------------| +| NuGet `SharedMemoryStore` | `1.0.1` | Not applicable | layout `1.2` | `1` | +| CMake `SharedMemoryStore` | `0.1.0` | provides C ABI `1.0` | layout `1.2` | `1` | +| Python `shared-memory-store` | `0.1.0` | requires C ABI `1.0` | layout `1.2` | `1` | + +The authoritative machine-readable declaration is +[`protocol/compatibility.json`](../protocol/compatibility.json). Release +evidence for a target OS and ordered runtime pair must still be recorded; a +metadata entry alone is not proof that a validation run completed. + +## Build and Pack the Managed Package ```powershell dotnet restore @@ -40,13 +118,18 @@ package to its symbol server alongside the primary package. ## Automated Publication -The [CI workflow](../.github/workflows/ci.yml) validates Linux and Windows on -pull requests and pushes to `main`. The manually triggered +The managed jobs in [CI](../.github/workflows/ci.yml) validate Linux and Windows +on pull requests and pushes to `main`. The manually triggered [release workflow](../.github/workflows/release.yml) performs the full release validation, including Docker, verifies that the version is unused, creates the package and symbols, publishes them to NuGet.org with trusted publishing, and creates the matching GitHub release and `v` tag. +That workflow publishes the managed NuGet artifact. The repository currently +defines native install artifacts and Python wheel builds, but registry +publication for a native archive or Python package is not implied until a +separate release path and its credentials are reviewed. + The one-time trusted-publishing policy and the exact release procedure are documented in [Release preparation](releases.md). @@ -66,6 +149,12 @@ It is expected to run with `pwsh` on Linux and Windows. This command is a maintainer validation path, not a requirement for ordinary package users. +The corresponding native clean consumer is part of +`scripts/validate-native.ps1`. Python clean consumption means installing the +built wheel into a fresh virtual environment, confirming the adjacent native +artifact exists, and running tests or the sample from a directory that cannot +import repository sources. + ## Package README Alignment The package-facing README is the repository [README.md](../README.md). Keep it @@ -83,6 +172,8 @@ aligned with: [Release preparation](releases.md) must agree on: - package version. +- distribution and registry being released. +- C ABI, layout, and resource-naming compatibility identities. - compatibility impact. - public API or behavior changes. - documentation-only changes. @@ -90,9 +181,9 @@ aligned with: - validated platform scope. - migration notes for breaking changes. -Documentation-only changes are patch-level for an already published package -unless they change a public behavior, layout, lifecycle, support, security, or -compatibility promise. +Documentation-only changes are patch-level for an already published +distribution unless they change a public behavior, ABI, layout, lifecycle, +support, security, or compatibility promise. ## License and Source Metadata diff --git a/docs/portability.md b/docs/portability.md index 77813fb..c05c3e4 100644 --- a/docs/portability.md +++ b/docs/portability.md @@ -1,37 +1,45 @@ # Portability -The current package targets `.NET 10` and supports ordinary same-host runtime -and development workflows on Linux and Windows. Same-host Linux Docker -containers are supported when deployment configuration exposes the required -shared-memory, synchronization, owner-liveness, permission, and capacity -capabilities. The shared-memory layout is the interoperability contract for -future C++ and Python implementations, but those bindings are not currently -implemented. +The repository contains .NET 10, C++20, and Python 3.10+ implementations for +ordinary same-host workflows on 64-bit little-endian Linux and Windows. +Same-host Linux Docker participation requires deployment configuration that +exposes compatible shared-memory, synchronization, owner-liveness, permission, +and capacity capabilities. Layout `1.2` and resource naming `1` are the common +interoperability boundary; similar public APIs alone are insufficient. Detailed sources: - [shared-memory-layout.md](../specs/001-frame-memory-store/contracts/shared-memory-layout.md) - [ingest-layout.md](../specs/003-zero-copy-ingest/contracts/ingest-layout.md) - [public-api-contract.md](../specs/005-api-production-readiness/contracts/public-api-contract.md) +- [protocol/README.md](../protocol/README.md) +- [resource-naming-v1.md](../protocol/resource-naming-v1.md) +- [compatibility.json](../protocol/compatibility.json) +- [interoperability.md](../specs/008-cpp-python-implementations/contracts/interoperability.md) ## Current Baseline -- Runtime package: `SharedMemoryStore` `1.0.1`. -- Target framework: `net10.0`. -- Supported host platforms: Linux and Windows. -- Supported container profile: Linux-based same-host Docker containers with - shared IPC and compatible owner-liveness, permissions, and shared-memory - capacity. -- Current language implementation: C#. -- Future audience: C++ and Python implementations or bindings that conform to - the documented layout and lifecycle contracts. +- Managed distribution: NuGet `SharedMemoryStore` `1.0.1`, targeting + `net10.0`. +- Native distribution: CMake `SharedMemoryStore` `0.1.0`, C++20, C ABI `1.0`. +- Python distribution: `shared-memory-store` `0.1.0`, Python 3.10 or newer, + using `ctypes` over the bundled native library. +- Shared identities: layout `1.2`, resource naming `1`, little-endian 64-bit + process model. +- Implementation targets: Linux and Windows. Release validation for each + distribution and ordered runtime pairing must be recorded separately. +- Managed supported container profile: Linux-based same-host Docker containers + with shared IPC and compatible owner-liveness, permissions, and shared-memory + capacity. Native/Python container claims require their own recorded + cross-runtime evidence. ## Layout Compatibility -Future implementations must use the same little-endian field encoding, 8-byte +Every implementation uses the same little-endian field encoding, 8-byte alignment, state-value assignments, key hashing, exact byte-key equality, slot lifecycle identity, lease registry, reservation progress, and remove/reuse -state machine. +state machine. Static fixtures pin exact record sizes, offsets, arithmetic, +hashes, status values, and resource-name vectors. Layout compatibility follows semantic versioning. A major layout-version change requires migration notes and contract-test updates. Minor compatible additions @@ -50,11 +58,27 @@ state, `SharedSlotMetadata.Reserved` stores bytes advanced by the producer, identifies the owner for explicit recovery. Commit must validate slot generation and exact progress before transitioning to `SlotPublished`. -Future implementations must treat writable reservation memory as valid only -while the slot remains pending and full lifecycle identity matches. +Every implementation treats writable reservation memory as valid only while +the slot remains pending and full lifecycle identity matches. Scatter/gather committed values are out of scope for this layout; segmented publish copies into one contiguous slot value. +## Language Distribution Boundaries + +The .NET implementation owns its managed protocol mechanisms and does not load +the native library. The native implementation owns one C++ protocol core and +the Windows/Linux adapters. Its exported C ABI uses explicit-width integers, +versioned structures, opaque handles, and caller-owned buffers. The public C++ +API adds move-only RAII and `std::span` views without making the C++ ABI the +Python boundary. + +The Python package calls C ABI `1.0` through standard-library `ctypes`. A wheel +places `shared_memory_store.dll` or `libshared_memory_store.so` beside the +Python modules. The loader does not search the current directory, `PATH`, or a +system library path, and it rejects incompatible ABI or protocol metadata. +Python lease views are read-only; reservation views are writable only for the +reservation lifetime. + ## Trusted Same-Host Boundary The direct ingest API exposes writable shared-memory bytes to producers before @@ -63,22 +87,27 @@ mapping and participate in the store. Deployment is responsible for OS ACLs, service identity, process isolation, and package distribution. SharedMemoryStore validates lifecycle state, slot generation, key ownership, and -reader visibility, but it does not defend against a malicious in-boundary writer -that intentionally corrupts mapped bytes, forges metadata, or ignores the public -API. Future implementations or bindings must document the same trust boundary. +reader visibility, but no distribution defends against a malicious in-boundary +writer that intentionally corrupts mapped bytes, forges metadata, or ignores +the public API. ## Platform Resource Model Windows uses named operating-system memory mappings and named synchronization. -An explicit `Global\` mapping name uses global synchronization as of `1.0.1`; -all participants in such a store must use `1.0.1` or later. Ordinary unqualified -and explicit `Local\` names retain session-local synchronization. +An explicit `Global\` mapping name uses global synchronization in managed +`1.0.1` and native `0.1.0`. All participants must implement compatible +resource-naming version `1` behavior. Ordinary unqualified and explicit +`Local\` names retain session-local synchronization. Linux uses deterministic files in a shared runtime memory location such as `/dev/shm`, with names derived from the public store name and a collision prevention hash. Docker containers participate in the Linux model only when their IPC and process-liveness configuration lets all participants see the same resources and classify owners safely. +The managed and native implementations must derive the same resources and +participate in the same lock. Python inherits the native behavior rather than +reimplementing it. + The Linux resource directory is owner-only (`0700`), and region, synchronization, owner, and lifecycle files are owner-only (`0600`). Cooperating host processes must therefore run as the same Unix identity. Containers must @@ -86,11 +115,12 @@ share a compatible identity as well as IPC and process-liveness namespaces. ## Unsupported Scenarios -- Current public docs do not claim C++ or Python bindings exist. - macOS is not currently supported. +- 32-bit processes, big-endian hosts, and architectures without recorded + conformance evidence are not current release targets. - Cross-host shared memory, distributed-cache behavior, persistence across host restart, Windows containers, and default-isolated Docker containers are not - supported by this package version. + supported by these distributions. - Platforms without reliable named mapping or owner-liveness checks return deterministic unsupported statuses for affected operations. - Application-specific schemas, including frame metadata, are not parsed by the @@ -102,8 +132,14 @@ share a compatible identity as well as IPC and process-liveness namespaces. - Public API, layout, lifecycle, error, diagnostics, and package metadata are compatibility contracts. +- NuGet, CMake, and Python versions advance independently. Compatibility is + determined from layout, resource naming, and C ABI declarations rather than + matching package version numbers. - Breaking public API or layout changes require semantic-version review, migration notes, and contract-test updates. +- A release must not convert target-platform metadata into a validation claim + until native tests, package consumption, and the relevant ordered runtime + pairs have passed on that platform. - Documentation that changes a compatibility promise must update [CHANGELOG.md](../CHANGELOG.md), [Maintainers](maintainers.md), and [Release preparation](releases.md). diff --git a/docs/releases.md b/docs/releases.md index 41fc258..2cfe3ed 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,9 +1,11 @@ # Release Preparation -This guide is the maintainer checklist for preparing a package release. It is -written for the current `1.0.1` package and should be updated when package -metadata, public API behavior, compatibility scope, support policy, security -reporting, documentation scope, or sample behavior changes. +This guide is the maintainer checklist for preparing independently versioned +managed, native, and Python releases. The current source identities are NuGet +`SharedMemoryStore` `1.0.1`, CMake `SharedMemoryStore` `0.1.0`, Python +`shared-memory-store` `0.1.0`, C ABI `1.0`, mapped layout `1.2`, and resource +naming `1`. Update this guide when any distribution, ABI, protocol, support, +security, documentation, or sample contract changes. ## Automated Release Process @@ -15,6 +17,12 @@ NuGet.org, and then publishes the GitHub release. CI separately validates the same commit on Linux and Windows through [ci.yml](../.github/workflows/ci.yml). +The existing workflow publishes the managed NuGet and GitHub release. Native +CMake installation and Python wheel construction exist in the repository, but +this guide does not claim a native registry or Python index publication path +until dedicated automation, credentials, artifact signing/provenance, and clean +install checks are reviewed. + Configure NuGet.org trusted publishing once before the first automated release: 1. Sign in to NuGet.org as the `rantri` owner and open **Trusted Publishing**. @@ -66,12 +74,28 @@ before publishing: - `RepositoryType` remains `git`. Add an owner-approved repository URL before a public package publication if the repository host is finalized. +For a native or Python release, also verify: + +- root `project(... VERSION ...)` and `[project].version` identify the intended + distribution versions. +- `SharedMemoryStoreConfig.cmake` declares native package, C ABI, layout, and + resource-naming versions. +- `shared_memory_store.__version__` and `pyproject.toml` agree. +- [`protocol/compatibility.json`](../protocol/compatibility.json) agrees with + every released distribution. +- wheel contents place the correct native library beside the Python modules and + the loader rejects an incompatible or misplaced artifact. +- target-platform metadata is not described as completed validation without a + corresponding recorded run. + ## Release Notes and Changelog Update [CHANGELOG.md](../CHANGELOG.md) in reverse chronological order. Each entry should identify: - package version. +- distribution ecosystem and artifact name. +- C ABI, layout, and resource-naming compatibility identities. - public API or behavior impact. - package metadata impact. - documentation-only changes. @@ -101,9 +125,16 @@ Compare public docs with these contracts: - [contention-configuration-contract.md](../specs/005-api-production-readiness/contracts/contention-configuration-contract.md) - [diagnostics-integration-contract.md](../specs/005-api-production-readiness/contracts/diagnostics-integration-contract.md) - [reservation-memory-contract.md](../specs/005-api-production-readiness/contracts/reservation-memory-contract.md) - -Confirm that docs do not claim current C++ or Python bindings, broad -macOS or cross-host support, unmeasured hardware performance, +- [protocol/README.md](../protocol/README.md) +- [native-c-api.md](../specs/008-cpp-python-implementations/contracts/native-c-api.md) +- [cpp-api.md](../specs/008-cpp-python-implementations/contracts/cpp-api.md) +- [python-api.md](../specs/008-cpp-python-implementations/contracts/python-api.md) +- [interoperability.md](../specs/008-cpp-python-implementations/contracts/interoperability.md) +- [packaging.md](../specs/008-cpp-python-implementations/contracts/packaging.md) + +Confirm that docs describe the delivered C++ and Python surfaces without +claiming registry publication, unrun platform/pair validation, broad macOS or +cross-host support, unmeasured hardware performance, application-specific frame parsing by the core store, hidden background work, persistence, Windows-container support, default-isolated Docker support, or cross-host cache behavior. @@ -116,6 +147,8 @@ Documentation-only changes still need release review: - Confirm examples and sample outputs match current source. - Confirm package metadata, `PackageReleaseNotes`, README, packaging guide, changelog, and release notes agree. +- Confirm CMake, Python, C ABI, layout, resource-naming, and compatibility + metadata versions agree. - Confirm compatibility wording did not change public behavior promises. - Confirm known limitations, platform scope, performance claims, support scope, and security reporting paths remain current. @@ -131,6 +164,9 @@ Documentation-only changes still need release review: - Confirm issue templates and the pull request template still route questions, bugs, documentation issues, feature requests, and security disclosures to the right place. +- Confirm reports can identify the affected managed, native, or Python + distribution and that all docs preserve the trusted same-host participant + boundary. ## Validation Commands @@ -149,11 +185,26 @@ dotnet test SharedMemoryStore.slnx -c Release dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/package pwsh ./scripts/validate-cross-platform.ps1 -SkipDocker pwsh ./scripts/validate-docker-shared-memory.ps1 +pwsh ./scripts/validate-native.ps1 -Configuration Release +python -m pip install build +python -m build --wheel +python -m venv artifacts/python-consumer +artifacts/python-consumer/Scripts/python -m pip install (Get-ChildItem dist/*.whl | Select-Object -First 1) +$env:SMS_TEST_INSTALLED_PACKAGE = '1' +artifacts/python-consumer/Scripts/python -m unittest discover -s tests/python -v +artifacts/python-consumer/Scripts/python samples/PythonBasicUsage/main.py +dotnet test tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj -c Release ``` +Use `bin/python` instead of `Scripts/python` on Linux. Configure the C++ and +Python agent paths required by the interoperability harness and record every +ordered pair that actually executes. A skipped or unavailable agent is not a +passing release result. + Expected result: documentation inventory, placeholder checks, internal links, -package metadata alignment, sample README contracts, public API/status drift -checks, sample commands, clean package consumption, tests, and pack all pass. +metadata alignment, sample contracts, managed regressions and package +consumption, native CTest/install/clean CMake consumption, installed-wheel +tests, samples, and every claimed runtime pair all pass. ## Linux, Windows, and Docker Support Notes @@ -222,6 +273,41 @@ dotnet run --project benchmarks/SharedMemoryStore.Benchmarks/SharedMemoryStore.B dotnet run --project benchmarks/SharedMemoryStore.Benchmarks/SharedMemoryStore.Benchmarks.csproj -c Release -- --filter *SegmentedPublish* ``` +## Native and Python 0.1.0 Preparation Notes (Unreleased) + +The repository now contains the implementation and packaging inputs for: + +- a C++20 native core with Windows/Linux adapters, fixed-width C ABI `1.0`, + move-only C++ RAII wrappers, CMake install/export rules, native tests, and a + basic sample. +- a Python 3.10+ `ctypes` package with context-managed stores, leases, and + reservations, package-adjacent native loading, wheel configuration, Python + tests, and a basic sample. +- canonical layout `1.2`, resource naming `1`, compatibility metadata, + conformance fixtures, test-only JSON-lines agents, and the ordered 3x3 core + exchange harness. + +These sibling distributions do not change the managed `1.0.1` public API, +status numbers, NuGet runtime dependencies, or mapped layout. Their `0.1.0` +versions are alpha lines and are not included in the NuGet package. + +Before publishing native or Python artifacts or describing a platform as +release-validated, capture all of the following: + +- native configure, compile, CTest, install, basic sample, and clean external + `find_package` consumption on Windows x64 and Linux x64. +- a wheel build and clean installed-wheel tests on both targets, including + proof that the bundled native library loads without repository or system + search-path fallback. +- every ordered .NET/C++/Python producer-consumer pair on each target, plus + mixed lease removal/reuse, reservation lifecycle, bounded contention, crash + recovery, and Linux owner cleanup. +- existing managed, documentation, Docker, package-consumption, security, and + vulnerability regression gates. + +No PyPI, native archive registry, or automatic native/Python publication is +claimed by this source change. Add and review those release channels separately. + ## 1.0.1 Production Hardening Notes The 2026-07-09 review completed the following release evidence: diff --git a/docs/samples.md b/docs/samples.md index 23f6278..db99cf8 100644 --- a/docs/samples.md +++ b/docs/samples.md @@ -13,6 +13,8 @@ related documentation, and non-goals. The README contract is defined in ## Prerequisites - .NET SDK compatible with `net10.0`. +- CMake 3.20 or newer and a C++20 compiler for the native sample. +- Python 3.10 or newer and an installed platform wheel for the Python sample. - PowerShell 7 (`pwsh`) when running repository validation scripts. - Linux or Windows for ordinary host samples. - Docker Engine or Docker Desktop when running the Docker shared-memory sample. @@ -30,6 +32,8 @@ of the success paths shown in sample output. | 3 | [samples/ZeroCopyIngest/README.md](../samples/ZeroCopyIngest/README.md) | You want direct reservation ingest, abort cleanup, segmented publish, and adapter examples. | `dotnet run --project samples/ZeroCopyIngest/ZeroCopyIngest.csproj -c Release` | | 4 | [samples/HostedServiceIntegration/README.md](../samples/HostedServiceIntegration/README.md) | You want an application-owned lifecycle and health wrapper without adding dependencies to the core package. | `dotnet run --project samples/HostedServiceIntegration/HostedServiceIntegration.csproj -c Release` | | 5 | [samples/DockerSharedMemory/README.md](../samples/DockerSharedMemory/README.md) | You want to validate same-host Docker containers sharing one store. | `pwsh ./scripts/validate-docker-shared-memory.ps1` | +| 6 | [samples/CppBasicUsage/README.md](../samples/CppBasicUsage/README.md) | You want the native C++20 RAII create, publish, lease, and release path. | Build with `SMS_BUILD_SAMPLES=ON` or consume the installed CMake package. | +| 7 | [samples/PythonBasicUsage/README.md](../samples/PythonBasicUsage/README.md) | You want context-managed Python access through an installed wheel and bundled native library. | `python samples/PythonBasicUsage/main.py` from the wheel environment. | ## Basic Usage @@ -113,6 +117,40 @@ Related guides: - [Lifecycle](lifecycle.md) for recovery and cleanup behavior. - [Diagnostics](diagnostics.md) for failure counters and recovery reports. +## C++ Basic Usage + +The C++ sample uses the installed public header and +`SharedMemoryStore::SharedMemoryStore` CMake target. It creates an isolated +store with `store_options::create`, publishes opaque bytes, acquires a move-only +`value_lease`, reads its zero-copy `std::span`, and releases it. + +Build it with the repository: + +```powershell +cmake -S . -B artifacts/native-build -DSMS_BUILD_SAMPLES=ON +cmake --build artifacts/native-build --config Release --target shared_memory_store_cpp_sample +``` + +Its standalone `CMakeLists.txt` also demonstrates +`find_package(SharedMemoryStore CONFIG REQUIRED)` against an installed native +package. See [Packaging](packaging.md) for the install and clean-consumer path. + +## Python Basic Usage + +The Python sample imports the installed `shared_memory_store` package, creates +an isolated store, publishes payload and descriptor bytes, reads a read-only +lease view, removes the value, and inspects diagnostics. It demonstrates +context-managed lifetimes over the packaged C ABI rather than a pure-Python +protocol implementation. + +```powershell +artifacts/python-consumer/Scripts/python samples/PythonBasicUsage/main.py +``` + +Use `bin/python` on Linux. Run it from a wheel environment; adding repository +sources to `PYTHONPATH` does not supply the adjacent native library required by +the loader. + ## Validation Build all samples through the solution: @@ -137,4 +175,12 @@ pwsh ./scripts/validate-package-consumption.ps1 dotnet test SharedMemoryStore.slnx -c Release dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/package pwsh ./scripts/validate-docker-shared-memory.ps1 +pwsh ./scripts/validate-native.ps1 -Configuration Release +$env:SMS_TEST_INSTALLED_PACKAGE = '1' +artifacts/python-consumer/Scripts/python -m unittest discover -s tests/python -v +dotnet test tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj -c Release ``` + +Use `bin/python` for the installed Python validation on Linux. The +interoperability suite skips or fails when required agent artifacts are absent; +release evidence must identify which runtime pairs actually ran. diff --git a/protocol/README.md b/protocol/README.md new file mode 100644 index 0000000..ff30066 --- /dev/null +++ b/protocol/README.md @@ -0,0 +1,63 @@ +# SharedMemoryStore Protocol + +This directory is the language-neutral compatibility boundary for the managed, +native, and Python distributions. It specifies the bytes in a mapped region and +the operating-system resources used to find, synchronize, own, and clean up that +region. Public language APIs may be idiomatic, but they must not reinterpret +these contracts. + +## Current protocol identities + +| Contract | Current identity | Canonical definition | +|---|---:|---| +| Mapped layout | `1.2` | [layout-v1.2.md](layout-v1.2.md) | +| Platform resource naming | `1` | [resource-naming-v1.md](resource-naming-v1.md) | +| Conformance manifest | `1` | [fixtures/v1.2/manifest.json](fixtures/v1.2/manifest.json) | +| Native C ABI | `1.0` | [native-c-api.md](../specs/008-cpp-python-implementations/contracts/native-c-api.md) | + +Package versions are independent of all four identities. A package release +must declare the layout versions it can create and open, the resource-naming +version it implements, and its C ABI range when applicable. + +## Layout version boundary + +New stores created by this repository use layout major `1`, minor `2`. The +advertised read and create boundary for this feature is exactly `1.2`. + +The two header numbers are not sufficient evidence of compatibility. An opener +must also validate the magic, major version, header and record sizes, configured +capacities, calculated section offsets and lengths, total-region bounds, and +every state value it may observe. A mapping with the right version numbers but +a different shape is incompatible. Likewise, a previously released minor +version is not implicitly readable: layout `1.2` enlarged the index, slot, and +lease records to add reuse epochs, so earlier record shapes are rejected rather +than partially interpreted. + +Any change to mapped bytes, field meaning, state transitions, hashing, probing, +or visibility rules requires a new layout identity and updated fixtures. A +major version is required for a protocol redesign that cannot safely coexist +with the major-1 validation model. A minor increment still requires explicit +compatibility evidence; it does not promise that all older mapped shapes can be +opened. + +Resource naming is versioned separately because two implementations with +identical mapped records still cannot interoperate if they derive different +mapping, lock, or owner resources. A resource-name or lock-participation change +requires a new resource-naming version and live cross-runtime tests. + +The C ABI is also independent. ABI-only additions do not change mapped bytes, +and a layout change does not by itself authorize an ABI break. + +## Canonical evidence + +The JSON manifest pins exact record sizes and offsets, numeric states, public +status values, open modes, layout arithmetic, FNV-1a hashes, and platform-name +vectors. It stores 64-bit hashes as fixed-width hexadecimal strings so JSON +number precision cannot alter them. Representative mapped-region binaries are +offline conformance inputs only; they must never be opened as live mappings +because their owner process identifiers and platform lifecycle resources are +not live. + +Changes to an executable constant or algorithm must update the narrative, +manifest, every language's static conformance tests, and the cross-runtime +matrix in one review. diff --git a/protocol/compatibility.json b/protocol/compatibility.json new file mode 100644 index 0000000..72b43c7 --- /dev/null +++ b/protocol/compatibility.json @@ -0,0 +1,61 @@ +{ + "schema_version": 1, + "shared_protocol": { + "layout": { "major": 1, "minor": 2 }, + "resource_naming": 1, + "byte_order": "little", + "required_pointer_width": 64 + }, + "distributions": [ + { + "name": "SharedMemoryStore", + "ecosystem": "NuGet", + "version": "1.0.1", + "creates_layouts": ["1.2"], + "reads_layouts": ["1.2"], + "resource_naming": 1, + "validated_targets": ["windows-x86_64", "linux-x86_64"] + }, + { + "name": "SharedMemoryStore", + "ecosystem": "CMake", + "version": "0.1.0", + "c_abi": "1.0", + "creates_layouts": ["1.2"], + "reads_layouts": ["1.2"], + "resource_naming": 1, + "validated_targets": ["windows-x86_64", "linux-x86_64"] + }, + { + "name": "shared-memory-store", + "ecosystem": "Python", + "version": "0.1.0", + "requires_c_abi": "1.0", + "creates_layouts": ["1.2"], + "reads_layouts": ["1.2"], + "resource_naming": 1, + "validated_targets": ["windows-x86_64", "linux-x86_64"] + } + ], + "interoperability": { + "ordered_pairs": [ + "dotnet->dotnet", + "dotnet->cpp", + "dotnet->python", + "cpp->dotnet", + "cpp->cpp", + "cpp->python", + "python->dotnet", + "python->cpp", + "python->python" + ], + "required_scenarios": [ + "binary-publish-acquire-release", + "remove-final-release-reuse", + "reservation-commit-abort", + "bounded-lock-contention", + "explicit-crash-recovery", + "linux-owner-cleanup" + ] + } +} diff --git a/protocol/fixtures/v1.2/empty.bin b/protocol/fixtures/v1.2/empty.bin new file mode 100644 index 0000000000000000000000000000000000000000..0874b2f6fce594aef7ba7cc1b19f3ba4018b7b4e GIT binary patch literal 1016 zcmWIc4K`$CU|?VZ;srqbgBeHw0f=G&Voo4t1!6%U<^W;?AO?X2V3L8M0i+oS6hH<5 z!2&SJz;FP{zX9dXfSM}-_7Es&~qYjib sh*1YhBE+a8CC$LnAuRnN`-v2D7C_VQ2`EjjIV+(0A7C*D*-UD)0ib6bv;Y7A literal 0 HcmV?d00001 diff --git a/protocol/fixtures/v1.2/empty.snapshot.json b/protocol/fixtures/v1.2/empty.snapshot.json new file mode 100644 index 0000000..172b069 --- /dev/null +++ b/protocol/fixtures/v1.2/empty.snapshot.json @@ -0,0 +1,90 @@ +{ + "format_version": 1, + "fixture": "empty", + "offline_only": true, + "protocol": { + "layout_major": 1, + "layout_minor": 2, + "byte_order": "little" + }, + "header": { + "magic_hex": "31534d53", + "header_length": 160, + "total_bytes": 1016, + "slot_count": 3, + "lease_record_count": 4, + "max_key_bytes": 9, + "max_descriptor_bytes": 5, + "max_value_bytes": 17, + "index_entry_count": 8, + "index_entry_size": 48, + "index_offset": 160, + "index_length": 384, + "lease_registry_offset": 544, + "lease_registry_length": 160, + "slot_metadata_offset": 704, + "slot_metadata_length": 216, + "descriptor_storage_offset": 920, + "descriptor_storage_length": 24, + "payload_storage_offset": 944, + "payload_storage_length": 72, + "store_id_hex": "0102030405060708", + "store_state": 1, + "sequence": 0 + }, + "index_entries": [], + "lease_records": [], + "slots": [ + { + "slot_index": 0, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 920, + "payload_offset": 944, + "committed_sequence": 0 + }, + { + "slot_index": 1, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 928, + "payload_offset": 968, + "committed_sequence": 0 + }, + { + "slot_index": 2, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 936, + "payload_offset": 992, + "committed_sequence": 0 + } + ] +} diff --git a/protocol/fixtures/v1.2/generate-fixtures.ps1 b/protocol/fixtures/v1.2/generate-fixtures.ps1 new file mode 100644 index 0000000..0c17272 --- /dev/null +++ b/protocol/fixtures/v1.2/generate-fixtures.ps1 @@ -0,0 +1,402 @@ +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' +$fixtureRoot = $PSScriptRoot +$utf8WithoutBom = [System.Text.UTF8Encoding]::new($false) + +function ConvertFrom-Hex { + param([Parameter(Mandatory)][string]$Hex) + + if (($Hex.Length % 2) -ne 0) { + throw "Hex input must contain an even number of characters: '$Hex'." + } + + $result = [byte[]]::new($Hex.Length / 2) + for ($index = 0; $index -lt $result.Length; $index++) { + $result[$index] = [Convert]::ToByte($Hex.Substring($index * 2, 2), 16) + } + + return ,$result +} + +function ConvertTo-Hex { + param([Parameter(Mandatory)][AllowEmptyCollection()][byte[]]$Bytes) + + return [Convert]::ToHexString($Bytes).ToLowerInvariant() +} + +function Set-Bytes { + param( + [Parameter(Mandatory)][byte[]]$Buffer, + [Parameter(Mandatory)][int]$Offset, + [Parameter(Mandatory)][byte[]]$Value + ) + + [Buffer]::BlockCopy($Value, 0, $Buffer, $Offset, $Value.Length) +} + +function Set-Int32 { + param([byte[]]$Buffer, [int]$Offset, [int]$Value) + + $bytes = [BitConverter]::GetBytes($Value) + if (-not [BitConverter]::IsLittleEndian) { + [Array]::Reverse($bytes) + } + + Set-Bytes $Buffer $Offset $bytes +} + +function Set-Int64 { + param([byte[]]$Buffer, [int]$Offset, [long]$Value) + + $bytes = [BitConverter]::GetBytes($Value) + if (-not [BitConverter]::IsLittleEndian) { + [Array]::Reverse($bytes) + } + + Set-Bytes $Buffer $Offset $bytes +} + +function Set-UInt64 { + param([byte[]]$Buffer, [int]$Offset, [UInt64]$Value) + + $bytes = [BitConverter]::GetBytes($Value) + if (-not [BitConverter]::IsLittleEndian) { + [Array]::Reverse($bytes) + } + + Set-Bytes $Buffer $Offset $bytes +} + +function Get-Int32 { + param([byte[]]$Buffer, [int]$Offset) + + return [BitConverter]::ToInt32($Buffer, $Offset) +} + +function Get-Int64 { + param([byte[]]$Buffer, [int]$Offset) + + return [BitConverter]::ToInt64($Buffer, $Offset) +} + +function Get-UInt64 { + param([byte[]]$Buffer, [int]$Offset) + + return [BitConverter]::ToUInt64($Buffer, $Offset) +} + +function Get-Bytes { + param([byte[]]$Buffer, [int]$Offset, [int]$Length) + + $result = [byte[]]::new($Length) + if ($Length -gt 0) { + [Buffer]::BlockCopy($Buffer, $Offset, $result, 0, $Length) + } + + return ,$result +} + +function New-CanonicalRegion { + $region = [byte[]]::new(1016) + + # Store header: layout v1.2, the alignment vector from manifest.json. + Set-Int32 $region 0 0x31534d53 + Set-Int32 $region 4 1 + Set-Int32 $region 8 2 + Set-Int32 $region 12 160 + Set-Int64 $region 16 1016 + Set-Int32 $region 24 3 + Set-Int32 $region 28 4 + Set-Int32 $region 32 9 + Set-Int32 $region 36 5 + Set-Int32 $region 40 17 + Set-Int32 $region 44 8 + Set-Int32 $region 48 48 + Set-Int64 $region 56 160 + Set-Int64 $region 64 384 + Set-Int64 $region 72 544 + Set-Int64 $region 80 160 + Set-Int64 $region 88 704 + Set-Int64 $region 96 216 + Set-Int64 $region 104 920 + Set-Int64 $region 112 24 + Set-Int64 $region 120 944 + Set-Int64 $region 128 72 + Set-Int64 $region 136 0x0102030405060708 + Set-Int32 $region 144 1 + Set-Int32 $region 148 0 + Set-Int64 $region 152 0 + + # Free lease records retain their canonical IDs and use -1 as the slot sentinel. + for ($recordIndex = 0; $recordIndex -lt 4; $recordIndex++) { + $recordOffset = 544 + ($recordIndex * 40) + Set-Int32 $region ($recordOffset + 4) $recordIndex + Set-Int32 $region ($recordOffset + 8) -1 + } + + # Free slots start at lifecycle identity (generation 1, reuse epoch 0). + for ($slotIndex = 0; $slotIndex -lt 3; $slotIndex++) { + $slotOffset = 704 + ($slotIndex * 72) + Set-Int32 $region ($slotOffset + 4) 1 + Set-Int64 $region ($slotOffset + 48) (920 + ($slotIndex * 8)) + Set-Int64 $region ($slotOffset + 56) (944 + ($slotIndex * 24)) + } + + return ,$region +} + +function Set-IndexEntry { + param( + [byte[]]$Region, + [int]$EntryIndex, + [int]$State, + [UInt64]$KeyHash, + [int]$SlotIndex, + [int]$Generation, + [long]$ReuseEpoch, + [byte[]]$Key + ) + + $entryOffset = 160 + ($EntryIndex * 48) + Set-Int32 $Region $entryOffset $State + Set-Int32 $Region ($entryOffset + 4) $Key.Length + Set-UInt64 $Region ($entryOffset + 8) $KeyHash + Set-Int32 $Region ($entryOffset + 16) $SlotIndex + Set-Int32 $Region ($entryOffset + 20) $Generation + Set-Int64 $Region ($entryOffset + 24) $ReuseEpoch + Set-Bytes $Region ($entryOffset + 32) $Key +} + +function Set-Slot { + param( + [byte[]]$Region, + [int]$SlotIndex, + [int]$State, + [int]$Generation, + [long]$ReuseEpoch, + [int]$UsageCount, + [byte[]]$Key, + [byte[]]$Descriptor, + [byte[]]$Payload, + [int]$PublisherProcessId, + [int]$Reserved, + [UInt64]$KeyHash, + [long]$CommittedSequence + ) + + $slotOffset = 704 + ($SlotIndex * 72) + $descriptorOffset = 920 + ($SlotIndex * 8) + $payloadOffset = 944 + ($SlotIndex * 24) + Set-Int32 $Region $slotOffset $State + Set-Int32 $Region ($slotOffset + 4) $Generation + Set-Int64 $Region ($slotOffset + 8) $ReuseEpoch + Set-Int32 $Region ($slotOffset + 16) $UsageCount + Set-Int32 $Region ($slotOffset + 20) $Key.Length + Set-Int32 $Region ($slotOffset + 24) $Descriptor.Length + Set-Int32 $Region ($slotOffset + 28) $Payload.Length + Set-Int32 $Region ($slotOffset + 32) $PublisherProcessId + Set-Int32 $Region ($slotOffset + 36) $Reserved + Set-UInt64 $Region ($slotOffset + 40) $KeyHash + Set-Int64 $Region ($slotOffset + 48) $descriptorOffset + Set-Int64 $Region ($slotOffset + 56) $payloadOffset + Set-Int64 $Region ($slotOffset + 64) $CommittedSequence + Set-Bytes $Region $descriptorOffset $Descriptor + Set-Bytes $Region $payloadOffset $Payload +} + +function Set-Lease { + param( + [byte[]]$Region, + [int]$RecordId, + [int]$SlotIndex, + [int]$Generation, + [long]$ReuseEpoch, + [int]$OwnerProcessId, + [long]$AcquireSequence + ) + + $recordOffset = 544 + ($RecordId * 40) + Set-Int32 $Region $recordOffset 1 + Set-Int32 $Region ($recordOffset + 4) $RecordId + Set-Int32 $Region ($recordOffset + 8) $SlotIndex + Set-Int32 $Region ($recordOffset + 12) $Generation + Set-Int64 $Region ($recordOffset + 16) $ReuseEpoch + Set-Int32 $Region ($recordOffset + 24) $OwnerProcessId + Set-Int32 $Region ($recordOffset + 28) 0 + Set-Int64 $Region ($recordOffset + 32) $AcquireSequence +} + +function Get-IndexStateName([int]$State) { + return @('Empty', 'Occupied', 'Tombstone')[$State] +} + +function Get-SlotStateName([int]$State) { + return @('Free', 'Publishing', 'Published', 'RemoveRequested', 'Reclaiming')[$State] +} + +function Get-LeaseStateName([int]$State) { + return @('Free', 'Active', 'Released', 'Abandoned')[$State] +} + +function Get-NormalizedSnapshot { + param([string]$FixtureName, [byte[]]$Region) + + $indexEntries = @() + for ($entryIndex = 0; $entryIndex -lt 8; $entryIndex++) { + $offset = 160 + ($entryIndex * 48) + $state = Get-Int32 $Region $offset + if ($state -eq 0) { + continue + } + + $keyLength = Get-Int32 $Region ($offset + 4) + $indexEntries += [ordered]@{ + entry_index = $entryIndex + state = $state + state_name = Get-IndexStateName $state + key_hex = ConvertTo-Hex (Get-Bytes $Region ($offset + 32) $keyLength) + key_hash_hex = '{0:x16}' -f (Get-UInt64 $Region ($offset + 8)) + slot_index = Get-Int32 $Region ($offset + 16) + slot_generation = Get-Int32 $Region ($offset + 20) + slot_reuse_epoch = Get-Int64 $Region ($offset + 24) + } + } + + $leaseRecords = @() + for ($recordIndex = 0; $recordIndex -lt 4; $recordIndex++) { + $offset = 544 + ($recordIndex * 40) + $state = Get-Int32 $Region $offset + if ($state -eq 0) { + continue + } + + $leaseRecords += [ordered]@{ + record_id = Get-Int32 $Region ($offset + 4) + state = $state + state_name = Get-LeaseStateName $state + slot_index = Get-Int32 $Region ($offset + 8) + slot_generation = Get-Int32 $Region ($offset + 12) + slot_reuse_epoch = Get-Int64 $Region ($offset + 16) + owner_process_id = Get-Int32 $Region ($offset + 24) + acquire_sequence = Get-Int64 $Region ($offset + 32) + } + } + + $slots = @() + for ($slotIndex = 0; $slotIndex -lt 3; $slotIndex++) { + $offset = 704 + ($slotIndex * 72) + $state = Get-Int32 $Region $offset + $descriptorLength = Get-Int32 $Region ($offset + 24) + $payloadLength = Get-Int32 $Region ($offset + 28) + $descriptorOffset = Get-Int64 $Region ($offset + 48) + $payloadOffset = Get-Int64 $Region ($offset + 56) + $slots += [ordered]@{ + slot_index = $slotIndex + state = $state + state_name = Get-SlotStateName $state + generation = Get-Int32 $Region ($offset + 4) + reuse_epoch = Get-Int64 $Region ($offset + 8) + usage_count = Get-Int32 $Region ($offset + 16) + key_length = Get-Int32 $Region ($offset + 20) + descriptor_hex = ConvertTo-Hex (Get-Bytes $Region $descriptorOffset $descriptorLength) + payload_hex = ConvertTo-Hex (Get-Bytes $Region $payloadOffset $payloadLength) + publisher_process_id = Get-Int32 $Region ($offset + 32) + reservation_bytes_written = Get-Int32 $Region ($offset + 36) + key_hash_hex = '{0:x16}' -f (Get-UInt64 $Region ($offset + 40)) + descriptor_offset = $descriptorOffset + payload_offset = $payloadOffset + committed_sequence = Get-Int64 $Region ($offset + 64) + } + } + + return [ordered]@{ + format_version = 1 + fixture = $FixtureName + offline_only = $true + protocol = [ordered]@{ + layout_major = Get-Int32 $Region 4 + layout_minor = Get-Int32 $Region 8 + byte_order = 'little' + } + header = [ordered]@{ + magic_hex = '{0:x8}' -f [UInt32](Get-Int32 $Region 0) + header_length = Get-Int32 $Region 12 + total_bytes = Get-Int64 $Region 16 + slot_count = Get-Int32 $Region 24 + lease_record_count = Get-Int32 $Region 28 + max_key_bytes = Get-Int32 $Region 32 + max_descriptor_bytes = Get-Int32 $Region 36 + max_value_bytes = Get-Int32 $Region 40 + index_entry_count = Get-Int32 $Region 44 + index_entry_size = Get-Int32 $Region 48 + index_offset = Get-Int64 $Region 56 + index_length = Get-Int64 $Region 64 + lease_registry_offset = Get-Int64 $Region 72 + lease_registry_length = Get-Int64 $Region 80 + slot_metadata_offset = Get-Int64 $Region 88 + slot_metadata_length = Get-Int64 $Region 96 + descriptor_storage_offset = Get-Int64 $Region 104 + descriptor_storage_length = Get-Int64 $Region 112 + payload_storage_offset = Get-Int64 $Region 120 + payload_storage_length = Get-Int64 $Region 128 + store_id_hex = '{0:x16}' -f [UInt64](Get-Int64 $Region 136) + store_state = Get-Int32 $Region 144 + sequence = Get-Int64 $Region 152 + } + index_entries = $indexEntries + lease_records = $leaseRecords + slots = $slots + } +} + +$binaryKey = ConvertFrom-Hex '0001ff80' +$binaryHash = [Convert]::ToUInt64('4653dd7f9a76930d', 16) +$helloKey = ConvertFrom-Hex '68656c6c6f' +$helloHash = [Convert]::ToUInt64('a430d84680aabd0b', 16) +$singleFfKey = ConvertFrom-Hex 'ff' +$singleFfHash = [Convert]::ToUInt64('af64724c8602eb6e', 16) + +$fixtureNames = @('empty', 'published', 'pending-reservation', 'pending-removal', 'reused-slot') +foreach ($fixtureName in $fixtureNames) { + $region = New-CanonicalRegion + + switch ($fixtureName) { + 'published' { + Set-Int64 $region 152 1 + Set-IndexEntry $region 5 1 $binaryHash 0 1 0 $binaryKey + Set-Slot $region 0 2 1 0 0 $binaryKey (ConvertFrom-Hex 'd0007f') (ConvertFrom-Hex '000102ff80') 4242 0 $binaryHash 1 + } + 'pending-reservation' { + Set-IndexEntry $region 3 1 $helloHash 1 1 0 $helloKey + Set-Slot $region 1 1 1 0 0 $helloKey (ConvertFrom-Hex 'aa00') (ConvertFrom-Hex '10002000000000') 4343 3 $helloHash 0 + } + 'pending-removal' { + Set-Int64 $region 152 9 + Set-IndexEntry $region 6 1 $singleFfHash 2 1 0 $singleFfKey + Set-Slot $region 2 3 1 0 1 $singleFfKey (ConvertFrom-Hex 'beef') (ConvertFrom-Hex 'de00adbe') 4444 0 $singleFfHash 8 + Set-Lease $region 2 2 1 0 5555 9 + } + 'reused-slot' { + Set-Int64 $region 152 2 + Set-IndexEntry $region 3 2 $helloHash 0 1 0 $helloKey + Set-IndexEntry $region 5 1 $binaryHash 0 2 0 $binaryKey + Set-Slot $region 0 2 2 0 0 $binaryKey (ConvertFrom-Hex '0102') (ConvertFrom-Hex '99008877') 4646 0 $binaryHash 2 + } + } + + $binaryPath = Join-Path $fixtureRoot "$fixtureName.bin" + [IO.File]::WriteAllBytes($binaryPath, $region) + + $snapshot = Get-NormalizedSnapshot $fixtureName $region + $jsonOptions = [Text.Json.JsonSerializerOptions]::new() + $jsonOptions.WriteIndented = $true + $json = [Text.Json.JsonSerializer]::Serialize($snapshot, $jsonOptions) + "`n" + [IO.File]::WriteAllText( + (Join-Path $fixtureRoot "$fixtureName.snapshot.json"), + $json.Replace("`r`n", "`n"), + $utf8WithoutBom) +} + +Write-Host "Generated $($fixtureNames.Count) deterministic layout-v1.2 fixture pairs in $fixtureRoot." diff --git a/protocol/fixtures/v1.2/manifest.json b/protocol/fixtures/v1.2/manifest.json new file mode 100644 index 0000000..31ebea0 --- /dev/null +++ b/protocol/fixtures/v1.2/manifest.json @@ -0,0 +1,678 @@ +{ + "format_version": 1, + "protocol": { + "layout_major": 1, + "layout_minor": 2, + "resource_naming_version": 1, + "byte_order": "little", + "max_field_alignment": 8, + "magic": { + "ascii": "SMS1", + "integer_hex": "31534d53", + "integer_value": 827542867, + "little_endian_bytes_hex": "534d5331" + } + }, + "records": { + "store_header": { + "size": 160, + "fields": { + "magic": { "type": "int32", "offset": 0 }, + "layout_major_version": { "type": "int32", "offset": 4 }, + "layout_minor_version": { "type": "int32", "offset": 8 }, + "header_length": { "type": "int32", "offset": 12 }, + "total_bytes": { "type": "int64", "offset": 16 }, + "slot_count": { "type": "int32", "offset": 24 }, + "lease_record_count": { "type": "int32", "offset": 28 }, + "max_key_bytes": { "type": "int32", "offset": 32 }, + "max_descriptor_bytes": { "type": "int32", "offset": 36 }, + "max_value_bytes": { "type": "int32", "offset": 40 }, + "index_entry_count": { "type": "int32", "offset": 44 }, + "index_entry_size": { "type": "int32", "offset": 48 }, + "index_offset": { "type": "int64", "offset": 56 }, + "index_length": { "type": "int64", "offset": 64 }, + "lease_registry_offset": { "type": "int64", "offset": 72 }, + "lease_registry_length": { "type": "int64", "offset": 80 }, + "slot_metadata_offset": { "type": "int64", "offset": 88 }, + "slot_metadata_length": { "type": "int64", "offset": 96 }, + "descriptor_storage_offset": { "type": "int64", "offset": 104 }, + "descriptor_storage_length": { "type": "int64", "offset": 112 }, + "payload_storage_offset": { "type": "int64", "offset": 120 }, + "payload_storage_length": { "type": "int64", "offset": 128 }, + "store_id": { "type": "int64", "offset": 136 }, + "store_state": { "type": "int32", "offset": 144 }, + "reserved": { "type": "int32", "offset": 148 }, + "sequence": { "type": "int64", "offset": 152 } + }, + "padding": [ + { "offset": 52, "length": 4 } + ] + }, + "index_entry_header": { + "size": 32, + "inline_key_offset": 32, + "fields": { + "state": { "type": "int32", "offset": 0 }, + "key_length": { "type": "int32", "offset": 4 }, + "key_hash": { "type": "uint64", "offset": 8 }, + "slot_index": { "type": "int32", "offset": 16 }, + "slot_generation": { "type": "int32", "offset": 20 }, + "slot_reuse_epoch": { "type": "int64", "offset": 24 } + }, + "padding": [] + }, + "slot_metadata": { + "size": 72, + "fields": { + "state": { "type": "int32", "offset": 0 }, + "generation": { "type": "int32", "offset": 4 }, + "reuse_epoch": { "type": "int64", "offset": 8 }, + "usage_count": { "type": "int32", "offset": 16 }, + "key_length": { "type": "int32", "offset": 20 }, + "descriptor_length": { "type": "int32", "offset": 24 }, + "value_length": { "type": "int32", "offset": 28 }, + "publisher_process_id": { "type": "int32", "offset": 32 }, + "reserved": { "type": "int32", "offset": 36 }, + "key_hash": { "type": "uint64", "offset": 40 }, + "descriptor_offset": { "type": "int64", "offset": 48 }, + "payload_offset": { "type": "int64", "offset": 56 }, + "committed_sequence": { "type": "int64", "offset": 64 } + }, + "padding": [] + }, + "lease_record": { + "size": 40, + "fields": { + "state": { "type": "int32", "offset": 0 }, + "lease_record_id": { "type": "int32", "offset": 4 }, + "slot_index": { "type": "int32", "offset": 8 }, + "slot_generation": { "type": "int32", "offset": 12 }, + "slot_reuse_epoch": { "type": "int64", "offset": 16 }, + "owner_process_id": { "type": "int32", "offset": 24 }, + "reserved": { "type": "int32", "offset": 28 }, + "acquire_sequence": { "type": "int64", "offset": 32 } + }, + "padding": [] + } + }, + "states": { + "store": { + "Initializing": 0, + "Ready": 1, + "Disposing": 2, + "Corrupt": 3, + "Unsupported": 4 + }, + "index": { + "Empty": 0, + "Occupied": 1, + "Tombstone": 2 + }, + "slot": { + "Free": 0, + "Publishing": 1, + "Published": 2, + "RemoveRequested": 3, + "Reclaiming": 4 + }, + "lease": { + "Free": 0, + "Active": 1, + "Released": 2, + "Abandoned": 3 + } + }, + "open_modes": { + "CreateNew": 0, + "OpenExisting": 1, + "CreateOrOpen": 2 + }, + "status_values": { + "store_open_status": { + "Success": 0, + "AlreadyExists": 1, + "NotFound": 2, + "InvalidOptions": 3, + "IncompatibleLayout": 4, + "UnsupportedPlatform": 5, + "InsufficientCapacity": 6, + "AccessDenied": 7, + "MappingFailed": 8, + "StoreBusy": 9, + "OperationCanceled": 10 + }, + "store_status": { + "Success": 0, + "DuplicateKey": 1, + "NotFound": 2, + "KeyTooLarge": 3, + "ValueTooLarge": 4, + "DescriptorTooLarge": 5, + "StoreFull": 6, + "LeaseTableFull": 7, + "InvalidLease": 8, + "LeaseAlreadyReleased": 9, + "RemovePending": 10, + "UnsupportedPlatform": 11, + "StoreDisposed": 12, + "CorruptStore": 13, + "AccessDenied": 14, + "UnknownFailure": 15, + "InvalidReservation": 16, + "ReservationIncomplete": 17, + "ReservationAlreadyCompleted": 18, + "ReservationWriteOutOfRange": 19, + "InvalidKey": 20, + "StoreBusy": 21, + "OperationCanceled": 22 + } + }, + "fnv1a_64": { + "offset_basis_hex": "cbf29ce484222325", + "prime_hex": "00000100000001b3", + "vectors": [ + { + "name": "empty", + "bytes_hex": "", + "valid_store_key": false, + "expected_hash_hex": "cbf29ce484222325" + }, + { + "name": "single-zero", + "bytes_hex": "00", + "valid_store_key": true, + "expected_hash_hex": "af63bd4c8601b7df" + }, + { + "name": "single-ff", + "bytes_hex": "ff", + "valid_store_key": true, + "expected_hash_hex": "af64724c8602eb6e" + }, + { + "name": "hello-ascii", + "bytes_hex": "68656c6c6f", + "valid_store_key": true, + "expected_hash_hex": "a430d84680aabd0b" + }, + { + "name": "foobar-ascii", + "bytes_hex": "666f6f626172", + "valid_store_key": true, + "expected_hash_hex": "85944171f73967e8" + }, + { + "name": "binary-with-high-bits", + "bytes_hex": "0001ff80", + "valid_store_key": true, + "expected_hash_hex": "4653dd7f9a76930d" + }, + { + "name": "library-name-ascii", + "bytes_hex": "5368617265644d656d6f727953746f7265", + "valid_store_key": true, + "expected_hash_hex": "9a497256b7ae6c50" + }, + { + "name": "utf8-cafe-emoji", + "bytes_hex": "636166c3a9f09f9880", + "valid_store_key": true, + "expected_hash_hex": "ab0ae80e9faaf1f4" + } + ] + }, + "layout_calculation": { + "vectors": [ + { + "name": "minimum-capacities", + "input": { + "slot_count": 1, + "max_value_bytes": 1, + "max_descriptor_bytes": 0, + "max_key_bytes": 1, + "lease_record_count": 1 + }, + "expected": { + "header_length": 160, + "index_entry_count": 4, + "index_entry_size": 40, + "index_offset": 160, + "index_length": 160, + "lease_registry_offset": 320, + "lease_registry_length": 40, + "slot_metadata_offset": 360, + "slot_metadata_length": 72, + "descriptor_stride": 8, + "descriptor_storage_offset": 432, + "descriptor_storage_length": 8, + "payload_stride": 8, + "payload_storage_offset": 440, + "payload_storage_length": 8, + "required_bytes": 448 + } + }, + { + "name": "alignment-and-next-power-of-two", + "input": { + "slot_count": 3, + "max_value_bytes": 17, + "max_descriptor_bytes": 5, + "max_key_bytes": 9, + "lease_record_count": 4 + }, + "expected": { + "header_length": 160, + "index_entry_count": 8, + "index_entry_size": 48, + "index_offset": 160, + "index_length": 384, + "lease_registry_offset": 544, + "lease_registry_length": 160, + "slot_metadata_offset": 704, + "slot_metadata_length": 216, + "descriptor_stride": 8, + "descriptor_storage_offset": 920, + "descriptor_storage_length": 24, + "payload_stride": 24, + "payload_storage_offset": 944, + "payload_storage_length": 72, + "required_bytes": 1016 + } + }, + { + "name": "ordinary-store", + "input": { + "slot_count": 4, + "max_value_bytes": 1024, + "max_descriptor_bytes": 16, + "max_key_bytes": 64, + "lease_record_count": 8 + }, + "expected": { + "header_length": 160, + "index_entry_count": 8, + "index_entry_size": 96, + "index_offset": 160, + "index_length": 768, + "lease_registry_offset": 928, + "lease_registry_length": 320, + "slot_metadata_offset": 1248, + "slot_metadata_length": 288, + "descriptor_stride": 16, + "descriptor_storage_offset": 1536, + "descriptor_storage_length": 64, + "payload_stride": 1024, + "payload_storage_offset": 1600, + "payload_storage_length": 4096, + "required_bytes": 5696 + } + }, + { + "name": "non-power-of-two-slots", + "input": { + "slot_count": 17, + "max_value_bytes": 4097, + "max_descriptor_bytes": 257, + "max_key_bytes": 255, + "lease_record_count": 33 + }, + "expected": { + "header_length": 160, + "index_entry_count": 64, + "index_entry_size": 288, + "index_offset": 160, + "index_length": 18432, + "lease_registry_offset": 18592, + "lease_registry_length": 1320, + "slot_metadata_offset": 19912, + "slot_metadata_length": 1224, + "descriptor_stride": 264, + "descriptor_storage_offset": 21136, + "descriptor_storage_length": 4488, + "payload_stride": 4104, + "payload_storage_offset": 25624, + "payload_storage_length": 69768, + "required_bytes": 95392 + } + }, + { + "name": "large-store", + "input": { + "slot_count": 512, + "max_value_bytes": 1048576, + "max_descriptor_bytes": 256, + "max_key_bytes": 128, + "lease_record_count": 1024 + }, + "expected": { + "header_length": 160, + "index_entry_count": 1024, + "index_entry_size": 160, + "index_offset": 160, + "index_length": 163840, + "lease_registry_offset": 164000, + "lease_registry_length": 40960, + "slot_metadata_offset": 204960, + "slot_metadata_length": 36864, + "descriptor_stride": 256, + "descriptor_storage_offset": 241824, + "descriptor_storage_length": 131072, + "payload_stride": 1048576, + "payload_storage_offset": 372896, + "payload_storage_length": 536870912, + "required_bytes": 537243808 + } + } + ], + "error_vectors": [ + { + "name": "slot-count-must-be-positive", + "input": { + "slot_count": 0, + "max_value_bytes": 1, + "max_descriptor_bytes": 0, + "max_key_bytes": 1, + "lease_record_count": 1 + }, + "expected_error": "invalid_argument" + }, + { + "name": "value-capacity-must-be-positive", + "input": { + "slot_count": 1, + "max_value_bytes": 0, + "max_descriptor_bytes": 0, + "max_key_bytes": 1, + "lease_record_count": 1 + }, + "expected_error": "invalid_argument" + }, + { + "name": "descriptor-capacity-must-not-be-negative", + "input": { + "slot_count": 1, + "max_value_bytes": 1, + "max_descriptor_bytes": -1, + "max_key_bytes": 1, + "lease_record_count": 1 + }, + "expected_error": "invalid_argument" + }, + { + "name": "key-capacity-must-be-positive", + "input": { + "slot_count": 1, + "max_value_bytes": 1, + "max_descriptor_bytes": 0, + "max_key_bytes": 0, + "lease_record_count": 1 + }, + "expected_error": "invalid_argument" + }, + { + "name": "lease-count-must-be-positive", + "input": { + "slot_count": 1, + "max_value_bytes": 1, + "max_descriptor_bytes": 0, + "max_key_bytes": 1, + "lease_record_count": 0 + }, + "expected_error": "invalid_argument" + }, + { + "name": "index-count-overflow", + "input": { + "slot_count": 536870913, + "max_value_bytes": 1, + "max_descriptor_bytes": 0, + "max_key_bytes": 1, + "lease_record_count": 1 + }, + "expected_error": "arithmetic_overflow" + }, + { + "name": "index-stride-overflow", + "input": { + "slot_count": 1, + "max_value_bytes": 1, + "max_descriptor_bytes": 0, + "max_key_bytes": 2147483647, + "lease_record_count": 1 + }, + "expected_error": "arithmetic_overflow" + }, + { + "name": "descriptor-stride-overflow", + "input": { + "slot_count": 1, + "max_value_bytes": 1, + "max_descriptor_bytes": 2147483647, + "max_key_bytes": 1, + "lease_record_count": 1 + }, + "expected_error": "arithmetic_overflow" + }, + { + "name": "payload-stride-overflow", + "input": { + "slot_count": 1, + "max_value_bytes": 2147483647, + "max_descriptor_bytes": 0, + "max_key_bytes": 1, + "lease_record_count": 1 + }, + "expected_error": "arithmetic_overflow" + } + ] + }, + "resource_naming": { + "version": 1, + "maximum_public_name_utf16_code_units": 240, + "maximum_linux_readable_utf16_code_units": 80, + "linux_directory_mode_octal": "0700", + "linux_file_mode_octal": "0600", + "vectors": [ + { + "name": "simple-with-dot", + "public_name": "sms.compatibility", + "utf16_code_units": 17, + "expected": { + "windows_region_name": "sms.compatibility", + "windows_synchronization_name": "Local\\SharedMemoryStore-sms_compatibility", + "linux_sha256_prefix_hex": "251220bcba0b63e6", + "linux_fragment": "sms-sms.compatibility-251220bcba0b63e6", + "linux_files": { + "region": "sms-sms.compatibility-251220bcba0b63e6.region", + "synchronization": "sms-sms.compatibility-251220bcba0b63e6.lock", + "owners": "sms-sms.compatibility-251220bcba0b63e6.owners", + "lifecycle": "sms-sms.compatibility-251220bcba0b63e6.lifecycle" + } + } + }, + { + "name": "separator", + "public_name": "store/name", + "utf16_code_units": 10, + "expected": { + "windows_region_name": "store/name", + "windows_synchronization_name": "Local\\SharedMemoryStore-store_name", + "linux_sha256_prefix_hex": "549a0c43d4d76e02", + "linux_fragment": "sms-store_name-549a0c43d4d76e02", + "linux_files": { + "region": "sms-store_name-549a0c43d4d76e02.region", + "synchronization": "sms-store_name-549a0c43d4d76e02.lock", + "owners": "sms-store_name-549a0c43d4d76e02.owners", + "lifecycle": "sms-store_name-549a0c43d4d76e02.lifecycle" + } + } + }, + { + "name": "sanitized-collision-kept-distinct", + "public_name": "store?name", + "utf16_code_units": 10, + "expected": { + "windows_region_name": "store?name", + "windows_synchronization_name": "Local\\SharedMemoryStore-store_name", + "linux_sha256_prefix_hex": "0f08216b745495cd", + "linux_fragment": "sms-store_name-0f08216b745495cd", + "linux_files": { + "region": "sms-store_name-0f08216b745495cd.region", + "synchronization": "sms-store_name-0f08216b745495cd.lock", + "owners": "sms-store_name-0f08216b745495cd.owners", + "lifecycle": "sms-store_name-0f08216b745495cd.lifecycle" + } + } + }, + { + "name": "empty-readable-fallback", + "public_name": " ..//?? ", + "utf16_code_units": 8, + "expected": { + "windows_region_name": " ..//?? ", + "windows_synchronization_name": "Local\\SharedMemoryStore-________", + "linux_sha256_prefix_hex": "86a089f14047e794", + "linux_fragment": "sms-store-86a089f14047e794", + "linux_files": { + "region": "sms-store-86a089f14047e794.region", + "synchronization": "sms-store-86a089f14047e794.lock", + "owners": "sms-store-86a089f14047e794.owners", + "lifecycle": "sms-store-86a089f14047e794.lifecycle" + } + } + }, + { + "name": "global-scope", + "public_name": "Global\\sms.compatibility", + "utf16_code_units": 24, + "expected": { + "windows_region_name": "Global\\sms.compatibility", + "windows_synchronization_name": "Global\\SharedMemoryStore-Global_sms_compatibility", + "linux_sha256_prefix_hex": "54e26013eb7f992d", + "linux_fragment": "sms-Global_sms.compatibility-54e26013eb7f992d", + "linux_files": { + "region": "sms-Global_sms.compatibility-54e26013eb7f992d.region", + "synchronization": "sms-Global_sms.compatibility-54e26013eb7f992d.lock", + "owners": "sms-Global_sms.compatibility-54e26013eb7f992d.owners", + "lifecycle": "sms-Global_sms.compatibility-54e26013eb7f992d.lifecycle" + } + } + }, + { + "name": "case-insensitive-global-scope", + "public_name": "gLoBaL\\A/B", + "utf16_code_units": 10, + "expected": { + "windows_region_name": "gLoBaL\\A/B", + "windows_synchronization_name": "Global\\SharedMemoryStore-gLoBaL_A_B", + "linux_sha256_prefix_hex": "08f24e66b8baf050", + "linux_fragment": "sms-gLoBaL_A_B-08f24e66b8baf050", + "linux_files": { + "region": "sms-gLoBaL_A_B-08f24e66b8baf050.region", + "synchronization": "sms-gLoBaL_A_B-08f24e66b8baf050.lock", + "owners": "sms-gLoBaL_A_B-08f24e66b8baf050.owners", + "lifecycle": "sms-gLoBaL_A_B-08f24e66b8baf050.lifecycle" + } + } + }, + { + "name": "explicit-local-scope", + "public_name": "Local\\A.B", + "utf16_code_units": 9, + "expected": { + "windows_region_name": "Local\\A.B", + "windows_synchronization_name": "Local\\SharedMemoryStore-Local_A_B", + "linux_sha256_prefix_hex": "2ed6fbf4a40d3d0b", + "linux_fragment": "sms-Local_A.B-2ed6fbf4a40d3d0b", + "linux_files": { + "region": "sms-Local_A.B-2ed6fbf4a40d3d0b.region", + "synchronization": "sms-Local_A.B-2ed6fbf4a40d3d0b.lock", + "owners": "sms-Local_A.B-2ed6fbf4a40d3d0b.owners", + "lifecycle": "sms-Local_A.B-2ed6fbf4a40d3d0b.lifecycle" + } + } + }, + { + "name": "unicode-and-supplementary-scalar", + "public_name": "café/共享/😀", + "utf16_code_units": 10, + "expected": { + "windows_region_name": "café/共享/😀", + "windows_synchronization_name": "Local\\SharedMemoryStore-café_共享___", + "linux_sha256_prefix_hex": "0f903cf0f516f93b", + "linux_fragment": "sms-caf-0f903cf0f516f93b", + "linux_files": { + "region": "sms-caf-0f903cf0f516f93b.region", + "synchronization": "sms-caf-0f903cf0f516f93b.lock", + "owners": "sms-caf-0f903cf0f516f93b.owners", + "lifecycle": "sms-caf-0f903cf0f516f93b.lifecycle" + } + } + }, + { + "name": "maximum-length-and-readable-truncation", + "public_name": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "utf16_code_units": 240, + "expected": { + "windows_region_name": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "windows_synchronization_name": "Local\\SharedMemoryStore-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "linux_sha256_prefix_hex": "718df6704d8a0610", + "linux_fragment": "sms-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-718df6704d8a0610", + "linux_files": { + "region": "sms-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-718df6704d8a0610.region", + "synchronization": "sms-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-718df6704d8a0610.lock", + "owners": "sms-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-718df6704d8a0610.owners", + "lifecycle": "sms-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-718df6704d8a0610.lifecycle" + } + } + } + ] + }, + "mapped_region_fixtures": [ + { + "name": "empty", + "binary_file": "empty.bin", + "binary_sha256_hex": "6b6e2a3ce5e491da504dfb46e142ce1170d8425b569da8d181b3c043d29e3176", + "snapshot_file": "empty.snapshot.json", + "snapshot_sha256_hex": "9649bdfd85c8048edd186d7af1fe3c2acb69ba93b91062a33a1f04834d82d623", + "byte_length": 1016, + "offline_only": true + }, + { + "name": "published", + "binary_file": "published.bin", + "binary_sha256_hex": "b1278352a3d76317a654e39d719337cdf5ae9ca64da51d32d0b71d1bde132027", + "snapshot_file": "published.snapshot.json", + "snapshot_sha256_hex": "889ca582151a24ead89e7a7beef79700606cb1f1b0f3cd9e50527d7b9e11f129", + "byte_length": 1016, + "offline_only": true + }, + { + "name": "pending-reservation", + "binary_file": "pending-reservation.bin", + "binary_sha256_hex": "ac44a4e923ad534e2e6aeb106f861b44c950b30ed2cba98def53375c3885fe33", + "snapshot_file": "pending-reservation.snapshot.json", + "snapshot_sha256_hex": "87d60e65b923c209d3422381ef30248fc20d4e3de572b429d6e724888ddba3dd", + "byte_length": 1016, + "offline_only": true + }, + { + "name": "pending-removal", + "binary_file": "pending-removal.bin", + "binary_sha256_hex": "984c79c6003467aa215581e306a3c358de56a825e16e577b159db470919eab76", + "snapshot_file": "pending-removal.snapshot.json", + "snapshot_sha256_hex": "25bb0b671718cd8b6ec80ba763ede197104f065a61109728a9e5309ace1e0d31", + "byte_length": 1016, + "offline_only": true + }, + { + "name": "reused-slot", + "binary_file": "reused-slot.bin", + "binary_sha256_hex": "1a54a08af9445040219ae9933207e39f0036884f8ebcace6c5c36adae05a3fd1", + "snapshot_file": "reused-slot.snapshot.json", + "snapshot_sha256_hex": "d7a4dc6514435f1d70b38a3437400302fa820bfa2eea8b196d67916aea6ced75", + "byte_length": 1016, + "offline_only": true + } + ] +} diff --git a/protocol/fixtures/v1.2/pending-removal.bin b/protocol/fixtures/v1.2/pending-removal.bin new file mode 100644 index 0000000000000000000000000000000000000000..383d550d9e01c52f081dc1cd6cbdfcf807c33c36 GIT binary patch literal 1016 zcmWIc4K`$CU|?VZ;srqbgBeHw0f=G&Voo4t1!6%U<^W;?AO?X2V3L8M0i+oS6hH<5 z!2&SJz;FP{zX9dXfSM}-d zSq}838@3A0l5*~ots4=ri0QO41kgdA@e|Lk^KwPi;IS( zLsnRs~X&H7%Tzt zk=>k;nv;`1_?%6#i~a)vAwxjnK#V$2(jZ11D2WiG4mtb@xe=BQVd)R19z7mG;-Gj2 xxs@G=zY9R}1T5YcK-2FDC=FBs2c)=X1yuh7Eard=Tm?53KSO{)0lxw?SpWc^D2o69 literal 0 HcmV?d00001 diff --git a/protocol/fixtures/v1.2/pending-reservation.snapshot.json b/protocol/fixtures/v1.2/pending-reservation.snapshot.json new file mode 100644 index 0000000..b7b627c --- /dev/null +++ b/protocol/fixtures/v1.2/pending-reservation.snapshot.json @@ -0,0 +1,101 @@ +{ + "format_version": 1, + "fixture": "pending-reservation", + "offline_only": true, + "protocol": { + "layout_major": 1, + "layout_minor": 2, + "byte_order": "little" + }, + "header": { + "magic_hex": "31534d53", + "header_length": 160, + "total_bytes": 1016, + "slot_count": 3, + "lease_record_count": 4, + "max_key_bytes": 9, + "max_descriptor_bytes": 5, + "max_value_bytes": 17, + "index_entry_count": 8, + "index_entry_size": 48, + "index_offset": 160, + "index_length": 384, + "lease_registry_offset": 544, + "lease_registry_length": 160, + "slot_metadata_offset": 704, + "slot_metadata_length": 216, + "descriptor_storage_offset": 920, + "descriptor_storage_length": 24, + "payload_storage_offset": 944, + "payload_storage_length": 72, + "store_id_hex": "0102030405060708", + "store_state": 1, + "sequence": 0 + }, + "index_entries": [ + { + "entry_index": 3, + "state": 1, + "state_name": "Occupied", + "key_hex": "68656c6c6f", + "key_hash_hex": "a430d84680aabd0b", + "slot_index": 1, + "slot_generation": 1, + "slot_reuse_epoch": 0 + } + ], + "lease_records": [], + "slots": [ + { + "slot_index": 0, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 920, + "payload_offset": 944, + "committed_sequence": 0 + }, + { + "slot_index": 1, + "state": 1, + "state_name": "Publishing", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 5, + "descriptor_hex": "aa00", + "payload_hex": "10002000000000", + "publisher_process_id": 4343, + "reservation_bytes_written": 3, + "key_hash_hex": "a430d84680aabd0b", + "descriptor_offset": 928, + "payload_offset": 968, + "committed_sequence": 0 + }, + { + "slot_index": 2, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 936, + "payload_offset": 992, + "committed_sequence": 0 + } + ] +} diff --git a/protocol/fixtures/v1.2/published.bin b/protocol/fixtures/v1.2/published.bin new file mode 100644 index 0000000000000000000000000000000000000000..1ae4079b4dae83ca80924735a2e8a434c94126bd GIT binary patch literal 1016 zcmWIc4K`$CU|?VZ;srqbgBeHw0f=G&Voo4t1!6%U<^W;?AO?X2V3L8M0i+oS6hH<5 z!2&SJz;FP{zX9dXfSM}-P^V$^|>1~KYDNraF(kXb0<1LT9k52P2AZYK!<2_S%l wCoCPp(jT&3WHv+>ZgK%M{hol*$SO%OX9ZOM12l6kFw`R(iC{A_Q6&KY04^FS2LJ#7 literal 0 HcmV?d00001 diff --git a/protocol/fixtures/v1.2/published.snapshot.json b/protocol/fixtures/v1.2/published.snapshot.json new file mode 100644 index 0000000..7448e03 --- /dev/null +++ b/protocol/fixtures/v1.2/published.snapshot.json @@ -0,0 +1,101 @@ +{ + "format_version": 1, + "fixture": "published", + "offline_only": true, + "protocol": { + "layout_major": 1, + "layout_minor": 2, + "byte_order": "little" + }, + "header": { + "magic_hex": "31534d53", + "header_length": 160, + "total_bytes": 1016, + "slot_count": 3, + "lease_record_count": 4, + "max_key_bytes": 9, + "max_descriptor_bytes": 5, + "max_value_bytes": 17, + "index_entry_count": 8, + "index_entry_size": 48, + "index_offset": 160, + "index_length": 384, + "lease_registry_offset": 544, + "lease_registry_length": 160, + "slot_metadata_offset": 704, + "slot_metadata_length": 216, + "descriptor_storage_offset": 920, + "descriptor_storage_length": 24, + "payload_storage_offset": 944, + "payload_storage_length": 72, + "store_id_hex": "0102030405060708", + "store_state": 1, + "sequence": 1 + }, + "index_entries": [ + { + "entry_index": 5, + "state": 1, + "state_name": "Occupied", + "key_hex": "0001ff80", + "key_hash_hex": "4653dd7f9a76930d", + "slot_index": 0, + "slot_generation": 1, + "slot_reuse_epoch": 0 + } + ], + "lease_records": [], + "slots": [ + { + "slot_index": 0, + "state": 2, + "state_name": "Published", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 4, + "descriptor_hex": "d0007f", + "payload_hex": "000102ff80", + "publisher_process_id": 4242, + "reservation_bytes_written": 0, + "key_hash_hex": "4653dd7f9a76930d", + "descriptor_offset": 920, + "payload_offset": 944, + "committed_sequence": 1 + }, + { + "slot_index": 1, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 928, + "payload_offset": 968, + "committed_sequence": 0 + }, + { + "slot_index": 2, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 936, + "payload_offset": 992, + "committed_sequence": 0 + } + ] +} diff --git a/protocol/fixtures/v1.2/reused-slot.bin b/protocol/fixtures/v1.2/reused-slot.bin new file mode 100644 index 0000000000000000000000000000000000000000..e48d23323d6b49485f287b9de735473f39ebfe8a GIT binary patch literal 1016 zcmWIc4K`$CU|?VZ;srqbgBeHw0f=G&Voo4t1!6%U<^W;?AO?X2V3L8M0i+oS6hH<5 z!2&SJz;FP{zX9dXfSM}-Bw^I#3cJqz+^jO85Zzpzs5UfoL@$AOQrh@Pws9So%ZOiyV&-UAV~w(DZu(N+YWz a#hevT{SVO0VMKNqlHoHMI?9o_WHSMYWG!g` literal 0 HcmV?d00001 diff --git a/protocol/fixtures/v1.2/reused-slot.snapshot.json b/protocol/fixtures/v1.2/reused-slot.snapshot.json new file mode 100644 index 0000000..133d39e --- /dev/null +++ b/protocol/fixtures/v1.2/reused-slot.snapshot.json @@ -0,0 +1,111 @@ +{ + "format_version": 1, + "fixture": "reused-slot", + "offline_only": true, + "protocol": { + "layout_major": 1, + "layout_minor": 2, + "byte_order": "little" + }, + "header": { + "magic_hex": "31534d53", + "header_length": 160, + "total_bytes": 1016, + "slot_count": 3, + "lease_record_count": 4, + "max_key_bytes": 9, + "max_descriptor_bytes": 5, + "max_value_bytes": 17, + "index_entry_count": 8, + "index_entry_size": 48, + "index_offset": 160, + "index_length": 384, + "lease_registry_offset": 544, + "lease_registry_length": 160, + "slot_metadata_offset": 704, + "slot_metadata_length": 216, + "descriptor_storage_offset": 920, + "descriptor_storage_length": 24, + "payload_storage_offset": 944, + "payload_storage_length": 72, + "store_id_hex": "0102030405060708", + "store_state": 1, + "sequence": 2 + }, + "index_entries": [ + { + "entry_index": 3, + "state": 2, + "state_name": "Tombstone", + "key_hex": "68656c6c6f", + "key_hash_hex": "a430d84680aabd0b", + "slot_index": 0, + "slot_generation": 1, + "slot_reuse_epoch": 0 + }, + { + "entry_index": 5, + "state": 1, + "state_name": "Occupied", + "key_hex": "0001ff80", + "key_hash_hex": "4653dd7f9a76930d", + "slot_index": 0, + "slot_generation": 2, + "slot_reuse_epoch": 0 + } + ], + "lease_records": [], + "slots": [ + { + "slot_index": 0, + "state": 2, + "state_name": "Published", + "generation": 2, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 4, + "descriptor_hex": "0102", + "payload_hex": "99008877", + "publisher_process_id": 4646, + "reservation_bytes_written": 0, + "key_hash_hex": "4653dd7f9a76930d", + "descriptor_offset": 920, + "payload_offset": 944, + "committed_sequence": 2 + }, + { + "slot_index": 1, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 928, + "payload_offset": 968, + "committed_sequence": 0 + }, + { + "slot_index": 2, + "state": 0, + "state_name": "Free", + "generation": 1, + "reuse_epoch": 0, + "usage_count": 0, + "key_length": 0, + "descriptor_hex": "", + "payload_hex": "", + "publisher_process_id": 0, + "reservation_bytes_written": 0, + "key_hash_hex": "0000000000000000", + "descriptor_offset": 936, + "payload_offset": 992, + "committed_sequence": 0 + } + ] +} diff --git a/protocol/layout-v1.2.md b/protocol/layout-v1.2.md new file mode 100644 index 0000000..3b75c15 --- /dev/null +++ b/protocol/layout-v1.2.md @@ -0,0 +1,191 @@ +# Mapped Layout 1.2 + +This document defines the canonical major-1, minor-2 mapped representation. All +multi-byte integers are little-endian, and signed integers use two's-complement +representation. Records use sequential field order with a maximum field +alignment of 8 bytes; the offsets below are part of the protocol and must be +asserted rather than inferred from a compiler ABI. Keys, descriptors, and +payloads are opaque bytes. + +The 32-bit magic integer is `0x31534d53`, whose little-endian bytes spell +`SMS1`. All offsets are absolute byte offsets from the beginning of the mapped +region. + +## Region order + +```text +store header +shared key index +lease registry +slot metadata table +descriptor storage +payload storage +``` + +There is no section directory outside the 160-byte header. Index entries have a +variable stride because the fixed 32-byte header is followed by inline key +capacity. Lease and slot records have fixed strides of 40 and 72 bytes. + +## Store header: 160 bytes + +| Field | Type | Offset | Meaning | +|---|---:|---:|---| +| `magic` | `int32` | 0 | `0x31534d53` | +| `layout_major_version` | `int32` | 4 | `1` | +| `layout_minor_version` | `int32` | 8 | `2` | +| `header_length` | `int32` | 12 | `160` | +| `total_bytes` | `int64` | 16 | Full mapped-region length supplied at creation | +| `slot_count` | `int32` | 24 | Number of slot records and storage strides | +| `lease_record_count` | `int32` | 28 | Number of lease records | +| `max_key_bytes` | `int32` | 32 | Inline key capacity in each index entry | +| `max_descriptor_bytes` | `int32` | 36 | Per-slot descriptor capacity | +| `max_value_bytes` | `int32` | 40 | Per-slot payload capacity | +| `index_entry_count` | `int32` | 44 | Power-of-two index capacity | +| `index_entry_size` | `int32` | 48 | Aligned fixed header plus key capacity | +| padding | 4 bytes | 52 | Must not be interpreted | +| `index_offset` | `int64` | 56 | Start of the index | +| `index_length` | `int64` | 64 | Index bytes | +| `lease_registry_offset` | `int64` | 72 | Start of lease records | +| `lease_registry_length` | `int64` | 80 | Lease-record bytes | +| `slot_metadata_offset` | `int64` | 88 | Start of slot records | +| `slot_metadata_length` | `int64` | 96 | Slot-record bytes | +| `descriptor_storage_offset` | `int64` | 104 | Start of descriptor strides | +| `descriptor_storage_length` | `int64` | 112 | Descriptor-storage bytes | +| `payload_storage_offset` | `int64` | 120 | Start of payload strides | +| `payload_storage_length` | `int64` | 128 | Payload-storage bytes | +| `store_id` | `int64` | 136 | Opaque identity assigned at initialization | +| `store_state` | `int32` | 144 | Store state value | +| `reserved` | `int32` | 148 | Reserved; readers must not attach meaning | +| `sequence` | `int64` | 152 | Monotonic commit/acquire sequence source | + +## Index entry header: 32 bytes + +| Field | Type | Offset | +|---|---:|---:| +| `state` | `int32` | 0 | +| `key_length` | `int32` | 4 | +| `key_hash` | `uint64` | 8 | +| `slot_index` | `int32` | 16 | +| `slot_generation` | `int32` | 20 | +| `slot_reuse_epoch` | `int64` | 24 | + +The inline key starts at offset 32 and occupies `max_key_bytes` bytes inside the +entry stride. Writers clear the full inline capacity, copy exactly `key_length` +bytes, and publish `Occupied` last. Exact equality requires both hash and bytes; +a hash match alone is never a key match. + +Keys use unsigned 64-bit FNV-1a with offset basis +`0xcbf29ce484222325` and prime `0x00000100000001b3`; multiplication wraps modulo +2^64 after every byte. Probe start is +`key_hash & (index_entry_count - 1)`, followed by linear probing with wraparound. +An `Empty` entry terminates lookup; a `Tombstone` does not. Insertions reuse the +first tombstone encountered. Removal marks every matching copy tombstone so an +interrupted index compaction cannot leave a stale duplicate. + +## Slot metadata: 72 bytes + +| Field | Type | Offset | Meaning | +|---|---:|---:|---| +| `state` | `int32` | 0 | Slot state value | +| `generation` | `int32` | 4 | Positive lifecycle generation | +| `reuse_epoch` | `int64` | 8 | Non-negative generation-rollover epoch | +| `usage_count` | `int32` | 16 | Active leases protecting this lifecycle | +| `key_length` | `int32` | 20 | Key bytes in the corresponding index entry | +| `descriptor_length` | `int32` | 24 | Committed or announced descriptor bytes | +| `value_length` | `int32` | 28 | Committed or announced payload bytes | +| `publisher_process_id` | `int32` | 32 | Reservation/publication owner PID | +| `reserved` | `int32` | 36 | Bytes advanced while `Publishing`; zero after commit | +| `key_hash` | `uint64` | 40 | FNV-1a hash of the key | +| `descriptor_offset` | `int64` | 48 | Absolute per-slot descriptor address | +| `payload_offset` | `int64` | 56 | Absolute per-slot payload address | +| `committed_sequence` | `int64` | 64 | Zero before commit; assigned before publication | + +The lifecycle identity is the pair `(generation, reuse_epoch)`, initially +`(1, 0)`. Reclaim increments generation. Reclaiming generation `2147483647` +sets generation to `1` and increments the epoch. The pair +`(2147483647, 9223372036854775807)` cannot advance and makes reuse unsafe; the +operation reports corruption rather than repeating an old identity. + +Descriptor address for slot `i` is +`descriptor_storage_offset + i * descriptor_stride`; payload address follows +the equivalent payload formula. Zero-length descriptors are valid, but their +physical stride is still 8 bytes. + +## Lease record: 40 bytes + +| Field | Type | Offset | +|---|---:|---:| +| `state` | `int32` | 0 | +| `lease_record_id` | `int32` | 4 | +| `slot_index` | `int32` | 8 | +| `slot_generation` | `int32` | 12 | +| `slot_reuse_epoch` | `int64` | 16 | +| `owner_process_id` | `int32` | 24 | +| `reserved` | `int32` | 28 | +| `acquire_sequence` | `int64` | 32 | + +An active lease is valid only when its record id, slot index, and complete +lifecycle pair still match. Publishing a record's `Active` state occurs after +its other fields are written. `Released` and `Abandoned` records may later be +overwritten for a new lease; stale process-local tokens must still be rejected +by their captured identity and token lifetime. + +## Numeric assignments and transitions + +| State family | Assignments | +|---|---| +| Store | `Initializing=0`, `Ready=1`, `Disposing=2`, `Corrupt=3`, `Unsupported=4` | +| Index | `Empty=0`, `Occupied=1`, `Tombstone=2` | +| Slot | `Free=0`, `Publishing=1`, `Published=2`, `RemoveRequested=3`, `Reclaiming=4` | +| Lease | `Free=0`, `Active=1`, `Released=2`, `Abandoned=3` | + +Slot flow is `Free -> Publishing -> Published`. Abort or authorized recovery +removes the pending index entry and returns `Publishing -> Free` without making +bytes visible. A remove with no leases removes the index entry and flows +`Published -> Reclaiming -> Free`. A remove with leases publishes +`Published -> RemoveRequested`; existing leases remain readable, no new lease +may be acquired, and the final release performs +`RemoveRequested -> Reclaiming -> Free`. + +For commit, descriptor and payload bytes and all non-state metadata are written +first, the sequence is assigned, and `Published` is stored last. Readers treat +only `Published` as newly acquirable and must recheck the state and lifecycle +after registering a lease. All shared mutations participate in the common +platform synchronization contract. + +## Checked layout calculation + +`align8(x) = (x + 7) & ~7`, with checked signed arithmetic. Let `S` be slot +count, `L` lease-record count, `K` maximum key bytes, `D` maximum descriptor +bytes, and `V` maximum value bytes: + +```text +header_length = 160 +index_entry_count = next_power_of_two(max(4, S * 2)) +index_entry_size = align8(32 + K) +index_offset = header_length +index_length = index_entry_count * index_entry_size +lease_registry_offset = align8(index_offset + index_length) +lease_registry_length = L * 40 +slot_metadata_offset = align8(lease_registry_offset + lease_registry_length) +slot_metadata_length = S * 72 +descriptor_stride = align8(max(1, D)) +descriptor_storage_offset = align8(slot_metadata_offset + slot_metadata_length) +descriptor_storage_length = S * descriptor_stride +payload_stride = align8(max(1, V)) +payload_storage_offset = align8(descriptor_storage_offset + descriptor_storage_length) +payload_storage_length = S * payload_stride +required_bytes = align8(payload_storage_offset + payload_storage_length) +``` + +`S`, `L`, `K`, and `V` must be positive; `D` may be zero. The index count must +be representable as a positive signed 32-bit power of two no greater than +2^30. Intermediate 32-bit operations used for counts and strides and all +64-bit offsets and lengths are checked. Overflow is an invalid layout, never +wraparound. `total_bytes` may exceed `required_bytes` but must be positive and +must not be smaller. + +Opening validates all stored dimensions, calculated offsets and lengths, and +monotonic in-bounds sections. A major-version mismatch, impossible arithmetic, +unknown unsafe shape, or section extending past `total_bytes` is incompatible; +no payload pointer may be formed from an unvalidated header. diff --git a/protocol/resource-naming-v1.md b/protocol/resource-naming-v1.md new file mode 100644 index 0000000..4c23a10 --- /dev/null +++ b/protocol/resource-naming-v1.md @@ -0,0 +1,117 @@ +# Platform Resource Naming Version 1 + +Mapped-layout compatibility is not enough for live interoperability. Every +participant must derive the same mapping, synchronization, ownership, and +lifecycle resources from the same public store name and must participate in the +same lock protocol. + +Public names are nonblank .NET-compatible strings of 1 through 240 UTF-16 code +units and must not contain NUL. No Unicode normalization is performed. Hashing +uses the UTF-8 encoding of the public name. Sanitization is defined over UTF-16 +code units to reproduce the managed baseline exactly, including two replacement +characters for a supplementary Unicode scalar represented by a surrogate pair. + +## Windows + +The memory-mapped region name is the public name unchanged. + +The synchronization name is: + +```text +SharedMemoryStore- +``` + +`scope` is `Global\` when the public name begins with `Global\` using an +ordinal, case-insensitive comparison; otherwise it is `Local\`. The scope text +inside the public name is not removed before sanitization. For each UTF-16 code +unit, keep a Unicode letter, a decimal digit, `-`, or `_`; replace every other +unit with `_`. Thus dots and separators are replaced, while BMP letters such as +`é` and `共` remain. The exact vectors are in the fixture manifest. + +All shared operations use the derived named mutex. An abandoned mutex is +treated as acquired so the caller can validate shared state under mutual +exclusion. No-wait, bounded, infinite, cancellation, access-denied, and disposed +outcomes are converted to public statuses. Closing one participant closes only +its mapping and mutex handles; Windows kernel object lifetime removes named +objects after the final handle closes. + +## Linux paths + +The root is `/dev/shm/SharedMemoryStore` when `/dev/shm` exists. Otherwise it is +`SharedMemoryStore` below the operating system temporary directory. The root +must be a real directory rather than a symbolic link or reparse point and is +forced to mode `0700`. + +The resource fragment is: + +```text +sms-- +``` + +To form `readable`, process UTF-16 code units in order. Keep only ASCII letters, +ASCII digits, `-`, `_`, and `.`; replace every other unit with `_`. Trim leading +and trailing `_` and `.`. Use `store` if nothing remains, then truncate to the +first 80 code units. `digest` is the lowercase hexadecimal encoding of the +first 8 bytes of SHA-256 over the UTF-8 public name. The digest is taken from the +unsanitized, untruncated public name and prevents sanitized-name collisions. + +Four paths use this fragment: + +| Suffix | Purpose | +|---|---| +| `.region` | Mapped data file | +| `.lock` | Ordinary store-operation synchronization | +| `.owners` | Live region-owner sidecar | +| `.lifecycle` | Serializes owner registration, stale cleanup, create/open, and final close | + +Every file is opened or created read/write with mode `0600`, and existing modes +are forced back to `0600` when touched. + +## Linux locking + +Both `.lock` and `.lifecycle` use a nonblocking POSIX record lock on byte range +`[0, 1)`, retried according to the caller's wait policy. This is the behavior +exposed by .NET `FileStream.Lock(0, 1)`; using `flock` is not compatible. Each +process must additionally serialize contenders through a process-local mutex +keyed by the absolute lock-file path because POSIX record locks alone do not +provide the required same-process ownership boundary. Release unlocks the byte +range before releasing the local mutex. + +Retries observe cancellation and bounded time using a monotonic clock. The +managed baseline retries at intervals no longer than 10 milliseconds or the +remaining timeout. A foreign implementation may use a shorter interval but +must preserve no-wait and bounded-wait outcomes. + +## Linux owner sidecar and cleanup + +One open store handle registers one owner line: + +```text +:: +``` + +The normal start token is `proc-` plus field 22 (`starttime`) from +`/proc//stat`. If procfs cannot be read, the managed fallback is `utc-` +plus the process start time in .NET UTC ticks. The unique token is a lowercase +32-hex-digit GUID without separators. Owner readers also tolerate legacy +PID-only lines conservatively, but writers emit all three fields. + +Owner updates occur while holding `.lifecycle`. A reader trims each line, splits +it into at most three colon-separated parts, and requires the first part to be a +positive decimal PID. It compares a start token only when all three parts are +present; legacy one- or two-part PID records therefore receive PID-only liveness +checking. The unique token is an ownership key, not a liveness value. Readers +discard blank, invalid-PID, and confirmed-dead records but conservatively retain +an owner when liveness cannot be determined. Updates write the complete set to +`.owners.tmp`, force mode `0600`, then atomically replace `.owners`; best-effort +cleanup removes a leftover temporary file. + +When no live owner remains, stale cleanup removes `.region`, `.lock`, `.owners`, +and `.owners.tmp`. The `.lifecycle` file is deliberately retained because it is +the rendezvous used while cleanup is in progress and by later openers. Closing a +non-final handle removes only its owner record. Closing the final live handle +performs the same stale-resource deletion while holding the lifecycle lock. + +The sidecar start token protects resource cleanup from PID reuse. Layout-1.2 +lease and reservation records themselves contain only a PID; their explicit +recovery policy remains the separate, conservative layout-1.2 contract. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d8e9da6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,48 @@ +[build-system] +requires = ["scikit-build-core>=1,<2"] +build-backend = "scikit_build_core.build" + +[project] +name = "shared-memory-store" +version = "0.1.0" +description = "Bounded, cross-process shared-memory values for Python" +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +authors = [{ name = "SharedMemoryStore contributors" }] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries", +] + +[project.urls] +Repository = "https://github.com/rantri/SharedMemoryStore" + +[tool.scikit-build] +minimum-version = "build-system.requires" +wheel.packages = [] +wheel.py-api = "py3" +install.components = ["Python"] +sdist.inclusion-mode = "explicit" +sdist.include = [ + "/pyproject.toml", + "/CMakeLists.txt", + "/cmake/**", + "/src/cpp/**", + "/src/python/shared_memory_store/*.py", + "/protocol/compatibility.json", + "/LICENSE", + "/README.md", +] + +[tool.scikit-build.cmake.define] +SMS_INSTALL = "ON" +SMS_BUILD_TESTS = "OFF" +SMS_BUILD_SAMPLES = "OFF" +SMS_PYTHON_INSTALL_DIR = "shared_memory_store" diff --git a/samples/CppBasicUsage/CMakeLists.txt b/samples/CppBasicUsage/CMakeLists.txt new file mode 100644 index 0000000..bf420f0 --- /dev/null +++ b/samples/CppBasicUsage/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.20) + +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(SharedMemoryStoreCppBasicUsage LANGUAGES CXX) + find_package(SharedMemoryStore CONFIG REQUIRED) +endif() + +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp") + add_executable(shared_memory_store_cpp_sample main.cpp) + target_compile_features(shared_memory_store_cpp_sample PRIVATE cxx_std_20) + set_target_properties(shared_memory_store_cpp_sample + PROPERTIES CXX_EXTENSIONS OFF) + target_link_libraries(shared_memory_store_cpp_sample + PRIVATE SharedMemoryStore::SharedMemoryStore) + + if(WIN32) + add_custom_command( + TARGET shared_memory_store_cpp_sample + POST_BUILD + COMMAND + "${CMAKE_COMMAND}" -E copy_if_different + "$" + "$" + VERBATIM) + endif() +endif() diff --git a/samples/CppBasicUsage/README.md b/samples/CppBasicUsage/README.md new file mode 100644 index 0000000..15df545 --- /dev/null +++ b/samples/CppBasicUsage/README.md @@ -0,0 +1,6 @@ +# C++ Basic Usage + +This sample builds with the repository when `SMS_BUILD_SAMPLES=ON`, or against +an installed package through `find_package(SharedMemoryStore CONFIG REQUIRED)`. +It creates a bounded store, publishes binary bytes, acquires a lease, reads the +zero-copy value view, and releases the lease. diff --git a/samples/CppBasicUsage/main.cpp b/samples/CppBasicUsage/main.cpp new file mode 100644 index 0000000..ead7ad1 --- /dev/null +++ b/samples/CppBasicUsage/main.cpp @@ -0,0 +1,38 @@ +#include + +#include +#include +#include +#include + +#if defined(_WIN32) +# define NOMINMAX +# include +#else +# include +#endif + +int main() { +#if defined(_WIN32) + const auto pid = GetCurrentProcessId(); +#else + const auto pid = getpid(); +#endif + using namespace shared_memory_store; + auto options = store_options::create( + "sms-cpp-sample-" + std::to_string(pid), 2, 64, 16, 16, 4, + open_mode::create_new); + memory_store store; + if (const auto opened = memory_store::try_create_or_open(options, store); + opened != open_status::success) { + std::cerr << "open failed: " << static_cast(opened) << '\n'; + return 1; + } + const std::array key{std::byte{1}, std::byte{2}, std::byte{3}}; + const std::array payload{std::byte{7}, std::byte{8}, std::byte{9}}; + if (store.try_publish(key, payload) != status::success) return 2; + value_lease lease; + if (store.try_acquire(key, lease) != status::success) return 3; + std::cout << "value bytes: " << lease.value().size() << '\n'; + return lease.release() == status::success ? 0 : 4; +} diff --git a/samples/PythonBasicUsage/README.md b/samples/PythonBasicUsage/README.md new file mode 100644 index 0000000..21ed2b8 --- /dev/null +++ b/samples/PythonBasicUsage/README.md @@ -0,0 +1,12 @@ +# Python basic usage sample + +Build and install the platform wheel into a clean environment, then run: + +```powershell +python main.py +``` + +The sample creates an isolated named store, publishes binary payload and +descriptor bytes, reads them through an ownership-safe zero-copy lease, removes +the value, and takes a diagnostics snapshot. Run it against an installed wheel; +the repository does not place the Python source root on `sys.path` for it. diff --git a/samples/PythonBasicUsage/main.py b/samples/PythonBasicUsage/main.py new file mode 100644 index 0000000..b86bee1 --- /dev/null +++ b/samples/PythonBasicUsage/main.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import os +import uuid + +from shared_memory_store import MemoryStore, StoreOpenStatus, StoreOptions, StoreStatus + + +def require(actual: object, expected: object, operation: str) -> None: + if actual != expected: + raise RuntimeError(f"{operation} returned {actual!r}; expected {expected!r}") + + +def main() -> None: + options = StoreOptions.create( + f"sms-python-sample-{os.getpid()}-{uuid.uuid4().hex}", + slot_count=2, + max_value_bytes=64, + max_descriptor_bytes=16, + max_key_bytes=16, + lease_record_count=4, + enable_lease_recovery=True, + ) + open_status, store = MemoryStore.open(options) + require(open_status, StoreOpenStatus.SUCCESS, "open") + assert store is not None + + key = b"frame-1" + with store: + require(store.publish(key, b"hello from Python\x00", b"sample"), StoreStatus.SUCCESS, "publish") + acquire_status, lease = store.acquire(key) + require(acquire_status, StoreStatus.SUCCESS, "acquire") + assert lease is not None + with lease: + print(f"value={bytes(lease.value)!r} descriptor={bytes(lease.descriptor)!r}") + require(store.remove(key), StoreStatus.SUCCESS, "remove") + + diagnostics_status, diagnostics = store.diagnostics() + require(diagnostics_status, StoreStatus.SUCCESS, "diagnostics") + assert diagnostics is not None + print(f"free slots: {diagnostics.free_slot_count}/{diagnostics.slot_count}") + + +if __name__ == "__main__": + main() diff --git a/scripts/validate-docker-shared-memory.ps1 b/scripts/validate-docker-shared-memory.ps1 index d9804fb..c732fd3 100644 --- a/scripts/validate-docker-shared-memory.ps1 +++ b/scripts/validate-docker-shared-memory.ps1 @@ -54,7 +54,8 @@ function Invoke-Compose { ) $projectName = "smsdocker-" + [Guid]::NewGuid().ToString("N").Substring(0, 12) - $upArgs = @("compose", "-p", $projectName, "-f", $ComposeFile, "up", "--abort-on-container-exit", "--exit-code-from", $ExitCodeFrom) + $composeArgs = @("compose", "-p", $projectName, "-f", $ComposeFile) + $upArgs = $composeArgs + @("up", "--detach") if (-not $SkipComposeBuild) { $upArgs += "--build" } @@ -63,6 +64,21 @@ function Invoke-Compose { try { Invoke-CommandChecked "docker" $upArgs "docker compose up" + $containerId = (& docker @composeArgs "ps" "--quiet" $ExitCodeFrom | Select-Object -Last 1).Trim() + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($containerId)) { + throw "Could not resolve the '$ExitCodeFrom' Compose container for project '$projectName'." + } + + $waitOutput = & docker wait $containerId + if ($LASTEXITCODE -ne 0) { + throw "docker wait failed for service '$ExitCodeFrom' in project '$projectName'." + } + + & docker @composeArgs "logs" "--no-color" @Services + $containerExitCode = [int](($waitOutput | Select-Object -Last 1).Trim()) + if ($containerExitCode -ne 0) { + throw "Compose service '$ExitCodeFrom' failed with exit code $containerExitCode." + } } finally { & docker compose -p $projectName -f $ComposeFile down --volumes diff --git a/scripts/validate-interoperability.ps1 b/scripts/validate-interoperability.ps1 new file mode 100644 index 0000000..332e347 --- /dev/null +++ b/scripts/validate-interoperability.ps1 @@ -0,0 +1,138 @@ +[CmdletBinding()] +param( + [ValidateSet('Debug', 'Release')] + [string]$Configuration = 'Release', + [string]$BuildDirectory = 'artifacts/interop-native', + [string]$CMakeExecutable = 'cmake', + [string]$PythonExecutable = 'python', + [switch]$SkipBuild, + [switch]$Stress, + [ValidateRange(1, 100000)] + [int]$StressValueCount = 1000, + [ValidateRange(1, 100000)] + [int]$StressLifecycleCycleCount = 10000, + [switch]$Docker, + [switch]$SkipDockerBuild, + [string]$DockerImage = 'shared-memory-store-interop:local' +) + +$ErrorActionPreference = 'Stop' +$repositoryRoot = Split-Path -Parent $PSScriptRoot +$buildPath = [IO.Path]::GetFullPath((Join-Path $repositoryRoot $BuildDirectory)) +$testProject = Join-Path $repositoryRoot 'tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj' + +function Invoke-Checked { + param( + [Parameter(Mandatory)][string]$Command, + [Parameter(ValueFromRemainingArguments)][string[]]$Arguments + ) + + & $Command @Arguments + if ($LASTEXITCODE -ne 0) { + throw "Command failed with exit code $LASTEXITCODE`: $Command $($Arguments -join ' ')" + } +} + +function Find-NativeAgent { + param([string]$Root, [string]$BuildConfiguration) + + $fileName = if ($IsWindows) { 'sms_cpp_interop_agent.exe' } else { 'sms_cpp_interop_agent' } + $candidates = @( + (Join-Path $Root "tests/cpp/$BuildConfiguration/$fileName"), + (Join-Path $Root "tests/cpp/$fileName"), + (Join-Path $Root $fileName) + ) + $agent = $candidates | Where-Object { Test-Path -LiteralPath $_ -PathType Leaf } | Select-Object -First 1 + if ($null -eq $agent) { + throw "The C++ interoperability agent was not found under '$Root'." + } + + return [IO.Path]::GetFullPath($agent) +} + +if ($Docker) { + $dockerCommand = (Get-Command docker -ErrorAction Stop).Source + $dockerfile = Join-Path $repositoryRoot 'tests/SharedMemoryStore.InteropTests/Dockerfile' + if (-not $SkipDockerBuild) { + Invoke-Checked $dockerCommand 'build' '--file' $dockerfile '--tag' $DockerImage $repositoryRoot + } + + $runArguments = @( + 'run', '--rm', '--shm-size', '256m', + '--env', "SMS_RUN_INTEROP_STRESS=$([int]$Stress.IsPresent)", + '--env', "SMS_INTEROP_STRESS_VALUES=$StressValueCount", + '--env', "SMS_INTEROP_STRESS_LIFECYCLE_CYCLES=$StressLifecycleCycleCount", + $DockerImage + ) + Invoke-Checked $dockerCommand @runArguments + Write-Host 'Docker C# / C++ / Python interoperability validation passed.' + return +} + +$dotnet = (Get-Command dotnet -ErrorAction Stop).Source +$python = (Get-Command $PythonExecutable -ErrorAction Stop).Source + +if (-not $SkipBuild) { + $cmake = (Get-Command $CMakeExecutable -ErrorAction Stop).Source + $ctestName = if ($IsWindows) { 'ctest.exe' } else { 'ctest' } + $ctest = Join-Path (Split-Path -Parent $cmake) $ctestName + if (-not (Test-Path -LiteralPath $ctest -PathType Leaf)) { + $ctest = (Get-Command $ctestName -ErrorAction Stop).Source + } + + $configureArguments = @( + '-S', $repositoryRoot, + '-B', $buildPath, + '-DSMS_BUILD_TESTS=ON', + '-DSMS_BUILD_SAMPLES=OFF', + '-DSMS_INSTALL=ON', + '-DSMS_PYTHON_INSTALL_DIR=shared_memory_store' + ) + if (-not $IsWindows) { + $configureArguments += "-DCMAKE_BUILD_TYPE=$Configuration" + } + + Invoke-Checked $cmake @configureArguments + Invoke-Checked $cmake '--build' $buildPath '--config' $Configuration '--parallel' + Invoke-Checked $ctest '--test-dir' $buildPath '-C' $Configuration '--output-on-failure' + Invoke-Checked $cmake '--install' $buildPath '--config' $Configuration '--component' 'Python' '--prefix' (Join-Path $repositoryRoot 'src/python') + Invoke-Checked $dotnet 'build' $testProject '-c' $Configuration +} + +$nativeAgent = Find-NativeAgent $buildPath $Configuration +$nativeLibraryName = if ($IsWindows) { 'shared_memory_store.dll' } else { 'libshared_memory_store.so' } +$nativePythonLibrary = Join-Path $repositoryRoot "src/python/shared_memory_store/$nativeLibraryName" +if (-not (Test-Path -LiteralPath $nativePythonLibrary -PathType Leaf)) { + throw "The Python package-adjacent native library is missing: '$nativePythonLibrary'." +} + +$savedEnvironment = @{ + SMS_CPP_AGENT = [Environment]::GetEnvironmentVariable('SMS_CPP_AGENT', 'Process') + SMS_PYTHON_EXECUTABLE = [Environment]::GetEnvironmentVariable('SMS_PYTHON_EXECUTABLE', 'Process') + SMS_RUN_INTEROP_STRESS = [Environment]::GetEnvironmentVariable('SMS_RUN_INTEROP_STRESS', 'Process') + SMS_INTEROP_STRESS_VALUES = [Environment]::GetEnvironmentVariable('SMS_INTEROP_STRESS_VALUES', 'Process') + SMS_INTEROP_STRESS_LIFECYCLE_CYCLES = [Environment]::GetEnvironmentVariable('SMS_INTEROP_STRESS_LIFECYCLE_CYCLES', 'Process') +} + +try { + [Environment]::SetEnvironmentVariable('SMS_CPP_AGENT', $nativeAgent, 'Process') + [Environment]::SetEnvironmentVariable('SMS_PYTHON_EXECUTABLE', $python, 'Process') + [Environment]::SetEnvironmentVariable('SMS_RUN_INTEROP_STRESS', '0', 'Process') + [Environment]::SetEnvironmentVariable('SMS_INTEROP_STRESS_VALUES', $StressValueCount.ToString([Globalization.CultureInfo]::InvariantCulture), 'Process') + [Environment]::SetEnvironmentVariable('SMS_INTEROP_STRESS_LIFECYCLE_CYCLES', $StressLifecycleCycleCount.ToString([Globalization.CultureInfo]::InvariantCulture), 'Process') + + Write-Host 'Running the normal C# / C++ / Python interoperability suite.' + Invoke-Checked $dotnet 'test' $testProject '-c' $Configuration '--no-build' '--filter' 'FullyQualifiedName!~StressInteropTests' + if ($Stress) { + [Environment]::SetEnvironmentVariable('SMS_RUN_INTEROP_STRESS', '1', 'Process') + Write-Host "Running stress validation ($StressValueCount values per ordered pair; $StressLifecycleCycleCount mixed lifecycle cycles)." + Invoke-Checked $dotnet 'test' $testProject '-c' $Configuration '--no-build' '--filter' 'FullyQualifiedName~StressInteropTests' + } +} +finally { + foreach ($entry in $savedEnvironment.GetEnumerator()) { + [Environment]::SetEnvironmentVariable($entry.Key, $entry.Value, 'Process') + } +} + +Write-Host 'Host C# / C++ / Python interoperability validation passed.' diff --git a/scripts/validate-native.ps1 b/scripts/validate-native.ps1 new file mode 100644 index 0000000..2c7fa7e --- /dev/null +++ b/scripts/validate-native.ps1 @@ -0,0 +1,81 @@ +[CmdletBinding()] +param( + [ValidateSet('Debug', 'Release')] + [string]$Configuration = 'Release', + [string]$BuildDirectory = 'artifacts/native-build', + [string]$InstallDirectory = 'artifacts/native-install', + [string]$CMakeExecutable = 'cmake', + [switch]$SkipTests, + [switch]$SkipConsumer +) + +$ErrorActionPreference = 'Stop' +$repositoryRoot = Split-Path -Parent $PSScriptRoot +$buildPath = [System.IO.Path]::GetFullPath((Join-Path $repositoryRoot $BuildDirectory)) +$installPath = [System.IO.Path]::GetFullPath((Join-Path $repositoryRoot $InstallDirectory)) + +function Invoke-Checked { + param( + [Parameter(Mandatory)] + [string]$Command, + [Parameter(ValueFromRemainingArguments)] + [string[]]$Arguments + ) + + & $Command @Arguments + if ($LASTEXITCODE -ne 0) { + throw "Command failed with exit code $LASTEXITCODE`: $Command $($Arguments -join ' ')" + } +} + +$cmakeCommand = (Get-Command $CMakeExecutable -ErrorAction Stop).Source +$ctestName = if ($IsWindows) { 'ctest.exe' } else { 'ctest' } +$ctestCommand = Join-Path (Split-Path -Parent $cmakeCommand) $ctestName +if (-not (Test-Path -LiteralPath $ctestCommand)) { + $ctestCommand = (Get-Command $ctestName -ErrorAction Stop).Source +} + +$configureArguments = @( + '-S', $repositoryRoot, + '-B', $buildPath, + '-DSMS_BUILD_TESTS=ON', + '-DSMS_BUILD_SAMPLES=ON', + '-DSMS_BUILD_STATIC=ON' +) +if (-not $IsWindows) { + $configureArguments += "-DCMAKE_BUILD_TYPE=$Configuration" +} + +Invoke-Checked $cmakeCommand @configureArguments +Invoke-Checked $cmakeCommand '--build' $buildPath '--config' $Configuration '--parallel' + +if (-not $SkipTests) { + Invoke-Checked $ctestCommand '--test-dir' $buildPath '-C' $Configuration '--output-on-failure' +} + +Invoke-Checked $cmakeCommand '--install' $buildPath '--config' $Configuration '--prefix' $installPath + +if (-not $SkipConsumer) { + $consumerBuild = Join-Path $buildPath 'package-consumer' + $consumerArguments = @( + '-S', (Join-Path $repositoryRoot 'tests/cpp/package_consumer'), + '-B', $consumerBuild, + "-DCMAKE_PREFIX_PATH=$installPath" + ) + if (-not $IsWindows) { + $consumerArguments += "-DCMAKE_BUILD_TYPE=$Configuration" + } + + Invoke-Checked $cmakeCommand @consumerArguments + Invoke-Checked $cmakeCommand '--build' $consumerBuild '--config' $Configuration '--parallel' + $consumer = Get-ChildItem -LiteralPath $consumerBuild -Recurse -File | + Where-Object { $_.Name -in @('shared_memory_store_package_consumer', 'shared_memory_store_package_consumer.exe') } | + Select-Object -First 1 + if ($null -eq $consumer) { + throw 'The installed-package consumer executable was not produced.' + } + + Invoke-Checked $consumer.FullName +} + +Write-Host 'Native SharedMemoryStore validation passed.' diff --git a/scripts/validate-python.ps1 b/scripts/validate-python.ps1 new file mode 100644 index 0000000..7a46a5a --- /dev/null +++ b/scripts/validate-python.ps1 @@ -0,0 +1,248 @@ +[CmdletBinding()] +param( + [ValidateSet('Debug', 'Release')] + [string]$Configuration = 'Release', + [string]$ArtifactsDirectory = 'artifacts/python-validation', + [string]$CMakeExecutable = 'cmake', + [string]$PythonExecutable = 'python' +) + +$ErrorActionPreference = 'Stop' +$repositoryRoot = Split-Path -Parent $PSScriptRoot +$artifactsRoot = [IO.Path]::GetFullPath((Join-Path $repositoryRoot 'artifacts')) +$workPath = [IO.Path]::GetFullPath((Join-Path $repositoryRoot $ArtifactsDirectory)) + +function Invoke-Checked { + param( + [Parameter(Mandatory)][string]$Command, + [Parameter(ValueFromRemainingArguments)][string[]]$Arguments + ) + + & $Command @Arguments + if ($LASTEXITCODE -ne 0) { + throw "Command failed with exit code $LASTEXITCODE`: $Command $($Arguments -join ' ')" + } +} + +function Assert-ArtifactPath { + param([Parameter(Mandatory)][string]$Path) + + $fullPath = [IO.Path]::GetFullPath($Path) + $prefix = $artifactsRoot + [IO.Path]::DirectorySeparatorChar + if (-not $fullPath.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "Validation output must stay below '$artifactsRoot'; received '$fullPath'." + } + + return $fullPath +} + +function Reset-Directory { + param([Parameter(Mandatory)][string]$Path) + + $fullPath = Assert-ArtifactPath $Path + if (Test-Path -LiteralPath $fullPath) { + Remove-Item -LiteralPath $fullPath -Recurse -Force + } + New-Item -ItemType Directory -Path $fullPath -Force | Out-Null + return $fullPath +} + +function Get-VenvPython { + param([Parameter(Mandatory)][string]$EnvironmentPath) + + $relative = if ($IsWindows) { 'Scripts/python.exe' } else { 'bin/python' } + $candidate = Join-Path $EnvironmentPath $relative + if (-not (Test-Path -LiteralPath $candidate -PathType Leaf)) { + throw "The virtual-environment interpreter was not created: '$candidate'." + } + return $candidate +} + +function Assert-NativeSet { + param( + [Parameter(Mandatory)][string]$Root, + [Parameter(Mandatory)][string]$ExpectedName, + [Parameter(Mandatory)][string]$Description + ) + + $nativeFiles = @( + Get-ChildItem -LiteralPath $Root -Recurse -File | + Where-Object { $_.Extension -in @('.dll', '.so', '.dylib') } + ) + if ($nativeFiles.Count -ne 1 -or $nativeFiles[0].Name -cne $ExpectedName) { + $actual = if ($nativeFiles.Count -eq 0) { '' } else { ($nativeFiles.FullName -join ', ') } + throw "$Description must contain exactly '$ExpectedName' and no opposite-platform native binary; found $actual." + } +} + +function Assert-ArchiveContainsSuffix { + param( + [Parameter(Mandatory)][string[]]$Entries, + [Parameter(Mandatory)][string]$Suffix, + [Parameter(Mandatory)][string]$Description + ) + + $normalizedSuffix = $Suffix.Replace('\', '/') + if (-not ($Entries | Where-Object { $_.Replace('\', '/').EndsWith($normalizedSuffix, [StringComparison]::Ordinal) })) { + throw "$Description is missing required entry '*$normalizedSuffix'." + } +} + +$workPath = Reset-Directory $workPath +$nativeBuildPath = Join-Path $workPath 'native-build' +$stagedSourcePath = Join-Path $workPath 'source-package' +$buildEnvironmentPath = Join-Path $workPath 'build-environment' +$wheelEnvironmentPath = Join-Path $workPath 'wheel-environment' +$distributionPath = Join-Path $workPath 'dist' +$unrelatedRunPath = Join-Path $workPath 'unrelated-run-directory' +New-Item -ItemType Directory -Path $stagedSourcePath, $distributionPath, $unrelatedRunPath -Force | Out-Null + +$cmake = (Get-Command $CMakeExecutable -ErrorAction Stop).Source +$python = (Get-Command $PythonExecutable -ErrorAction Stop).Source +$nativeLibraryName = if ($IsWindows) { 'shared_memory_store.dll' } else { 'libshared_memory_store.so' } +$expectedPythonFiles = @('__init__.py', '_native.py', 'enums.py', 'store.py') + +$configureArguments = @( + '-S', $repositoryRoot, + '-B', $nativeBuildPath, + '-DSMS_BUILD_TESTS=OFF', + '-DSMS_BUILD_SAMPLES=OFF', + '-DSMS_INSTALL=ON', + '-DSMS_PYTHON_INSTALL_DIR=shared_memory_store' +) +if (-not $IsWindows) { + $configureArguments += "-DCMAKE_BUILD_TYPE=$Configuration" +} + +Invoke-Checked $cmake @configureArguments +Invoke-Checked $cmake '--build' $nativeBuildPath '--config' $Configuration '--parallel' +Invoke-Checked $cmake '--install' $nativeBuildPath '--config' $Configuration '--component' 'Python' '--prefix' $stagedSourcePath + +$stagedPackagePath = Join-Path $stagedSourcePath 'shared_memory_store' +foreach ($file in $expectedPythonFiles) { + $candidate = Join-Path $stagedPackagePath $file + if (-not (Test-Path -LiteralPath $candidate -PathType Leaf)) { + throw "The staged source package is missing '$candidate'." + } +} +Assert-NativeSet $stagedPackagePath $nativeLibraryName 'The staged source package' + +$savedPythonPath = [Environment]::GetEnvironmentVariable('PYTHONPATH', 'Process') +$savedInstalledGate = [Environment]::GetEnvironmentVariable('SMS_TEST_INSTALLED_PACKAGE', 'Process') +try { + [Environment]::SetEnvironmentVariable('PYTHONPATH', $stagedSourcePath, 'Process') + [Environment]::SetEnvironmentVariable('SMS_TEST_INSTALLED_PACKAGE', '0', 'Process') + Invoke-Checked $python '-m' 'unittest' 'discover' '-s' (Join-Path $repositoryRoot 'tests/python') '-v' +} +finally { + [Environment]::SetEnvironmentVariable('PYTHONPATH', $savedPythonPath, 'Process') + [Environment]::SetEnvironmentVariable('SMS_TEST_INSTALLED_PACKAGE', $savedInstalledGate, 'Process') +} + +Invoke-Checked $python '-m' 'venv' $buildEnvironmentPath +$buildPython = Get-VenvPython $buildEnvironmentPath +Invoke-Checked $buildPython '-m' 'pip' 'install' '--upgrade' 'pip' 'build' +Invoke-Checked $buildPython '-m' 'build' '--wheel' '--sdist' '--outdir' $distributionPath $repositoryRoot + +$wheels = @(Get-ChildItem -LiteralPath $distributionPath -Filter '*.whl' -File) +$sdists = @(Get-ChildItem -LiteralPath $distributionPath -Filter '*.tar.gz' -File) +if ($wheels.Count -ne 1) { + throw "Expected exactly one wheel in '$distributionPath'; found $($wheels.Count)." +} +if ($sdists.Count -ne 1) { + throw "Expected exactly one source distribution in '$distributionPath'; found $($sdists.Count)." +} +if ($wheels[0].Name -notmatch '-py3-none-' -or $wheels[0].Name -match '-any\.whl$') { + throw "The ctypes distribution must be a platform wheel with a generic Python 3 ABI tag: '$($wheels[0].Name)'." +} + +Add-Type -AssemblyName System.IO.Compression.FileSystem +$archive = [IO.Compression.ZipFile]::OpenRead($wheels[0].FullName) +try { + $wheelEntries = @($archive.Entries | ForEach-Object { $_.FullName }) +} +finally { + $archive.Dispose() +} +Assert-ArchiveContainsSuffix $wheelEntries "shared_memory_store/$nativeLibraryName" 'The wheel' +foreach ($file in $expectedPythonFiles) { + Assert-ArchiveContainsSuffix $wheelEntries "shared_memory_store/$file" 'The wheel' +} +$wheelNativeEntries = @($wheelEntries | Where-Object { $_ -match '\.(dll|so|dylib)$' }) +if ($wheelNativeEntries.Count -ne 1 -or -not $wheelNativeEntries[0].EndsWith("/$nativeLibraryName", [StringComparison]::Ordinal)) { + throw "The wheel must contain exactly one '$nativeLibraryName' native artifact; found $($wheelNativeEntries -join ', ')." +} + +$tar = (Get-Command tar -ErrorAction Stop).Source +$sdistEntries = @(& $tar '-tf' $sdists[0].FullName) +if ($LASTEXITCODE -ne 0) { + throw "Could not inspect source distribution '$($sdists[0].FullName)'." +} +foreach ($suffix in @( + 'pyproject.toml', + 'CMakeLists.txt', + 'src/cpp/CMakeLists.txt', + 'src/cpp/include/shared_memory_store/c_api.h', + 'src/python/shared_memory_store/__init__.py', + 'src/python/shared_memory_store/_native.py', + 'src/python/shared_memory_store/enums.py', + 'src/python/shared_memory_store/store.py', + 'protocol/compatibility.json', + 'LICENSE', + 'README.md' +)) { + Assert-ArchiveContainsSuffix $sdistEntries $suffix 'The source distribution' +} +$sdistNativeEntries = @($sdistEntries | Where-Object { $_ -match '\.(dll|so|dylib)$' }) +if ($sdistNativeEntries.Count -ne 0) { + throw "The source distribution must not contain compiled binaries; found $($sdistNativeEntries -join ', ')." +} + +# Prove the explicit source distribution contains everything needed to compile a +# wheel, independent of files that happen to exist in the checkout. +$sdistWheelPath = Join-Path $workPath 'sdist-wheel' +New-Item -ItemType Directory -Path $sdistWheelPath -Force | Out-Null +Invoke-Checked $buildPython '-m' 'pip' 'wheel' '--no-deps' '--wheel-dir' $sdistWheelPath $sdists[0].FullName +$sdistWheels = @(Get-ChildItem -LiteralPath $sdistWheelPath -Filter '*.whl' -File) +if ($sdistWheels.Count -ne 1) { + throw "Building the source distribution should produce exactly one wheel; found $($sdistWheels.Count)." +} +$sdistArchive = [IO.Compression.ZipFile]::OpenRead($sdistWheels[0].FullName) +try { + $sdistWheelEntries = @($sdistArchive.Entries | ForEach-Object { $_.FullName }) +} +finally { + $sdistArchive.Dispose() +} +$sdistWheelNativeEntries = @($sdistWheelEntries | Where-Object { $_ -match '\.(dll|so|dylib)$' }) +if ($sdistWheelNativeEntries.Count -ne 1 -or -not $sdistWheelNativeEntries[0].EndsWith("/$nativeLibraryName", [StringComparison]::Ordinal)) { + throw "The wheel built from the sdist must contain exactly one '$nativeLibraryName'; found $($sdistWheelNativeEntries -join ', ')." +} +foreach ($file in $expectedPythonFiles) { + Assert-ArchiveContainsSuffix $sdistWheelEntries "shared_memory_store/$file" 'The wheel built from the source distribution' +} + +Invoke-Checked $python '-m' 'venv' $wheelEnvironmentPath +$wheelPython = Get-VenvPython $wheelEnvironmentPath +Invoke-Checked $wheelPython '-m' 'pip' 'install' '--no-deps' $sdistWheels[0].FullName + +$savedPythonPath = [Environment]::GetEnvironmentVariable('PYTHONPATH', 'Process') +$savedInstalledGate = [Environment]::GetEnvironmentVariable('SMS_TEST_INSTALLED_PACKAGE', 'Process') +try { + [Environment]::SetEnvironmentVariable('PYTHONPATH', $null, 'Process') + [Environment]::SetEnvironmentVariable('SMS_TEST_INSTALLED_PACKAGE', '1', 'Process') + Push-Location $unrelatedRunPath + try { + Invoke-Checked $wheelPython '-m' 'unittest' 'discover' '-s' (Join-Path $repositoryRoot 'tests/python') '-v' + Invoke-Checked $wheelPython (Join-Path $repositoryRoot 'samples/PythonBasicUsage/main.py') + } + finally { + Pop-Location + } +} +finally { + [Environment]::SetEnvironmentVariable('PYTHONPATH', $savedPythonPath, 'Process') + [Environment]::SetEnvironmentVariable('SMS_TEST_INSTALLED_PACKAGE', $savedInstalledGate, 'Process') +} + +Write-Host "Python SharedMemoryStore validation passed for $($wheels[0].Name)." diff --git a/specs/003-zero-copy-ingest/contracts/ingest-layout.md b/specs/003-zero-copy-ingest/contracts/ingest-layout.md index 3aac450..959980c 100644 --- a/specs/003-zero-copy-ingest/contracts/ingest-layout.md +++ b/specs/003-zero-copy-ingest/contracts/ingest-layout.md @@ -29,8 +29,10 @@ state and visibility rules. - Layout major version remains `1` unless implementation requires incompatible section sizes, field ordering changes, or unsafe behavior for old clients. -- Layout minor version becomes `1` to document pending reservation progress in - existing slot metadata. +- Layout minor version is `2`. Minor `1` introduced pending reservation + progress in existing slot metadata; minor `2` subsequently added the reuse + epoch to index, slot, and lease records so lifecycle identities cannot repeat + when the 32-bit generation rolls over. - Opening a mapping with an unsupported higher major version fails with `IncompatibleLayout`. - Contract tests must verify header version constants and state numeric values. diff --git a/specs/008-cpp-python-implementations/checklists/requirements.md b/specs/008-cpp-python-implementations/checklists/requirements.md new file mode 100644 index 0000000..5c4b406 --- /dev/null +++ b/specs/008-cpp-python-implementations/checklists/requirements.md @@ -0,0 +1,36 @@ +# Specification Quality Checklist: Native and Python Implementations + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-07-10 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages are stated only as required consumer audiences) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification beyond the requested distribution audiences + +## Notes + +- Validation passed on the first review iteration. +- Python and C++ are feature audiences and compatibility targets; binding and + build mechanisms are intentionally deferred to the implementation plan. diff --git a/specs/008-cpp-python-implementations/contracts/cpp-api.md b/specs/008-cpp-python-implementations/contracts/cpp-api.md new file mode 100644 index 0000000..746e687 --- /dev/null +++ b/specs/008-cpp-python-implementations/contracts/cpp-api.md @@ -0,0 +1,27 @@ +# Contract: C++ API + +Namespace `shared_memory_store` exposes RAII wrappers over the C ABI. + +## Public Types + +- Strong enums for open mode, open status, and operation status with numeric + values identical to the C and managed contracts. +- `store_options` with a required-capacity factory and validation helpers. +- `wait_options` supporting default, no-wait, bounded, and infinite waits. +- Move-only `memory_store`, `value_lease`, and `value_reservation` types. +- Non-owning byte views represented with `std::span`. +- Versioned recovery reports and diagnostics value types. + +## Behavior + +- Ordinary operation failures return status/result objects and do not throw. +- Construction/configuration programmer errors may use standard exceptions in + the C++ wrapper; nothing throws across the C ABI. +- Destructors close handles and best-effort release/abort active tokens. +- Moving transfers ownership and leaves the source invalid. +- Lease payload and descriptor spans are read-only and invalid after release or + owning store close. +- Reservation spans are writable and invalid after advance, commit, abort, + recovery, or owning store close. +- String helpers encode store names as UTF-8; keys/descriptors/payloads remain + opaque bytes. diff --git a/specs/008-cpp-python-implementations/contracts/interoperability.md b/specs/008-cpp-python-implementations/contracts/interoperability.md new file mode 100644 index 0000000..cf487e6 --- /dev/null +++ b/specs/008-cpp-python-implementations/contracts/interoperability.md @@ -0,0 +1,44 @@ +# Contract: Cross-Runtime Interoperability + +## Static Conformance + +Every implementation must consume the same fixture manifest and prove: + +- exact record sizes and every field offset. +- numeric layout, state, status, and open-mode assignments. +- layout-calculation and overflow vectors. +- FNV-1a key hashes and exact byte equality. +- resource-name vectors including Unicode, punctuation, scope prefixes, and + maximum-length names. +- parsing and normalized description of empty, published, pending reservation, + pending removal, and reused-slot binary fixtures. + +## Live Agent Protocol + +Each runtime provides a test-only JSON-lines subprocess agent with equivalent +commands: open/create, publish, segmented publish, acquire/read/release, remove, +reserve/write/advance/commit/abort, recover, diagnostics, hold lock, close, and +crash. Binary arguments and results use base64; status names and numbers are +both emitted. Agents write protocol responses only, never library diagnostics. + +## Required Matrix + +For each supported OS, run every ordered producer-to-consumer pair among C#, +C++, and Python. The producer remains alive while the consumer opens the same +store. Verify bytes and then reverse mutation ownership through remove and +republish. + +Pair scenarios also cover: + +- pending reservation invisibility and duplicate blocking before commit. +- remove while a foreign lease is active, final release, and slot reuse. +- abrupt lease and reservation owner termination followed by explicit recovery. +- no-wait and bounded-wait contention against a foreign lock owner. +- mismatched capacity and incompatible layout rejection. +- three simultaneous Linux owners with non-final and final close cleanup. +- concurrent same-key publication with one success. + +## Fixture Safety + +Binary fixtures are offline parse/emit inputs, never live mappings. Live replay +would carry fake PIDs and omit platform ownership resources. diff --git a/specs/008-cpp-python-implementations/contracts/native-c-api.md b/specs/008-cpp-python-implementations/contracts/native-c-api.md new file mode 100644 index 0000000..22d9578 --- /dev/null +++ b/specs/008-cpp-python-implementations/contracts/native-c-api.md @@ -0,0 +1,62 @@ +# Contract: Native C ABI + +## ABI Rules + +- Exported symbols use `extern "C"` and the platform's ordinary C calling + convention. +- All integer fields use explicit-width C types. +- Every extensible input/output structure begins with `struct_size` and + `abi_version`. +- Stores, leases, and reservations are opaque pointers owned by matching close + or release functions. +- Byte pointers are paired with explicit lengths and may contain NUL bytes. +- A null pointer is allowed only when its paired length is zero or the parameter + is explicitly optional. +- No exception, C++ standard-library type, allocator ownership, or thread-local + error string crosses the ABI. +- Every operation returns the exact public status enum; optional diagnostics are + returned through caller-allocated versioned structures. + +## Required Symbol Groups + +### Version and layout + +- Query ABI and shared protocol versions. +- Calculate required mapped bytes with overflow detection. +- Return canonical record sizes and offsets for conformance tests. + +### Store lifecycle + +- Create/open from versioned options and bounded wait configuration. +- Close an opaque handle exactly once; null close is harmless. +- Query options/layout and diagnostics without exposing internal pointers. + +### Values and leases + +- Publish one contiguous payload and optional descriptor. +- Publish an array of byte segments as one committed payload. +- Acquire by key into an opaque lease. +- Query lease validity, descriptor pointer/length, and payload pointer/length. +- Release the lease with a status and destroy its process-local token. +- Remove a key with pending-removal behavior. + +### Reservations + +- Reserve announced payload length and immutable descriptor. +- Query validity, total, written, remaining, and writable pointer/length. +- Advance by an exact byte count, commit, or abort. +- Destroying a live reservation performs best-effort abort. + +### Recovery + +- Recover stale/current-process leases according to an explicit option and + return scanned, recovered, active, unsupported, and failed counts. +- Recover pending reservations under the equivalent explicit policy. + +## Threading and Lifetime + +One store handle is safe for concurrent calls. All shared mutations and lease +registry changes participate in the common platform lock. A byte pointer returned +from a lease remains valid only while that lease and store are live. A writable +reservation pointer remains valid only until the next reservation operation, +completion, recovery, identity change, or store close. diff --git a/specs/008-cpp-python-implementations/contracts/packaging.md b/specs/008-cpp-python-implementations/contracts/packaging.md new file mode 100644 index 0000000..8b2064f --- /dev/null +++ b/specs/008-cpp-python-implementations/contracts/packaging.md @@ -0,0 +1,31 @@ +# Contract: Packaging and Compatibility + +## C++ Distribution + +- CMake builds a shared library, optional static library, public C and C++ + headers, tests, and samples. +- Install rules use standard runtime, library, archive, and include destinations. +- An exported `SharedMemoryStore::SharedMemoryStore` target is consumable through + `find_package` from a clean external CMake project. +- Installed public artifacts declare native package, ABI, and layout versions. + +## Python Distribution + +- A PEP 517 source build compiles the native library and installs it beside the + Python modules. +- Platform wheels contain the shared library and require no compiler at install + time. +- Because the native library does not use the CPython ABI, one wheel per OS and + architecture may support all declared Python 3 versions. +- The wheel is tested after installation into a clean environment; importing + from the source tree must not mask missing packaged files. + +## Versioning + +- NuGet, native, and Python package versions advance independently. +- The C ABI has its own major/minor version. +- Every package release declares readable and creatable layout versions and the + resource-naming version. +- Breaking mapped-layout, resource, public API, or C ABI changes require + semantic-version review, migration notes, updated fixtures, and cross-version + compatibility tests. diff --git a/specs/008-cpp-python-implementations/contracts/python-api.md b/specs/008-cpp-python-implementations/contracts/python-api.md new file mode 100644 index 0000000..c5c03d7 --- /dev/null +++ b/specs/008-cpp-python-implementations/contracts/python-api.md @@ -0,0 +1,38 @@ +# Contract: Python API + +Package `shared_memory_store` exposes Pythonic lifetime wrappers over the native +C ABI with no third-party runtime dependency. + +## Public Types + +- `OpenMode`, `StoreOpenStatus`, and `StoreStatus` as `IntEnum` values identical + to the shared numeric contract. +- Immutable or keyword-oriented `StoreOptions`, `WaitOptions`, recovery reports, + and diagnostics snapshots. +- Context-managed `MemoryStore`, `ValueLease`, and `ValueReservation`. + +## Operations + +- `calculate_required_bytes(...)` and `StoreOptions.create(...)`. +- `MemoryStore.open(options, wait=...)` returns `(status, store_or_none)`. +- Store methods mirror publish, segmented publish, acquire, remove, reserve, + recovery, and diagnostics using explicit status results. +- `ValueLease.value` and `.descriptor` return read-only zero-copy `memoryview` + objects tied to the lease lifetime. +- A reservation exposes a writable zero-copy `memoryview` for its remaining + range plus `advance`, `commit`, and `abort`. +- Closing a context is idempotent. Finalizers are best-effort fallbacks and are + not the primary resource-management contract. + +## Input Rules + +Keys, descriptors, and payloads accept bytes-like objects and preserve exact +bytes. Store names are Python strings encoded as strict UTF-8. Mutable buffers +passed for immediate publication may be copied into the store before the call +returns; borrowed shared-memory views never outlive their owning token. + +## Native Loading + +The package loads only the shared library shipped beside its Python modules. +It does not search arbitrary current-working-directory libraries. Loader errors +identify the expected platform artifact and package location without printing. diff --git a/specs/008-cpp-python-implementations/data-model.md b/specs/008-cpp-python-implementations/data-model.md new file mode 100644 index 0000000..82f8b4a --- /dev/null +++ b/specs/008-cpp-python-implementations/data-model.md @@ -0,0 +1,114 @@ +# Data Model: Native and Python Implementations + +## Shared Protocol Version + +- `layout_major`: 1 +- `layout_minor`: 2 +- `resource_naming_version`: 1 +- `c_abi_major`: 1 +- `c_abi_minor`: starts at 0 and may gain backward-compatible functions or + trailing versioned-struct fields. + +Package versions are independent of these protocol versions. + +## Canonical Mapped Records + +All records are little-endian, sequential, and packed to a maximum alignment of +8 bytes. Every implementation must assert both total sizes and field offsets. + +| Record | Size | Key offsets | +|--------|-----:|-------------| +| Store header | 160 | magic 0, total bytes 16, index offset 56, store id 136, store state 144, sequence 152 | +| Index entry header | 32 | state 0, key length 4, hash 8, slot index 16, generation 20, reuse epoch 24 | +| Slot metadata | 72 | state 0, generation 4, reuse epoch 8, usage 16, publisher PID 32, hash 40, descriptor offset 48, payload offset 56, sequence 64 | +| Lease record | 40 | state 0, id 4, slot 8, generation 12, reuse epoch 16, owner PID 24, acquire sequence 32 | + +The index entry stride is `align8(32 + max_key_bytes)`. Section calculations +must exactly match the canonical algorithm in `StoreLayout`. + +## Store Options + +- UTF-8 public name, 1 through 240 Unicode scalar/code-unit-compatible input + characters with no NUL. +- open mode: create new, open existing, or create-or-open. +- total mapped bytes: positive and at least the calculated requirement. +- slot count, maximum value bytes, maximum key bytes, and lease records: positive. +- maximum descriptor bytes: non-negative. +- lease recovery enabled: process-local policy, not stored in the mapping. +- ABI struct size and ABI version: required for native forward compatibility. + +## Store Handle + +Owns one process-local mapping view, shared synchronization handle, platform +owner registration, operation gate, diagnostics counters, and disposed flag. +Closing one handle unregisters only that owner and does not mutate a live store's +header state. + +## Slot Lifecycle Identity + +`(generation: int32, reuse_epoch: int64)` uniquely identifies one use of a slot. +Initial value is `(1, 0)`. Reclaim increments generation; generation rollover +sets generation to 1 and increments reuse epoch. Exhausting both fields marks +the store corrupt rather than making stale tokens valid. + +## Slot State Machine + +```text +Free -> Publishing -> Published -> RemoveRequested -> Reclaiming -> Free + | | ^ + +--------------+----------------------------------+ + abort/recovery or unleased remove +``` + +- Publishing is invisible but owns its key in the index. +- Commit requires exact reservation progress and publishes metadata after bytes. +- RemoveRequested remains readable to existing leases but not acquirable anew. +- Reclaim removes every matching index entry before advancing identity. + +## Lease + +Contains an owning store reference, slot index, lifecycle identity, and lease +record id. It exposes read-only descriptor and payload bytes only while the +record remains active and the slot identity matches. Release is idempotently +status-returning; a second release returns the documented already-released +outcome. + +## Reservation + +Contains an owning store reference, slot index, lifecycle identity, announced +payload length, and shared progress. It exposes only the currently remaining +writable range. Commit requires progress equal to announced length. Abort, +commit, recovery, close, or identity change invalidates borrowed views. + +## Platform Resource Identity + +Windows: + +- region name: exact public name. +- mutex: scope plus `SharedMemoryStore-` and character-for-character sanitized + public name. + +Linux: + +- root: `/dev/shm/SharedMemoryStore` when available, otherwise the system temp + directory plus `SharedMemoryStore`. +- fragment: `sms--`. +- files: `.region`, `.lock`, `.owners`, `.lifecycle`. +- directory mode 0700 and files mode 0600. +- owner line: `::`. + +## Diagnostics Snapshot + +Shared facts include total bytes, slot capacity/state counts, active leases, +index occupancy/tombstones/probe observations, and compaction count. Runtime +local facts include last observed failure and per-status failure counters. +Snapshots never own a telemetry sink and never print. + +## Compatibility Matrix Entry + +- distribution name and version range. +- supported ABI range where applicable. +- readable/creatable layout versions. +- resource naming version. +- validated operating systems and architectures. +- ordered-pair interoperability evidence. diff --git a/specs/008-cpp-python-implementations/plan.md b/specs/008-cpp-python-implementations/plan.md new file mode 100644 index 0000000..43ea4e8 --- /dev/null +++ b/specs/008-cpp-python-implementations/plan.md @@ -0,0 +1,208 @@ +# Implementation Plan: Native and Python Implementations + +**Branch**: `codex/cpp-python-implementations` | **Date**: 2026-07-10 | **Spec**: [spec.md](spec.md) + +**Input**: Feature specification from `specs/008-cpp-python-implementations/spec.md` + +## Summary + +Deliver interoperable C++ and Python distributions in the existing monorepo +without changing the current C# API or layout-v1.2 mapped records. A C++20 core +will own the protocol algorithms and Linux/Windows mechanisms. A fixed-width, +opaque-handle C ABI will be the binary boundary; the public C++ API will wrap it +with RAII, and the Python package will call it through the standard library's +foreign-function interface. Shared protocol vectors, exact ABI assertions, and +mixed-process tests will make the on-memory and platform-resource contracts +executable across all three runtimes. + +## Technical Context + +**Language/Version**: Existing C# on .NET 10; C++20 core with a C-compatible ABI; +Python 3.10 or newer. + +**Primary Dependencies**: Runtime dependencies are the platform OS APIs, C++ +standard library, and Python standard library only. CMake 3.20 or newer and +scikit-build-core are build-only dependencies. Existing xUnit remains the +cross-process orchestration dependency for the repository test suite. + +**Storage**: Existing layout major 1, minor 2 named memory mapping. The mapped +region remains little-endian and contains the 160-byte header, open-addressed +key index, lease registry, 72-byte slot records, descriptor storage, and payload +storage. Linux also uses deterministic region, lock, owner, and lifecycle files; +Windows uses a named mapping and named mutex. + +**Testing**: Dependency-free C++ assertions run through CTest; Python `unittest`; +existing .NET unit/contract/integration tests; exact JSON and binary protocol +fixtures; and a new xUnit subprocess interoperability harness covering C#, +C++, and Python agents. + +**Target Platform**: Little-endian 64-bit Linux and Windows hosts, with x64 as +the required release-validation architecture. Linux-based same-host Docker +remains supported when IPC, PID, identity, permission, and capacity requirements +are met. The code uses fixed-width ABI fields so additional little-endian +architectures can be validated without changing the contract. + +**Project Type**: Monorepo containing independently consumable NuGet, CMake, and +Python library distributions plus shared protocol artifacts and tests. + +**Performance Goals**: Preserve caller-bounded waits within the selected limit +plus 250 milliseconds; pass 1,000-value ordered producer-consumer scenarios; +complete 10,000 mixed lifecycle/recovery cycles per supported environment; and +retain the existing managed long-running churn validation. + +**Constraints**: Preserve the existing C# public API and layout v1.2; never pass +C++ exceptions, standard-library types, platform-sized integers, or ownership +ambiguity across the C ABI; keep runtime dependencies minimal; no hidden workers, +global mutable configuration, direct console output, cross-host semantics, +persistence guarantee, or malicious-writer protection. + +**Scale/Scope**: Core create/open, publish, segmented publish, acquire/release, +remove/reuse, reservation/advance/commit/abort, explicit lease and reservation +recovery, diagnostics, bounded waits, platform ownership/cleanup, samples, +packaging, and all ordered language pairings. + +## Constitution Check + +*GATE: Passed before Phase 0 research and re-checked after Phase 1 design.* + +- Library and package first: PASS. The primary NuGet artifact remains intact; + native and Python outputs are independently consumable libraries rather than + application wrappers. +- Stable contracts and semantic versioning: PASS. Layout v1.2 remains unchanged, + the new C ABI is explicitly versioned, package versions remain independent, + and cross-runtime compatibility is covered by executable contracts. +- Test-driven production quality: PASS. Exact layout, API, ownership, lifetime, + contention, crash recovery, package consumption, and every ordered runtime + pairing have planned automated coverage. +- .NET 10 baseline and portable core: PASS. Existing .NET behavior stays the + baseline; new platform and language mechanisms conform to its documented + protocol rather than redefining it. +- Minimal, observable, dependency-conscious design: PASS. The native and Python + runtime layers add no third-party runtime dependencies, diagnostics remain + caller-controlled, and no hidden background work is introduced. + +## Project Structure + +### Documentation (this feature) + +```text +specs/008-cpp-python-implementations/ +|-- spec.md +|-- plan.md +|-- research.md +|-- data-model.md +|-- quickstart.md +|-- contracts/ +| |-- native-c-api.md +| |-- cpp-api.md +| |-- python-api.md +| |-- interoperability.md +| `-- packaging.md +|-- checklists/ +| `-- requirements.md +`-- tasks.md +``` + +### Source Code (repository root) + +```text +CMakeLists.txt +pyproject.toml +cmake/ +|-- SharedMemoryStoreConfig.cmake.in +`-- toolchain-independent package helpers +protocol/ +|-- README.md +|-- layout-v1.2.md +|-- resource-naming-v1.md +|-- compatibility.json +`-- fixtures/v1.2/ + |-- manifest.json + `-- representative mapped-region binaries +src/ +|-- SharedMemoryStore/ # Existing C# implementation, unchanged location +|-- cpp/ +| |-- include/shared_memory_store/ +| | |-- c_api.h +| | `-- store.hpp +| `-- src/ +| |-- c_api.cpp +| |-- store.cpp +| |-- layout.hpp +| |-- sha256.cpp +| |-- platform_linux.cpp +| `-- platform_windows.cpp +`-- python/ + `-- shared_memory_store/ + |-- __init__.py + |-- _native.py + |-- enums.py + `-- store.py +tests/ +|-- cpp/ +| |-- CMakeLists.txt +| `-- *_tests.cpp +|-- python/ +| `-- test_*.py +|-- SharedMemoryStore.InteropAgent/ # JSON-lines C# subprocess participant +`-- SharedMemoryStore.InteropTests/ # Cross-runtime orchestrator +samples/ +|-- CppBasicUsage/ +`-- PythonBasicUsage/ +scripts/ +|-- validate-native.ps1 +|-- validate-python.ps1 +`-- validate-interoperability.ps1 +``` + +**Structure Decision**: Keep the current C# source tree stable and add language +siblings beneath `src/`. Put the authoritative language-neutral protocol, +fixtures, and compatibility matrix at repository root because every distribution +depends on them. Keep a root `pyproject.toml` so a Python source distribution can +legally include the sibling native sources it must build. The C++ implementation +owns mechanisms; C++ and Python ergonomics depend on the same C ABI. + +## Dependency Direction + +```text +Python API --> ctypes declarations --> versioned C ABI --> C++ protocol core --> OS adapter +C++ RAII API --------------------------^ --> shared protocol fixtures +C# implementation -------------------------------------> shared protocol fixtures +Interop agents ----------------------------------------> public APIs only +``` + +The C ABI must not depend on Python or expose the C++ ABI. Platform adapters must +not know about Python or C++ ergonomic wrappers. All implementations depend on +the protocol; the protocol never depends on an implementation. + +## Change Impact Analysis + +- Layout or state change: updates the canonical protocol, all implementations, + every fixture, compatibility metadata, and full cross-runtime matrix. This is + a separately versioned compatibility event. +- Python-only API change: remains above the C ABI and affects Python tests and + packaging only. +- C++ ergonomic API change: remains in the RAII wrapper unless it requires a new + C ABI capability. +- Platform resource or locking change: affects the relevant C# and C++ adapters + and the cross-runtime ownership/contention tests, but not public wrappers. +- Throughput or allocation optimization: stays inside one core when layout, + state visibility, lifetimes, and status outcomes remain unchanged. + +## Complexity Tracking + +No constitution violations are planned. Multiple distributions and test agents +are required by the explicit cross-language interoperability goal; they remain +one repository because protocol changes require atomic review and validation. + +## Phase 0 Research Summary + +See [research.md](research.md). All technical unknowns are resolved. + +## Phase 1 Design Summary + +See [data-model.md](data-model.md), the contracts under [contracts/](contracts/), +and [quickstart.md](quickstart.md). The post-design constitution check remains +PASS: layout v1.2 is preserved, the new ABI is isolated and versioned, runtime +dependencies remain minimal, ownership is explicit, and validation covers both +offline bytes and live cross-process mechanisms. diff --git a/specs/008-cpp-python-implementations/quickstart.md b/specs/008-cpp-python-implementations/quickstart.md new file mode 100644 index 0000000..f58f313 --- /dev/null +++ b/specs/008-cpp-python-implementations/quickstart.md @@ -0,0 +1,57 @@ +# Quickstart: Validate Native and Python Implementations + +## Prerequisites + +- .NET SDK 10. +- Python 3.10 or newer. +- CMake 3.20 or newer and a C++20 compiler for the host platform. +- Docker Desktop or Docker Engine for the Linux container validation path. + +## Native Build and Tests + +```powershell +cmake -S . -B artifacts/native-build -DSMS_BUILD_TESTS=ON -DSMS_BUILD_SAMPLES=ON +cmake --build artifacts/native-build --config Release +ctest --test-dir artifacts/native-build -C Release --output-on-failure +``` + +Expected: ABI/layout, hashing, store lifecycle, lease, reservation, recovery, +platform-resource, and C++ RAII tests pass. + +## Python Source and Wheel Tests + +```powershell +python -m pip install --upgrade build +python -m build --wheel +python -m venv artifacts/python-consumer +artifacts/python-consumer/Scripts/python -m pip install (Get-ChildItem dist/*.whl | Select-Object -First 1) +artifacts/python-consumer/Scripts/python -m unittest discover -s tests/python -v +artifacts/python-consumer/Scripts/python samples/PythonBasicUsage/main.py +``` + +On Linux, use `bin/python` rather than `Scripts/python`. + +Expected: the installed package locates its bundled native library, passes its +public API/lifetime tests, and runs the basic sample without importing repository +sources accidentally. + +## Cross-Runtime Validation + +```powershell +dotnet build SharedMemoryStore.slnx -c Release +pwsh ./scripts/validate-interoperability.ps1 -Configuration Release +``` + +Expected: the ordered C#/C++/Python producer-consumer matrix passes on the host. +Use the script's Docker option to run the equivalent Linux matrix from Windows. + +## Existing Regression Gates + +```powershell +pwsh ./scripts/validate-docs.ps1 +dotnet test SharedMemoryStore.slnx -c Release +dotnet pack src/SharedMemoryStore/SharedMemoryStore.csproj -c Release -o artifacts/package +pwsh ./scripts/validate-package-consumption.ps1 +``` + +Expected: all existing managed behavior and package checks remain passing. diff --git a/specs/008-cpp-python-implementations/research.md b/specs/008-cpp-python-implementations/research.md new file mode 100644 index 0000000..fc6acba --- /dev/null +++ b/specs/008-cpp-python-implementations/research.md @@ -0,0 +1,125 @@ +# Research: Native and Python Implementations + +## Decision 1: Keep the implementations in one repository + +**Decision**: Add C++ and Python as independently packaged siblings of the +existing C# implementation, with one root protocol and interoperability suite. + +**Rationale**: The three distributions participate in the same mapped bytes, +locks, resource names, and lifecycle. A protocol change must update fixtures and +all participants atomically while the project has one owner. + +**Alternatives considered**: Separate language repositories were rejected for +the initial implementation because they would require synchronized protocol PRs +before ownership and release cadence justify that cost. Moving the C# source was +rejected because it creates unrelated churn. + +## Decision 2: One native core and one stable C ABI + +**Decision**: Implement the state machine and platform mechanisms once in C++20. +Expose fixed-width structs, enums, byte pointers, lengths, and opaque handles +through `extern "C"`. Build an ergonomic C++ RAII wrapper over that ABI. + +**Rationale**: A C ABI avoids exported standard-library types, exception +boundaries, compiler-specific name mangling, and ownership ambiguity. The C++ +wrapper remains pleasant for native users without becoming the Python contract. + +**Alternatives considered**: Exporting a C++ ABI was rejected as compiler and +standard-library fragile. Reimplementing the state machine separately in C and +C++ was rejected as duplication. + +## Decision 3: Python uses the standard library foreign-function interface + +**Decision**: Use Python `ctypes` over the C ABI, with context-managed store, +lease, and reservation objects. Support Python 3.10 or newer and create borrowed +`memoryview` objects whose Python owner retains the native lease or reservation. + +**Rationale**: `ctypes` loads C-compatible shared libraries without a CPython +extension or runtime dependency. A shared library that does not touch the Python +ABI can use one platform wheel across supported Python 3 versions. Current +official guidance explicitly documents both C-compatible calls and packaging a +CMake-built library beside a `ctypes` wrapper: +[Python ctypes](https://docs.python.org/3/library/ctypes.html), +[scikit-build-core ctypes guide](https://scikit-build-core.readthedocs.io/en/latest/guide/ctypes.html). + +**Alternatives considered**: pybind11/nanobind and the CPython C API were +rejected because they add a runtime-specific extension ABI and extra build +surface. A pure-Python protocol implementation was rejected because it would +duplicate atomic lifecycle, mapping, recovery, and platform-lock correctness. + +## Decision 4: Preserve layout v1.2 and make its hidden details canonical + +**Decision**: Do not change mapped records. Publish exact sizes, offsets, +alignment, state/status numbers, FNV-1a vectors, layout calculations, and binary +fixtures under `protocol/`. + +**Rationale**: Existing code and mappings use header 160 bytes, index header 32, +slot metadata 72, and lease record 40 with pack 8. Existing narrative contracts +omit some exact offsets and one ingest document still names stale minor version +1. Interoperability requires executable precision. + +**Alternatives considered**: A new layout adding process start tokens was +rejected for this feature because it would invalidate existing mappings and +expand the C# compatibility change. Lease/reservation recovery therefore keeps +the current PID-based liveness contract; Linux resource owner sidecars retain +their stronger PID-plus-start-token check. + +## Decision 5: Match the full platform-resource protocol + +**Decision**: On Windows use the exact public mapping name and derived +`Local\\`/`Global\\SharedMemoryStore-*` mutex. On Linux reproduce SHA-256-derived +paths, permissions, owner records, atomic owner-file replacement, lifecycle +locking, and nonblocking `fcntl` byte-range locks over `[0,1)` plus a +process-local per-path mutex. + +**Rationale**: Layout compatibility alone is insufficient. A foreign process +that omits owner registration can have its region deleted by a later C# opener; +using `flock` would not contend with .NET file-region locks. + +**Alternatives considered**: New native-only resource names and `flock` were +rejected because C# and native processes would not synchronize or share cleanup. + +## Decision 6: Use CMake and scikit-build-core only at build time + +**Decision**: Use target-based CMake installation/export for the native library. +Use root-level scikit-build-core packaging to build and place the shared library +inside the Python package. End users receive platform wheels and need no compiler. + +**Rationale**: CMake supports portable target installation and exports, while +Python packaging requires platform-specific wheels for bundled compiled code: +[CMake installation guide](https://cmake.org/cmake/help/latest/guide/tutorial/Installation%20Commands%20and%20Concepts.html), +[Python packaging flow](https://packaging.python.org/en/latest/flow/). + +**Alternatives considered**: A nested Python project was rejected because its +source distribution cannot naturally include sibling C++ sources. Hand-written +wheel construction was rejected as unnecessary packaging risk. + +## Decision 7: Use layered conformance and live interoperability tests + +**Decision**: Add static ABI assertions, canonical JSON/binary fixtures, +dependency-free C++ tests, Python unit tests, and JSON-lines subprocess agents +orchestrated by xUnit. Run the ordered 3x3 producer-consumer matrix and mixed +lease, reservation, removal, contention, ownership, and crash-recovery cases on +Windows and Linux. + +**Rationale**: Fixtures detect encoding drift; live tests detect incompatible +locks, cleanup, and memory-lifetime behavior that fixtures cannot exercise. + +**Alternatives considered**: Testing each implementation independently was +rejected because all could reproduce the same mistaken interpretation. Binary +fixtures alone were rejected because platform lifecycle is outside the mapping. + +## Decision 8: Validate with available and reproducible toolchains + +**Decision**: Make CMake/CTest the canonical build, provide PowerShell wrappers, +and add a Linux toolchain container for repeatable local validation. Validate +native Windows with a portable or installed standards-compliant compiler and in +Windows CI. + +**Rationale**: The current host has .NET 10, Python 3.14, Docker, and WSL g++ but +no native Windows compiler or CMake. The repository must make missing build +prerequisites explicit rather than silently skipping native validation. + +**Alternatives considered**: Treating uncompiled C++ as complete was rejected. +Installing a required system-wide compiler implicitly was rejected in favor of +workspace-local tooling or documented CI/toolchain prerequisites. diff --git a/specs/008-cpp-python-implementations/spec.md b/specs/008-cpp-python-implementations/spec.md new file mode 100644 index 0000000..8d6b356 --- /dev/null +++ b/specs/008-cpp-python-implementations/spec.md @@ -0,0 +1,240 @@ +# Feature Specification: Native and Python Implementations + +**Feature Branch**: `codex/cpp-python-implementations` + +**Created**: 2026-07-10 + +**Status**: Draft + +**Input**: User description: "Add interoperable Python and C++ implementations in the same repository, with a shared protocol boundary, and keep working until the complete cross-language feature works." + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Exchange Values Across Runtimes (Priority: P1) + +An application using any supported runtime can create or open a named store, +publish opaque keys, descriptors, and payloads, and exchange those values with +applications using either of the other supported runtimes on the same host. + +**Why this priority**: Cross-runtime exchange is the central user value. A +second implementation that cannot participate in the same store would be a +separate product rather than an interoperable SharedMemoryStore participant. + +**Independent Test**: Start one participant as the store creator and producer, +start a different participant as the consumer, and verify the consumer can +acquire, inspect, release, remove, and replace the value. Repeat for every +ordered producer-consumer pairing. + +**Acceptance Scenarios**: + +1. **Given** a store created by one supported runtime, **When** another runtime opens it with matching capacities, **Then** both participants observe the same store and values. +2. **Given** opaque keys and binary descriptors and payloads containing zero and non-text bytes, **When** one participant publishes them, **Then** every other participant reads the exact bytes without reinterpretation. +3. **Given** several participants racing to publish the same key, **When** their operations complete, **Then** exactly one succeeds and all others receive the documented duplicate outcome. + +--- + +### User Story 2 - Use the Complete Store Lifecycle (Priority: P1) + +Native and Python application developers can use the same create/open modes, +bounded waits, publication, acquisition, lease release, removal, reuse, +reservation, commit, abort, recovery, and status semantics already documented +for SharedMemoryStore. + +**Why this priority**: Partial lifecycle support would create unsafe combinations +in which one participant can create shared state that another cannot manage or +recover correctly. + +**Independent Test**: Run the public lifecycle contract suite against each +runtime independently, then run cross-runtime lease, removal, reservation, and +recovery scenarios. + +**Acceptance Scenarios**: + +1. **Given** a reader holding a lease, **When** another participant removes the key, **Then** removal remains pending until the lease is released and the slot is subsequently reusable. +2. **Given** an incomplete direct-write reservation, **When** the producer commits too early, aborts, or terminates, **Then** no partial value is exposed and an authorized participant can apply the documented recovery policy. +3. **Given** an unavailable shared lock, **When** a caller selects no-wait or a bounded wait, **Then** the operation returns the documented busy outcome within the selected bound. + +--- + +### User Story 3 - Install and Use Each Distribution Independently (Priority: P2) + +Developers can build, test, and consume the managed, native, and Python +distributions independently while still knowing which shared protocol versions +they can safely use together. + +**Why this priority**: Each ecosystem has its own release and consumption +workflow, but those independent releases must not obscure interoperability. + +**Independent Test**: Build each distribution from a clean checkout, consume it +from a minimal external sample, and verify that its advertised protocol version +matches the interoperability matrix. + +**Acceptance Scenarios**: + +1. **Given** a clean checkout with a supported toolchain, **When** a developer follows one distribution's documented build steps, **Then** only that distribution and its declared build dependencies are required. +2. **Given** independently versioned distributions, **When** a developer reviews compatibility information, **Then** the common on-memory protocol version is explicit and unambiguous. +3. **Given** a consuming application, **When** it uses a lease or writable reservation view, **Then** the distribution prevents or clearly detects use after release, abort, commit, recovery, or store closure. + +--- + +### User Story 4 - Diagnose Capacity and Lifecycle State Consistently (Priority: P3) + +Operators can inspect capacity, slot, lease, reservation, index-health, recovery, +and failure information without the libraries writing directly to application +output or starting hidden maintenance work. + +**Why this priority**: Consistent diagnostics make mixed-runtime deployments +supportable without coupling the core store to a logging or hosting framework. + +**Independent Test**: Produce known store states from multiple participants and +verify that diagnostics from every runtime report equivalent shared facts and +runtime-local failure information according to the documented contract. + +**Acceptance Scenarios**: + +1. **Given** published, pending-removal, and pending-reservation slots, **When** diagnostics are requested, **Then** every participant reports equivalent shared-state counts. +2. **Given** expected operation failures, **When** diagnostics are requested, **Then** failures are available to the caller and no library writes directly to console output. + +### Edge Cases + +- Empty keys, oversized keys, descriptors, or payloads return their documented + outcomes without mutating the store. +- Store options whose calculated layout overflows or exceeds the supplied region + are rejected before a mapping is used. +- Existing mappings with a different layout major version, record size, section + offset, capacity, or unsupported state are rejected deterministically. +- Names containing Unicode, separators, punctuation, scope prefixes, or the + maximum supported length resolve to the same platform resources in every + implementation. +- Abrupt participant termination does not cause a live store to be deleted while + another participant still owns it. +- Linux resource-owner records use the available process-start identity so PID + reuse does not cause deletion of a region that still has a live owner. +- Slot generation rollover advances the full lifecycle identity and never makes + an old lease or reservation valid again. +- Zero-length payloads and descriptors remain valid where the existing public + contract permits them. +- Closing a store invalidates borrowed views without closing or corrupting other + live participants. +- Unsupported operating systems and insufficient permissions produce documented + outcomes rather than undefined behavior. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The repository MUST deliver independently consumable managed, + native, and Python distributions under one shared compatibility policy. +- **FR-002**: Every distribution MUST conform to one canonical, versioned + on-memory layout, byte order, alignment, numeric state assignment, hashing, + probing, lifecycle identity, and visibility contract. +- **FR-003**: Every distribution MUST resolve a public store name to mutually + compatible platform mapping, synchronization, ownership, and lifecycle + resources on supported operating systems. +- **FR-004**: Native and Python consumers MUST be able to calculate required + capacity and create, create-or-open, or open an existing store with the same + validation and deterministic open outcomes as existing consumers. +- **FR-005**: Native and Python consumers MUST support publish, acquire, byte + access, lease release, remove, and slot reuse with status semantics equivalent + to the existing public contract. +- **FR-006**: Native and Python consumers MUST support announced-length writable + reservations, progress advancement, exact commit, abort, and invalidation of + completed reservation views. +- **FR-007**: Native and Python consumers MUST support segmented publication as + one contiguous committed value without exposing partially copied data. +- **FR-008**: Every distribution MUST preserve pending-removal semantics while + leases exist and reclaim the slot only after the final valid release. +- **FR-009**: Every distribution MUST support explicit, policy-controlled stale + lease and reservation recovery without reclaiming records owned by another + live participant. +- **FR-010**: Every distribution MUST expose caller-controlled bounded and + no-wait synchronization outcomes and MUST NOT introduce hidden retry workers. +- **FR-011**: Every distribution MUST expose diagnostics covering shared + capacity and lifecycle facts plus its own observed failures, without direct + console output or a required telemetry framework. +- **FR-012**: Borrowed payload, descriptor, and writable reservation memory MUST + remain valid only for the documented lifetime of its owning lease, + reservation, and store handle. +- **FR-013**: The existing managed public API and behavior MUST remain compatible + unless a separately reviewed semantic-version change is documented. +- **FR-014**: Distribution versions MAY advance independently, but each release + MUST declare the shared protocol versions it can open and produce. +- **FR-015**: Automated conformance tests MUST pin exact record sizes, field + offsets, state and status numbers, layout arithmetic, hash vectors, resource + naming vectors, and representative binary fixtures. +- **FR-016**: Automated interoperability tests MUST cover every ordered + producer-consumer pairing and mixed-runtime lease, removal, reservation, + contention, owner-lifecycle, and recovery scenarios on each supported host + platform. +- **FR-017**: Each distribution MUST include a minimal runnable example and a + clean-consumer validation path. +- **FR-018**: Runtime libraries MUST avoid global mutable configuration, hidden + background work, direct console output, and undeclared runtime dependencies. +- **FR-019**: The security documentation MUST preserve the trusted same-host + participant boundary and MUST NOT imply protection from malicious writers + that can legitimately access the shared resources. +- **FR-020**: Cross-host sharing, persistence guarantees, application-schema + parsing, and distributed-cache behavior MUST remain outside this feature. + +### Key Entities + +- **Shared Protocol Version**: The major and minor compatibility identity for + mapped bytes, state transitions, hashing, synchronization participation, and + resource ownership behavior. +- **Distribution**: One independently versioned consumable package exposing the + store contract to a supported application runtime. +- **Store Handle**: A process-local owner of one mapped-region view and its + synchronization and lifecycle resources. +- **Lease**: A bounded-lifetime read capability tied to one slot lifecycle + identity and one active shared lease record. +- **Reservation**: A bounded-lifetime write capability for an announced payload, + tied to one pending slot lifecycle identity. +- **Conformance Fixture**: A versioned set of canonical values and mapped bytes + used to prove that implementations interpret the protocol identically. +- **Compatibility Matrix**: The declared relationship between independently + released distribution versions and supported shared protocol versions. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: All nine ordered producer-consumer combinations exchange at least + 1,000 values containing arbitrary key, descriptor, and payload bytes without + a byte mismatch or undocumented status. +- **SC-002**: Every supported participant passes 100% of the shared conformance + vectors for layout, hashing, resource naming, lifecycle, and status semantics. +- **SC-003**: Mixed-participant contention tests complete within the + caller-selected wait limit plus 250 milliseconds in at least 99.9% of 10,000 + bounded-wait operations. +- **SC-004**: Each supported host environment completes 10,000 mixed-participant + removal, final-release, reservation-abort, and recovery cycles without stale + handle acceptance, partial-value visibility, or live-resource deletion. +- **SC-005**: A developer can build and run the basic example for any one + distribution from a clean checkout by following that distribution's + documentation, with every required dependency and command stated explicitly. +- **SC-006**: Existing managed contract, unit, integration, sample, documentation, + package-consumption, build, test, and pack validations remain passing. +- **SC-007**: Reviewers can determine whether any two released distributions are + interoperable from one compatibility matrix without comparing source code. + +## Assumptions + +- The implementations participate in the same named store and are not merely + separate APIs with similar behavior. +- Linux and Windows remain the supported same-host targets. Linux-based + same-host containers remain a deployment profile rather than a new protocol. +- The existing layout major version 1, minor version 2 is the initial shared + baseline; this feature does not intentionally alter its mapped record layout. +- Lease and reservation recovery preserves the current layout-v1.2 PID-based + liveness semantics; adding a process-start identity to those records requires + a separately versioned layout change. +- Native and Python distributions may expose ecosystem-appropriate names and + lifetime constructs while preserving equivalent outcomes and byte semantics. +- Python consumption may rely on a native component shipped with or built for + the Python distribution; a separately maintained copy of the state machine is + not required. +- Build-only tooling is allowed when documented and license-compatible, but the + runtime dependency surface remains minimal and deliberate. +- Performance claims remain bounded to measured same-host scenarios and do not + promise persistence, cross-host sharing, or resistance to malicious in-scope + writers. diff --git a/specs/008-cpp-python-implementations/tasks.md b/specs/008-cpp-python-implementations/tasks.md new file mode 100644 index 0000000..d1b9c67 --- /dev/null +++ b/specs/008-cpp-python-implementations/tasks.md @@ -0,0 +1,273 @@ +--- + +description: "Dependency-ordered implementation tasks for native and Python interoperability" +--- + +# Tasks: Native and Python Implementations + +**Input**: Design documents from `specs/008-cpp-python-implementations/` + +**Prerequisites**: plan.md, spec.md, research.md, data-model.md, contracts/, quickstart.md + +**Tests**: The specification explicitly requires automated conformance, +interoperability, lifecycle, packaging, and regression tests. Test tasks precede +their corresponding implementation tasks. + +**Organization**: Tasks are grouped by user story so each slice has an explicit +independent validation checkpoint. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel with adjacent tasks because it owns different files +- **[Story]**: Maps the task to a prioritized user story in spec.md +- Every task names its primary file or directory + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Establish build, package, source, test, and protocol roots without +moving the existing C# implementation. + +- [x] T001 Create the root native build and install skeleton in CMakeLists.txt and cmake/SharedMemoryStoreConfig.cmake.in +- [x] T002 [P] Create the C++ public/internal source skeleton in src/cpp/include/shared_memory_store/ and src/cpp/src/ +- [x] T003 [P] Create the Python source-package skeleton in pyproject.toml and src/python/shared_memory_store/ +- [x] T004 [P] Create native, Python, and interoperability test roots in tests/cpp/, tests/python/, tests/SharedMemoryStore.InteropAgent/, and tests/SharedMemoryStore.InteropTests/ +- [x] T005 [P] Create protocol and sample roots in protocol/, samples/CppBasicUsage/, and samples/PythonBasicUsage/ + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Make layout v1.2, status values, resource names, and ownership rules +precise before implementing any runtime behavior. + +**Critical**: No user-story implementation begins until this phase passes its +static conformance checks. + +- [x] T006 Correct the stale layout-minor statement and publish canonical version boundaries in specs/003-zero-copy-ingest/contracts/ingest-layout.md and protocol/README.md +- [x] T007 [P] Document exact mapped records, lifecycle states, and layout calculations in protocol/layout-v1.2.md +- [x] T008 [P] Document Windows/Linux naming, locking, permissions, owner sidecars, and cleanup in protocol/resource-naming-v1.md +- [x] T009 Create canonical layout, hash, status, and resource-name vectors in protocol/fixtures/v1.2/manifest.json +- [x] T010 Define fixed-width enums, versioned structs, opaque handles, export macros, and required symbols in src/cpp/include/shared_memory_store/c_api.h +- [x] T011 Implement packed records, static ABI assertions, checked layout arithmetic, FNV-1a hashing, UTF conversion, and SHA-256 helpers in src/cpp/src/layout.hpp and src/cpp/src/protocol.cpp +- [x] T012 Add failing-then-passing static conformance tests for every manifest vector in tests/cpp/protocol_tests.cpp and tests/python/test_protocol_manifest.py + +**Checkpoint**: The native records compile to exact sizes and all offline +language-neutral vectors agree with the managed baseline. + +--- + +## Phase 3: User Story 1 - Exchange Values Across Runtimes (Priority: P1) — MVP + +**Goal**: C#, C++, and Python can create/open the same store and exchange, +acquire, release, remove, and replace exact bytes. + +**Independent Test**: Run every ordered producer-consumer pairing for create, +publish, acquire/read/release, remove, and republish with arbitrary binary bytes. + +### Tests for User Story 1 + +- [x] T013 [P] [US1] Add native create/open and publish/acquire/remove/reuse contract tests in tests/cpp/store_tests.cpp +- [x] T014 [P] [US1] Add C ABI null, bounds, ownership, and status contract tests in tests/cpp/c_api_tests.cpp +- [x] T015 [P] [US1] Add Python basic API and context-manager tests in tests/python/test_store.py +- [x] T016 [P] [US1] Add JSON-lines agent protocol tests in tests/SharedMemoryStore.InteropTests/AgentProtocolTests.cs +- [x] T017 [US1] Add the ordered 3x3 core exchange matrix tests in tests/SharedMemoryStore.InteropTests/CoreExchangeMatrixTests.cs + +### Implementation for User Story 1 + +- [x] T018 [P] [US1] Implement Windows named mapping, mutex, UTF-8 naming, error mapping, and process-local gating in src/cpp/src/platform_windows.cpp +- [x] T019 [P] [US1] Implement Linux mapped files, fcntl byte locks, per-path local mutexes, permissions, owner sidecars, and cleanup in src/cpp/src/platform_linux.cpp +- [x] T020 [US1] Implement option validation, mapping initialization/validation, key index, slots, leases, publish, acquire, release, remove, and reuse in src/cpp/src/store.cpp +- [x] T021 [US1] Implement store/lease opaque-handle operations and exception containment in src/cpp/src/c_api.cpp +- [x] T022 [P] [US1] Implement move-only C++ store and lease wrappers in src/cpp/include/shared_memory_store/store.hpp +- [x] T023 [US1] Implement ctypes declarations, enums, loader, store, and lease wrappers in src/python/shared_memory_store/ +- [x] T024 [US1] Implement equivalent C#, C++, and Python JSON-lines participants in tests/SharedMemoryStore.InteropAgent/, tests/cpp/interop_agent.cpp, and tests/python/interop_agent.py + +**Checkpoint**: User Story 1 works independently on Windows and Linux; exact +bytes pass through all nine ordered runtime pairings. + +--- + +## Phase 4: User Story 2 - Complete Store Lifecycle (Priority: P1) + +**Goal**: All runtimes support bounded waits, segmented publication, direct +reservations, pending removal, crash recovery, and safe token invalidation. + +**Independent Test**: Run mixed-runtime pending-reservation visibility, +foreign-lease removal, final release/reuse, bounded contention, crash recovery, +and three-owner cleanup scenarios. + +### Tests for User Story 2 + +- [x] T025 [P] [US2] Add native reservation, segmented publish, bounded-wait, recovery, and lifecycle tests in tests/cpp/lifecycle_tests.cpp +- [x] T026 [P] [US2] Add Python reservation memoryview, segmented publish, timeout, recovery, and invalidation tests in tests/python/test_lifecycle.py +- [x] T027 [P] [US2] Add mixed-runtime lease/remove/reuse and reservation/commit/abort tests in tests/SharedMemoryStore.InteropTests/MixedLifecycleTests.cs +- [x] T028 [P] [US2] Add mixed-runtime lock contention, crash recovery, and Linux owner-cleanup tests in tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs + +### Implementation for User Story 2 + +- [x] T029 [US2] Implement bounded/infinite waits, segmented publish, reservations, progress, commit, abort, and token lifetime control in src/cpp/src/store.cpp +- [x] T030 [US2] Implement PID-based lease/reservation recovery and reports compatible with layout v1.2 in src/cpp/src/store.cpp +- [x] T031 [US2] Extend the C ABI with segments, reservation, wait, recovery, and borrowed-buffer functions in src/cpp/include/shared_memory_store/c_api.h and src/cpp/src/c_api.cpp +- [x] T032 [P] [US2] Extend the C++ RAII API with reservation, wait, segmented publish, and recovery types in src/cpp/include/shared_memory_store/store.hpp +- [x] T033 [US2] Extend Python with owned memoryviews, reservations, waits, segments, and recovery in src/python/shared_memory_store/store.py and src/python/shared_memory_store/_native.py +- [x] T034 [US2] Extend all three interop agents and orchestration helpers with lifecycle, contention, recovery, close, and crash commands in tests/cpp/interop_agent.cpp, tests/python/interop_agent.py, tests/SharedMemoryStore.InteropAgent/Program.cs, and tests/SharedMemoryStore.InteropTests/TestSupport/ + +**Checkpoint**: Mixed-runtime lifecycle and crash behavior matches the existing +public contract without partial visibility, stale token acceptance, or live +resource deletion. + +--- + +## Phase 5: User Story 3 - Independent Distribution Consumption (Priority: P2) + +**Goal**: Native and Python users can build/install one distribution cleanly and +see an explicit compatibility declaration. + +**Independent Test**: Install the CMake package into an external native sample +and install a built wheel into a clean virtual environment; run both without +repository-source imports. + +### Tests for User Story 3 + +- [x] T035 [P] [US3] Add a clean external find_package consumer test in tests/cpp/package_consumer/ +- [x] T036 [P] [US3] Add installed-wheel loading and package-content tests in tests/python/test_installed_package.py + +### Implementation for User Story 3 + +- [x] T037 [US3] Complete CMake targets, visibility, install/export, package config, CTest, and sample integration in CMakeLists.txt and cmake/SharedMemoryStoreConfig.cmake.in +- [x] T038 [US3] Configure scikit-build-core to bundle the correct native shared library and platform wheel tag in pyproject.toml +- [x] T039 [P] [US3] Implement minimal native and Python consumer samples in samples/CppBasicUsage/main.cpp and samples/PythonBasicUsage/main.py +- [x] T040 [P] [US3] Publish package/ABI/layout/resource compatibility metadata in protocol/compatibility.json +- [x] T041 [US3] Add reproducible native, wheel, and clean-consumer wrappers in scripts/validate-native.ps1 and scripts/validate-python.ps1 + +**Checkpoint**: CMake install consumption and Python wheel consumption pass from +clean locations with no undeclared runtime dependency. + +--- + +## Phase 6: User Story 4 - Consistent Diagnostics (Priority: P3) + +**Goal**: Native and Python callers inspect equivalent shared capacity and +lifecycle facts plus caller-owned local failure accounting. + +**Independent Test**: Create known mixed-runtime store states and compare shared +diagnostic fields from every participant while verifying no library console I/O. + +### Tests for User Story 4 + +- [x] T042 [P] [US4] Add native and C ABI diagnostics contract tests in tests/cpp/diagnostics_tests.cpp +- [x] T043 [P] [US4] Add Python and mixed-runtime diagnostics comparison tests in tests/python/test_diagnostics.py and tests/SharedMemoryStore.InteropTests/DiagnosticsInteropTests.cs + +### Implementation for User Story 4 + +- [x] T044 [US4] Implement shared counts, index health, compaction, recovery counters, and local failure accounting in src/cpp/src/diagnostics.cpp and src/cpp/src/store.cpp +- [x] T045 [P] [US4] Expose versioned diagnostics value types through the C and C++ APIs in src/cpp/include/shared_memory_store/c_api.h and src/cpp/include/shared_memory_store/store.hpp +- [x] T046 [US4] Expose immutable Python diagnostics snapshots and agent responses in src/python/shared_memory_store/store.py and tests/python/interop_agent.py + +**Checkpoint**: Shared facts agree across runtimes and all diagnostics remain +caller-controlled. + +--- + +## Phase 7: Polish & Cross-Cutting Validation + +**Purpose**: Finish protocol fixtures, documentation, CI, stress evidence, and +all existing release gates. + +- [x] T047 Generate representative offline v1.2 binary fixtures and normalized snapshots in protocol/fixtures/v1.2/ and add their hashes to protocol/fixtures/v1.2/manifest.json +- [x] T048 Add the host/Docker all-language orchestration and stress entry point in scripts/validate-interoperability.ps1 and tests/SharedMemoryStore.InteropTests/StressInteropTests.cs +- [x] T049 [P] Add Windows and Linux native/Python/interoperability CI jobs in .github/workflows/ci.yml +- [x] T050 [P] Update README.md, docs/architecture.md, docs/portability.md, docs/getting-started.md, docs/packaging.md, docs/samples.md, docs/maintainers.md, and docs/releases.md for delivered native/Python support +- [x] T051 [P] Update CHANGELOG.md, src/SharedMemoryStore/SharedMemoryStore.csproj release notes, SECURITY.md, and SUPPORT.md with compatibility and trusted-boundary impact +- [x] T052 Run protocol, native, Python, wheel, CMake-consumer, and full cross-runtime validation from specs/008-cpp-python-implementations/quickstart.md and fix every failure +- [x] T053 Run pwsh ./scripts/validate-docs.ps1, dotnet build/test/pack, package consumption, samples, cross-platform, and Docker regression gates and fix every failure +- [x] T054 Audit git diff/status for generated artifacts, accidental binaries, public API drift, missing licenses, and user-owned changes without committing or pushing + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- Setup has no dependencies. +- Foundational depends on Setup and blocks every user story. +- User Story 1 depends on Foundational and creates the usable cross-runtime MVP. +- User Story 2 depends on the store/lease core from User Story 1. +- User Story 3 depends on the native and Python public surfaces from User Stories + 1 and 2 but is independently verifiable through clean consumers. +- User Story 4 depends on shared core state from User Stories 1 and 2 and can be + implemented in parallel with packaging after those phases. +- Polish depends on all desired story phases. + +### User Story Dependency Graph + +```text +Setup -> Foundation -> US1 -> US2 ->+-> US3 -+ + +-> US4 -+-> Polish +``` + +### Parallel Opportunities + +- Setup skeletons T002-T005 own separate directories. +- Foundational protocol narratives T007-T008 can proceed in parallel. +- US1 platform adapters T018-T019 and public-wrapper tests T013-T016 are + independent before store-core integration. +- US2 native, Python, mixed-lifecycle, and recovery tests T025-T028 are separate. +- After US2, packaging US3 and diagnostics US4 can proceed concurrently. +- Documentation, CI, and release metadata T049-T051 own separate files. + +## Parallel Examples + +### User Story 1 + +```text +Task T018: Implement Windows platform adapter. +Task T019: Implement Linux platform adapter. +Task T015: Define Python public behavior tests. +Task T016: Define language-neutral agent protocol tests. +``` + +### User Story 2 + +```text +Task T025: Native lifecycle tests. +Task T026: Python lifecycle tests. +Task T027: Mixed lease/reservation tests. +Task T028: Contention/recovery/owner tests. +``` + +### User Stories 3 and 4 + +```text +Task group T035-T041: Distribution packaging and consumption. +Task group T042-T046: Diagnostics contracts and implementation. +``` + +## Implementation Strategy + +### MVP First + +1. Complete Setup and Foundational phases. +2. Complete User Story 1. +3. Validate all nine basic ordered runtime pairings on Windows and Linux. +4. Do not claim feature completion until lifecycle, packaging, diagnostics, and + regression phases also pass. + +### Incremental Delivery + +1. Canonical protocol and ABI. +2. Core cross-runtime byte exchange. +3. Complete lifecycle and recovery. +4. Clean ecosystem consumption. +5. Diagnostics and stress evidence. +6. Documentation and existing release gates. + +## Notes + +- Tests are written before the corresponding implementation and must fail for + the expected missing behavior before they pass. +- The coarse shared lock is preserved until conformance is proven; optimization + must not change visibility or lifecycle semantics. +- Layout v1.2 recovery remains PID-based; adding process-start identity is a + future versioned layout feature. +- Do not commit or push any change for this user request. diff --git a/src/SharedMemoryStore/Interop/LinuxSharedMemoryRegion.cs b/src/SharedMemoryStore/Interop/LinuxSharedMemoryRegion.cs index 2d69025..9964b55 100644 --- a/src/SharedMemoryStore/Interop/LinuxSharedMemoryRegion.cs +++ b/src/SharedMemoryStore/Interop/LinuxSharedMemoryRegion.cs @@ -88,7 +88,7 @@ private static StoreOpenStatus CreateRegion( try { stream.SetLength(options.TotalBytes); - return CreateMappedRegion(resourceName, options, stream, out region); + return CreateMappedRegion(resourceName, options.TotalBytes, stream, out region); } catch { @@ -122,12 +122,12 @@ private static StoreOpenStatus OpenExistingRegion( return StoreOpenStatus.IncompatibleLayout; } - return CreateMappedRegion(resourceName, options, stream, out region); + return CreateMappedRegion(resourceName, stream.Length, stream, out region); } private static StoreOpenStatus CreateMappedRegion( PlatformResourceName resourceName, - SharedMemoryStoreOptions options, + long mappingCapacity, FileStream stream, out MemoryMappedStoreRegion? region) { @@ -142,16 +142,16 @@ private static StoreOpenStatus CreateMappedRegion( mapping = MemoryMappedFile.CreateFromFile( stream, mapName: null, - capacity: options.TotalBytes, + capacity: mappingCapacity, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, leaveOpen: false); - accessor = mapping.CreateViewAccessor(0, options.TotalBytes, MemoryMappedFileAccess.ReadWrite); + accessor = mapping.CreateViewAccessor(0, mappingCapacity, MemoryMappedFileAccess.ReadWrite); candidate = MemoryMappedStoreRegion.Create( mapping, accessor, - options.TotalBytes, + mappingCapacity, () => { if (ownerRegistered) diff --git a/src/SharedMemoryStore/Interop/SharedStorePlatform.cs b/src/SharedMemoryStore/Interop/SharedStorePlatform.cs index 5be4904..109a310 100644 --- a/src/SharedMemoryStore/Interop/SharedStorePlatform.cs +++ b/src/SharedMemoryStore/Interop/SharedStorePlatform.cs @@ -48,6 +48,11 @@ public static StoreOpenStatus TryOpenRegion( out MemoryMappedStoreRegion? region) { region = null; + if (!Environment.Is64BitProcess || !BitConverter.IsLittleEndian) + { + return StoreOpenStatus.UnsupportedPlatform; + } + var resourceName = PlatformResourceName.Create(options.Name); if (OperatingSystem.IsWindows()) { diff --git a/src/SharedMemoryStore/SharedMemoryStore.csproj b/src/SharedMemoryStore/SharedMemoryStore.csproj index 19f159b..c86ddac 100644 --- a/src/SharedMemoryStore/SharedMemoryStore.csproj +++ b/src/SharedMemoryStore/SharedMemoryStore.csproj @@ -19,7 +19,7 @@ git https://github.com/rantri/SharedMemoryStore README.md - Linux, Windows, and same-host Docker support hardening: fixes bounded waits, crash-safe ownership and index maintenance, private Linux resource permissions, layout overflow validation, and cleanup reliability while preserving the 1.0.0 public API and layout. + Linux, Windows, and same-host Docker support hardening remains the managed 1.0.1 package scope. The repository also adds independently versioned native C++ and Python sibling distributions over layout 1.2 and C ABI 1.0; they are not included in the NuGet package, whose public API and runtime dependency surface remain unchanged. true snupkg SharedMemoryStore diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt new file mode 100644 index 0000000..ac999a8 --- /dev/null +++ b/src/cpp/CMakeLists.txt @@ -0,0 +1,141 @@ +set( + _sms_library_sources + "${CMAKE_CURRENT_SOURCE_DIR}/src/protocol.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/store.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/c_api.cpp") + +if(WIN32) + set(_sms_platform_source + "${CMAKE_CURRENT_SOURCE_DIR}/src/platform_windows.cpp") + set(_sms_platform_definition SMS_PLATFORM_WINDOWS=1) +elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(_sms_platform_source + "${CMAKE_CURRENT_SOURCE_DIR}/src/platform_linux.cpp") + set(_sms_platform_definition SMS_PLATFORM_LINUX=1) +else() + message( + FATAL_ERROR + "SharedMemoryStore currently supports only Windows and Linux; detected '${CMAKE_SYSTEM_NAME}'." + ) +endif() + +list(APPEND _sms_library_sources "${_sms_platform_source}") + +set( + _sms_public_headers + "${CMAKE_CURRENT_SOURCE_DIR}/include/shared_memory_store/c_api.h" + "${CMAKE_CURRENT_SOURCE_DIR}/include/shared_memory_store/store.hpp") + +function(_sms_configure_library target_name) + target_sources( + ${target_name} + PRIVATE ${_sms_library_sources} ${_sms_public_headers}) + + target_compile_features(${target_name} PUBLIC cxx_std_20) + set_target_properties( + ${target_name} + PROPERTIES CXX_EXTENSIONS OFF + POSITION_INDEPENDENT_CODE ON + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN YES + LINKER_LANGUAGE CXX) + + target_include_directories( + ${target_name} + PUBLIC $ + $ + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + + target_compile_definitions( + ${target_name} + PRIVATE SMS_BUILDING_LIBRARY=1 ${_sms_platform_definition}) + + if(WIN32) + target_compile_definitions(${target_name} PRIVATE WIN32_LEAN_AND_MEAN + NOMINMAX) + if(MINGW) + # Wheels and native archives must not depend on an unbundled + # MinGW C++ runtime or unwinder DLL. + target_link_options(${target_name} PRIVATE -static) + endif() + else() + target_compile_definitions(${target_name} PRIVATE _FILE_OFFSET_BITS=64) + endif() + + target_link_libraries(${target_name} PRIVATE Threads::Threads) + + if(MSVC) + target_compile_options(${target_name} PRIVATE /W4 /permissive- /EHsc) + else() + target_compile_options(${target_name} PRIVATE -Wall -Wextra -Wpedantic) + endif() +endfunction() + +add_library(SharedMemoryStore SHARED) +add_library(SharedMemoryStore::SharedMemoryStore ALIAS SharedMemoryStore) +_sms_configure_library(SharedMemoryStore) +set_target_properties( + SharedMemoryStore + PROPERTIES EXPORT_NAME SharedMemoryStore + OUTPUT_NAME shared_memory_store + WINDOWS_EXPORT_ALL_SYMBOLS OFF) +if(WIN32) + set_target_properties(SharedMemoryStore PROPERTIES PREFIX "") +endif() + +# Native installations use the ABI major in the soname. Python wheels need a +# stable, unversioned filename for the standard-library loader. +if(NOT DEFINED SKBUILD AND SMS_PYTHON_INSTALL_DIR STREQUAL "") + set_target_properties( + SharedMemoryStore + PROPERTIES VERSION "${PROJECT_VERSION}" + SOVERSION 1) +endif() + +if(SMS_BUILD_STATIC) + add_library(SharedMemoryStoreStatic STATIC) + add_library(SharedMemoryStore::SharedMemoryStoreStatic ALIAS + SharedMemoryStoreStatic) + _sms_configure_library(SharedMemoryStoreStatic) + target_compile_definitions(SharedMemoryStoreStatic PUBLIC SMS_STATIC=1) + set_target_properties( + SharedMemoryStoreStatic + PROPERTIES EXPORT_NAME SharedMemoryStoreStatic + OUTPUT_NAME shared_memory_store_static) +endif() + +if(SMS_INSTALL) + install( + TARGETS SharedMemoryStore + EXPORT SharedMemoryStoreTargets + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT Runtime + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT Runtime + NAMELINK_COMPONENT Development + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT Development + INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + + if(SMS_BUILD_STATIC) + install( + TARGETS SharedMemoryStoreStatic + EXPORT SharedMemoryStoreTargets + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" + COMPONENT Development + INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + endif() + + install( + DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT Development + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.hpp") + + if(NOT SMS_PYTHON_INSTALL_DIR STREQUAL "") + install( + TARGETS SharedMemoryStore + RUNTIME DESTINATION "${SMS_PYTHON_INSTALL_DIR}" COMPONENT Python + LIBRARY DESTINATION "${SMS_PYTHON_INSTALL_DIR}" COMPONENT Python + NAMELINK_SKIP) + endif() +endif() diff --git a/src/cpp/include/shared_memory_store/c_api.h b/src/cpp/include/shared_memory_store/c_api.h new file mode 100644 index 0000000..a6a3bf8 --- /dev/null +++ b/src/cpp/include/shared_memory_store/c_api.h @@ -0,0 +1,332 @@ +#ifndef SHARED_MEMORY_STORE_C_API_H +#define SHARED_MEMORY_STORE_C_API_H + +#include +#include + +#if defined(SMS_STATIC) +# define SMS_API +# define SMS_CALL +#elif defined(_WIN32) +# if defined(SMS_BUILDING_LIBRARY) +# define SMS_API __declspec(dllexport) +# else +# define SMS_API __declspec(dllimport) +# endif +# define SMS_CALL __cdecl +#else +# if defined(SMS_BUILDING_LIBRARY) +# define SMS_API __attribute__((visibility("default"))) +# else +# define SMS_API +# endif +# define SMS_CALL +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define SMS_C_ABI_VERSION 0x00010000u +#define SMS_LAYOUT_MAJOR_VERSION 1 +#define SMS_LAYOUT_MINOR_VERSION 2 +#define SMS_RESOURCE_NAMING_VERSION 1 +#define SMS_WAIT_INFINITE (-1LL) + +typedef int32_t sms_open_mode; +enum sms_open_mode_values { + SMS_OPEN_MODE_CREATE_NEW = 0, + SMS_OPEN_MODE_OPEN_EXISTING = 1, + SMS_OPEN_MODE_CREATE_OR_OPEN = 2 +}; + +typedef int32_t sms_open_status; +enum sms_open_status_values { + SMS_OPEN_SUCCESS = 0, + SMS_OPEN_ALREADY_EXISTS = 1, + SMS_OPEN_NOT_FOUND = 2, + SMS_OPEN_INVALID_OPTIONS = 3, + SMS_OPEN_INCOMPATIBLE_LAYOUT = 4, + SMS_OPEN_UNSUPPORTED_PLATFORM = 5, + SMS_OPEN_INSUFFICIENT_CAPACITY = 6, + SMS_OPEN_ACCESS_DENIED = 7, + SMS_OPEN_MAPPING_FAILED = 8, + SMS_OPEN_STORE_BUSY = 9, + SMS_OPEN_OPERATION_CANCELED = 10 +}; + +typedef int32_t sms_status; +enum sms_status_values { + SMS_STATUS_SUCCESS = 0, + SMS_STATUS_DUPLICATE_KEY = 1, + SMS_STATUS_NOT_FOUND = 2, + SMS_STATUS_KEY_TOO_LARGE = 3, + SMS_STATUS_VALUE_TOO_LARGE = 4, + SMS_STATUS_DESCRIPTOR_TOO_LARGE = 5, + SMS_STATUS_STORE_FULL = 6, + SMS_STATUS_LEASE_TABLE_FULL = 7, + SMS_STATUS_INVALID_LEASE = 8, + SMS_STATUS_LEASE_ALREADY_RELEASED = 9, + SMS_STATUS_REMOVE_PENDING = 10, + SMS_STATUS_UNSUPPORTED_PLATFORM = 11, + SMS_STATUS_STORE_DISPOSED = 12, + SMS_STATUS_CORRUPT_STORE = 13, + SMS_STATUS_ACCESS_DENIED = 14, + SMS_STATUS_UNKNOWN_FAILURE = 15, + SMS_STATUS_INVALID_RESERVATION = 16, + SMS_STATUS_RESERVATION_INCOMPLETE = 17, + SMS_STATUS_RESERVATION_ALREADY_COMPLETED = 18, + SMS_STATUS_RESERVATION_WRITE_OUT_OF_RANGE = 19, + SMS_STATUS_INVALID_KEY = 20, + SMS_STATUS_STORE_BUSY = 21, + SMS_STATUS_OPERATION_CANCELED = 22 +}; + +typedef struct sms_store sms_store; +typedef struct sms_lease sms_lease; +typedef struct sms_reservation sms_reservation; + +typedef struct sms_bytes { + const uint8_t* data; + uint64_t length; +} sms_bytes; + +typedef struct sms_mutable_bytes { + uint8_t* data; + uint64_t length; +} sms_mutable_bytes; + +typedef struct sms_wait_options { + uint32_t struct_size; + uint32_t abi_version; + int64_t timeout_milliseconds; +} sms_wait_options; + +typedef struct sms_store_options { + uint32_t struct_size; + uint32_t abi_version; + const char* name_utf8; + uint64_t name_length; + int32_t open_mode; + int64_t total_bytes; + int32_t slot_count; + int32_t max_value_bytes; + int32_t max_descriptor_bytes; + int32_t max_key_bytes; + int32_t lease_record_count; + uint8_t enable_lease_recovery; + uint8_t reserved[7]; +} sms_store_options; + +typedef struct sms_segment { + const uint8_t* data; + uint64_t length; +} sms_segment; + +typedef struct sms_recovery_report { + uint32_t struct_size; + uint32_t abi_version; + int32_t scanned_count; + int32_t recovered_count; + int32_t active_count; + int32_t unsupported_count; + int32_t failed_count; + int32_t reserved; +} sms_recovery_report; + +typedef struct sms_diagnostics { + uint32_t struct_size; + uint32_t abi_version; + int64_t total_bytes; + int32_t slot_count; + int32_t free_slot_count; + int32_t published_slot_count; + int32_t pending_removal_count; + int32_t active_lease_count; + int32_t active_reservation_count; + int32_t index_entry_count; + int32_t occupied_index_entry_count; + int32_t tombstone_index_entry_count; + int32_t empty_index_entry_count; + int32_t usable_index_capacity; + int32_t last_observed_probe_length; + int32_t max_observed_probe_length; + int32_t last_failure_status; + int64_t aborted_reservation_count; + int64_t recovered_lease_count; + int64_t active_lease_recovery_count; + int64_t unsupported_lease_recovery_count; + int64_t failed_lease_recovery_count; + int64_t recovered_reservation_count; + int64_t active_reservation_recovery_count; + int64_t unsupported_reservation_recovery_count; + int64_t failed_reservation_recovery_count; + int64_t capacity_pressure_count; + int64_t index_compaction_count; + int64_t failure_counts[23]; +} sms_diagnostics; + +typedef struct sms_protocol_info { + uint32_t struct_size; + uint32_t abi_version; + int32_t layout_major; + int32_t layout_minor; + int32_t resource_naming_version; + int32_t store_header_size; + int32_t index_entry_header_size; + int32_t slot_metadata_size; + int32_t lease_record_size; +} sms_protocol_info; + +typedef int32_t sms_layout_field; +enum sms_layout_field_values { + SMS_LAYOUT_FIELD_HEADER_MAGIC = 0, + SMS_LAYOUT_FIELD_HEADER_INDEX_OFFSET = 1, + SMS_LAYOUT_FIELD_HEADER_STORE_STATE = 2, + SMS_LAYOUT_FIELD_HEADER_SEQUENCE = 3, + SMS_LAYOUT_FIELD_INDEX_STATE = 100, + SMS_LAYOUT_FIELD_INDEX_KEY_HASH = 101, + SMS_LAYOUT_FIELD_INDEX_REUSE_EPOCH = 102, + SMS_LAYOUT_FIELD_SLOT_STATE = 200, + SMS_LAYOUT_FIELD_SLOT_REUSE_EPOCH = 201, + SMS_LAYOUT_FIELD_SLOT_USAGE_COUNT = 202, + SMS_LAYOUT_FIELD_SLOT_KEY_HASH = 203, + SMS_LAYOUT_FIELD_SLOT_COMMITTED_SEQUENCE = 204, + SMS_LAYOUT_FIELD_LEASE_STATE = 300, + SMS_LAYOUT_FIELD_LEASE_REUSE_EPOCH = 301, + SMS_LAYOUT_FIELD_LEASE_OWNER_PROCESS_ID = 302, + SMS_LAYOUT_FIELD_LEASE_ACQUIRE_SEQUENCE = 303 +}; + +typedef struct sms_store_layout { + uint32_t struct_size; + uint32_t abi_version; + int64_t total_bytes; + int32_t slot_count; + int32_t lease_record_count; + int32_t max_value_bytes; + int32_t max_descriptor_bytes; + int32_t max_key_bytes; + int32_t header_length; + int32_t index_entry_count; + int32_t index_entry_size; + int64_t index_offset; + int64_t index_length; + int64_t lease_registry_offset; + int64_t lease_registry_length; + int64_t slot_metadata_offset; + int64_t slot_metadata_length; + int32_t descriptor_stride; + int32_t payload_stride; + int64_t descriptor_storage_offset; + int64_t descriptor_storage_length; + int64_t payload_storage_offset; + int64_t payload_storage_length; + int64_t required_bytes; +} sms_store_layout; + +SMS_API uint32_t SMS_CALL sms_abi_version(void); +SMS_API sms_status SMS_CALL sms_get_protocol_info(sms_protocol_info* info); +SMS_API sms_status SMS_CALL sms_get_layout_field_offset(sms_layout_field field, uint32_t* offset); + +SMS_API sms_open_status SMS_CALL sms_calculate_required_bytes( + int32_t slot_count, + int32_t max_value_bytes, + int32_t max_descriptor_bytes, + int32_t max_key_bytes, + int32_t lease_record_count, + int64_t* required_bytes); + +SMS_API sms_open_status SMS_CALL sms_open_store( + const sms_store_options* options, + const sms_wait_options* wait_options, + sms_store** store); + +SMS_API void SMS_CALL sms_close_store(sms_store* store); +SMS_API sms_status SMS_CALL sms_get_store_layout( + sms_store* store, + const sms_wait_options* wait_options, + sms_store_layout* layout); + +SMS_API sms_status SMS_CALL sms_publish( + sms_store* store, + sms_bytes key, + sms_bytes value, + sms_bytes descriptor, + const sms_wait_options* wait_options); + +SMS_API sms_status SMS_CALL sms_publish_segments( + sms_store* store, + sms_bytes key, + const sms_segment* segments, + uint64_t segment_count, + sms_bytes descriptor, + const sms_wait_options* wait_options, + int64_t* copied_bytes); + +SMS_API sms_status SMS_CALL sms_acquire( + sms_store* store, + sms_bytes key, + const sms_wait_options* wait_options, + sms_lease** lease); + +SMS_API int32_t SMS_CALL sms_lease_is_valid(const sms_lease* lease); +SMS_API sms_bytes SMS_CALL sms_lease_value(const sms_lease* lease); +SMS_API sms_bytes SMS_CALL sms_lease_descriptor(const sms_lease* lease); +SMS_API sms_status SMS_CALL sms_release_lease( + sms_lease* lease, + const sms_wait_options* wait_options); +SMS_API void SMS_CALL sms_destroy_lease(sms_lease* lease); + +SMS_API sms_status SMS_CALL sms_remove( + sms_store* store, + sms_bytes key, + const sms_wait_options* wait_options); + +SMS_API sms_status SMS_CALL sms_reserve( + sms_store* store, + sms_bytes key, + int32_t payload_length, + sms_bytes descriptor, + const sms_wait_options* wait_options, + sms_reservation** reservation); + +SMS_API int32_t SMS_CALL sms_reservation_is_valid(const sms_reservation* reservation); +SMS_API int32_t SMS_CALL sms_reservation_payload_length(const sms_reservation* reservation); +SMS_API int32_t SMS_CALL sms_reservation_bytes_written(const sms_reservation* reservation); +SMS_API int32_t SMS_CALL sms_reservation_remaining_bytes(const sms_reservation* reservation); +SMS_API sms_mutable_bytes SMS_CALL sms_reservation_buffer(sms_reservation* reservation, int32_t size_hint); +SMS_API sms_status SMS_CALL sms_advance_reservation( + sms_reservation* reservation, + int32_t byte_count, + const sms_wait_options* wait_options); +SMS_API sms_status SMS_CALL sms_commit_reservation( + sms_reservation* reservation, + const sms_wait_options* wait_options); +SMS_API sms_status SMS_CALL sms_abort_reservation( + sms_reservation* reservation, + const sms_wait_options* wait_options); +SMS_API void SMS_CALL sms_destroy_reservation(sms_reservation* reservation); + +SMS_API sms_status SMS_CALL sms_recover_leases( + sms_store* store, + int32_t recover_current_process, + const sms_wait_options* wait_options, + sms_recovery_report* report); + +SMS_API sms_status SMS_CALL sms_recover_reservations( + sms_store* store, + int32_t recover_current_process, + const sms_wait_options* wait_options, + sms_recovery_report* report); + +SMS_API sms_status SMS_CALL sms_get_diagnostics( + sms_store* store, + const sms_wait_options* wait_options, + sms_diagnostics* diagnostics); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/cpp/include/shared_memory_store/store.hpp b/src/cpp/include/shared_memory_store/store.hpp new file mode 100644 index 0000000..57c952c --- /dev/null +++ b/src/cpp/include/shared_memory_store/store.hpp @@ -0,0 +1,347 @@ +#pragma once + +#include "c_api.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace shared_memory_store { + +enum class open_mode : std::int32_t { + create_new = SMS_OPEN_MODE_CREATE_NEW, + open_existing = SMS_OPEN_MODE_OPEN_EXISTING, + create_or_open = SMS_OPEN_MODE_CREATE_OR_OPEN +}; + +enum class open_status : std::int32_t { + success = SMS_OPEN_SUCCESS, + already_exists = SMS_OPEN_ALREADY_EXISTS, + not_found = SMS_OPEN_NOT_FOUND, + invalid_options = SMS_OPEN_INVALID_OPTIONS, + incompatible_layout = SMS_OPEN_INCOMPATIBLE_LAYOUT, + unsupported_platform = SMS_OPEN_UNSUPPORTED_PLATFORM, + insufficient_capacity = SMS_OPEN_INSUFFICIENT_CAPACITY, + access_denied = SMS_OPEN_ACCESS_DENIED, + mapping_failed = SMS_OPEN_MAPPING_FAILED, + store_busy = SMS_OPEN_STORE_BUSY, + operation_canceled = SMS_OPEN_OPERATION_CANCELED +}; + +enum class status : std::int32_t { + success = SMS_STATUS_SUCCESS, + duplicate_key = SMS_STATUS_DUPLICATE_KEY, + not_found = SMS_STATUS_NOT_FOUND, + key_too_large = SMS_STATUS_KEY_TOO_LARGE, + value_too_large = SMS_STATUS_VALUE_TOO_LARGE, + descriptor_too_large = SMS_STATUS_DESCRIPTOR_TOO_LARGE, + store_full = SMS_STATUS_STORE_FULL, + lease_table_full = SMS_STATUS_LEASE_TABLE_FULL, + invalid_lease = SMS_STATUS_INVALID_LEASE, + lease_already_released = SMS_STATUS_LEASE_ALREADY_RELEASED, + remove_pending = SMS_STATUS_REMOVE_PENDING, + unsupported_platform = SMS_STATUS_UNSUPPORTED_PLATFORM, + store_disposed = SMS_STATUS_STORE_DISPOSED, + corrupt_store = SMS_STATUS_CORRUPT_STORE, + access_denied = SMS_STATUS_ACCESS_DENIED, + unknown_failure = SMS_STATUS_UNKNOWN_FAILURE, + invalid_reservation = SMS_STATUS_INVALID_RESERVATION, + reservation_incomplete = SMS_STATUS_RESERVATION_INCOMPLETE, + reservation_already_completed = SMS_STATUS_RESERVATION_ALREADY_COMPLETED, + reservation_write_out_of_range = SMS_STATUS_RESERVATION_WRITE_OUT_OF_RANGE, + invalid_key = SMS_STATUS_INVALID_KEY, + store_busy = SMS_STATUS_STORE_BUSY, + operation_canceled = SMS_STATUS_OPERATION_CANCELED +}; + +struct wait_options { + std::int64_t timeout_milliseconds{1000}; + static constexpr wait_options defaults() noexcept { return {1000}; } + static constexpr wait_options no_wait() noexcept { return {0}; } + static constexpr wait_options infinite() noexcept { return {SMS_WAIT_INFINITE}; } +}; + +struct store_options { + std::string name; + open_mode mode{open_mode::create_or_open}; + std::int64_t total_bytes{}; + std::int32_t slot_count{}; + std::int32_t max_value_bytes{}; + std::int32_t max_descriptor_bytes{}; + std::int32_t max_key_bytes{}; + std::int32_t lease_record_count{}; + bool enable_lease_recovery{}; + + static std::int64_t calculate_required_bytes(std::int32_t slots, std::int32_t max_value, + std::int32_t max_descriptor, std::int32_t max_key, + std::int32_t leases) { + std::int64_t required{}; + if (sms_calculate_required_bytes(slots, max_value, max_descriptor, max_key, leases, &required) != SMS_OPEN_SUCCESS) + throw std::invalid_argument("SharedMemoryStore capacities are invalid."); + return required; + } + + static store_options create(std::string name, std::int32_t slots, std::int32_t max_value, + std::int32_t max_descriptor, std::int32_t max_key, + std::int32_t leases, open_mode mode = open_mode::create_or_open, + bool recovery = false) { + store_options result{std::move(name), mode, 0, slots, max_value, max_descriptor, max_key, leases, recovery}; + result.total_bytes = calculate_required_bytes(slots, max_value, max_descriptor, max_key, leases); + return result; + } +}; + +struct recovery_report { + std::int32_t scanned_count{}; + std::int32_t recovered_count{}; + std::int32_t active_count{}; + std::int32_t unsupported_count{}; + std::int32_t failed_count{}; +}; + +class diagnostics_snapshot { +public: + std::int64_t total_bytes() const noexcept { return value_.total_bytes; } + std::int32_t slot_count() const noexcept { return value_.slot_count; } + std::int32_t free_slot_count() const noexcept { return value_.free_slot_count; } + std::int32_t published_slot_count() const noexcept { return value_.published_slot_count; } + std::int32_t pending_removal_count() const noexcept { return value_.pending_removal_count; } + std::int32_t active_lease_count() const noexcept { return value_.active_lease_count; } + std::int32_t active_reservation_count() const noexcept { return value_.active_reservation_count; } + std::int32_t index_entry_count() const noexcept { return value_.index_entry_count; } + std::int32_t occupied_index_entry_count() const noexcept { return value_.occupied_index_entry_count; } + std::int32_t tombstone_index_entry_count() const noexcept { return value_.tombstone_index_entry_count; } + std::int32_t usable_index_capacity() const noexcept { return value_.usable_index_capacity; } + status last_failure_status() const noexcept { return static_cast(value_.last_failure_status); } + std::int64_t failure_count(status value) const noexcept { + const auto index = static_cast(value); + return index >= 0 && index < 23 ? value_.failure_counts[index] : 0; + } + const sms_diagnostics& native() const noexcept { return value_; } +private: + friend class memory_store; + sms_diagnostics value_{}; +}; + +namespace detail { +inline sms_wait_options native_wait(wait_options value) noexcept { + return {sizeof(sms_wait_options), SMS_C_ABI_VERSION, value.timeout_milliseconds}; +} +inline sms_bytes bytes(std::span value) noexcept { + return {reinterpret_cast(value.data()), static_cast(value.size())}; +} +inline std::span byte_span(sms_bytes value) noexcept { + return {reinterpret_cast(value.data), static_cast(value.length)}; +} +} // namespace detail + +class value_lease { +public: + value_lease() noexcept = default; + ~value_lease() { reset(); } + value_lease(const value_lease&) = delete; + value_lease& operator=(const value_lease&) = delete; + value_lease(value_lease&& other) noexcept : handle_(std::exchange(other.handle_, nullptr)) {} + value_lease& operator=(value_lease&& other) noexcept { + if (this != &other) { reset(); handle_ = std::exchange(other.handle_, nullptr); } + return *this; + } + bool valid() const noexcept { return handle_ && sms_lease_is_valid(handle_) != 0; } + std::span value() const noexcept { + return handle_ ? detail::byte_span(sms_lease_value(handle_)) : std::span{}; + } + std::span descriptor() const noexcept { + return handle_ ? detail::byte_span(sms_lease_descriptor(handle_)) : std::span{}; + } + status release(wait_options wait = wait_options::defaults()) noexcept { + if (!handle_) return status::invalid_lease; + const auto native = detail::native_wait(wait); + return static_cast(sms_release_lease(handle_, &native)); + } + void reset() noexcept { if (handle_) sms_destroy_lease(std::exchange(handle_, nullptr)); } +private: + friend class memory_store; + explicit value_lease(sms_lease* handle) noexcept : handle_(handle) {} + sms_lease* handle_{}; +}; + +class value_reservation { +public: + value_reservation() noexcept = default; + ~value_reservation() { reset(); } + value_reservation(const value_reservation&) = delete; + value_reservation& operator=(const value_reservation&) = delete; + value_reservation(value_reservation&& other) noexcept : handle_(std::exchange(other.handle_, nullptr)) {} + value_reservation& operator=(value_reservation&& other) noexcept { + if (this != &other) { reset(); handle_ = std::exchange(other.handle_, nullptr); } + return *this; + } + bool valid() const noexcept { return handle_ && sms_reservation_is_valid(handle_) != 0; } + std::int32_t payload_length() const noexcept { return handle_ ? sms_reservation_payload_length(handle_) : 0; } + std::int32_t bytes_written() const noexcept { return handle_ ? sms_reservation_bytes_written(handle_) : 0; } + std::int32_t remaining_bytes() const noexcept { + return handle_ ? sms_reservation_remaining_bytes(handle_) : 0; + } + std::span buffer(std::int32_t size_hint = 0) noexcept { + if (!handle_) return {}; + const auto value = sms_reservation_buffer(handle_, size_hint); + return {reinterpret_cast(value.data), static_cast(value.length)}; + } + status advance(std::int32_t count, wait_options wait = wait_options::defaults()) noexcept { + if (!handle_) return status::invalid_reservation; + const auto native = detail::native_wait(wait); + return static_cast(sms_advance_reservation(handle_, count, &native)); + } + status commit(wait_options wait = wait_options::defaults()) noexcept { + if (!handle_) return status::invalid_reservation; + const auto native = detail::native_wait(wait); + return static_cast(sms_commit_reservation(handle_, &native)); + } + status abort(wait_options wait = wait_options::defaults()) noexcept { + if (!handle_) return status::invalid_reservation; + const auto native = detail::native_wait(wait); + return static_cast(sms_abort_reservation(handle_, &native)); + } + void reset() noexcept { if (handle_) sms_destroy_reservation(std::exchange(handle_, nullptr)); } +private: + friend class memory_store; + explicit value_reservation(sms_reservation* handle) noexcept : handle_(handle) {} + sms_reservation* handle_{}; +}; + +class memory_store { +public: + memory_store() noexcept = default; + ~memory_store() { close(); } + memory_store(const memory_store&) = delete; + memory_store& operator=(const memory_store&) = delete; + memory_store(memory_store&& other) noexcept : handle_(std::exchange(other.handle_, nullptr)) {} + memory_store& operator=(memory_store&& other) noexcept { + if (this != &other) { close(); handle_ = std::exchange(other.handle_, nullptr); } + return *this; + } + + static open_status try_create_or_open(const store_options& options, memory_store& result, + wait_options wait = wait_options::defaults()) noexcept { + result.close(); + sms_store_options native{}; + native.struct_size = sizeof(native); + native.abi_version = SMS_C_ABI_VERSION; + native.name_utf8 = options.name.data(); + native.name_length = options.name.size(); + native.open_mode = static_cast(options.mode); + native.total_bytes = options.total_bytes; + native.slot_count = options.slot_count; + native.max_value_bytes = options.max_value_bytes; + native.max_descriptor_bytes = options.max_descriptor_bytes; + native.max_key_bytes = options.max_key_bytes; + native.lease_record_count = options.lease_record_count; + native.enable_lease_recovery = options.enable_lease_recovery ? 1 : 0; + const auto native_wait = detail::native_wait(wait); + return static_cast(sms_open_store(&native, &native_wait, &result.handle_)); + } + + bool valid() const noexcept { return handle_ != nullptr; } + void close() noexcept { if (handle_) sms_close_store(std::exchange(handle_, nullptr)); } + + status try_publish(std::span key, std::span value, + std::span descriptor = {}, + wait_options wait = wait_options::defaults()) noexcept { + const auto native = detail::native_wait(wait); + return static_cast(sms_publish(handle_, detail::bytes(key), detail::bytes(value), + detail::bytes(descriptor), &native)); + } + + status try_publish_segments(std::span key, + std::span> segments, + std::span descriptor, std::int64_t& copied, + wait_options wait = wait_options::defaults()) noexcept { + try { + std::vector native_segments; + native_segments.reserve(segments.size()); + for (const auto value : segments) + native_segments.push_back({reinterpret_cast(value.data()), + static_cast(value.size())}); + const auto native = detail::native_wait(wait); + return static_cast(sms_publish_segments(handle_, detail::bytes(key), native_segments.data(), + native_segments.size(), detail::bytes(descriptor), + &native, &copied)); + } catch (...) { + copied = 0; + return status::unknown_failure; + } + } + + status try_acquire(std::span key, value_lease& lease, + wait_options wait = wait_options::defaults()) noexcept { + lease.reset(); + sms_lease* handle{}; + const auto native = detail::native_wait(wait); + const auto result = static_cast(sms_acquire(handle_, detail::bytes(key), &native, &handle)); + if (result == status::success) lease.handle_ = handle; + return result; + } + + status try_remove(std::span key, + wait_options wait = wait_options::defaults()) noexcept { + const auto native = detail::native_wait(wait); + return static_cast(sms_remove(handle_, detail::bytes(key), &native)); + } + + status try_reserve(std::span key, std::int32_t payload_length, + std::span descriptor, value_reservation& reservation, + wait_options wait = wait_options::defaults()) noexcept { + reservation.reset(); + sms_reservation* handle{}; + const auto native = detail::native_wait(wait); + const auto result = static_cast(sms_reserve(handle_, detail::bytes(key), payload_length, + detail::bytes(descriptor), &native, &handle)); + if (result == status::success) reservation.handle_ = handle; + return result; + } + + status try_recover_leases(bool recover_current_process, recovery_report& report, + wait_options wait = wait_options::defaults()) noexcept { + sms_recovery_report native{}; + native.struct_size = sizeof(native); + native.abi_version = SMS_C_ABI_VERSION; + const auto native_wait = detail::native_wait(wait); + const auto result = static_cast(sms_recover_leases(handle_, recover_current_process ? 1 : 0, + &native_wait, &native)); + report = {native.scanned_count, native.recovered_count, native.active_count, + native.unsupported_count, native.failed_count}; + return result; + } + + status try_recover_reservations(bool recover_current_process, recovery_report& report, + wait_options wait = wait_options::defaults()) noexcept { + sms_recovery_report native{}; + native.struct_size = sizeof(native); + native.abi_version = SMS_C_ABI_VERSION; + const auto native_wait = detail::native_wait(wait); + const auto result = static_cast(sms_recover_reservations(handle_, recover_current_process ? 1 : 0, + &native_wait, &native)); + report = {native.scanned_count, native.recovered_count, native.active_count, + native.unsupported_count, native.failed_count}; + return result; + } + + status try_get_diagnostics(diagnostics_snapshot& snapshot, + wait_options wait = wait_options::defaults()) noexcept { + snapshot.value_ = {}; + snapshot.value_.struct_size = sizeof(snapshot.value_); + snapshot.value_.abi_version = SMS_C_ABI_VERSION; + const auto native = detail::native_wait(wait); + return static_cast(sms_get_diagnostics(handle_, &native, &snapshot.value_)); + } + +private: + sms_store* handle_{}; +}; + +} // namespace shared_memory_store diff --git a/src/cpp/src/c_api.cpp b/src/cpp/src/c_api.cpp new file mode 100644 index 0000000..b9de663 --- /dev/null +++ b/src/cpp/src/c_api.cpp @@ -0,0 +1,443 @@ +#include "internal.hpp" + +#include +#include + +using sms::detail::Diagnostics; +using sms::detail::Layout; +using sms::detail::LifecycleId; +using sms::detail::Options; +using sms::detail::RecoveryReport; +using sms::detail::Store; +using sms::detail::Wait; + +struct sms_store { std::shared_ptr implementation; }; +struct sms_lease { + std::shared_ptr store; + std::int32_t slot{-1}; + LifecycleId lifecycle{}; + std::int32_t lease_id{-1}; +}; +struct sms_reservation { + std::shared_ptr store; + std::int32_t slot{-1}; + LifecycleId lifecycle{}; +}; + +static_assert(sizeof(sms_open_mode) == 4); +static_assert(sizeof(sms_open_status) == 4); +static_assert(sizeof(sms_status) == 4); +static_assert(sizeof(sms_bytes) == 16 && offsetof(sms_bytes, length) == 8); +static_assert(sizeof(sms_mutable_bytes) == 16 && offsetof(sms_mutable_bytes, length) == 8); +static_assert(sizeof(sms_segment) == 16 && offsetof(sms_segment, length) == 8); +static_assert(sizeof(sms_wait_options) == 16 && offsetof(sms_wait_options, timeout_milliseconds) == 8); +static_assert(sizeof(sms_store_options) == 72 && offsetof(sms_store_options, total_bytes) == 32); +static_assert(sizeof(sms_recovery_report) == 32 && offsetof(sms_recovery_report, scanned_count) == 8); +static_assert(sizeof(sms_diagnostics) == 344 && offsetof(sms_diagnostics, failure_counts) == 160); +static_assert(sizeof(sms_protocol_info) == 36); +static_assert(sizeof(sms_store_layout) == 144 && offsetof(sms_store_layout, required_bytes) == 136); + +namespace { + +bool abi_compatible(std::uint32_t version) noexcept { + return (version >> 16) == (SMS_C_ABI_VERSION >> 16); +} + +bool read_wait(const sms_wait_options* input, Wait& result) noexcept { + result = Wait{1000}; + if (!input) return true; + if (input->struct_size < sizeof(sms_wait_options) || !abi_compatible(input->abi_version) || + input->timeout_milliseconds < SMS_WAIT_INFINITE) return false; + result.milliseconds = input->timeout_milliseconds; + return true; +} + +bool valid_bytes(sms_bytes value) noexcept { + return value.length <= std::numeric_limits::max() && + (value.length == 0 || value.data != nullptr); +} + +std::span as_span(sms_bytes value) noexcept { + return {value.data, static_cast(value.length)}; +} + +void fill_report(sms_recovery_report& destination, const RecoveryReport& source) noexcept { + destination.struct_size = sizeof(destination); + destination.abi_version = SMS_C_ABI_VERSION; + destination.scanned_count = source.scanned; + destination.recovered_count = source.recovered; + destination.active_count = source.active; + destination.unsupported_count = source.unsupported; + destination.failed_count = source.failed; + destination.reserved = 0; +} + +} // namespace + +extern "C" { + +uint32_t SMS_CALL sms_abi_version(void) { return SMS_C_ABI_VERSION; } + +sms_status SMS_CALL sms_get_protocol_info(sms_protocol_info* info) { + if (!info || info->struct_size < sizeof(*info) || !abi_compatible(info->abi_version)) + return SMS_STATUS_UNKNOWN_FAILURE; + *info = {}; + info->struct_size = sizeof(*info); + info->abi_version = SMS_C_ABI_VERSION; + info->layout_major = SMS_LAYOUT_MAJOR_VERSION; + info->layout_minor = SMS_LAYOUT_MINOR_VERSION; + info->resource_naming_version = SMS_RESOURCE_NAMING_VERSION; + info->store_header_size = sizeof(sms::detail::StoreHeader); + info->index_entry_header_size = sizeof(sms::detail::IndexEntryHeader); + info->slot_metadata_size = sizeof(sms::detail::SlotMetadata); + info->lease_record_size = sizeof(sms::detail::LeaseRecord); + return SMS_STATUS_SUCCESS; +} + +sms_status SMS_CALL sms_get_layout_field_offset(sms_layout_field field, uint32_t* offset) { + if (!offset) return SMS_STATUS_UNKNOWN_FAILURE; + switch (field) { + case SMS_LAYOUT_FIELD_HEADER_MAGIC: *offset = offsetof(sms::detail::StoreHeader, Magic); break; + case SMS_LAYOUT_FIELD_HEADER_INDEX_OFFSET: *offset = offsetof(sms::detail::StoreHeader, IndexOffset); break; + case SMS_LAYOUT_FIELD_HEADER_STORE_STATE: *offset = offsetof(sms::detail::StoreHeader, StoreState); break; + case SMS_LAYOUT_FIELD_HEADER_SEQUENCE: *offset = offsetof(sms::detail::StoreHeader, Sequence); break; + case SMS_LAYOUT_FIELD_INDEX_STATE: *offset = offsetof(sms::detail::IndexEntryHeader, State); break; + case SMS_LAYOUT_FIELD_INDEX_KEY_HASH: *offset = offsetof(sms::detail::IndexEntryHeader, KeyHash); break; + case SMS_LAYOUT_FIELD_INDEX_REUSE_EPOCH: *offset = offsetof(sms::detail::IndexEntryHeader, SlotReuseEpoch); break; + case SMS_LAYOUT_FIELD_SLOT_STATE: *offset = offsetof(sms::detail::SlotMetadata, State); break; + case SMS_LAYOUT_FIELD_SLOT_REUSE_EPOCH: *offset = offsetof(sms::detail::SlotMetadata, ReuseEpoch); break; + case SMS_LAYOUT_FIELD_SLOT_USAGE_COUNT: *offset = offsetof(sms::detail::SlotMetadata, UsageCount); break; + case SMS_LAYOUT_FIELD_SLOT_KEY_HASH: *offset = offsetof(sms::detail::SlotMetadata, KeyHash); break; + case SMS_LAYOUT_FIELD_SLOT_COMMITTED_SEQUENCE: *offset = offsetof(sms::detail::SlotMetadata, CommittedSequence); break; + case SMS_LAYOUT_FIELD_LEASE_STATE: *offset = offsetof(sms::detail::LeaseRecord, State); break; + case SMS_LAYOUT_FIELD_LEASE_REUSE_EPOCH: *offset = offsetof(sms::detail::LeaseRecord, SlotReuseEpoch); break; + case SMS_LAYOUT_FIELD_LEASE_OWNER_PROCESS_ID: *offset = offsetof(sms::detail::LeaseRecord, OwnerProcessId); break; + case SMS_LAYOUT_FIELD_LEASE_ACQUIRE_SEQUENCE: *offset = offsetof(sms::detail::LeaseRecord, AcquireSequence); break; + default: *offset = 0; return SMS_STATUS_UNKNOWN_FAILURE; + } + return SMS_STATUS_SUCCESS; +} + +sms_open_status SMS_CALL sms_calculate_required_bytes( + int32_t slot_count, int32_t max_value_bytes, int32_t max_descriptor_bytes, + int32_t max_key_bytes, int32_t lease_record_count, int64_t* required_bytes) { + if (!required_bytes) return SMS_OPEN_INVALID_OPTIONS; + *required_bytes = 0; + Layout layout{}; + if (!Layout::calculate(0, slot_count, max_value_bytes, max_descriptor_bytes, + max_key_bytes, lease_record_count, layout)) return SMS_OPEN_INVALID_OPTIONS; + *required_bytes = layout.required_bytes; + return SMS_OPEN_SUCCESS; +} + +sms_open_status SMS_CALL sms_open_store(const sms_store_options* options, + const sms_wait_options* wait_options, + sms_store** store) { + if (!store) return SMS_OPEN_INVALID_OPTIONS; + *store = nullptr; + Wait wait{}; + if (!options || options->struct_size < sizeof(*options) || !abi_compatible(options->abi_version) || + !read_wait(wait_options, wait) || options->name_length > std::numeric_limits::max() || + (options->name_length > 0 && !options->name_utf8)) + return SMS_OPEN_INVALID_OPTIONS; + try { + Options native{}; + native.name.assign(options->name_utf8 ? options->name_utf8 : "", + static_cast(options->name_length)); + native.open_mode = static_cast(options->open_mode); + native.total_bytes = options->total_bytes; + native.slot_count = options->slot_count; + native.max_value_bytes = options->max_value_bytes; + native.max_descriptor_bytes = options->max_descriptor_bytes; + native.max_key_bytes = options->max_key_bytes; + native.lease_record_count = options->lease_record_count; + native.enable_lease_recovery = options->enable_lease_recovery != 0; + std::shared_ptr implementation; + const auto status = Store::open(native, wait, implementation); + if (status != SMS_OPEN_SUCCESS) return status; + auto* handle = new (std::nothrow) sms_store{implementation}; + if (!handle) { implementation->close(); return SMS_OPEN_MAPPING_FAILED; } + *store = handle; + return SMS_OPEN_SUCCESS; + } catch (...) { + return SMS_OPEN_MAPPING_FAILED; + } +} + +void SMS_CALL sms_close_store(sms_store* store) { + if (!store) return; + if (store->implementation) store->implementation->close(); + delete store; +} + +sms_status SMS_CALL sms_get_store_layout(sms_store* store, const sms_wait_options* wait_options, + sms_store_layout* layout) { + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!layout || layout->struct_size < sizeof(*layout) || !abi_compatible(layout->abi_version) || + !read_wait(wait_options, wait)) return SMS_STATUS_UNKNOWN_FAILURE; + Layout native{}; + const auto status = store->implementation->get_layout(wait, native); + if (status != SMS_STATUS_SUCCESS) return status; + *layout = {}; + layout->struct_size = sizeof(*layout); + layout->abi_version = SMS_C_ABI_VERSION; + layout->total_bytes = native.total_bytes; + layout->slot_count = native.slot_count; + layout->lease_record_count = native.lease_record_count; + layout->max_value_bytes = native.max_value_bytes; + layout->max_descriptor_bytes = native.max_descriptor_bytes; + layout->max_key_bytes = native.max_key_bytes; + layout->header_length = native.header_length; + layout->index_entry_count = native.index_entry_count; + layout->index_entry_size = native.index_entry_size; + layout->index_offset = native.index_offset; + layout->index_length = native.index_length; + layout->lease_registry_offset = native.lease_registry_offset; + layout->lease_registry_length = native.lease_registry_length; + layout->slot_metadata_offset = native.slot_metadata_offset; + layout->slot_metadata_length = native.slot_metadata_length; + layout->descriptor_stride = native.descriptor_stride; + layout->payload_stride = native.payload_stride; + layout->descriptor_storage_offset = native.descriptor_storage_offset; + layout->descriptor_storage_length = native.descriptor_storage_length; + layout->payload_storage_offset = native.payload_storage_offset; + layout->payload_storage_length = native.payload_storage_length; + layout->required_bytes = native.required_bytes; + return SMS_STATUS_SUCCESS; +} + +sms_status SMS_CALL sms_publish(sms_store* store, sms_bytes key, sms_bytes value, + sms_bytes descriptor, const sms_wait_options* wait_options) { + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!read_wait(wait_options, wait) || !valid_bytes(key) || !valid_bytes(value) || !valid_bytes(descriptor)) + return SMS_STATUS_UNKNOWN_FAILURE; + return store->implementation->publish(as_span(key), as_span(value), as_span(descriptor), wait); +} + +sms_status SMS_CALL sms_publish_segments(sms_store* store, sms_bytes key, + const sms_segment* segments, uint64_t segment_count, + sms_bytes descriptor, const sms_wait_options* wait_options, + int64_t* copied_bytes) { + if (copied_bytes) *copied_bytes = 0; + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!copied_bytes || !read_wait(wait_options, wait) || !valid_bytes(key) || !valid_bytes(descriptor) || + segment_count > std::numeric_limits::max() || + (segment_count > 0 && !segments)) return SMS_STATUS_UNKNOWN_FAILURE; + const auto count = static_cast(segment_count); + for (std::size_t index = 0; index < count; ++index) + if (segments[index].length > std::numeric_limits::max() || + (segments[index].length > 0 && !segments[index].data)) return SMS_STATUS_UNKNOWN_FAILURE; + return store->implementation->publish_segments(as_span(key), {segments, count}, as_span(descriptor), + wait, *copied_bytes); +} + +sms_status SMS_CALL sms_acquire(sms_store* store, sms_bytes key, + const sms_wait_options* wait_options, sms_lease** lease) { + if (!lease) return SMS_STATUS_INVALID_LEASE; + *lease = nullptr; + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!read_wait(wait_options, wait) || !valid_bytes(key)) return SMS_STATUS_UNKNOWN_FAILURE; + std::int32_t slot{}, lease_id{}; LifecycleId lifecycle{}; + const auto status = store->implementation->acquire(as_span(key), wait, slot, lifecycle, lease_id); + if (status != SMS_STATUS_SUCCESS) return status; + auto* handle = new (std::nothrow) sms_lease{store->implementation, slot, lifecycle, lease_id}; + if (!handle) { + store->implementation->release_lease(slot, lifecycle, lease_id, Wait{1000}); + return SMS_STATUS_UNKNOWN_FAILURE; + } + *lease = handle; + return SMS_STATUS_SUCCESS; +} + +int32_t SMS_CALL sms_lease_is_valid(const sms_lease* lease) { + return lease && lease->store && lease->store->lease_valid(lease->slot, lease->lifecycle, lease->lease_id) ? 1 : 0; +} + +sms_bytes SMS_CALL sms_lease_value(const sms_lease* lease) { + if (!lease || !lease->store) return {}; + const auto value = lease->store->lease_value(lease->slot, lease->lifecycle, lease->lease_id); + return {value.data(), value.size()}; +} + +sms_bytes SMS_CALL sms_lease_descriptor(const sms_lease* lease) { + if (!lease || !lease->store) return {}; + const auto value = lease->store->lease_descriptor(lease->slot, lease->lifecycle, lease->lease_id); + return {value.data(), value.size()}; +} + +sms_status SMS_CALL sms_release_lease(sms_lease* lease, const sms_wait_options* wait_options) { + Wait wait{}; + if (!lease || !lease->store) return SMS_STATUS_INVALID_LEASE; + if (!read_wait(wait_options, wait)) return SMS_STATUS_UNKNOWN_FAILURE; + return lease->store->release_lease(lease->slot, lease->lifecycle, lease->lease_id, wait); +} + +void SMS_CALL sms_destroy_lease(sms_lease* lease) { + if (!lease) return; + if (lease->store && lease->store->lease_valid(lease->slot, lease->lifecycle, lease->lease_id)) + lease->store->release_lease(lease->slot, lease->lifecycle, lease->lease_id, Wait{1000}); + delete lease; +} + +sms_status SMS_CALL sms_remove(sms_store* store, sms_bytes key, const sms_wait_options* wait_options) { + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!read_wait(wait_options, wait) || !valid_bytes(key)) return SMS_STATUS_UNKNOWN_FAILURE; + return store->implementation->remove(as_span(key), wait); +} + +sms_status SMS_CALL sms_reserve(sms_store* store, sms_bytes key, int32_t payload_length, + sms_bytes descriptor, const sms_wait_options* wait_options, + sms_reservation** reservation) { + if (!reservation) return SMS_STATUS_INVALID_RESERVATION; + *reservation = nullptr; + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!read_wait(wait_options, wait) || !valid_bytes(key) || !valid_bytes(descriptor)) + return SMS_STATUS_UNKNOWN_FAILURE; + std::int32_t slot{}; LifecycleId lifecycle{}; + const auto status = store->implementation->reserve(as_span(key), payload_length, as_span(descriptor), + wait, slot, lifecycle); + if (status != SMS_STATUS_SUCCESS) return status; + auto* handle = new (std::nothrow) sms_reservation{store->implementation, slot, lifecycle}; + if (!handle) { + store->implementation->abort_reservation(slot, lifecycle, false, Wait{1000}); + return SMS_STATUS_UNKNOWN_FAILURE; + } + *reservation = handle; + return SMS_STATUS_SUCCESS; +} + +int32_t SMS_CALL sms_reservation_is_valid(const sms_reservation* reservation) { + return reservation && reservation->store && + reservation->store->reservation_valid(reservation->slot, reservation->lifecycle) ? 1 : 0; +} + +int32_t SMS_CALL sms_reservation_payload_length(const sms_reservation* reservation) { + return reservation && reservation->store + ? reservation->store->reservation_payload_length(reservation->slot, reservation->lifecycle) : 0; +} + +int32_t SMS_CALL sms_reservation_bytes_written(const sms_reservation* reservation) { + return reservation && reservation->store + ? reservation->store->reservation_bytes_written(reservation->slot, reservation->lifecycle) : 0; +} + +int32_t SMS_CALL sms_reservation_remaining_bytes(const sms_reservation* reservation) { + if (!reservation || !reservation->store) return 0; + const auto total = reservation->store->reservation_payload_length(reservation->slot, reservation->lifecycle); + const auto written = reservation->store->reservation_bytes_written(reservation->slot, reservation->lifecycle); + return std::max(0, total - written); +} + +sms_mutable_bytes SMS_CALL sms_reservation_buffer(sms_reservation* reservation, int32_t size_hint) { + if (!reservation || !reservation->store) return {}; + const auto value = reservation->store->reservation_buffer(reservation->slot, reservation->lifecycle, size_hint); + return {value.data(), value.size()}; +} + +sms_status SMS_CALL sms_advance_reservation(sms_reservation* reservation, int32_t byte_count, + const sms_wait_options* wait_options) { + Wait wait{}; + if (!reservation || !reservation->store) return SMS_STATUS_INVALID_RESERVATION; + if (!read_wait(wait_options, wait)) return SMS_STATUS_UNKNOWN_FAILURE; + return reservation->store->advance_reservation(reservation->slot, reservation->lifecycle, byte_count, wait); +} + +sms_status SMS_CALL sms_commit_reservation(sms_reservation* reservation, + const sms_wait_options* wait_options) { + Wait wait{}; + if (!reservation || !reservation->store) return SMS_STATUS_INVALID_RESERVATION; + if (!read_wait(wait_options, wait)) return SMS_STATUS_UNKNOWN_FAILURE; + return reservation->store->commit_reservation(reservation->slot, reservation->lifecycle, wait); +} + +sms_status SMS_CALL sms_abort_reservation(sms_reservation* reservation, + const sms_wait_options* wait_options) { + Wait wait{}; + if (!reservation || !reservation->store) return SMS_STATUS_INVALID_RESERVATION; + if (!read_wait(wait_options, wait)) return SMS_STATUS_UNKNOWN_FAILURE; + return reservation->store->abort_reservation(reservation->slot, reservation->lifecycle, true, wait); +} + +void SMS_CALL sms_destroy_reservation(sms_reservation* reservation) { + if (!reservation) return; + if (reservation->store && reservation->store->reservation_valid(reservation->slot, reservation->lifecycle)) + reservation->store->abort_reservation(reservation->slot, reservation->lifecycle, false, Wait{1000}); + delete reservation; +} + +sms_status SMS_CALL sms_recover_leases(sms_store* store, int32_t recover_current_process, + const sms_wait_options* wait_options, sms_recovery_report* report) { + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!report || report->struct_size < sizeof(*report) || !abi_compatible(report->abi_version) || + !read_wait(wait_options, wait)) return SMS_STATUS_UNKNOWN_FAILURE; + RecoveryReport native{}; + const auto status = store->implementation->recover_leases(recover_current_process != 0, wait, native); + fill_report(*report, native); + return status; +} + +sms_status SMS_CALL sms_recover_reservations(sms_store* store, int32_t recover_current_process, + const sms_wait_options* wait_options, + sms_recovery_report* report) { + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!report || report->struct_size < sizeof(*report) || !abi_compatible(report->abi_version) || + !read_wait(wait_options, wait)) return SMS_STATUS_UNKNOWN_FAILURE; + RecoveryReport native{}; + const auto status = store->implementation->recover_reservations(recover_current_process != 0, wait, native); + fill_report(*report, native); + return status; +} + +sms_status SMS_CALL sms_get_diagnostics(sms_store* store, const sms_wait_options* wait_options, + sms_diagnostics* diagnostics) { + Wait wait{}; + if (!store || !store->implementation) return SMS_STATUS_STORE_DISPOSED; + if (!diagnostics || diagnostics->struct_size < sizeof(*diagnostics) || + !abi_compatible(diagnostics->abi_version) || !read_wait(wait_options, wait)) + return SMS_STATUS_UNKNOWN_FAILURE; + Diagnostics native{}; + const auto status = store->implementation->diagnostics(wait, native); + if (status != SMS_STATUS_SUCCESS) return status; + *diagnostics = {}; + diagnostics->struct_size = sizeof(*diagnostics); + diagnostics->abi_version = SMS_C_ABI_VERSION; + diagnostics->total_bytes = native.total_bytes; + diagnostics->slot_count = native.slot_count; + diagnostics->free_slot_count = native.free_slots; + diagnostics->published_slot_count = native.published_slots; + diagnostics->pending_removal_count = native.pending_removal; + diagnostics->active_lease_count = native.active_leases; + diagnostics->active_reservation_count = native.active_reservations; + diagnostics->index_entry_count = native.index_entries; + diagnostics->occupied_index_entry_count = native.occupied_index_entries; + diagnostics->tombstone_index_entry_count = native.tombstone_index_entries; + diagnostics->empty_index_entry_count = native.empty_index_entries; + diagnostics->usable_index_capacity = native.empty_index_entries + native.tombstone_index_entries; + diagnostics->last_observed_probe_length = native.last_probe; + diagnostics->max_observed_probe_length = native.max_probe; + diagnostics->last_failure_status = native.last_failure; + diagnostics->aborted_reservation_count = native.aborted_reservations; + diagnostics->recovered_lease_count = native.recovered_leases; + diagnostics->active_lease_recovery_count = native.active_lease_recoveries; + diagnostics->unsupported_lease_recovery_count = native.unsupported_lease_recoveries; + diagnostics->failed_lease_recovery_count = native.failed_lease_recoveries; + diagnostics->recovered_reservation_count = native.recovered_reservations; + diagnostics->active_reservation_recovery_count = native.active_reservation_recoveries; + diagnostics->unsupported_reservation_recovery_count = native.unsupported_reservation_recoveries; + diagnostics->failed_reservation_recovery_count = native.failed_reservation_recoveries; + diagnostics->capacity_pressure_count = native.capacity_pressure; + diagnostics->index_compaction_count = native.index_compactions; + for (std::size_t index = 0; index < native.failures.size(); ++index) + diagnostics->failure_counts[index] = native.failures[index]; + return SMS_STATUS_SUCCESS; +} + +} // extern "C" diff --git a/src/cpp/src/internal.hpp b/src/cpp/src/internal.hpp new file mode 100644 index 0000000..5528515 --- /dev/null +++ b/src/cpp/src/internal.hpp @@ -0,0 +1,421 @@ +#pragma once + +#include "shared_memory_store/c_api.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sms::detail { + +static_assert(sizeof(void*) == 8, "SharedMemoryStore native ABI requires a 64-bit target."); +static_assert( + std::endian::native == std::endian::little, + "SharedMemoryStore mapped layout requires a little-endian target."); + +constexpr std::int32_t magic = 0x31534D53; +constexpr std::int32_t alignment = 8; + +constexpr std::int32_t store_initializing = 0; +constexpr std::int32_t store_ready = 1; +constexpr std::int32_t store_disposing = 2; +constexpr std::int32_t store_corrupt = 3; +constexpr std::int32_t store_unsupported = 4; + +constexpr std::int32_t index_empty = 0; +constexpr std::int32_t index_occupied = 1; +constexpr std::int32_t index_tombstone = 2; + +constexpr std::int32_t slot_free = 0; +constexpr std::int32_t slot_publishing = 1; +constexpr std::int32_t slot_published = 2; +constexpr std::int32_t slot_remove_requested = 3; +constexpr std::int32_t slot_reclaiming = 4; + +constexpr std::int32_t lease_free = 0; +constexpr std::int32_t lease_active = 1; +constexpr std::int32_t lease_released = 2; +constexpr std::int32_t lease_abandoned = 3; + +#pragma pack(push, 8) +struct StoreHeader { + std::int32_t Magic; + std::int32_t LayoutMajorVersion; + std::int32_t LayoutMinorVersion; + std::int32_t HeaderLength; + std::int64_t TotalBytes; + std::int32_t SlotCount; + std::int32_t LeaseRecordCount; + std::int32_t MaxKeyBytes; + std::int32_t MaxDescriptorBytes; + std::int32_t MaxValueBytes; + std::int32_t IndexEntryCount; + std::int32_t IndexEntrySize; + std::int64_t IndexOffset; + std::int64_t IndexLength; + std::int64_t LeaseRegistryOffset; + std::int64_t LeaseRegistryLength; + std::int64_t SlotMetadataOffset; + std::int64_t SlotMetadataLength; + std::int64_t DescriptorStorageOffset; + std::int64_t DescriptorStorageLength; + std::int64_t PayloadStorageOffset; + std::int64_t PayloadStorageLength; + std::int64_t StoreId; + std::int32_t StoreState; + std::int32_t Reserved; + std::int64_t Sequence; +}; + +struct IndexEntryHeader { + std::int32_t State; + std::int32_t KeyLength; + std::uint64_t KeyHash; + std::int32_t SlotIndex; + std::int32_t SlotGeneration; + std::int64_t SlotReuseEpoch; +}; + +struct SlotMetadata { + std::int32_t State; + std::int32_t Generation; + std::int64_t ReuseEpoch; + std::int32_t UsageCount; + std::int32_t KeyLength; + std::int32_t DescriptorLength; + std::int32_t ValueLength; + std::int32_t PublisherProcessId; + std::int32_t Reserved; + std::uint64_t KeyHash; + std::int64_t DescriptorOffset; + std::int64_t PayloadOffset; + std::int64_t CommittedSequence; +}; + +struct LeaseRecord { + std::int32_t State; + std::int32_t LeaseRecordId; + std::int32_t SlotIndex; + std::int32_t SlotGeneration; + std::int64_t SlotReuseEpoch; + std::int32_t OwnerProcessId; + std::int32_t Reserved; + std::int64_t AcquireSequence; +}; +#pragma pack(pop) + +static_assert(sizeof(StoreHeader) == 160); +static_assert(offsetof(StoreHeader, IndexOffset) == 56); +static_assert(offsetof(StoreHeader, StoreId) == 136); +static_assert(offsetof(StoreHeader, StoreState) == 144); +static_assert(offsetof(StoreHeader, Sequence) == 152); +static_assert(sizeof(IndexEntryHeader) == 32); +static_assert(offsetof(IndexEntryHeader, KeyHash) == 8); +static_assert(offsetof(IndexEntryHeader, SlotReuseEpoch) == 24); +static_assert(sizeof(SlotMetadata) == 72); +static_assert(offsetof(SlotMetadata, ReuseEpoch) == 8); +static_assert(offsetof(SlotMetadata, UsageCount) == 16); +static_assert(offsetof(SlotMetadata, PublisherProcessId) == 32); +static_assert(offsetof(SlotMetadata, KeyHash) == 40); +static_assert(offsetof(SlotMetadata, CommittedSequence) == 64); +static_assert(sizeof(LeaseRecord) == 40); +static_assert(offsetof(LeaseRecord, SlotReuseEpoch) == 16); +static_assert(offsetof(LeaseRecord, OwnerProcessId) == 24); +static_assert(offsetof(LeaseRecord, AcquireSequence) == 32); + +struct Layout { + std::int64_t total_bytes{}; + std::int32_t slot_count{}; + std::int32_t lease_record_count{}; + std::int32_t max_value_bytes{}; + std::int32_t max_descriptor_bytes{}; + std::int32_t max_key_bytes{}; + std::int32_t header_length{}; + std::int32_t index_entry_count{}; + std::int32_t index_entry_size{}; + std::int64_t index_offset{}; + std::int64_t index_length{}; + std::int64_t lease_registry_offset{}; + std::int64_t lease_registry_length{}; + std::int64_t slot_metadata_offset{}; + std::int64_t slot_metadata_length{}; + std::int32_t descriptor_stride{}; + std::int64_t descriptor_storage_offset{}; + std::int64_t descriptor_storage_length{}; + std::int32_t payload_stride{}; + std::int64_t payload_storage_offset{}; + std::int64_t payload_storage_length{}; + std::int64_t required_bytes{}; + + static bool calculate( + std::int64_t total_bytes, + std::int32_t slot_count, + std::int32_t max_value_bytes, + std::int32_t max_descriptor_bytes, + std::int32_t max_key_bytes, + std::int32_t lease_record_count, + Layout& result) noexcept; + + bool matches(const StoreHeader& header) const noexcept; + bool bounds_valid(const StoreHeader& header) const noexcept; +}; + +struct Options { + std::string name; + sms_open_mode open_mode{SMS_OPEN_MODE_CREATE_OR_OPEN}; + std::int64_t total_bytes{}; + std::int32_t slot_count{}; + std::int32_t max_value_bytes{}; + std::int32_t max_descriptor_bytes{}; + std::int32_t max_key_bytes{}; + std::int32_t lease_record_count{}; + bool enable_lease_recovery{}; +}; + +struct Wait { + std::int64_t milliseconds{1000}; + bool valid() const noexcept { return milliseconds >= -1; } + bool infinite() const noexcept { return milliseconds == -1; } +}; + +struct LifecycleId { + std::int32_t generation{}; + std::int64_t reuse_epoch{}; + + bool valid() const noexcept { return generation > 0 && reuse_epoch >= 0; } + bool matches(std::int32_t g, std::int64_t e) const noexcept { + return generation == g && reuse_epoch == e; + } + bool advance(LifecycleId& next) const noexcept; +}; + +struct ResourceName { + std::string public_name; + std::string fragment; + std::string linux_region_path; + std::string linux_lock_path; + std::string linux_owners_path; + std::string linux_lifecycle_path; +#if defined(_WIN32) + std::wstring windows_region_name; + std::wstring windows_lock_name; +#endif +}; + +std::uint64_t hash_key(std::span key) noexcept; +std::array sha256(std::span data) noexcept; +bool valid_utf8(std::string_view value) noexcept; +std::size_t utf16_length(std::string_view value) noexcept; +bool utf8_whitespace_only(std::string_view value) noexcept; +bool make_resource_name(std::string_view public_name, ResourceName& result) noexcept; +std::int32_t current_process_id() noexcept; + +template +inline T load_acquire(T& value) noexcept { + return std::atomic_ref(value).load(std::memory_order_acquire); +} + +template +inline void store_release(T& target, T value) noexcept { + std::atomic_ref(target).store(value, std::memory_order_release); +} + +template +inline T increment(T& target) noexcept { + return std::atomic_ref(target).fetch_add(static_cast(1), std::memory_order_acq_rel) + static_cast(1); +} + +template +inline T decrement(T& target) noexcept { + return std::atomic_ref(target).fetch_sub(static_cast(1), std::memory_order_acq_rel) - static_cast(1); +} + +class MappedRegion { +public: + virtual ~MappedRegion() = default; + virtual std::uint8_t* data() noexcept = 0; + virtual std::int64_t size() const noexcept = 0; + virtual void close() noexcept = 0; +}; + +class SharedLock { +public: + virtual ~SharedLock() = default; + virtual sms_status acquire(const Wait& wait) noexcept = 0; + virtual void release() noexcept = 0; +}; + +struct PlatformOpenResult { + sms_open_status status{SMS_OPEN_MAPPING_FAILED}; + std::unique_ptr region; + std::unique_ptr lock; +}; + +PlatformOpenResult platform_open(const ResourceName& resource, const Options& options, const Wait& wait) noexcept; +enum class OwnerKind { current, live, stale, unsupported }; +OwnerKind classify_process(std::int32_t process_id) noexcept; + +struct RecoveryReport { + std::int32_t scanned{}; + std::int32_t recovered{}; + std::int32_t active{}; + std::int32_t unsupported{}; + std::int32_t failed{}; +}; + +struct Diagnostics { + std::int64_t total_bytes{}; + std::int32_t slot_count{}; + std::int32_t free_slots{}; + std::int32_t published_slots{}; + std::int32_t pending_removal{}; + std::int32_t active_leases{}; + std::int32_t active_reservations{}; + std::int32_t index_entries{}; + std::int32_t occupied_index_entries{}; + std::int32_t tombstone_index_entries{}; + std::int32_t empty_index_entries{}; + std::int32_t last_probe{}; + std::int32_t max_probe{}; + sms_status last_failure{SMS_STATUS_SUCCESS}; + std::int64_t aborted_reservations{}; + std::int64_t recovered_leases{}; + std::int64_t active_lease_recoveries{}; + std::int64_t unsupported_lease_recoveries{}; + std::int64_t failed_lease_recoveries{}; + std::int64_t recovered_reservations{}; + std::int64_t active_reservation_recoveries{}; + std::int64_t unsupported_reservation_recoveries{}; + std::int64_t failed_reservation_recoveries{}; + std::int64_t capacity_pressure{}; + std::int64_t index_compactions{}; + std::array failures{}; +}; + +class Store : public std::enable_shared_from_this { +public: + static sms_open_status open(const Options& options, const Wait& wait, std::shared_ptr& result) noexcept; + ~Store(); + + Store(const Store&) = delete; + Store& operator=(const Store&) = delete; + + sms_status publish(std::span key, + std::span value, + std::span descriptor, + const Wait& wait) noexcept; + sms_status publish_segments(std::span key, + std::span segments, + std::span descriptor, + const Wait& wait, + std::int64_t& copied) noexcept; + sms_status acquire(std::span key, + const Wait& wait, + std::int32_t& slot, + LifecycleId& lifecycle, + std::int32_t& lease_id) noexcept; + sms_status release_lease(std::int32_t slot, LifecycleId lifecycle, + std::int32_t lease_id, const Wait& wait) noexcept; + bool lease_valid(std::int32_t slot, LifecycleId lifecycle, std::int32_t lease_id) noexcept; + std::span lease_value(std::int32_t slot, LifecycleId lifecycle, + std::int32_t lease_id) noexcept; + std::span lease_descriptor(std::int32_t slot, LifecycleId lifecycle, + std::int32_t lease_id) noexcept; + sms_status remove(std::span key, const Wait& wait) noexcept; + + sms_status reserve(std::span key, std::int32_t payload_length, + std::span descriptor, const Wait& wait, + std::int32_t& slot, LifecycleId& lifecycle) noexcept; + bool reservation_valid(std::int32_t slot, LifecycleId lifecycle) noexcept; + std::int32_t reservation_payload_length(std::int32_t slot, LifecycleId lifecycle) noexcept; + std::int32_t reservation_bytes_written(std::int32_t slot, LifecycleId lifecycle) noexcept; + std::span reservation_buffer(std::int32_t slot, LifecycleId lifecycle, + std::int32_t size_hint) noexcept; + sms_status advance_reservation(std::int32_t slot, LifecycleId lifecycle, + std::int32_t count, const Wait& wait) noexcept; + sms_status commit_reservation(std::int32_t slot, LifecycleId lifecycle, + const Wait& wait) noexcept; + sms_status abort_reservation(std::int32_t slot, LifecycleId lifecycle, + bool count_abort, const Wait& wait) noexcept; + sms_status recover_leases(bool recover_current, const Wait& wait, RecoveryReport& report) noexcept; + sms_status recover_reservations(bool recover_current, const Wait& wait, RecoveryReport& report) noexcept; + sms_status diagnostics(const Wait& wait, Diagnostics& result) noexcept; + sms_status get_layout(const Wait& wait, Layout& result) noexcept; + void close() noexcept; + +private: + Store(std::unique_ptr region, std::unique_ptr lock, + Layout layout, bool recovery_enabled) noexcept; + + class Guard { + public: + Guard(Store& store, const Wait& wait) noexcept; + ~Guard(); + bool acquired() const noexcept { return acquired_; } + sms_status status() const noexcept { return status_; } + private: + Store& store_; + bool local_acquired_{}; + bool acquired_{}; + sms_status status_{SMS_STATUS_UNKNOWN_FAILURE}; + }; + + sms_open_status initialize_or_validate(const Options& options) noexcept; + void initialize_header() noexcept; + sms_status ensure_ready() const noexcept; + sms_status validate_key(std::span key) const noexcept; + sms_status validate_value(std::span key, std::size_t value_length, + std::size_t descriptor_length, bool reservation) const noexcept; + sms_status record(sms_status status) noexcept; + + StoreHeader& header() noexcept; + IndexEntryHeader& index_entry(std::int32_t index) noexcept; + std::uint8_t* index_key(std::int32_t index) noexcept; + SlotMetadata& slot(std::int32_t index) noexcept; + LeaseRecord& lease(std::int32_t index) noexcept; + + bool index_find(std::span key, std::uint64_t hash, + std::int32_t& slot_index, LifecycleId& lifecycle) noexcept; + bool index_insert(std::span key, std::uint64_t hash, + std::int32_t slot_index, LifecycleId lifecycle) noexcept; + bool index_remove_slot(std::int32_t slot_index, LifecycleId lifecycle, + std::uint64_t hash) noexcept; + void write_index(std::int32_t index, std::span key, + std::uint64_t hash, std::int32_t slot_index, LifecycleId lifecycle) noexcept; + void record_probe(std::int32_t probes) noexcept; + bool reserve_slot(std::int32_t& slot_index) noexcept; + bool activate_lease(std::int32_t slot_index, LifecycleId lifecycle, + std::int64_t sequence, std::int32_t& lease_id) noexcept; + sms_status request_remove(std::int32_t slot_index, LifecycleId lifecycle) noexcept; + sms_status reclaim(std::int32_t slot_index) noexcept; + sms_status reclaim_after_release(std::int32_t slot_index, LifecycleId lifecycle) noexcept; + void abort_slot(std::int32_t slot_index) noexcept; + void maybe_compact_index() noexcept; + bool compact_index() noexcept; + + std::unique_ptr region_; + std::unique_ptr lock_; + Layout layout_; + bool recovery_enabled_{}; + std::timed_mutex gate_; + std::atomic closed_{false}; + std::atomic next_slot_{0}; + std::atomic next_lease_{0}; + std::atomic last_probe_{0}; + std::atomic max_probe_{0}; + std::mutex diagnostics_gate_; + Diagnostics local_diagnostics_{}; +}; + +} // namespace sms::detail diff --git a/src/cpp/src/platform_linux.cpp b/src/cpp/src/platform_linux.cpp new file mode 100644 index 0000000..b0eb02c --- /dev/null +++ b/src/cpp/src/platform_linux.cpp @@ -0,0 +1,412 @@ +#include "internal.hpp" + +#if !defined(_WIN32) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sms::detail { +namespace { + +using clock_type = std::chrono::steady_clock; + +class FileState { +public: + explicit FileState(std::string path) : path_(std::move(path)) { + fd_ = ::open(path_.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (fd_ >= 0) ::fchmod(fd_, 0600); + } + ~FileState() { if (fd_ >= 0) ::close(fd_); } + int fd() const noexcept { return fd_; } + std::timed_mutex mutex; +private: + std::string path_; + int fd_{-1}; +}; + +std::mutex file_states_gate; +std::unordered_map> file_states; + +std::shared_ptr get_file_state(const std::string& raw_path) { + std::error_code error; + auto path = std::filesystem::absolute(raw_path, error).lexically_normal().string(); + if (error) path = raw_path; + std::lock_guard guard(file_states_gate); + if (auto found = file_states[path].lock()) return found; + auto created = std::make_shared(path); + file_states[path] = created; + return created; +} + +class LinuxFileLock final : public SharedLock { +public: + explicit LinuxFileLock(std::shared_ptr state) : state_(std::move(state)) {} + ~LinuxFileLock() override { release(); } + + bool usable() const noexcept { return state_ && state_->fd() >= 0; } + + sms_status acquire(const Wait& wait) noexcept override { + if (!usable()) return errno == EACCES ? SMS_STATUS_ACCESS_DENIED : SMS_STATUS_UNKNOWN_FAILURE; + if (held_) return SMS_STATUS_SUCCESS; + const auto started = clock_type::now(); + if (wait.infinite()) { + state_->mutex.lock(); + local_held_ = true; + } else if (wait.milliseconds == 0) { + local_held_ = state_->mutex.try_lock(); + } else { + local_held_ = state_->mutex.try_lock_for(std::chrono::milliseconds(wait.milliseconds)); + } + if (!local_held_) return SMS_STATUS_STORE_BUSY; + + for (;;) { + struct flock request{}; + request.l_type = F_WRLCK; + request.l_whence = SEEK_SET; + request.l_start = 0; + request.l_len = 1; + if (::fcntl(state_->fd(), F_SETLK, &request) == 0) { + held_ = true; + return SMS_STATUS_SUCCESS; + } + const auto error = errno; + if (error != EACCES && error != EAGAIN) { + release(); + if (error == EACCES || error == EPERM) return SMS_STATUS_ACCESS_DENIED; + if (error == ENOSYS || error == ENOTSUP) return SMS_STATUS_UNSUPPORTED_PLATFORM; + return SMS_STATUS_UNKNOWN_FAILURE; + } + if (!wait.infinite()) { + const auto elapsed = std::chrono::duration_cast(clock_type::now() - started); + if (elapsed.count() >= wait.milliseconds) { + release(); + return SMS_STATUS_STORE_BUSY; + } + const auto remaining = std::chrono::milliseconds(wait.milliseconds) - elapsed; + std::this_thread::sleep_for(std::min(std::chrono::milliseconds(10), remaining)); + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + } + + void release() noexcept override { + if (held_ && usable()) { + struct flock request{}; + request.l_type = F_UNLCK; + request.l_whence = SEEK_SET; + request.l_start = 0; + request.l_len = 1; + ::fcntl(state_->fd(), F_SETLK, &request); + held_ = false; + } + if (local_held_) { + state_->mutex.unlock(); + local_held_ = false; + } + } + +private: + std::shared_ptr state_; + bool local_held_{}; + bool held_{}; +}; + +std::unique_ptr open_lock(const std::string& path) { + auto result = std::make_unique(get_file_state(path)); + return result->usable() ? std::move(result) : nullptr; +} + +bool ensure_directory(const std::string& child_path) noexcept { + try { + const auto directory = std::filesystem::path(child_path).parent_path(); + struct stat information{}; + if (::lstat(directory.c_str(), &information) == 0) { + if (S_ISLNK(information.st_mode) || !S_ISDIR(information.st_mode)) { + errno = EACCES; + return false; + } + } else { + if (errno != ENOENT || ::mkdir(directory.c_str(), 0700) != 0) return false; + } + return ::chmod(directory.c_str(), 0700) == 0; + } catch (...) { + return false; + } +} + +bool exists(const std::string& path) noexcept { + struct stat value{}; + return ::stat(path.c_str(), &value) == 0; +} + +std::string process_start_token(std::int32_t pid) noexcept { + try { + std::ifstream input("/proc/" + std::to_string(pid) + "/stat"); + std::string stat; + std::getline(input, stat); + const auto command_end = stat.rfind(')'); + if (command_end == std::string::npos || command_end + 2 >= stat.size()) return {}; + std::istringstream fields(stat.substr(command_end + 2)); + std::string value; + for (int index = 0; fields >> value; ++index) { + if (index == 19) return "proc-" + value; + } + } catch (...) { + } + return {}; +} + +bool process_live(std::int32_t pid, std::string_view start_token) noexcept { + if (pid <= 0) return false; + if (::kill(pid, 0) != 0 && errno == ESRCH) return false; + if (start_token.empty()) return true; + const auto observed = process_start_token(pid); + return observed.empty() || observed == start_token; +} + +bool parse_owner(std::string_view line, std::int32_t& pid, std::string& token) noexcept { + try { + const auto first = line.find(':'); + const auto pid_text = line.substr(0, first); + std::size_t used{}; + const auto parsed = std::stoll(std::string(pid_text), &used, 10); + if (used != pid_text.size() || parsed < std::numeric_limits::min() || + parsed > std::numeric_limits::max()) return false; + pid = static_cast(parsed); + token.clear(); + if (first != std::string_view::npos) { + const auto second = line.find(':', first + 1); + if (second != std::string_view::npos) token = std::string(line.substr(first + 1, second - first - 1)); + } + return true; + } catch (...) { + return false; + } +} + +std::vector read_live_owners(const std::string& path) { + std::vector owners; + std::ifstream input(path); + std::string line; + while (std::getline(input, line)) { + while (!line.empty() && (line.back() == '\r' || line.back() == '\n' || line.back() == ' ' || line.back() == '\t')) line.pop_back(); + const auto first = line.find_first_not_of(" \t"); + if (first == std::string::npos) continue; + line.erase(0, first); + std::int32_t pid{}; + std::string token; + if (parse_owner(line, pid, token) && process_live(pid, token)) owners.push_back(line); + } + return owners; +} + +bool write_owners(const std::string& path, const std::vector& owners) noexcept { + const auto temporary = path + ".tmp"; + const auto fd = ::open(temporary.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); + if (fd < 0) return false; + bool success = ::fchmod(fd, 0600) == 0; + for (const auto& owner : owners) { + const auto line = owner + "\n"; + std::size_t offset{}; + while (success && offset < line.size()) { + const auto written = ::write(fd, line.data() + offset, line.size() - offset); + if (written <= 0) success = false; + else offset += static_cast(written); + } + } + if (success) success = ::fsync(fd) == 0; + ::close(fd); + if (success) success = ::rename(temporary.c_str(), path.c_str()) == 0; + if (!success) ::unlink(temporary.c_str()); + return success; +} + +void delete_stale(const ResourceName& resource) noexcept { + ::unlink(resource.linux_region_path.c_str()); + ::unlink(resource.linux_lock_path.c_str()); + ::unlink(resource.linux_owners_path.c_str()); + ::unlink((resource.linux_owners_path + ".tmp").c_str()); +} + +std::string random_hex() { + std::random_device random; + constexpr char hex[] = "0123456789abcdef"; + std::string result(32, '0'); + for (auto& value : result) value = hex[random() & 15U]; + return result; +} + +std::string create_owner_record() { + const auto pid = current_process_id(); + return std::to_string(pid) + ":" + process_start_token(pid) + ":" + random_hex(); +} + +void release_owner(const ResourceName& resource, const std::string& owner) noexcept { + try { + auto lifecycle = open_lock(resource.linux_lifecycle_path); + if (!lifecycle || lifecycle->acquire(Wait{-1}) != SMS_STATUS_SUCCESS) return; + auto owners = read_live_owners(resource.linux_owners_path); + owners.erase(std::remove(owners.begin(), owners.end(), owner), owners.end()); + if (owners.empty()) delete_stale(resource); + else write_owners(resource.linux_owners_path, owners); + lifecycle->release(); + } catch (...) { + } +} + +class LinuxRegion final : public MappedRegion { +public: + LinuxRegion(int fd, std::uint8_t* data, std::int64_t size, ResourceName resource, std::string owner) + : fd_(fd), data_(data), size_(size), resource_(std::move(resource)), owner_(std::move(owner)) {} + ~LinuxRegion() override { close(); } + std::uint8_t* data() noexcept override { return data_; } + std::int64_t size() const noexcept override { return size_; } + void close() noexcept override { + if (closed_) return; + closed_ = true; + if (data_ && data_ != MAP_FAILED) ::munmap(data_, static_cast(size_)); + data_ = nullptr; + if (fd_ >= 0) ::close(fd_); + fd_ = -1; + release_owner(resource_, owner_); + } +private: + int fd_{-1}; + std::uint8_t* data_{}; + std::int64_t size_{}; + ResourceName resource_; + std::string owner_; + bool closed_{}; +}; + +sms_open_status map_lock_status(sms_status status) noexcept { + switch (status) { + case SMS_STATUS_SUCCESS: return SMS_OPEN_SUCCESS; + case SMS_STATUS_STORE_BUSY: return SMS_OPEN_STORE_BUSY; + case SMS_STATUS_OPERATION_CANCELED: return SMS_OPEN_OPERATION_CANCELED; + case SMS_STATUS_ACCESS_DENIED: return SMS_OPEN_ACCESS_DENIED; + case SMS_STATUS_UNSUPPORTED_PLATFORM: return SMS_OPEN_UNSUPPORTED_PLATFORM; + default: return SMS_OPEN_MAPPING_FAILED; + } +} + +} // namespace + +PlatformOpenResult platform_open(const ResourceName& resource, const Options& options, const Wait& wait) noexcept { + PlatformOpenResult result{}; + try { + if (!ensure_directory(resource.linux_region_path)) { + result.status = (errno == EACCES || errno == EPERM) ? SMS_OPEN_ACCESS_DENIED : SMS_OPEN_MAPPING_FAILED; + return result; + } + auto lifecycle = open_lock(resource.linux_lifecycle_path); + if (!lifecycle) { + result.status = (errno == EACCES || errno == EPERM) ? SMS_OPEN_ACCESS_DENIED : SMS_OPEN_MAPPING_FAILED; + return result; + } + const auto lock_status = lifecycle->acquire(wait); + if (lock_status != SMS_STATUS_SUCCESS) { + result.status = map_lock_status(lock_status); + return result; + } + + auto owners = read_live_owners(resource.linux_owners_path); + auto live_resource = exists(resource.linux_region_path) && !owners.empty(); + if (!live_resource) delete_stale(resource); + if (options.open_mode == SMS_OPEN_MODE_CREATE_NEW && live_resource) { + lifecycle->release(); + result.status = SMS_OPEN_ALREADY_EXISTS; + return result; + } + if (options.open_mode == SMS_OPEN_MODE_OPEN_EXISTING && !live_resource) { + lifecycle->release(); + result.status = SMS_OPEN_NOT_FOUND; + return result; + } + + const bool create = !live_resource; + const auto flags = O_RDWR | O_CLOEXEC | (create ? (O_CREAT | O_EXCL) : 0); + const auto fd = ::open(resource.linux_region_path.c_str(), flags, 0600); + if (fd < 0) { + lifecycle->release(); + result.status = errno == EEXIST ? SMS_OPEN_ALREADY_EXISTS : + (errno == EACCES || errno == EPERM ? SMS_OPEN_ACCESS_DENIED : SMS_OPEN_MAPPING_FAILED); + return result; + } + ::fchmod(fd, 0600); + if (create) { + if (::ftruncate(fd, options.total_bytes) != 0) { + ::close(fd); delete_stale(resource); lifecycle->release(); + result.status = errno == EACCES || errno == EPERM ? SMS_OPEN_ACCESS_DENIED : SMS_OPEN_MAPPING_FAILED; + return result; + } + } else { + struct stat information{}; + if (::fstat(fd, &information) != 0 || information.st_size < options.total_bytes) { + ::close(fd); lifecycle->release(); + result.status = SMS_OPEN_INCOMPATIBLE_LAYOUT; + return result; + } + } + if (options.total_bytes <= 0 || static_cast(options.total_bytes) > std::numeric_limits::max()) { + ::close(fd); lifecycle->release(); result.status = SMS_OPEN_INVALID_OPTIONS; return result; + } + auto* mapped = static_cast(::mmap(nullptr, static_cast(options.total_bytes), + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); + if (mapped == MAP_FAILED) { + ::close(fd); if (create) delete_stale(resource); lifecycle->release(); + result.status = errno == EACCES || errno == EPERM ? SMS_OPEN_ACCESS_DENIED : SMS_OPEN_MAPPING_FAILED; + return result; + } + + const auto owner = create_owner_record(); + owners = read_live_owners(resource.linux_owners_path); + owners.push_back(owner); + if (!write_owners(resource.linux_owners_path, owners)) { + ::munmap(mapped, static_cast(options.total_bytes)); + ::close(fd); if (create) delete_stale(resource); lifecycle->release(); + result.status = errno == EACCES || errno == EPERM ? SMS_OPEN_ACCESS_DENIED : SMS_OPEN_MAPPING_FAILED; + return result; + } + lifecycle->release(); + + auto shared_lock = open_lock(resource.linux_lock_path); + if (!shared_lock) { + release_owner(resource, owner); + ::munmap(mapped, static_cast(options.total_bytes)); + ::close(fd); + result.status = errno == EACCES || errno == EPERM ? SMS_OPEN_ACCESS_DENIED : SMS_OPEN_MAPPING_FAILED; + return result; + } + result.region = std::make_unique(fd, mapped, options.total_bytes, resource, owner); + result.lock = std::move(shared_lock); + result.status = SMS_OPEN_SUCCESS; + return result; + } catch (...) { + result.status = SMS_OPEN_MAPPING_FAILED; + return result; + } +} + +OwnerKind classify_process(std::int32_t pid) noexcept { + if (pid <= 0) return OwnerKind::stale; + if (pid == current_process_id()) return OwnerKind::current; + if (::kill(pid, 0) == 0 || errno == EPERM) return OwnerKind::live; + return errno == ESRCH ? OwnerKind::stale : OwnerKind::unsupported; +} + +} // namespace sms::detail + +#endif diff --git a/src/cpp/src/platform_windows.cpp b/src/cpp/src/platform_windows.cpp new file mode 100644 index 0000000..f258a43 --- /dev/null +++ b/src/cpp/src/platform_windows.cpp @@ -0,0 +1,154 @@ +#include "internal.hpp" + +#if defined(_WIN32) + +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include + +namespace sms::detail { +namespace { + +sms_open_status map_windows_open_error(DWORD error) noexcept { + switch (error) { + case ERROR_FILE_NOT_FOUND: + case ERROR_INVALID_NAME: + return error == ERROR_FILE_NOT_FOUND ? SMS_OPEN_NOT_FOUND : SMS_OPEN_INVALID_OPTIONS; + case ERROR_ACCESS_DENIED: + case ERROR_PRIVILEGE_NOT_HELD: + return SMS_OPEN_ACCESS_DENIED; + case ERROR_NOT_SUPPORTED: + case ERROR_CALL_NOT_IMPLEMENTED: + return SMS_OPEN_UNSUPPORTED_PLATFORM; + case ERROR_INVALID_PARAMETER: + return SMS_OPEN_INVALID_OPTIONS; + default: + return SMS_OPEN_MAPPING_FAILED; + } +} + +class WindowsRegion final : public MappedRegion { +public: + WindowsRegion(HANDLE mapping, std::uint8_t* data, std::int64_t size) + : mapping_(mapping), data_(data), size_(size) {} + ~WindowsRegion() override { close(); } + std::uint8_t* data() noexcept override { return data_; } + std::int64_t size() const noexcept override { return size_; } + void close() noexcept override { + if (data_) UnmapViewOfFile(data_); + data_ = nullptr; + if (mapping_) CloseHandle(mapping_); + mapping_ = nullptr; + } +private: + HANDLE mapping_{}; + std::uint8_t* data_{}; + std::int64_t size_{}; +}; + +class WindowsLock final : public SharedLock { +public: + explicit WindowsLock(HANDLE mutex) : mutex_(mutex) {} + ~WindowsLock() override { + release(); + if (mutex_) CloseHandle(mutex_); + } + sms_status acquire(const Wait& wait) noexcept override { + if (!mutex_) return SMS_STATUS_STORE_DISPOSED; + DWORD timeout = INFINITE; + if (!wait.infinite()) { + timeout = wait.milliseconds >= static_cast(INFINITE - 1) + ? INFINITE - 1 + : static_cast(wait.milliseconds); + } + const auto result = WaitForSingleObject(mutex_, timeout); + if (result == WAIT_OBJECT_0 || result == WAIT_ABANDONED) { + held_ = true; + return SMS_STATUS_SUCCESS; + } + if (result == WAIT_TIMEOUT) return SMS_STATUS_STORE_BUSY; + const auto error = GetLastError(); + return error == ERROR_ACCESS_DENIED ? SMS_STATUS_ACCESS_DENIED : SMS_STATUS_UNKNOWN_FAILURE; + } + void release() noexcept override { + if (held_ && mutex_) { + ReleaseMutex(mutex_); + held_ = false; + } + } +private: + HANDLE mutex_{}; + bool held_{}; +}; + +} // namespace + +PlatformOpenResult platform_open(const ResourceName& resource, const Options& options, const Wait&) noexcept { + PlatformOpenResult result{}; + HANDLE mapping{}; + if (options.open_mode == SMS_OPEN_MODE_OPEN_EXISTING) { + mapping = OpenFileMappingW(FILE_MAP_ALL_ACCESS, FALSE, resource.windows_region_name.c_str()); + if (!mapping) { + result.status = map_windows_open_error(GetLastError()); + return result; + } + } else { + const auto unsigned_size = static_cast(options.total_bytes); + mapping = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, + static_cast(unsigned_size >> 32), + static_cast(unsigned_size), + resource.windows_region_name.c_str()); + if (!mapping) { + result.status = map_windows_open_error(GetLastError()); + return result; + } + if (options.open_mode == SMS_OPEN_MODE_CREATE_NEW && GetLastError() == ERROR_ALREADY_EXISTS) { + CloseHandle(mapping); + result.status = SMS_OPEN_ALREADY_EXISTS; + return result; + } + } + + auto* data = static_cast(MapViewOfFile( + mapping, FILE_MAP_ALL_ACCESS, 0, 0, static_cast(options.total_bytes))); + if (!data) { + const auto error = GetLastError(); + CloseHandle(mapping); + result.status = error == ERROR_MAPPED_ALIGNMENT ? SMS_OPEN_INVALID_OPTIONS : map_windows_open_error(error); + return result; + } + + const auto mutex = CreateMutexW(nullptr, FALSE, resource.windows_lock_name.c_str()); + if (!mutex) { + const auto error = GetLastError(); + UnmapViewOfFile(data); + CloseHandle(mapping); + result.status = map_windows_open_error(error); + return result; + } + result.region = std::make_unique(mapping, data, options.total_bytes); + result.lock = std::make_unique(mutex); + result.status = SMS_OPEN_SUCCESS; + return result; +} + +OwnerKind classify_process(std::int32_t pid) noexcept { + if (pid <= 0) return OwnerKind::stale; + if (pid == current_process_id()) return OwnerKind::current; + const auto process = OpenProcess(SYNCHRONIZE | PROCESS_QUERY_LIMITED_INFORMATION, FALSE, + static_cast(pid)); + if (!process) { + const auto error = GetLastError(); + return error == ERROR_INVALID_PARAMETER ? OwnerKind::stale : OwnerKind::unsupported; + } + const auto wait = WaitForSingleObject(process, 0); + CloseHandle(process); + if (wait == WAIT_OBJECT_0) return OwnerKind::stale; + if (wait == WAIT_TIMEOUT) return OwnerKind::live; + return OwnerKind::unsupported; +} + +} // namespace sms::detail + +#endif diff --git a/src/cpp/src/protocol.cpp b/src/cpp/src/protocol.cpp new file mode 100644 index 0000000..41b9ae4 --- /dev/null +++ b/src/cpp/src/protocol.cpp @@ -0,0 +1,394 @@ +#include "internal.hpp" + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +#else +# include +#endif + +namespace sms::detail { +namespace { + +constexpr std::uint64_t fnv_offset = 14695981039346656037ULL; +constexpr std::uint64_t fnv_prime = 1099511628211ULL; + +bool checked_add(std::int64_t a, std::int64_t b, std::int64_t& result) noexcept { + if (a < 0 || b < 0 || a > std::numeric_limits::max() - b) { + return false; + } + result = a + b; + return true; +} + +bool checked_mul(std::int64_t a, std::int64_t b, std::int64_t& result) noexcept { + if (a < 0 || b < 0 || (a != 0 && b > std::numeric_limits::max() / a)) { + return false; + } + result = a * b; + return true; +} + +bool align8(std::int64_t value, std::int64_t& result) noexcept { + std::int64_t expanded{}; + if (!checked_add(value, alignment - 1, expanded)) { + return false; + } + result = expanded & ~static_cast(alignment - 1); + return true; +} + +bool next_power_of_two(std::int64_t value, std::int32_t& result) noexcept { + if (value <= 0 || value > (1LL << 30)) { + return false; + } + std::int32_t current = 1; + while (current < value) { + current <<= 1; + } + result = current; + return true; +} + +bool next_utf8(std::string_view text, std::size_t& index, std::uint32_t& cp) noexcept { + if (index >= text.size()) { + return false; + } + const auto first = static_cast(text[index++]); + if (first < 0x80) { + cp = first; + return true; + } + int continuation{}; + std::uint32_t value{}; + std::uint32_t minimum{}; + if ((first & 0xE0) == 0xC0) { + continuation = 1; + value = first & 0x1F; + minimum = 0x80; + } else if ((first & 0xF0) == 0xE0) { + continuation = 2; + value = first & 0x0F; + minimum = 0x800; + } else if ((first & 0xF8) == 0xF0) { + continuation = 3; + value = first & 0x07; + minimum = 0x10000; + } else { + return false; + } + if (index + static_cast(continuation) > text.size()) { + return false; + } + for (int i = 0; i < continuation; ++i) { + const auto part = static_cast(text[index++]); + if ((part & 0xC0) != 0x80) { + return false; + } + value = (value << 6) | (part & 0x3F); + } + if (value < minimum || value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF)) { + return false; + } + cp = value; + return true; +} + +std::string lowercase_hex(std::span bytes) { + std::ostringstream output; + output << std::hex << std::setfill('0'); + for (auto byte : bytes) { + output << std::setw(2) << static_cast(byte); + } + return output.str(); +} + +constexpr std::array sha_k{ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; + +constexpr std::uint32_t rotr(std::uint32_t value, int bits) noexcept { + return (value >> bits) | (value << (32 - bits)); +} + +#if defined(_WIN32) +bool utf8_to_wide(std::string_view value, std::wstring& result) noexcept { + if (value.empty()) { + result.clear(); + return true; + } + if (value.size() > static_cast(std::numeric_limits::max())) { + return false; + } + const auto needed = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, value.data(), + static_cast(value.size()), nullptr, 0); + if (needed <= 0) { + return false; + } + result.resize(static_cast(needed)); + return MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, value.data(), + static_cast(value.size()), result.data(), needed) == needed; +} + +bool starts_global(const std::wstring& value) noexcept { + constexpr wchar_t prefix[] = L"Global\\"; + if (value.size() < 7) { + return false; + } + return CompareStringOrdinal(value.data(), 7, prefix, 7, TRUE) == CSTR_EQUAL; +} +#endif + +} // namespace + +bool Layout::calculate(std::int64_t total, std::int32_t slots, std::int32_t max_value, + std::int32_t max_descriptor, std::int32_t max_key, + std::int32_t leases, Layout& result) noexcept { + if (slots <= 0 || max_value <= 0 || max_descriptor < 0 || max_key <= 0 || leases <= 0) { + return false; + } + Layout value{}; + value.total_bytes = total; + value.slot_count = slots; + value.lease_record_count = leases; + value.max_value_bytes = max_value; + value.max_descriptor_bytes = max_descriptor; + value.max_key_bytes = max_key; + + std::int64_t temp{}; + if (!align8(sizeof(StoreHeader), temp) || temp > std::numeric_limits::max()) return false; + value.header_length = static_cast(temp); + if (!next_power_of_two(std::max(4, static_cast(slots) * 2), value.index_entry_count)) return false; + if (!checked_add(sizeof(IndexEntryHeader), max_key, temp) || !align8(temp, temp) || + temp > std::numeric_limits::max()) return false; + value.index_entry_size = static_cast(temp); + value.index_offset = value.header_length; + if (!checked_mul(value.index_entry_count, value.index_entry_size, value.index_length) || + !checked_add(value.index_offset, value.index_length, temp) || !align8(temp, value.lease_registry_offset) || + !checked_mul(leases, sizeof(LeaseRecord), value.lease_registry_length) || + !checked_add(value.lease_registry_offset, value.lease_registry_length, temp) || !align8(temp, value.slot_metadata_offset) || + !checked_mul(slots, sizeof(SlotMetadata), value.slot_metadata_length) || + !align8(std::max(1, max_descriptor), temp) || temp > std::numeric_limits::max()) return false; + value.descriptor_stride = static_cast(temp); + if (!checked_add(value.slot_metadata_offset, value.slot_metadata_length, temp) || !align8(temp, value.descriptor_storage_offset) || + !checked_mul(slots, value.descriptor_stride, value.descriptor_storage_length) || + !align8(std::max(1, max_value), temp) || temp > std::numeric_limits::max()) return false; + value.payload_stride = static_cast(temp); + if (!checked_add(value.descriptor_storage_offset, value.descriptor_storage_length, temp) || !align8(temp, value.payload_storage_offset) || + !checked_mul(slots, value.payload_stride, value.payload_storage_length) || + !checked_add(value.payload_storage_offset, value.payload_storage_length, temp) || !align8(temp, value.required_bytes)) return false; + result = value; + return true; +} + +bool Layout::matches(const StoreHeader& h) const noexcept { + return h.HeaderLength == header_length && h.TotalBytes == total_bytes && h.SlotCount == slot_count && + h.LeaseRecordCount == lease_record_count && h.MaxKeyBytes == max_key_bytes && + h.MaxDescriptorBytes == max_descriptor_bytes && h.MaxValueBytes == max_value_bytes && + h.IndexEntryCount == index_entry_count && h.IndexEntrySize == index_entry_size && + h.IndexOffset == index_offset && h.IndexLength == index_length && + h.LeaseRegistryOffset == lease_registry_offset && h.LeaseRegistryLength == lease_registry_length && + h.SlotMetadataOffset == slot_metadata_offset && h.SlotMetadataLength == slot_metadata_length && + h.DescriptorStorageOffset == descriptor_storage_offset && h.DescriptorStorageLength == descriptor_storage_length && + h.PayloadStorageOffset == payload_storage_offset && h.PayloadStorageLength == payload_storage_length; +} + +bool Layout::bounds_valid(const StoreHeader& h) const noexcept { + auto within = [&](std::int64_t offset, std::int64_t length, std::int64_t minimum) { + std::int64_t end{}; + return offset >= minimum && checked_add(offset, length, end) && end <= h.TotalBytes; + }; + std::int64_t index_end{}, lease_end{}, slot_end{}, descriptor_end{}; + return within(h.IndexOffset, h.IndexLength, h.HeaderLength) && + checked_add(h.IndexOffset, h.IndexLength, index_end) && + within(h.LeaseRegistryOffset, h.LeaseRegistryLength, index_end) && + checked_add(h.LeaseRegistryOffset, h.LeaseRegistryLength, lease_end) && + within(h.SlotMetadataOffset, h.SlotMetadataLength, lease_end) && + checked_add(h.SlotMetadataOffset, h.SlotMetadataLength, slot_end) && + within(h.DescriptorStorageOffset, h.DescriptorStorageLength, slot_end) && + checked_add(h.DescriptorStorageOffset, h.DescriptorStorageLength, descriptor_end) && + within(h.PayloadStorageOffset, h.PayloadStorageLength, descriptor_end); +} + +bool LifecycleId::advance(LifecycleId& next) const noexcept { + if (generation == std::numeric_limits::max()) { + if (reuse_epoch == std::numeric_limits::max()) return false; + next = {1, reuse_epoch + 1}; + } else { + next = {generation + 1, reuse_epoch}; + } + return true; +} + +std::uint64_t hash_key(std::span key) noexcept { + auto hash = fnv_offset; + for (auto byte : key) { + hash ^= byte; + hash *= fnv_prime; + } + return hash; +} + +std::array sha256(std::span input) noexcept { + std::array state{0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + const auto total = input.size() + 1 + 8; + const auto padded = ((total + 63) / 64) * 64; + std::vector bytes(padded, 0); + std::copy(input.begin(), input.end(), bytes.begin()); + bytes[input.size()] = 0x80; + const auto bits = static_cast(input.size()) * 8; + for (int i = 0; i < 8; ++i) bytes[padded - 1 - i] = static_cast(bits >> (i * 8)); + + for (std::size_t offset = 0; offset < padded; offset += 64) { + std::array w{}; + for (int i = 0; i < 16; ++i) { + const auto p = offset + static_cast(i) * 4; + w[i] = (static_cast(bytes[p]) << 24) | + (static_cast(bytes[p + 1]) << 16) | + (static_cast(bytes[p + 2]) << 8) | bytes[p + 3]; + } + for (int i = 16; i < 64; ++i) { + const auto s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + const auto s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + auto a = state[0], b = state[1], c = state[2], d = state[3]; + auto e = state[4], f = state[5], g = state[6], h = state[7]; + for (int i = 0; i < 64; ++i) { + const auto s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + const auto choose = (e & f) ^ (~e & g); + const auto temp1 = h + s1 + choose + sha_k[i] + w[i]; + const auto s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + const auto majority = (a & b) ^ (a & c) ^ (b & c); + const auto temp2 = s0 + majority; + h = g; g = f; f = e; e = d + temp1; + d = c; c = b; b = a; a = temp1 + temp2; + } + state[0] += a; state[1] += b; state[2] += c; state[3] += d; + state[4] += e; state[5] += f; state[6] += g; state[7] += h; + } + std::array result{}; + for (std::size_t i = 0; i < state.size(); ++i) { + result[i * 4] = static_cast(state[i] >> 24); + result[i * 4 + 1] = static_cast(state[i] >> 16); + result[i * 4 + 2] = static_cast(state[i] >> 8); + result[i * 4 + 3] = static_cast(state[i]); + } + return result; +} + +bool valid_utf8(std::string_view value) noexcept { + std::size_t index{}; + std::uint32_t cp{}; + while (index < value.size()) if (!next_utf8(value, index, cp)) return false; + return true; +} + +std::size_t utf16_length(std::string_view value) noexcept { + std::size_t index{}, count{}; + std::uint32_t cp{}; + while (index < value.size()) { + if (!next_utf8(value, index, cp)) return std::numeric_limits::max(); + count += cp > 0xFFFF ? 2 : 1; + } + return count; +} + +bool utf8_whitespace_only(std::string_view value) noexcept { + if (value.empty()) return true; + std::size_t index{}; + std::uint32_t cp{}; + while (index < value.size()) { + if (!next_utf8(value, index, cp)) return false; + const bool whitespace = (cp >= 0x09 && cp <= 0x0D) || cp == 0x20 || cp == 0x85 || + cp == 0xA0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || + cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000; + if (!whitespace) return false; + } + return true; +} + +bool make_resource_name(std::string_view name, ResourceName& result) noexcept { + if (!valid_utf8(name)) return false; + std::string readable; + readable.reserve(name.size()); + std::size_t index{}; + std::uint32_t cp{}; + while (index < name.size()) { + next_utf8(name, index, cp); + if (cp < 128 && ((cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z') || + (cp >= '0' && cp <= '9') || cp == '-' || cp == '_' || cp == '.')) { + readable.push_back(static_cast(cp)); + } else { + readable.push_back('_'); + if (cp > 0xFFFF) readable.push_back('_'); + } + } + const auto first = readable.find_first_not_of("_."); + if (first == std::string::npos) { + readable = "store"; + } else { + const auto last = readable.find_last_not_of("_."); + readable = readable.substr(first, last - first + 1); + } + if (readable.size() > 80) readable.resize(80); + const auto digest = sha256(std::span( + reinterpret_cast(name.data()), name.size())); + const auto fragment = "sms-" + readable + "-" + lowercase_hex(std::span(digest.data(), 8)); + + std::filesystem::path root; + std::error_code error; + if (std::filesystem::is_directory(std::filesystem::path("/dev/shm"), error)) { + root = "/dev/shm"; + } else { + root = std::filesystem::temp_directory_path(error); + if (error) root = std::filesystem::path("/tmp"); + } + const auto directory = root / "SharedMemoryStore"; + result.public_name = std::string(name); + result.fragment = fragment; + result.linux_region_path = (directory / (fragment + ".region")).string(); + result.linux_lock_path = (directory / (fragment + ".lock")).string(); + result.linux_owners_path = (directory / (fragment + ".owners")).string(); + result.linux_lifecycle_path = (directory / (fragment + ".lifecycle")).string(); +#if defined(_WIN32) + if (!utf8_to_wide(name, result.windows_region_name)) return false; + std::wstring sanitized; + sanitized.reserve(result.windows_region_name.size()); + for (const auto ch : result.windows_region_name) { + WORD type{}; + const bool alpha_numeric = GetStringTypeW(CT_CTYPE1, &ch, 1, &type) != 0 && + (type & (C1_ALPHA | C1_DIGIT)) != 0; + sanitized.push_back(alpha_numeric || ch == L'-' || ch == L'_' ? ch : L'_'); + } + result.windows_lock_name = (starts_global(result.windows_region_name) ? L"Global\\" : L"Local\\") + + std::wstring(L"SharedMemoryStore-") + sanitized; +#endif + return true; +} + +std::int32_t current_process_id() noexcept { +#if defined(_WIN32) + return static_cast(GetCurrentProcessId()); +#else + return static_cast(getpid()); +#endif +} + +} // namespace sms::detail diff --git a/src/cpp/src/store.cpp b/src/cpp/src/store.cpp new file mode 100644 index 0000000..41f9de3 --- /dev/null +++ b/src/cpp/src/store.cpp @@ -0,0 +1,910 @@ +#include "internal.hpp" + +#include +#include +#include + +namespace sms::detail { +namespace { + +sms_open_status to_open_status(sms_status status) noexcept { + switch (status) { + case SMS_STATUS_SUCCESS: return SMS_OPEN_SUCCESS; + case SMS_STATUS_STORE_BUSY: return SMS_OPEN_STORE_BUSY; + case SMS_STATUS_OPERATION_CANCELED: return SMS_OPEN_OPERATION_CANCELED; + case SMS_STATUS_ACCESS_DENIED: return SMS_OPEN_ACCESS_DENIED; + case SMS_STATUS_UNSUPPORTED_PLATFORM: return SMS_OPEN_UNSUPPORTED_PLATFORM; + default: return SMS_OPEN_MAPPING_FAILED; + } +} + +Wait remaining_wait(const Wait& wait, std::chrono::steady_clock::time_point started) noexcept { + if (wait.infinite()) return wait; + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - started).count(); + return Wait{std::max(0, wait.milliseconds - elapsed)}; +} + +} // namespace + +Store::Store(std::unique_ptr region, std::unique_ptr lock, + Layout layout, bool recovery_enabled) noexcept + : region_(std::move(region)), lock_(std::move(lock)), layout_(layout), recovery_enabled_(recovery_enabled) { + local_diagnostics_.total_bytes = layout.total_bytes; + local_diagnostics_.slot_count = layout.slot_count; + local_diagnostics_.index_entries = layout.index_entry_count; +} + +Store::~Store() { close(); } + +sms_open_status Store::open(const Options& options, const Wait& wait, std::shared_ptr& result) noexcept { + result.reset(); + try { + if (!wait.valid() || utf8_whitespace_only(options.name) || options.name.find('\0') != std::string::npos || + !valid_utf8(options.name) || utf16_length(options.name) > 240 || + options.open_mode < SMS_OPEN_MODE_CREATE_NEW || options.open_mode > SMS_OPEN_MODE_CREATE_OR_OPEN || + options.total_bytes <= 0) { + return SMS_OPEN_INVALID_OPTIONS; + } + Layout layout{}; + if (!Layout::calculate(options.total_bytes, options.slot_count, options.max_value_bytes, + options.max_descriptor_bytes, options.max_key_bytes, + options.lease_record_count, layout)) { + return SMS_OPEN_INVALID_OPTIONS; + } + if (options.total_bytes < layout.required_bytes) return SMS_OPEN_INSUFFICIENT_CAPACITY; + ResourceName resource{}; + if (!make_resource_name(options.name, resource)) return SMS_OPEN_INVALID_OPTIONS; + + const auto started = std::chrono::steady_clock::now(); + auto platform = platform_open(resource, options, wait); + if (platform.status != SMS_OPEN_SUCCESS || !platform.region || !platform.lock) return platform.status; + auto candidate = std::shared_ptr(new Store(std::move(platform.region), std::move(platform.lock), + layout, options.enable_lease_recovery)); + const auto left = remaining_wait(wait, started); + sms_open_status initialize{}; + { + Guard guard(*candidate, left); + if (!guard.acquired()) { + return to_open_status(guard.status()); + } + initialize = candidate->initialize_or_validate(options); + } + if (initialize != SMS_OPEN_SUCCESS) { + candidate->close(); + return initialize; + } + result = std::move(candidate); + return SMS_OPEN_SUCCESS; + } catch (const std::bad_alloc&) { + return SMS_OPEN_MAPPING_FAILED; + } catch (...) { + return SMS_OPEN_MAPPING_FAILED; + } +} + +Store::Guard::Guard(Store& store, const Wait& wait) noexcept : store_(store) { + if (!wait.valid()) { status_ = SMS_STATUS_UNKNOWN_FAILURE; return; } + if (store_.closed_.load(std::memory_order_acquire)) { status_ = SMS_STATUS_STORE_DISPOSED; return; } + const auto started = std::chrono::steady_clock::now(); + if (wait.infinite()) { + store_.gate_.lock(); + local_acquired_ = true; + } else if (wait.milliseconds == 0) { + local_acquired_ = store_.gate_.try_lock(); + } else { + local_acquired_ = store_.gate_.try_lock_for(std::chrono::milliseconds(wait.milliseconds)); + } + if (!local_acquired_) { status_ = SMS_STATUS_STORE_BUSY; return; } + if (store_.closed_.load(std::memory_order_acquire) || !store_.lock_) { + store_.gate_.unlock(); + local_acquired_ = false; + status_ = SMS_STATUS_STORE_DISPOSED; + return; + } + status_ = store_.lock_->acquire(remaining_wait(wait, started)); + if (status_ != SMS_STATUS_SUCCESS) { + store_.gate_.unlock(); + local_acquired_ = false; + return; + } + acquired_ = true; +} + +Store::Guard::~Guard() { + if (acquired_ && store_.lock_) store_.lock_->release(); + if (local_acquired_) store_.gate_.unlock(); +} + +sms_open_status Store::initialize_or_validate(const Options& options) noexcept { + auto& h = header(); + if (options.open_mode == SMS_OPEN_MODE_CREATE_NEW || h.Magic == 0) { + if (options.open_mode == SMS_OPEN_MODE_OPEN_EXISTING) return SMS_OPEN_INCOMPATIBLE_LAYOUT; + initialize_header(); + return SMS_OPEN_SUCCESS; + } + if (h.Magic != magic || h.LayoutMajorVersion != SMS_LAYOUT_MAJOR_VERSION || + !layout_.matches(h) || !layout_.bounds_valid(h)) { + return SMS_OPEN_INCOMPATIBLE_LAYOUT; + } + return load_acquire(h.StoreState) == store_unsupported ? SMS_OPEN_UNSUPPORTED_PLATFORM : SMS_OPEN_SUCCESS; +} + +void Store::initialize_header() noexcept { + std::memset(region_->data(), 0, static_cast(layout_.required_bytes)); + auto& h = header(); + h.Magic = magic; + h.LayoutMajorVersion = SMS_LAYOUT_MAJOR_VERSION; + h.LayoutMinorVersion = SMS_LAYOUT_MINOR_VERSION; + h.HeaderLength = layout_.header_length; + h.TotalBytes = layout_.total_bytes; + h.SlotCount = layout_.slot_count; + h.LeaseRecordCount = layout_.lease_record_count; + h.MaxKeyBytes = layout_.max_key_bytes; + h.MaxDescriptorBytes = layout_.max_descriptor_bytes; + h.MaxValueBytes = layout_.max_value_bytes; + h.IndexEntryCount = layout_.index_entry_count; + h.IndexEntrySize = layout_.index_entry_size; + h.IndexOffset = layout_.index_offset; + h.IndexLength = layout_.index_length; + h.LeaseRegistryOffset = layout_.lease_registry_offset; + h.LeaseRegistryLength = layout_.lease_registry_length; + h.SlotMetadataOffset = layout_.slot_metadata_offset; + h.SlotMetadataLength = layout_.slot_metadata_length; + h.DescriptorStorageOffset = layout_.descriptor_storage_offset; + h.DescriptorStorageLength = layout_.descriptor_storage_length; + h.PayloadStorageOffset = layout_.payload_storage_offset; + h.PayloadStorageLength = layout_.payload_storage_length; + h.StoreId = static_cast(std::chrono::system_clock::now().time_since_epoch().count()) ^ current_process_id(); + h.Sequence = 0; + + for (std::int32_t index = 0; index < layout_.slot_count; ++index) { + auto& value = slot(index); + value.State = slot_free; + value.Generation = 1; + value.ReuseEpoch = 0; + value.DescriptorOffset = layout_.descriptor_storage_offset + static_cast(index) * layout_.descriptor_stride; + value.PayloadOffset = layout_.payload_storage_offset + static_cast(index) * layout_.payload_stride; + } + for (std::int32_t index = 0; index < layout_.lease_record_count; ++index) { + auto& value = lease(index); + value.State = lease_free; + value.LeaseRecordId = index; + value.SlotIndex = -1; + } + store_release(h.StoreState, store_ready); +} + +sms_status Store::ensure_ready() const noexcept { + if (closed_.load(std::memory_order_acquire) || !region_) return SMS_STATUS_STORE_DISPOSED; + const auto& h = *reinterpret_cast(region_->data()); + switch (load_acquire(const_cast(h.StoreState))) { + case store_ready: return SMS_STATUS_SUCCESS; + case store_unsupported: return SMS_STATUS_UNSUPPORTED_PLATFORM; + case store_corrupt: return SMS_STATUS_CORRUPT_STORE; + default: return SMS_STATUS_UNKNOWN_FAILURE; + } +} + +sms_status Store::validate_key(std::span key) const noexcept { + if (key.empty()) return SMS_STATUS_INVALID_KEY; + return key.size() > static_cast(layout_.max_key_bytes) + ? SMS_STATUS_KEY_TOO_LARGE : SMS_STATUS_SUCCESS; +} + +sms_status Store::validate_value(std::span key, std::size_t value_length, + std::size_t descriptor_length, bool) const noexcept { + const auto key_status = validate_key(key); + if (key_status != SMS_STATUS_SUCCESS) return key_status; + if (value_length > static_cast(layout_.max_value_bytes)) return SMS_STATUS_VALUE_TOO_LARGE; + if (descriptor_length > static_cast(layout_.max_descriptor_bytes)) return SMS_STATUS_DESCRIPTOR_TOO_LARGE; + return SMS_STATUS_SUCCESS; +} + +sms_status Store::record(sms_status status) noexcept { + if (status == SMS_STATUS_SUCCESS) return status; + if (status == SMS_STATUS_CORRUPT_STORE && region_) store_release(header().StoreState, store_corrupt); + std::lock_guard guard(diagnostics_gate_); + const auto index = static_cast(status); + if (index >= 0 && index < static_cast(local_diagnostics_.failures.size())) { + ++local_diagnostics_.failures[static_cast(index)]; + } + local_diagnostics_.last_failure = status; + if (status == SMS_STATUS_STORE_FULL || status == SMS_STATUS_LEASE_TABLE_FULL) ++local_diagnostics_.capacity_pressure; + return status; +} + +StoreHeader& Store::header() noexcept { return *reinterpret_cast(region_->data()); } +IndexEntryHeader& Store::index_entry(std::int32_t index) noexcept { + return *reinterpret_cast(region_->data() + layout_.index_offset + + static_cast(index) * layout_.index_entry_size); +} +std::uint8_t* Store::index_key(std::int32_t index) noexcept { + return region_->data() + layout_.index_offset + static_cast(index) * layout_.index_entry_size + sizeof(IndexEntryHeader); +} +SlotMetadata& Store::slot(std::int32_t index) noexcept { + return *reinterpret_cast(region_->data() + layout_.slot_metadata_offset + + static_cast(index) * sizeof(SlotMetadata)); +} +LeaseRecord& Store::lease(std::int32_t index) noexcept { + return *reinterpret_cast(region_->data() + layout_.lease_registry_offset + + static_cast(index) * sizeof(LeaseRecord)); +} + +void Store::record_probe(std::int32_t probes) noexcept { + last_probe_.store(probes, std::memory_order_release); + auto current = max_probe_.load(std::memory_order_acquire); + while (probes > current && !max_probe_.compare_exchange_weak(current, probes, std::memory_order_acq_rel)) {} +} + +bool Store::index_find(std::span key, std::uint64_t hash, + std::int32_t& slot_index, LifecycleId& lifecycle) noexcept { + slot_index = -1; + lifecycle = {}; + const auto mask = layout_.index_entry_count - 1; + const auto start = static_cast(hash & static_cast(mask)); + std::int32_t probes{}; + for (std::int32_t step = 0; step < layout_.index_entry_count; ++step) { + ++probes; + const auto index = (start + step) & mask; + auto& entry = index_entry(index); + const auto state = load_acquire(entry.State); + if (state == index_empty) { record_probe(probes); return false; } + if (state == index_occupied && entry.KeyHash == hash && entry.KeyLength >= 0 && + entry.KeyLength <= layout_.max_key_bytes && static_cast(entry.KeyLength) == key.size() && + std::memcmp(index_key(index), key.data(), key.size()) == 0) { + slot_index = entry.SlotIndex; + lifecycle = {entry.SlotGeneration, entry.SlotReuseEpoch}; + record_probe(probes); + return true; + } + } + record_probe(probes); + return false; +} + +void Store::write_index(std::int32_t index, std::span key, std::uint64_t hash, + std::int32_t slot_index, LifecycleId lifecycle) noexcept { + auto& entry = index_entry(index); + store_release(entry.State, index_tombstone); + entry.KeyLength = static_cast(key.size()); + entry.KeyHash = hash; + entry.SlotIndex = slot_index; + entry.SlotGeneration = lifecycle.generation; + entry.SlotReuseEpoch = lifecycle.reuse_epoch; + std::memset(index_key(index), 0, static_cast(layout_.max_key_bytes)); + if (!key.empty()) std::memcpy(index_key(index), key.data(), key.size()); + store_release(entry.State, index_occupied); +} + +bool Store::index_insert(std::span key, std::uint64_t hash, + std::int32_t slot_index, LifecycleId lifecycle) noexcept { + const auto mask = layout_.index_entry_count - 1; + const auto start = static_cast(hash & static_cast(mask)); + std::int32_t tombstone = -1, probes{}; + for (std::int32_t step = 0; step < layout_.index_entry_count; ++step) { + ++probes; + const auto index = (start + step) & mask; + auto& entry = index_entry(index); + const auto state = load_acquire(entry.State); + if (state == index_occupied) { + if (entry.KeyHash == hash && entry.KeyLength >= 0 && + static_cast(entry.KeyLength) == key.size() && + std::memcmp(index_key(index), key.data(), key.size()) == 0) { + record_probe(probes); return false; + } + } else if (state == index_tombstone) { + if (tombstone < 0) tombstone = index; + } else { + write_index(tombstone >= 0 ? tombstone : index, key, hash, slot_index, lifecycle); + record_probe(probes); return true; + } + } + if (tombstone >= 0) { + write_index(tombstone, key, hash, slot_index, lifecycle); + record_probe(probes); return true; + } + record_probe(probes); + return false; +} + +bool Store::index_remove_slot(std::int32_t slot_index, LifecycleId lifecycle, std::uint64_t hash) noexcept { + const auto mask = layout_.index_entry_count - 1; + const auto start = static_cast(hash & static_cast(mask)); + bool removed{}; + std::int32_t probes{}; + for (std::int32_t step = 0; step < layout_.index_entry_count; ++step) { + ++probes; + const auto index = (start + step) & mask; + auto& entry = index_entry(index); + const auto state = load_acquire(entry.State); + if (state == index_empty) { record_probe(probes); return removed; } + if (state == index_occupied && entry.KeyHash == hash && entry.SlotIndex == slot_index && + lifecycle.matches(entry.SlotGeneration, entry.SlotReuseEpoch)) { + store_release(entry.State, index_tombstone); + removed = true; + } + } + record_probe(probes); + return removed; +} + +bool Store::reserve_slot(std::int32_t& slot_index) noexcept { + const auto start = ++next_slot_; + for (std::int32_t step = 0; step < layout_.slot_count; ++step) { + const auto candidate = static_cast((start + static_cast(step)) % + static_cast(layout_.slot_count)); + auto& value = slot(candidate); + if (load_acquire(value.State) != slot_free) continue; + store_release(value.State, slot_publishing); + value.UsageCount = 0; + value.PublisherProcessId = current_process_id(); + value.Reserved = 0; + value.KeyHash = 0; + value.KeyLength = 0; + value.DescriptorLength = 0; + value.ValueLength = 0; + value.CommittedSequence = 0; + slot_index = candidate; + return true; + } + slot_index = -1; + return false; +} + +void Store::abort_slot(std::int32_t index) noexcept { + auto& value = slot(index); + value.KeyHash = 0; + value.KeyLength = 0; + value.ValueLength = 0; + value.DescriptorLength = 0; + value.UsageCount = 0; + value.PublisherProcessId = 0; + value.Reserved = 0; + value.CommittedSequence = 0; + store_release(value.State, slot_free); +} + +bool Store::activate_lease(std::int32_t slot_index, LifecycleId lifecycle, + std::int64_t sequence, std::int32_t& lease_id) noexcept { + const auto start = ++next_lease_; + for (std::int32_t step = 0; step < layout_.lease_record_count; ++step) { + const auto candidate = static_cast((start + static_cast(step)) % + static_cast(layout_.lease_record_count)); + auto& record = lease(candidate); + if (load_acquire(record.State) == lease_active) continue; + record.LeaseRecordId = candidate; + record.SlotIndex = slot_index; + record.SlotGeneration = lifecycle.generation; + record.SlotReuseEpoch = lifecycle.reuse_epoch; + record.OwnerProcessId = current_process_id(); + record.AcquireSequence = sequence; + store_release(record.State, lease_active); + lease_id = candidate; + return true; + } + lease_id = -1; + return false; +} + +sms_status Store::publish(std::span key, std::span value, + std::span descriptor, const Wait& wait) noexcept { + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); + if (status != SMS_STATUS_SUCCESS) return record(status); + status = validate_value(key, value.size(), descriptor.size(), false); + if (status != SMS_STATUS_SUCCESS) return record(status); + const auto hash = hash_key(key); + std::int32_t existing{}; LifecycleId ignored{}; + if (index_find(key, hash, existing, ignored) && existing >= 0 && existing < layout_.slot_count) { + const auto state = load_acquire(slot(existing).State); + if (state == slot_published || state == slot_publishing || state == slot_remove_requested) + return record(SMS_STATUS_DUPLICATE_KEY); + } + std::int32_t index{}; + if (!reserve_slot(index)) return record(SMS_STATUS_STORE_FULL); + auto& target = slot(index); + const LifecycleId lifecycle{target.Generation, target.ReuseEpoch}; + if (!descriptor.empty()) std::memcpy(region_->data() + target.DescriptorOffset, descriptor.data(), descriptor.size()); + if (!value.empty()) std::memcpy(region_->data() + target.PayloadOffset, value.data(), value.size()); + if (!index_insert(key, hash, index, lifecycle)) { + abort_slot(index); + return record(SMS_STATUS_DUPLICATE_KEY); + } + target.KeyHash = hash; + target.KeyLength = static_cast(key.size()); + target.DescriptorLength = static_cast(descriptor.size()); + target.ValueLength = static_cast(value.size()); + target.PublisherProcessId = current_process_id(); + target.Reserved = 0; + target.CommittedSequence = increment(header().Sequence); + store_release(target.State, slot_published); + return SMS_STATUS_SUCCESS; +} + +sms_status Store::publish_segments(std::span key, std::span segments, + std::span descriptor, const Wait& wait, + std::int64_t& copied) noexcept { + copied = 0; + std::size_t total{}; + for (const auto& segment : segments) { + if (segment.length > 0 && !segment.data) return record(SMS_STATUS_UNKNOWN_FAILURE); + if (segment.length > std::numeric_limits::max() - total) return record(SMS_STATUS_VALUE_TOO_LARGE); + total += segment.length; + } + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); + if (status != SMS_STATUS_SUCCESS) return record(status); + status = validate_value(key, total, descriptor.size(), false); + if (status != SMS_STATUS_SUCCESS) return record(status); + const auto hash = hash_key(key); + std::int32_t existing{}; LifecycleId ignored{}; + if (index_find(key, hash, existing, ignored) && existing >= 0 && existing < layout_.slot_count) { + const auto state = load_acquire(slot(existing).State); + if (state == slot_published || state == slot_publishing || state == slot_remove_requested) + return record(SMS_STATUS_DUPLICATE_KEY); + } + std::int32_t index{}; + if (!reserve_slot(index)) return record(SMS_STATUS_STORE_FULL); + auto& target = slot(index); + const LifecycleId lifecycle{target.Generation, target.ReuseEpoch}; + if (!descriptor.empty()) std::memcpy(region_->data() + target.DescriptorOffset, descriptor.data(), descriptor.size()); + auto* output = region_->data() + target.PayloadOffset; + for (const auto& segment : segments) { + if (segment.length) std::memcpy(output + copied, segment.data, segment.length); + copied += static_cast(segment.length); + } + if (!index_insert(key, hash, index, lifecycle)) { abort_slot(index); return record(SMS_STATUS_DUPLICATE_KEY); } + target.KeyHash = hash; + target.KeyLength = static_cast(key.size()); + target.DescriptorLength = static_cast(descriptor.size()); + target.ValueLength = static_cast(total); + target.PublisherProcessId = current_process_id(); + target.Reserved = 0; + target.CommittedSequence = increment(header().Sequence); + store_release(target.State, slot_published); + return SMS_STATUS_SUCCESS; +} + +sms_status Store::acquire(std::span key, const Wait& wait, + std::int32_t& slot_index, LifecycleId& lifecycle, std::int32_t& lease_id) noexcept { + slot_index = -1; lifecycle = {}; lease_id = -1; + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); + if (status != SMS_STATUS_SUCCESS) return record(status); + status = validate_key(key); + if (status != SMS_STATUS_SUCCESS) return record(status); + const auto hash = hash_key(key); + if (!index_find(key, hash, slot_index, lifecycle) || slot_index < 0 || slot_index >= layout_.slot_count) + return record(SMS_STATUS_NOT_FOUND); + auto& target = slot(slot_index); + if (load_acquire(target.State) != slot_published || !lifecycle.matches(target.Generation, target.ReuseEpoch)) + return record(SMS_STATUS_NOT_FOUND); + const auto sequence = increment(header().Sequence); + if (!activate_lease(slot_index, lifecycle, sequence, lease_id)) return record(SMS_STATUS_LEASE_TABLE_FULL); + increment(target.UsageCount); + if (load_acquire(target.State) != slot_published || !lifecycle.matches(target.Generation, target.ReuseEpoch)) { + auto& activated = lease(lease_id); + store_release(activated.State, lease_released); + decrement(target.UsageCount); + return record(SMS_STATUS_NOT_FOUND); + } + return SMS_STATUS_SUCCESS; +} + +bool Store::lease_valid(std::int32_t slot_index, LifecycleId lifecycle, std::int32_t lease_id) noexcept { + Guard guard(*this, Wait{1000}); + if (!guard.acquired() || lease_id < 0 || lease_id >= layout_.lease_record_count || + slot_index < 0 || slot_index >= layout_.slot_count) return false; + auto& record = lease(lease_id); + auto& target = slot(slot_index); + return load_acquire(record.State) == lease_active && record.SlotIndex == slot_index && + lifecycle.matches(record.SlotGeneration, record.SlotReuseEpoch) && + lifecycle.matches(target.Generation, target.ReuseEpoch) && + (load_acquire(target.State) == slot_published || load_acquire(target.State) == slot_remove_requested); +} + +std::span Store::lease_value(std::int32_t slot_index, LifecycleId lifecycle, + std::int32_t lease_id) noexcept { + Guard guard(*this, Wait{1000}); + if (!guard.acquired() || lease_id < 0 || lease_id >= layout_.lease_record_count || + slot_index < 0 || slot_index >= layout_.slot_count) return {}; + auto& record = lease(lease_id); auto& target = slot(slot_index); + const auto state = load_acquire(target.State); + if (load_acquire(record.State) != lease_active || record.SlotIndex != slot_index || + !lifecycle.matches(record.SlotGeneration, record.SlotReuseEpoch) || + !lifecycle.matches(target.Generation, target.ReuseEpoch) || + (state != slot_published && state != slot_remove_requested) || target.ValueLength < 0) return {}; + return {region_->data() + target.PayloadOffset, static_cast(target.ValueLength)}; +} + +std::span Store::lease_descriptor(std::int32_t slot_index, LifecycleId lifecycle, + std::int32_t lease_id) noexcept { + Guard guard(*this, Wait{1000}); + if (!guard.acquired() || lease_id < 0 || lease_id >= layout_.lease_record_count || + slot_index < 0 || slot_index >= layout_.slot_count) return {}; + auto& record = lease(lease_id); auto& target = slot(slot_index); + const auto state = load_acquire(target.State); + if (load_acquire(record.State) != lease_active || record.SlotIndex != slot_index || + !lifecycle.matches(record.SlotGeneration, record.SlotReuseEpoch) || + !lifecycle.matches(target.Generation, target.ReuseEpoch) || + (state != slot_published && state != slot_remove_requested) || target.DescriptorLength < 0) return {}; + return {region_->data() + target.DescriptorOffset, static_cast(target.DescriptorLength)}; +} + +sms_status Store::release_lease(std::int32_t slot_index, LifecycleId lifecycle, + std::int32_t lease_id, const Wait& wait) noexcept { + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + if (lease_id < 0 || lease_id >= layout_.lease_record_count) return record(SMS_STATUS_INVALID_LEASE); + auto& record_value = lease(lease_id); + const auto state = load_acquire(record_value.State); + if (state == lease_released || state == lease_abandoned) return record(SMS_STATUS_LEASE_ALREADY_RELEASED); + if (state != lease_active || record_value.SlotIndex != slot_index || + !lifecycle.matches(record_value.SlotGeneration, record_value.SlotReuseEpoch) || + slot_index < 0 || slot_index >= layout_.slot_count) return record(SMS_STATUS_INVALID_LEASE); + auto& target = slot(slot_index); + if (!lifecycle.matches(target.Generation, target.ReuseEpoch)) return record(SMS_STATUS_INVALID_LEASE); + store_release(record_value.State, lease_released); + const auto remaining = decrement(target.UsageCount); + if (remaining < 0) { store_release(target.State, slot_free); return record(SMS_STATUS_CORRUPT_STORE); } + const auto result = remaining == 0 ? reclaim_after_release(slot_index, lifecycle) : SMS_STATUS_SUCCESS; + if (result == SMS_STATUS_SUCCESS) maybe_compact_index(); + return result == SMS_STATUS_SUCCESS ? result : record(result); +} + +sms_status Store::reclaim_after_release(std::int32_t slot_index, LifecycleId lifecycle) noexcept { + auto& target = slot(slot_index); + if (!lifecycle.matches(target.Generation, target.ReuseEpoch)) return SMS_STATUS_INVALID_LEASE; + if (load_acquire(target.State) == slot_remove_requested && load_acquire(target.UsageCount) == 0) { + if (!index_remove_slot(slot_index, lifecycle, target.KeyHash)) return SMS_STATUS_CORRUPT_STORE; + return reclaim(slot_index); + } + return SMS_STATUS_SUCCESS; +} + +sms_status Store::reclaim(std::int32_t slot_index) noexcept { + auto& target = slot(slot_index); + store_release(target.State, slot_reclaiming); + LifecycleId next{}; + if (!LifecycleId{target.Generation, target.ReuseEpoch}.advance(next)) return SMS_STATUS_CORRUPT_STORE; + target.KeyHash = 0; target.KeyLength = 0; target.ValueLength = 0; target.DescriptorLength = 0; + target.PublisherProcessId = 0; target.UsageCount = 0; target.Reserved = 0; target.CommittedSequence = 0; + target.Generation = next.generation; target.ReuseEpoch = next.reuse_epoch; + store_release(target.State, slot_free); + return SMS_STATUS_SUCCESS; +} + +sms_status Store::request_remove(std::int32_t slot_index, LifecycleId lifecycle) noexcept { + auto& target = slot(slot_index); + const auto state = load_acquire(target.State); + if (state == slot_remove_requested) return SMS_STATUS_REMOVE_PENDING; + if (state != slot_published || !lifecycle.matches(target.Generation, target.ReuseEpoch)) return SMS_STATUS_NOT_FOUND; + if (load_acquire(target.UsageCount) > 0) { + store_release(target.State, slot_remove_requested); + return SMS_STATUS_REMOVE_PENDING; + } + if (!index_remove_slot(slot_index, lifecycle, target.KeyHash)) return SMS_STATUS_CORRUPT_STORE; + return reclaim(slot_index); +} + +sms_status Store::remove(std::span key, const Wait& wait) noexcept { + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); + if (status != SMS_STATUS_SUCCESS) return record(status); + status = validate_key(key); + if (status != SMS_STATUS_SUCCESS) return record(status); + std::int32_t index{}; LifecycleId lifecycle{}; + if (!index_find(key, hash_key(key), index, lifecycle) || index < 0 || index >= layout_.slot_count) + return record(SMS_STATUS_NOT_FOUND); + status = request_remove(index, lifecycle); + if (status == SMS_STATUS_SUCCESS) maybe_compact_index(); + return status == SMS_STATUS_SUCCESS ? status : record(status); +} + +sms_status Store::reserve(std::span key, std::int32_t payload_length, + std::span descriptor, const Wait& wait, + std::int32_t& slot_index, LifecycleId& lifecycle) noexcept { + slot_index = -1; lifecycle = {}; + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); + if (status != SMS_STATUS_SUCCESS) return record(status); + if (payload_length < 0) return record(SMS_STATUS_VALUE_TOO_LARGE); + status = validate_value(key, static_cast(payload_length), descriptor.size(), true); + if (status != SMS_STATUS_SUCCESS) return record(status); + const auto hash = hash_key(key); + std::int32_t existing{}; LifecycleId ignored{}; + if (index_find(key, hash, existing, ignored) && existing >= 0 && existing < layout_.slot_count) { + const auto state = load_acquire(slot(existing).State); + if (state == slot_published || state == slot_publishing || state == slot_remove_requested) + return record(SMS_STATUS_DUPLICATE_KEY); + } + if (!reserve_slot(slot_index)) return record(SMS_STATUS_STORE_FULL); + auto& target = slot(slot_index); + lifecycle = {target.Generation, target.ReuseEpoch}; + target.KeyHash = hash; + target.KeyLength = static_cast(key.size()); + target.DescriptorLength = static_cast(descriptor.size()); + target.ValueLength = payload_length; + target.PublisherProcessId = current_process_id(); + target.Reserved = 0; + target.CommittedSequence = 0; + if (!descriptor.empty()) std::memcpy(region_->data() + target.DescriptorOffset, descriptor.data(), descriptor.size()); + if (!index_insert(key, hash, slot_index, lifecycle)) { + abort_slot(slot_index); slot_index = -1; lifecycle = {}; + return record(SMS_STATUS_DUPLICATE_KEY); + } + return SMS_STATUS_SUCCESS; +} + +bool Store::reservation_valid(std::int32_t index, LifecycleId lifecycle) noexcept { + Guard guard(*this, Wait{1000}); + if (!guard.acquired() || index < 0 || index >= layout_.slot_count) return false; + auto& target = slot(index); + return load_acquire(target.State) == slot_publishing && lifecycle.matches(target.Generation, target.ReuseEpoch); +} + +std::int32_t Store::reservation_payload_length(std::int32_t index, LifecycleId lifecycle) noexcept { + Guard guard(*this, Wait{1000}); + if (!guard.acquired() || index < 0 || index >= layout_.slot_count) return 0; + auto& target = slot(index); + return load_acquire(target.State) == slot_publishing && lifecycle.matches(target.Generation, target.ReuseEpoch) + ? target.ValueLength : 0; +} + +std::int32_t Store::reservation_bytes_written(std::int32_t index, LifecycleId lifecycle) noexcept { + Guard guard(*this, Wait{1000}); + if (!guard.acquired() || index < 0 || index >= layout_.slot_count) return 0; + auto& target = slot(index); + return load_acquire(target.State) == slot_publishing && lifecycle.matches(target.Generation, target.ReuseEpoch) + ? target.Reserved : 0; +} + +std::span Store::reservation_buffer(std::int32_t index, LifecycleId lifecycle, + std::int32_t size_hint) noexcept { + Guard guard(*this, Wait{1000}); + if (!guard.acquired() || index < 0 || index >= layout_.slot_count) return {}; + auto& target = slot(index); + if (load_acquire(target.State) != slot_publishing || !lifecycle.matches(target.Generation, target.ReuseEpoch)) return {}; + const auto remaining = target.ValueLength - target.Reserved; + if (remaining <= 0 || size_hint < 0 || size_hint > remaining) return {}; + return {region_->data() + target.PayloadOffset + target.Reserved, static_cast(remaining)}; +} + +sms_status Store::advance_reservation(std::int32_t index, LifecycleId lifecycle, + std::int32_t count, const Wait& wait) noexcept { + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); if (status != SMS_STATUS_SUCCESS) return record(status); + if (index < 0 || index >= layout_.slot_count) return record(SMS_STATUS_INVALID_RESERVATION); + auto& target = slot(index); + if (!lifecycle.matches(target.Generation, target.ReuseEpoch)) return record(SMS_STATUS_INVALID_RESERVATION); + if (load_acquire(target.State) != slot_publishing) return record(SMS_STATUS_RESERVATION_ALREADY_COMPLETED); + const auto remaining = target.ValueLength - target.Reserved; + if (count < 0 || count > remaining) return record(SMS_STATUS_RESERVATION_WRITE_OUT_OF_RANGE); + target.Reserved += count; + return SMS_STATUS_SUCCESS; +} + +sms_status Store::commit_reservation(std::int32_t index, LifecycleId lifecycle, const Wait& wait) noexcept { + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); if (status != SMS_STATUS_SUCCESS) return record(status); + if (index < 0 || index >= layout_.slot_count) return record(SMS_STATUS_INVALID_RESERVATION); + auto& target = slot(index); + if (!lifecycle.matches(target.Generation, target.ReuseEpoch)) return record(SMS_STATUS_INVALID_RESERVATION); + if (load_acquire(target.State) != slot_publishing) return record(SMS_STATUS_RESERVATION_ALREADY_COMPLETED); + if (target.Reserved != target.ValueLength) return record(SMS_STATUS_RESERVATION_INCOMPLETE); + target.PublisherProcessId = current_process_id(); + target.Reserved = 0; + target.CommittedSequence = increment(header().Sequence); + store_release(target.State, slot_published); + return SMS_STATUS_SUCCESS; +} + +sms_status Store::abort_reservation(std::int32_t index, LifecycleId lifecycle, + bool count_abort, const Wait& wait) noexcept { + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); if (status != SMS_STATUS_SUCCESS) return record(status); + if (index < 0 || index >= layout_.slot_count) return record(SMS_STATUS_INVALID_RESERVATION); + auto& target = slot(index); + if (!lifecycle.matches(target.Generation, target.ReuseEpoch)) return record(SMS_STATUS_INVALID_RESERVATION); + if (load_acquire(target.State) != slot_publishing) return record(SMS_STATUS_RESERVATION_ALREADY_COMPLETED); + if (!index_remove_slot(index, lifecycle, target.KeyHash)) return record(SMS_STATUS_CORRUPT_STORE); + status = reclaim(index); + if (status != SMS_STATUS_SUCCESS) return record(status); + if (count_abort) { std::lock_guard local(diagnostics_gate_); ++local_diagnostics_.aborted_reservations; } + maybe_compact_index(); + return SMS_STATUS_SUCCESS; +} + +sms_status Store::recover_leases(bool recover_current, const Wait& wait, RecoveryReport& report) noexcept { + report = {}; + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); if (status != SMS_STATUS_SUCCESS) return record(status); + if (!recovery_enabled_) { + report.scanned = layout_.lease_record_count; + report.unsupported = layout_.lease_record_count; + return record(SMS_STATUS_UNSUPPORTED_PLATFORM); + } + for (std::int32_t index = 0; index < layout_.lease_record_count; ++index) { + ++report.scanned; + auto& value = lease(index); + if (load_acquire(value.State) != lease_active) continue; + if (value.SlotIndex < 0 || value.SlotIndex >= layout_.slot_count) { ++report.failed; continue; } + const auto owner = classify_process(value.OwnerProcessId); + if (owner == OwnerKind::unsupported) { ++report.unsupported; continue; } + if (owner == OwnerKind::live || (owner == OwnerKind::current && !recover_current)) { ++report.active; continue; } + auto& target = slot(value.SlotIndex); + const LifecycleId lifecycle{value.SlotGeneration, value.SlotReuseEpoch}; + if (!lifecycle.valid() || !lifecycle.matches(target.Generation, target.ReuseEpoch) || + load_acquire(target.UsageCount) <= 0) { ++report.failed; continue; } + store_release(value.State, lease_abandoned); + const auto remaining = decrement(target.UsageCount); + if (remaining == 0 && reclaim_after_release(value.SlotIndex, lifecycle) != SMS_STATUS_SUCCESS) { + ++report.failed; continue; + } + ++report.recovered; + } + if (report.recovered > 0) maybe_compact_index(); + { + std::lock_guard local(diagnostics_gate_); + local_diagnostics_.recovered_leases += report.recovered; + local_diagnostics_.active_lease_recoveries += report.active; + local_diagnostics_.unsupported_lease_recoveries += report.unsupported; + local_diagnostics_.failed_lease_recoveries += report.failed; + } + return SMS_STATUS_SUCCESS; +} + +sms_status Store::recover_reservations(bool recover_current, const Wait& wait, RecoveryReport& report) noexcept { + report = {}; + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + auto status = ensure_ready(); if (status != SMS_STATUS_SUCCESS) return record(status); + for (std::int32_t index = 0; index < layout_.slot_count; ++index) { + auto& target = slot(index); + if (load_acquire(target.State) != slot_publishing) continue; + ++report.scanned; + const auto owner = classify_process(target.PublisherProcessId); + if (owner == OwnerKind::unsupported) { ++report.unsupported; continue; } + if (owner == OwnerKind::live || (owner == OwnerKind::current && !recover_current)) { ++report.active; continue; } + const LifecycleId lifecycle{target.Generation, target.ReuseEpoch}; + if (!index_remove_slot(index, lifecycle, target.KeyHash) || reclaim(index) != SMS_STATUS_SUCCESS) { + ++report.failed; continue; + } + ++report.recovered; + } + if (report.recovered > 0) maybe_compact_index(); + { + std::lock_guard local(diagnostics_gate_); + local_diagnostics_.recovered_reservations += report.recovered; + local_diagnostics_.active_reservation_recoveries += report.active; + local_diagnostics_.unsupported_reservation_recoveries += report.unsupported; + local_diagnostics_.failed_reservation_recoveries += report.failed; + } + return SMS_STATUS_SUCCESS; +} + +bool Store::compact_index() noexcept { + bool compacted{}; + const auto mask = layout_.index_entry_count - 1; + auto clear = [&](std::int32_t index) { + auto& entry = index_entry(index); + store_release(entry.State, index_empty); + entry.KeyLength = 0; entry.KeyHash = 0; entry.SlotIndex = -1; + entry.SlotGeneration = 0; entry.SlotReuseEpoch = 0; + std::memset(index_key(index), 0, static_cast(layout_.max_key_bytes)); + }; + for (std::int32_t pass = 0; pass < layout_.index_entry_count; ++pass) { + bool changed{}; + for (std::int32_t initial = 0; initial < layout_.index_entry_count; ++initial) { + if (load_acquire(index_entry(initial).State) != index_tombstone) continue; + auto hole = initial; + auto scan = (hole + 1) & mask; + bool closed{}; + for (std::int32_t step = 0; step < layout_.index_entry_count; ++step) { + auto& candidate = index_entry(scan); + const auto state = load_acquire(candidate.State); + if (state == index_empty) { + clear(hole); changed = compacted = closed = true; break; + } + if (state == index_occupied && candidate.KeyLength >= 0 && candidate.KeyLength <= layout_.max_key_bytes) { + const auto home = static_cast(candidate.KeyHash & static_cast(mask)); + const auto distance_hole = (hole - home) & mask; + const auto distance_candidate = (scan - home) & mask; + if (distance_hole < distance_candidate) { + const auto key = std::span( + index_key(scan), static_cast(candidate.KeyLength)); + write_index(hole, key, candidate.KeyHash, candidate.SlotIndex, + {candidate.SlotGeneration, candidate.SlotReuseEpoch}); + store_release(candidate.State, index_tombstone); + hole = scan; + } + } + scan = (scan + 1) & mask; + } + (void)closed; + } + if (!changed) break; + } + return compacted; +} + +void Store::maybe_compact_index() noexcept { + std::int32_t tombstones{}, empty{}; + for (std::int32_t index = 0; index < layout_.index_entry_count; ++index) { + switch (load_acquire(index_entry(index).State)) { + case index_occupied: break; + case index_tombstone: ++tombstones; break; + default: ++empty; break; + } + } + if (tombstones == 0) return; + const auto tombstone_pressure = static_cast(tombstones) / layout_.index_entry_count >= 0.35 || empty == 0; + const auto probe_pressure = max_probe_.load(std::memory_order_acquire) >= std::max(1, (layout_.index_entry_count * 3) / 4); + if ((tombstone_pressure || probe_pressure) && compact_index()) { + std::lock_guard local(diagnostics_gate_); + ++local_diagnostics_.index_compactions; + } +} + +sms_status Store::diagnostics(const Wait& wait, Diagnostics& result) noexcept { + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + result = {}; + { + std::lock_guard local(diagnostics_gate_); + result = local_diagnostics_; + } + result.total_bytes = layout_.total_bytes; + result.slot_count = layout_.slot_count; + result.index_entries = layout_.index_entry_count; + for (std::int32_t index = 0; index < layout_.slot_count; ++index) { + switch (load_acquire(slot(index).State)) { + case slot_free: ++result.free_slots; break; + case slot_published: ++result.published_slots; break; + case slot_remove_requested: ++result.pending_removal; break; + case slot_publishing: ++result.active_reservations; break; + } + } + for (std::int32_t index = 0; index < layout_.lease_record_count; ++index) + if (load_acquire(lease(index).State) == lease_active) ++result.active_leases; + for (std::int32_t index = 0; index < layout_.index_entry_count; ++index) { + switch (load_acquire(index_entry(index).State)) { + case index_occupied: ++result.occupied_index_entries; break; + case index_tombstone: ++result.tombstone_index_entries; break; + default: ++result.empty_index_entries; break; + } + } + result.last_probe = last_probe_.load(std::memory_order_acquire); + result.max_probe = max_probe_.load(std::memory_order_acquire); + return SMS_STATUS_SUCCESS; +} + +sms_status Store::get_layout(const Wait& wait, Layout& result) noexcept { + Guard guard(*this, wait); + if (!guard.acquired()) return record(guard.status()); + const auto status = ensure_ready(); + if (status != SMS_STATUS_SUCCESS) return record(status); + result = layout_; + return SMS_STATUS_SUCCESS; +} + +void Store::close() noexcept { + if (closed_.exchange(true, std::memory_order_acq_rel)) return; + gate_.lock(); + if (region_) region_->close(); + region_.reset(); + lock_.reset(); + gate_.unlock(); +} + +} // namespace sms::detail diff --git a/src/python/shared_memory_store/__init__.py b/src/python/shared_memory_store/__init__.py new file mode 100644 index 0000000..7581e4a --- /dev/null +++ b/src/python/shared_memory_store/__init__.py @@ -0,0 +1,45 @@ +"""Interoperable named shared-memory values for Python.""" + +from __future__ import annotations + +from ._native import ( + ABI_VERSION, + LAYOUT_MAJOR_VERSION, + LAYOUT_MINOR_VERSION, + RESOURCE_NAMING_VERSION, + native_library_path, +) +from .enums import OpenMode, StoreOpenStatus, StoreStatus +from .store import ( + DiagnosticsSnapshot, + MemoryStore, + RecoveryReport, + StoreOptions, + ValueLease, + ValueReservation, + WaitOptions, + calculate_required_bytes, +) + + +__version__ = "0.1.0" + +__all__ = [ + "__version__", + "ABI_VERSION", + "LAYOUT_MAJOR_VERSION", + "LAYOUT_MINOR_VERSION", + "RESOURCE_NAMING_VERSION", + "OpenMode", + "StoreOpenStatus", + "StoreStatus", + "WaitOptions", + "StoreOptions", + "RecoveryReport", + "DiagnosticsSnapshot", + "MemoryStore", + "ValueLease", + "ValueReservation", + "calculate_required_bytes", + "native_library_path", +] diff --git a/src/python/shared_memory_store/_native.py b/src/python/shared_memory_store/_native.py new file mode 100644 index 0000000..8798c00 --- /dev/null +++ b/src/python/shared_memory_store/_native.py @@ -0,0 +1,416 @@ +"""Private ``ctypes`` declarations and deterministic native-library loader.""" + +from __future__ import annotations + +import atexit +import ctypes +from contextlib import ExitStack +from importlib.resources import as_file, files +from pathlib import Path +import sys +import threading +from typing import Optional + + +ABI_VERSION = 0x00010000 +LAYOUT_MAJOR_VERSION = 1 +LAYOUT_MINOR_VERSION = 2 +RESOURCE_NAMING_VERSION = 1 +WAIT_INFINITE = -1 +STATUS_COUNT = 23 + +UInt8Pointer = ctypes.POINTER(ctypes.c_uint8) + + +class Bytes(ctypes.Structure): + _fields_ = [("data", UInt8Pointer), ("length", ctypes.c_uint64)] + + +class MutableBytes(ctypes.Structure): + _fields_ = [("data", UInt8Pointer), ("length", ctypes.c_uint64)] + + +class WaitOptions(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("abi_version", ctypes.c_uint32), + ("timeout_milliseconds", ctypes.c_int64), + ] + + +class StoreOptions(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("abi_version", ctypes.c_uint32), + ("name_utf8", ctypes.c_char_p), + ("name_length", ctypes.c_uint64), + ("open_mode", ctypes.c_int32), + ("total_bytes", ctypes.c_int64), + ("slot_count", ctypes.c_int32), + ("max_value_bytes", ctypes.c_int32), + ("max_descriptor_bytes", ctypes.c_int32), + ("max_key_bytes", ctypes.c_int32), + ("lease_record_count", ctypes.c_int32), + ("enable_lease_recovery", ctypes.c_uint8), + ("reserved", ctypes.c_uint8 * 7), + ] + + +class Segment(ctypes.Structure): + _fields_ = [("data", UInt8Pointer), ("length", ctypes.c_uint64)] + + +class RecoveryReport(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("abi_version", ctypes.c_uint32), + ("scanned_count", ctypes.c_int32), + ("recovered_count", ctypes.c_int32), + ("active_count", ctypes.c_int32), + ("unsupported_count", ctypes.c_int32), + ("failed_count", ctypes.c_int32), + ("reserved", ctypes.c_int32), + ] + + +class Diagnostics(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("abi_version", ctypes.c_uint32), + ("total_bytes", ctypes.c_int64), + ("slot_count", ctypes.c_int32), + ("free_slot_count", ctypes.c_int32), + ("published_slot_count", ctypes.c_int32), + ("pending_removal_count", ctypes.c_int32), + ("active_lease_count", ctypes.c_int32), + ("active_reservation_count", ctypes.c_int32), + ("index_entry_count", ctypes.c_int32), + ("occupied_index_entry_count", ctypes.c_int32), + ("tombstone_index_entry_count", ctypes.c_int32), + ("empty_index_entry_count", ctypes.c_int32), + ("usable_index_capacity", ctypes.c_int32), + ("last_observed_probe_length", ctypes.c_int32), + ("max_observed_probe_length", ctypes.c_int32), + ("last_failure_status", ctypes.c_int32), + ("aborted_reservation_count", ctypes.c_int64), + ("recovered_lease_count", ctypes.c_int64), + ("active_lease_recovery_count", ctypes.c_int64), + ("unsupported_lease_recovery_count", ctypes.c_int64), + ("failed_lease_recovery_count", ctypes.c_int64), + ("recovered_reservation_count", ctypes.c_int64), + ("active_reservation_recovery_count", ctypes.c_int64), + ("unsupported_reservation_recovery_count", ctypes.c_int64), + ("failed_reservation_recovery_count", ctypes.c_int64), + ("capacity_pressure_count", ctypes.c_int64), + ("index_compaction_count", ctypes.c_int64), + ("failure_counts", ctypes.c_int64 * STATUS_COUNT), + ] + + +class ProtocolInfo(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("abi_version", ctypes.c_uint32), + ("layout_major", ctypes.c_int32), + ("layout_minor", ctypes.c_int32), + ("resource_naming_version", ctypes.c_int32), + ("store_header_size", ctypes.c_int32), + ("index_entry_header_size", ctypes.c_int32), + ("slot_metadata_size", ctypes.c_int32), + ("lease_record_size", ctypes.c_int32), + ] + + +class StoreLayout(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("abi_version", ctypes.c_uint32), + ("total_bytes", ctypes.c_int64), + ("slot_count", ctypes.c_int32), + ("lease_record_count", ctypes.c_int32), + ("max_value_bytes", ctypes.c_int32), + ("max_descriptor_bytes", ctypes.c_int32), + ("max_key_bytes", ctypes.c_int32), + ("header_length", ctypes.c_int32), + ("index_entry_count", ctypes.c_int32), + ("index_entry_size", ctypes.c_int32), + ("index_offset", ctypes.c_int64), + ("index_length", ctypes.c_int64), + ("lease_registry_offset", ctypes.c_int64), + ("lease_registry_length", ctypes.c_int64), + ("slot_metadata_offset", ctypes.c_int64), + ("slot_metadata_length", ctypes.c_int64), + ("descriptor_stride", ctypes.c_int32), + ("payload_stride", ctypes.c_int32), + ("descriptor_storage_offset", ctypes.c_int64), + ("descriptor_storage_length", ctypes.c_int64), + ("payload_storage_offset", ctypes.c_int64), + ("payload_storage_length", ctypes.c_int64), + ("required_bytes", ctypes.c_int64), + ] + + +_LIBRARY_LOCK = threading.Lock() +_LIBRARY: Optional[ctypes.CDLL] = None +_LIBRARY_PATH: Optional[Path] = None +_RESOURCE_CONTEXTS = ExitStack() +atexit.register(_RESOURCE_CONTEXTS.close) + + +def _library_filename() -> str: + if sys.platform == "win32": + return "shared_memory_store.dll" + if sys.platform.startswith("linux"): + return "libshared_memory_store.so" + raise OSError( + "SharedMemoryStore supports native loading only on Windows and Linux; " + f"the current platform is {sys.platform!r}." + ) + + +def _bundled_library_path(filename: str) -> Path: + resource = files("shared_memory_store").joinpath(filename) + if not resource.is_file(): + raise OSError( + f"The packaged native library {filename!r} is missing from the " + "shared_memory_store package. Build or install a platform wheel; the " + "loader never searches the current directory or system library path." + ) + return Path(_RESOURCE_CONTEXTS.enter_context(as_file(resource))) + + +def _configure_signatures(lib: ctypes.CDLL) -> None: + store_handle = ctypes.c_void_p + lease_handle = ctypes.c_void_p + reservation_handle = ctypes.c_void_p + + lib.sms_abi_version.argtypes = [] + lib.sms_abi_version.restype = ctypes.c_uint32 + lib.sms_get_protocol_info.argtypes = [ctypes.POINTER(ProtocolInfo)] + lib.sms_get_protocol_info.restype = ctypes.c_int32 + lib.sms_get_layout_field_offset.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] + lib.sms_get_layout_field_offset.restype = ctypes.c_int32 + lib.sms_calculate_required_bytes.argtypes = [ + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_int32, + ctypes.POINTER(ctypes.c_int64), + ] + lib.sms_calculate_required_bytes.restype = ctypes.c_int32 + lib.sms_open_store.argtypes = [ + ctypes.POINTER(StoreOptions), + ctypes.POINTER(WaitOptions), + ctypes.POINTER(store_handle), + ] + lib.sms_open_store.restype = ctypes.c_int32 + lib.sms_close_store.argtypes = [store_handle] + lib.sms_close_store.restype = None + lib.sms_get_store_layout.argtypes = [ + store_handle, + ctypes.POINTER(WaitOptions), + ctypes.POINTER(StoreLayout), + ] + lib.sms_get_store_layout.restype = ctypes.c_int32 + lib.sms_publish.argtypes = [store_handle, Bytes, Bytes, Bytes, ctypes.POINTER(WaitOptions)] + lib.sms_publish.restype = ctypes.c_int32 + lib.sms_publish_segments.argtypes = [ + store_handle, + Bytes, + ctypes.POINTER(Segment), + ctypes.c_uint64, + Bytes, + ctypes.POINTER(WaitOptions), + ctypes.POINTER(ctypes.c_int64), + ] + lib.sms_publish_segments.restype = ctypes.c_int32 + lib.sms_acquire.argtypes = [ + store_handle, + Bytes, + ctypes.POINTER(WaitOptions), + ctypes.POINTER(lease_handle), + ] + lib.sms_acquire.restype = ctypes.c_int32 + lib.sms_lease_is_valid.argtypes = [lease_handle] + lib.sms_lease_is_valid.restype = ctypes.c_int32 + lib.sms_lease_value.argtypes = [lease_handle] + lib.sms_lease_value.restype = Bytes + lib.sms_lease_descriptor.argtypes = [lease_handle] + lib.sms_lease_descriptor.restype = Bytes + lib.sms_release_lease.argtypes = [lease_handle, ctypes.POINTER(WaitOptions)] + lib.sms_release_lease.restype = ctypes.c_int32 + lib.sms_destroy_lease.argtypes = [lease_handle] + lib.sms_destroy_lease.restype = None + lib.sms_remove.argtypes = [store_handle, Bytes, ctypes.POINTER(WaitOptions)] + lib.sms_remove.restype = ctypes.c_int32 + lib.sms_reserve.argtypes = [ + store_handle, + Bytes, + ctypes.c_int32, + Bytes, + ctypes.POINTER(WaitOptions), + ctypes.POINTER(reservation_handle), + ] + lib.sms_reserve.restype = ctypes.c_int32 + lib.sms_reservation_is_valid.argtypes = [reservation_handle] + lib.sms_reservation_is_valid.restype = ctypes.c_int32 + lib.sms_reservation_payload_length.argtypes = [reservation_handle] + lib.sms_reservation_payload_length.restype = ctypes.c_int32 + lib.sms_reservation_bytes_written.argtypes = [reservation_handle] + lib.sms_reservation_bytes_written.restype = ctypes.c_int32 + lib.sms_reservation_remaining_bytes.argtypes = [reservation_handle] + lib.sms_reservation_remaining_bytes.restype = ctypes.c_int32 + lib.sms_reservation_buffer.argtypes = [reservation_handle, ctypes.c_int32] + lib.sms_reservation_buffer.restype = MutableBytes + lib.sms_advance_reservation.argtypes = [ + reservation_handle, + ctypes.c_int32, + ctypes.POINTER(WaitOptions), + ] + lib.sms_advance_reservation.restype = ctypes.c_int32 + lib.sms_commit_reservation.argtypes = [reservation_handle, ctypes.POINTER(WaitOptions)] + lib.sms_commit_reservation.restype = ctypes.c_int32 + lib.sms_abort_reservation.argtypes = [reservation_handle, ctypes.POINTER(WaitOptions)] + lib.sms_abort_reservation.restype = ctypes.c_int32 + lib.sms_destroy_reservation.argtypes = [reservation_handle] + lib.sms_destroy_reservation.restype = None + lib.sms_recover_leases.argtypes = [ + store_handle, + ctypes.c_int32, + ctypes.POINTER(WaitOptions), + ctypes.POINTER(RecoveryReport), + ] + lib.sms_recover_leases.restype = ctypes.c_int32 + lib.sms_recover_reservations.argtypes = [ + store_handle, + ctypes.c_int32, + ctypes.POINTER(WaitOptions), + ctypes.POINTER(RecoveryReport), + ] + lib.sms_recover_reservations.restype = ctypes.c_int32 + lib.sms_get_diagnostics.argtypes = [ + store_handle, + ctypes.POINTER(WaitOptions), + ctypes.POINTER(Diagnostics), + ] + lib.sms_get_diagnostics.restype = ctypes.c_int32 + + +def _verify_contract(lib: ctypes.CDLL, path: Path) -> None: + actual_abi = int(lib.sms_abi_version()) + if actual_abi >> 16 != ABI_VERSION >> 16 or actual_abi < ABI_VERSION: + raise ImportError( + f"Native SharedMemoryStore ABI 0x{actual_abi:08x} at {path} is not " + f"compatible with required ABI 0x{ABI_VERSION:08x}." + ) + + info = ProtocolInfo() + info.struct_size = ctypes.sizeof(ProtocolInfo) + info.abi_version = ABI_VERSION + status = int(lib.sms_get_protocol_info(ctypes.byref(info))) + expected = ( + LAYOUT_MAJOR_VERSION, + LAYOUT_MINOR_VERSION, + RESOURCE_NAMING_VERSION, + 160, + 32, + 72, + 40, + ) + actual = ( + info.layout_major, + info.layout_minor, + info.resource_naming_version, + info.store_header_size, + info.index_entry_header_size, + info.slot_metadata_size, + info.lease_record_size, + ) + if status != 0 or actual != expected: + raise ImportError( + f"Native SharedMemoryStore protocol metadata at {path} is incompatible: " + f"status={status}, expected={expected}, actual={actual}." + ) + + expected_offsets = { + 0: 0, + 1: 56, + 2: 144, + 3: 152, + 100: 0, + 101: 8, + 102: 24, + 200: 0, + 201: 8, + 202: 16, + 203: 40, + 204: 64, + 300: 0, + 301: 16, + 302: 24, + 303: 32, + } + for field, expected_offset in expected_offsets.items(): + actual_offset = ctypes.c_uint32() + offset_status = int(lib.sms_get_layout_field_offset(field, ctypes.byref(actual_offset))) + if offset_status != 0 or actual_offset.value != expected_offset: + raise ImportError( + f"Native SharedMemoryStore layout field {field} at {path} is incompatible: " + f"status={offset_status}, expected offset={expected_offset}, " + f"actual offset={actual_offset.value}." + ) + + +def library() -> ctypes.CDLL: + """Return the validated process-wide native ABI library.""" + + global _LIBRARY, _LIBRARY_PATH + if _LIBRARY is not None: + return _LIBRARY + with _LIBRARY_LOCK: + if _LIBRARY is not None: + return _LIBRARY + filename = _library_filename() + path = _bundled_library_path(filename) + try: + candidate = ctypes.CDLL(str(path.resolve(strict=True))) + _configure_signatures(candidate) + _verify_contract(candidate, path) + except (AttributeError, OSError) as error: + raise OSError( + f"Unable to load the SharedMemoryStore native library from {path}. " + f"Expected platform artifact {filename!r}." + ) from error + _LIBRARY = candidate + _LIBRARY_PATH = path + return candidate + + +def native_library_path() -> Path: + """Return the exact loaded artifact path, primarily for package diagnostics.""" + + library() + assert _LIBRARY_PATH is not None + return _LIBRARY_PATH + + +__all__ = [ + "ABI_VERSION", + "LAYOUT_MAJOR_VERSION", + "LAYOUT_MINOR_VERSION", + "RESOURCE_NAMING_VERSION", + "WAIT_INFINITE", + "STATUS_COUNT", + "Bytes", + "MutableBytes", + "WaitOptions", + "StoreOptions", + "Segment", + "RecoveryReport", + "Diagnostics", + "ProtocolInfo", + "StoreLayout", + "library", + "native_library_path", +] diff --git a/src/python/shared_memory_store/enums.py b/src/python/shared_memory_store/enums.py new file mode 100644 index 0000000..7644ba2 --- /dev/null +++ b/src/python/shared_memory_store/enums.py @@ -0,0 +1,61 @@ +"""Numeric public contracts shared by every SharedMemoryStore runtime.""" + +from __future__ import annotations + +from enum import IntEnum + + +class OpenMode(IntEnum): + """Controls whether a named store is created, opened, or either.""" + + CREATE_NEW = 0 + OPEN_EXISTING = 1 + CREATE_OR_OPEN = 2 + + + +class StoreOpenStatus(IntEnum): + """Outcome of opening a named store.""" + + SUCCESS = 0 + ALREADY_EXISTS = 1 + NOT_FOUND = 2 + INVALID_OPTIONS = 3 + INCOMPATIBLE_LAYOUT = 4 + UNSUPPORTED_PLATFORM = 5 + INSUFFICIENT_CAPACITY = 6 + ACCESS_DENIED = 7 + MAPPING_FAILED = 8 + STORE_BUSY = 9 + OPERATION_CANCELED = 10 + + + +class StoreStatus(IntEnum): + """Outcome of a store lifecycle operation.""" + + SUCCESS = 0 + DUPLICATE_KEY = 1 + NOT_FOUND = 2 + KEY_TOO_LARGE = 3 + VALUE_TOO_LARGE = 4 + DESCRIPTOR_TOO_LARGE = 5 + STORE_FULL = 6 + LEASE_TABLE_FULL = 7 + INVALID_LEASE = 8 + LEASE_ALREADY_RELEASED = 9 + REMOVE_PENDING = 10 + UNSUPPORTED_PLATFORM = 11 + STORE_DISPOSED = 12 + CORRUPT_STORE = 13 + ACCESS_DENIED = 14 + UNKNOWN_FAILURE = 15 + INVALID_RESERVATION = 16 + RESERVATION_INCOMPLETE = 17 + RESERVATION_ALREADY_COMPLETED = 18 + RESERVATION_WRITE_OUT_OF_RANGE = 19 + INVALID_KEY = 20 + STORE_BUSY = 21 + OPERATION_CANCELED = 22 + +__all__ = ["OpenMode", "StoreOpenStatus", "StoreStatus"] diff --git a/src/python/shared_memory_store/store.py b/src/python/shared_memory_store/store.py new file mode 100644 index 0000000..5899b00 --- /dev/null +++ b/src/python/shared_memory_store/store.py @@ -0,0 +1,890 @@ +"""Pythonic ownership wrappers over the versioned SharedMemoryStore C ABI.""" + +from __future__ import annotations + +import ctypes +from dataclasses import dataclass +import sys +import threading +from typing import Any, ClassVar, Iterable, Optional +import weakref + +from . import _native +from .enums import OpenMode, StoreOpenStatus, StoreStatus + + +_INT32_MIN = -(1 << 31) +_INT32_MAX = (1 << 31) - 1 +_INT64_MAX = (1 << 63) - 1 + + +def _require_integer(value: object, name: str, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value < minimum or value > maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}") + return value + + +def _fits_integer(value: object, minimum: int, maximum: int) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and minimum <= value <= maximum + + +def _store_status(value: int) -> StoreStatus: + try: + return StoreStatus(value) + except ValueError: + return StoreStatus.UNKNOWN_FAILURE + + +def _open_status(value: int) -> StoreOpenStatus: + try: + return StoreOpenStatus(value) + except ValueError: + return StoreOpenStatus.MAPPING_FAILED + + +@dataclass(frozen=True, slots=True) +class WaitOptions: + """Caller-controlled shared-lock wait policy in milliseconds.""" + + timeout_milliseconds: int = 1000 + + DEFAULT: ClassVar["WaitOptions"] + NO_WAIT: ClassVar["WaitOptions"] + INFINITE: ClassVar["WaitOptions"] + + def __post_init__(self) -> None: + _require_integer( + self.timeout_milliseconds, + "timeout_milliseconds", + _native.WAIT_INFINITE, + _INT64_MAX, + ) + + @classmethod + def default(cls) -> "WaitOptions": + return cls.DEFAULT + + @classmethod + def defaults(cls) -> "WaitOptions": + return cls.DEFAULT + + @classmethod + def no_wait(cls) -> "WaitOptions": + return cls.NO_WAIT + + @classmethod + def infinite(cls) -> "WaitOptions": + return cls.INFINITE + + +WaitOptions.DEFAULT = WaitOptions(1000) +WaitOptions.NO_WAIT = WaitOptions(0) +WaitOptions.INFINITE = WaitOptions(_native.WAIT_INFINITE) + + +def _native_wait(value: WaitOptions) -> _native.WaitOptions: + if not isinstance(value, WaitOptions): + raise TypeError("wait must be a WaitOptions instance") + result = _native.WaitOptions() + result.struct_size = ctypes.sizeof(_native.WaitOptions) + result.abi_version = _native.ABI_VERSION + result.timeout_milliseconds = value.timeout_milliseconds + return result + + +def calculate_required_bytes( + *, + slot_count: int, + max_value_bytes: int, + max_descriptor_bytes: int, + max_key_bytes: int, + lease_record_count: int, +) -> int: + """Calculate the exact mapped capacity for the supplied protocol limits.""" + + arguments = { + "slot_count": slot_count, + "max_value_bytes": max_value_bytes, + "max_descriptor_bytes": max_descriptor_bytes, + "max_key_bytes": max_key_bytes, + "lease_record_count": lease_record_count, + } + for name, value in arguments.items(): + _require_integer(value, name, _INT32_MIN, _INT32_MAX) + + required = ctypes.c_int64() + status = _open_status( + int( + _native.library().sms_calculate_required_bytes( + slot_count, + max_value_bytes, + max_descriptor_bytes, + max_key_bytes, + lease_record_count, + ctypes.byref(required), + ) + ) + ) + if status is not StoreOpenStatus.SUCCESS: + raise ValueError(f"invalid SharedMemoryStore capacities ({status.name})") + return int(required.value) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class StoreOptions: + """Immutable capacities and open behavior for one named store handle.""" + + name: str + total_bytes: int + slot_count: int + max_value_bytes: int + max_descriptor_bytes: int + max_key_bytes: int + lease_record_count: int + open_mode: OpenMode = OpenMode.CREATE_OR_OPEN + enable_lease_recovery: bool = False + + @classmethod + def create( + cls, + name: str, + *, + slot_count: int, + max_value_bytes: int, + max_descriptor_bytes: int, + max_key_bytes: int, + lease_record_count: int, + open_mode: OpenMode = OpenMode.CREATE_OR_OPEN, + enable_lease_recovery: bool = False, + ) -> "StoreOptions": + total_bytes = calculate_required_bytes( + slot_count=slot_count, + max_value_bytes=max_value_bytes, + max_descriptor_bytes=max_descriptor_bytes, + max_key_bytes=max_key_bytes, + lease_record_count=lease_record_count, + ) + return cls( + name=name, + open_mode=open_mode, + total_bytes=total_bytes, + slot_count=slot_count, + max_value_bytes=max_value_bytes, + max_descriptor_bytes=max_descriptor_bytes, + max_key_bytes=max_key_bytes, + lease_record_count=lease_record_count, + enable_lease_recovery=enable_lease_recovery, + ) + + +@dataclass(frozen=True, slots=True) +class RecoveryReport: + scanned_count: int + recovered_count: int + active_count: int + unsupported_count: int + failed_count: int + + +@dataclass(frozen=True, slots=True) +class DiagnosticsSnapshot: + total_bytes: int + slot_count: int + free_slot_count: int + published_slot_count: int + pending_removal_count: int + active_lease_count: int + active_reservation_count: int + index_entry_count: int + occupied_index_entry_count: int + tombstone_index_entry_count: int + empty_index_entry_count: int + usable_index_capacity: int + last_observed_probe_length: int + max_observed_probe_length: int + last_failure_status: StoreStatus + aborted_reservation_count: int + recovered_lease_count: int + active_lease_recovery_count: int + unsupported_lease_recovery_count: int + failed_lease_recovery_count: int + recovered_reservation_count: int + active_reservation_recovery_count: int + unsupported_reservation_recovery_count: int + failed_reservation_recovery_count: int + capacity_pressure_count: int + index_compaction_count: int + failure_counts: tuple[int, ...] + + def failure_count(self, status: StoreStatus) -> int: + try: + index = int(StoreStatus(status)) + except (TypeError, ValueError): + return 0 + return self.failure_counts[index] if 0 <= index < len(self.failure_counts) else 0 + + get_failure_count = failure_count + + +def _coerce_bytes(value: Any, name: str) -> bytes: + try: + view = memoryview(value) + except TypeError as error: + raise TypeError(f"{name} must support the buffer protocol") from error + try: + return view.tobytes() + finally: + view.release() + + +class _InputBuffer: + __slots__ = ("_array", "native") + + def __init__(self, value: Any, name: str) -> None: + data = _coerce_bytes(value, name) + if data: + array = (ctypes.c_uint8 * len(data)).from_buffer_copy(data) + pointer = ctypes.cast(array, _native.UInt8Pointer) + self._array: Optional[ctypes.Array[Any]] = array + else: + pointer = _native.UInt8Pointer() + self._array = None + self.native = _native.Bytes(pointer, len(data)) + + def segment(self) -> _native.Segment: + return _native.Segment(self.native.data, self.native.length) + + +def _borrowed_view( + pointer: _native.UInt8Pointer, + length: int, + owner: Any, + *, + readonly: bool, +) -> memoryview: + if length < 0 or length > sys.maxsize: + raise RuntimeError(f"native library returned an invalid byte length: {length}") + array_type = ctypes.c_uint8 * length + if length: + address = ctypes.cast(pointer, ctypes.c_void_p).value + if not address: + raise RuntimeError("native library returned a null pointer for a non-empty byte view") + array = array_type.from_address(address) + else: + array = array_type() + # A memoryview retains its ctypes exporter; the exporter retains the token; + # the token retains the store. This makes the native lifetime explicit. + array._shared_memory_store_owner = owner # type: ignore[attr-defined] + view = memoryview(array).cast("B") + if readonly: + result = view.toreadonly() + view.release() + return result + return view + + +class _ViewOwner: + __slots__ = ("_view_refs",) + + def _initialize_views(self) -> None: + self._view_refs: list[weakref.ReferenceType[memoryview]] = [] + + def _track_view(self, view: memoryview) -> memoryview: + self._view_refs.append(weakref.ref(view)) + return view + + def _release_views(self) -> None: + references, self._view_refs = self._view_refs, [] + for reference in references: + view = reference() + if view is not None: + try: + view.release() + except (BufferError, ValueError): + pass + + +class MemoryStore: + """Context-managed owner of one process-local native store handle.""" + + __slots__ = ("_handle", "_lib", "_lock", "_children", "__weakref__") + + def __init__(self, handle: Optional[ctypes.c_void_p] = None, lib: Any = None) -> None: + if (handle is None) != (lib is None): + raise ValueError("handle and lib must either both be supplied or both be omitted") + self._handle = handle + self._lib = lib + self._lock = threading.RLock() + self._children: weakref.WeakSet[Any] = weakref.WeakSet() + + @classmethod + def open( + cls, + options: StoreOptions, + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> tuple[StoreOpenStatus, Optional["MemoryStore"]]: + if not isinstance(options, StoreOptions): + raise TypeError("options must be a StoreOptions instance") + native_wait = _native_wait(wait) + if not isinstance(options.name, str): + return StoreOpenStatus.INVALID_OPTIONS, None + try: + name_utf8 = options.name.encode("utf-8", errors="strict") + except UnicodeEncodeError: + return StoreOpenStatus.INVALID_OPTIONS, None + integer_fields = ( + (options.total_bytes, _INT64_MAX), + (options.slot_count, _INT32_MAX), + (options.max_value_bytes, _INT32_MAX), + (options.max_descriptor_bytes, _INT32_MAX), + (options.max_key_bytes, _INT32_MAX), + (options.lease_record_count, _INT32_MAX), + ) + if not all(_fits_integer(value, _INT32_MIN if maximum == _INT32_MAX else -(1 << 63), maximum) + for value, maximum in integer_fields): + return StoreOpenStatus.INVALID_OPTIONS, None + if not (type(options.open_mode) is int or isinstance(options.open_mode, OpenMode)) or not _fits_integer( + options.open_mode, _INT32_MIN, _INT32_MAX + ): + return StoreOpenStatus.INVALID_OPTIONS, None + open_mode = int(options.open_mode) + if not isinstance(options.enable_lease_recovery, bool): + return StoreOpenStatus.INVALID_OPTIONS, None + + native = _native.StoreOptions() + native.struct_size = ctypes.sizeof(_native.StoreOptions) + native.abi_version = _native.ABI_VERSION + native.name_utf8 = name_utf8 + native.name_length = len(name_utf8) + native.open_mode = open_mode + native.total_bytes = options.total_bytes + native.slot_count = options.slot_count + native.max_value_bytes = options.max_value_bytes + native.max_descriptor_bytes = options.max_descriptor_bytes + native.max_key_bytes = options.max_key_bytes + native.lease_record_count = options.lease_record_count + native.enable_lease_recovery = 1 if options.enable_lease_recovery else 0 + + lib = _native.library() + handle = ctypes.c_void_p() + status = _open_status(int(lib.sms_open_store(ctypes.byref(native), ctypes.byref(native_wait), ctypes.byref(handle)))) + if status is not StoreOpenStatus.SUCCESS or not handle.value: + return status, None + return status, cls(handle, lib) + + @property + def is_open(self) -> bool: + with self._lock: + return self._handle is not None + + @property + def is_valid(self) -> bool: + return self.is_open + + def __enter__(self) -> "MemoryStore": + if not self.is_open: + raise RuntimeError("the MemoryStore is closed") + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.close() + + def close(self) -> None: + with self._lock: + handle = self._handle + if handle is None: + return + for child in list(self._children): + child._close_from_store_locked() + self._children.clear() + self._handle = None + self._lib.sms_close_store(handle) + + def __del__(self) -> None: + try: + self.close() + except BaseException: + pass + + def publish( + self, + key: Any, + value: Any, + descriptor: Any = b"", + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> StoreStatus: + native_wait = _native_wait(wait) + with self._lock: + if self._handle is None: + return StoreStatus.STORE_DISPOSED + native_key = _InputBuffer(key, "key") + native_value = _InputBuffer(value, "value") + native_descriptor = _InputBuffer(descriptor, "descriptor") + return _store_status( + int( + self._lib.sms_publish( + self._handle, + native_key.native, + native_value.native, + native_descriptor.native, + ctypes.byref(native_wait), + ) + ) + ) + + def publish_segments( + self, + key: Any, + segments: Iterable[Any], + descriptor: Any = b"", + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> tuple[StoreStatus, int]: + native_wait = _native_wait(wait) + with self._lock: + if self._handle is None: + return StoreStatus.STORE_DISPOSED, 0 + native_key = _InputBuffer(key, "key") + native_descriptor = _InputBuffer(descriptor, "descriptor") + try: + buffers = [_InputBuffer(value, f"segments[{index}]") for index, value in enumerate(segments)] + except TypeError as error: + if "not iterable" in str(error): + raise TypeError("segments must be an iterable of bytes-like objects") from error + raise + if len(buffers) > (1 << 64) - 1: + raise OverflowError("too many payload segments") + if buffers: + array = (_native.Segment * len(buffers))(*(buffer.segment() for buffer in buffers)) + pointer = array + else: + array = None + pointer = None + copied = ctypes.c_int64() + status = _store_status( + int( + self._lib.sms_publish_segments( + self._handle, + native_key.native, + pointer, + len(buffers), + native_descriptor.native, + ctypes.byref(native_wait), + ctypes.byref(copied), + ) + ) + ) + return status, int(copied.value) + + def acquire( + self, + key: Any, + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> tuple[StoreStatus, Optional["ValueLease"]]: + native_wait = _native_wait(wait) + with self._lock: + if self._handle is None: + return StoreStatus.STORE_DISPOSED, None + native_key = _InputBuffer(key, "key") + handle = ctypes.c_void_p() + status = _store_status( + int(self._lib.sms_acquire(self._handle, native_key.native, ctypes.byref(native_wait), ctypes.byref(handle))) + ) + if status is not StoreStatus.SUCCESS or not handle.value: + return status, None + lease = ValueLease(self, handle) + self._children.add(lease) + return status, lease + + def remove( + self, + key: Any, + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> StoreStatus: + native_wait = _native_wait(wait) + with self._lock: + if self._handle is None: + return StoreStatus.STORE_DISPOSED + native_key = _InputBuffer(key, "key") + return _store_status(int(self._lib.sms_remove(self._handle, native_key.native, ctypes.byref(native_wait)))) + + def reserve( + self, + key: Any, + payload_length: int, + descriptor: Any = b"", + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> tuple[StoreStatus, Optional["ValueReservation"]]: + native_wait = _native_wait(wait) + if isinstance(payload_length, bool) or not isinstance(payload_length, int): + raise TypeError("payload_length must be an integer") + if payload_length < _INT32_MIN or payload_length > _INT32_MAX: + return StoreStatus.VALUE_TOO_LARGE, None + with self._lock: + if self._handle is None: + return StoreStatus.STORE_DISPOSED, None + native_key = _InputBuffer(key, "key") + native_descriptor = _InputBuffer(descriptor, "descriptor") + handle = ctypes.c_void_p() + status = _store_status( + int( + self._lib.sms_reserve( + self._handle, + native_key.native, + payload_length, + native_descriptor.native, + ctypes.byref(native_wait), + ctypes.byref(handle), + ) + ) + ) + if status is not StoreStatus.SUCCESS or not handle.value: + return status, None + reservation = ValueReservation(self, handle) + self._children.add(reservation) + return status, reservation + + def recover_leases( + self, + recover_current_process: bool = False, + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> tuple[StoreStatus, RecoveryReport]: + return self._recover("sms_recover_leases", recover_current_process, wait) + + def recover_reservations( + self, + recover_current_process: bool = False, + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> tuple[StoreStatus, RecoveryReport]: + return self._recover("sms_recover_reservations", recover_current_process, wait) + + def _recover( + self, + function_name: str, + recover_current_process: bool, + wait: WaitOptions, + ) -> tuple[StoreStatus, RecoveryReport]: + if not isinstance(recover_current_process, bool): + raise TypeError("recover_current_process must be a bool") + native_wait = _native_wait(wait) + native_report = _native.RecoveryReport() + native_report.struct_size = ctypes.sizeof(_native.RecoveryReport) + native_report.abi_version = _native.ABI_VERSION + with self._lock: + if self._handle is None: + return StoreStatus.STORE_DISPOSED, RecoveryReport(0, 0, 0, 0, 0) + function = getattr(self._lib, function_name) + status = _store_status( + int( + function( + self._handle, + 1 if recover_current_process else 0, + ctypes.byref(native_wait), + ctypes.byref(native_report), + ) + ) + ) + if status is StoreStatus.SUCCESS and recover_current_process: + for child in list(self._children): + child._invalidate_views_locked() + report = RecoveryReport( + native_report.scanned_count, + native_report.recovered_count, + native_report.active_count, + native_report.unsupported_count, + native_report.failed_count, + ) + return status, report + + def diagnostics( + self, + *, + wait: WaitOptions = WaitOptions.DEFAULT, + ) -> tuple[StoreStatus, Optional[DiagnosticsSnapshot]]: + native_wait = _native_wait(wait) + native = _native.Diagnostics() + native.struct_size = ctypes.sizeof(_native.Diagnostics) + native.abi_version = _native.ABI_VERSION + with self._lock: + if self._handle is None: + return StoreStatus.STORE_DISPOSED, None + status = _store_status( + int(self._lib.sms_get_diagnostics(self._handle, ctypes.byref(native_wait), ctypes.byref(native))) + ) + if status is not StoreStatus.SUCCESS: + return status, None + return status, DiagnosticsSnapshot( + total_bytes=native.total_bytes, + slot_count=native.slot_count, + free_slot_count=native.free_slot_count, + published_slot_count=native.published_slot_count, + pending_removal_count=native.pending_removal_count, + active_lease_count=native.active_lease_count, + active_reservation_count=native.active_reservation_count, + index_entry_count=native.index_entry_count, + occupied_index_entry_count=native.occupied_index_entry_count, + tombstone_index_entry_count=native.tombstone_index_entry_count, + empty_index_entry_count=native.empty_index_entry_count, + usable_index_capacity=native.usable_index_capacity, + last_observed_probe_length=native.last_observed_probe_length, + max_observed_probe_length=native.max_observed_probe_length, + last_failure_status=_store_status(native.last_failure_status), + aborted_reservation_count=native.aborted_reservation_count, + recovered_lease_count=native.recovered_lease_count, + active_lease_recovery_count=native.active_lease_recovery_count, + unsupported_lease_recovery_count=native.unsupported_lease_recovery_count, + failed_lease_recovery_count=native.failed_lease_recovery_count, + recovered_reservation_count=native.recovered_reservation_count, + active_reservation_recovery_count=native.active_reservation_recovery_count, + unsupported_reservation_recovery_count=native.unsupported_reservation_recovery_count, + failed_reservation_recovery_count=native.failed_reservation_recovery_count, + capacity_pressure_count=native.capacity_pressure_count, + index_compaction_count=native.index_compaction_count, + failure_counts=tuple(int(value) for value in native.failure_counts), + ) + + get_diagnostics = diagnostics + + +class ValueLease(_ViewOwner): + """Owning token for read-only, zero-copy value and descriptor views. + + Directly returned views are actively released with this token. Any slice or + derived view created by the caller is subject to the same lifetime and must + not be retained beyond release, recovery, or store close. + """ + + __slots__ = ("_store", "_handle", "_lock", "__weakref__") + + def __init__(self, store: MemoryStore, handle: ctypes.c_void_p) -> None: + self._store = store + self._handle: Optional[ctypes.c_void_p] = handle + self._lock = threading.RLock() + self._initialize_views() + + def _valid_locked(self) -> bool: + if self._handle is None: + return False + valid = bool(self._store._lib.sms_lease_is_valid(self._handle)) + if not valid: + self._release_views() + return valid + + @property + def is_valid(self) -> bool: + with self._store._lock: + with self._lock: + return self._valid_locked() + + @property + def value(self) -> memoryview: + return self._bytes_view("sms_lease_value") + + @property + def descriptor(self) -> memoryview: + return self._bytes_view("sms_lease_descriptor") + + def _bytes_view(self, function_name: str) -> memoryview: + with self._store._lock: + with self._lock: + if not self._valid_locked(): + raise RuntimeError("the value lease is no longer valid") + native = getattr(self._store._lib, function_name)(self._handle) + view = _borrowed_view(native.data, int(native.length), self, readonly=True) + return self._track_view(view) + + def release(self, *, wait: WaitOptions = WaitOptions.DEFAULT) -> StoreStatus: + native_wait = _native_wait(wait) + with self._store._lock: + with self._lock: + self._release_views() + if self._handle is None: + return StoreStatus.INVALID_LEASE + return _store_status( + int(self._store._lib.sms_release_lease(self._handle, ctypes.byref(native_wait))) + ) + + def __enter__(self) -> "ValueLease": + if not self.is_valid: + raise RuntimeError("the value lease is no longer valid") + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.close() + + def close(self) -> None: + with self._store._lock: + self._close_from_store_locked() + self._store._children.discard(self) + + def _close_from_store_locked(self) -> None: + with self._lock: + self._release_views() + if self._handle is not None: + handle, self._handle = self._handle, None + self._store._lib.sms_destroy_lease(handle) + + def _invalidate_views_locked(self) -> None: + with self._lock: + self._release_views() + + def __del__(self) -> None: + try: + self.close() + except BaseException: + pass + + +class ValueReservation(_ViewOwner): + """Owning token for an announced-length, zero-copy writable payload. + + A writable view is immediate-use memory and is released by the next + reservation operation. Caller-created slices share that same lifetime. + """ + + __slots__ = ("_store", "_handle", "_lock", "__weakref__") + + def __init__(self, store: MemoryStore, handle: ctypes.c_void_p) -> None: + self._store = store + self._handle: Optional[ctypes.c_void_p] = handle + self._lock = threading.RLock() + self._initialize_views() + + def _valid_locked(self) -> bool: + if self._handle is None: + return False + valid = bool(self._store._lib.sms_reservation_is_valid(self._handle)) + if not valid: + self._release_views() + return valid + + @property + def is_valid(self) -> bool: + with self._store._lock: + with self._lock: + return self._valid_locked() + + @property + def payload_length(self) -> int: + with self._store._lock: + with self._lock: + if not self._valid_locked(): + return 0 + return int(self._store._lib.sms_reservation_payload_length(self._handle)) + + @property + def bytes_written(self) -> int: + with self._store._lock: + with self._lock: + if not self._valid_locked(): + return 0 + return int(self._store._lib.sms_reservation_bytes_written(self._handle)) + + @property + def remaining_bytes(self) -> int: + with self._store._lock: + with self._lock: + if not self._valid_locked(): + return 0 + return max(0, int(self._store._lib.sms_reservation_remaining_bytes(self._handle))) + + def buffer(self, size_hint: int = 0) -> memoryview: + size_hint = _require_integer(size_hint, "size_hint", _INT32_MIN, _INT32_MAX) + with self._store._lock: + with self._lock: + self._release_views() + if not self._valid_locked(): + raise RuntimeError("the value reservation is no longer valid") + native = self._store._lib.sms_reservation_buffer(self._handle, size_hint) + view = _borrowed_view(native.data, int(native.length), self, readonly=False) + return self._track_view(view) + + @property + def view(self) -> memoryview: + return self.buffer() + + def advance(self, byte_count: int, *, wait: WaitOptions = WaitOptions.DEFAULT) -> StoreStatus: + native_wait = _native_wait(wait) + if isinstance(byte_count, bool) or not isinstance(byte_count, int): + raise TypeError("byte_count must be an integer") + if byte_count < _INT32_MIN or byte_count > _INT32_MAX: + return StoreStatus.RESERVATION_WRITE_OUT_OF_RANGE + with self._store._lock: + with self._lock: + self._release_views() + if self._handle is None: + return StoreStatus.INVALID_RESERVATION + return _store_status( + int( + self._store._lib.sms_advance_reservation( + self._handle, byte_count, ctypes.byref(native_wait) + ) + ) + ) + + def commit(self, *, wait: WaitOptions = WaitOptions.DEFAULT) -> StoreStatus: + return self._complete("sms_commit_reservation", wait) + + def abort(self, *, wait: WaitOptions = WaitOptions.DEFAULT) -> StoreStatus: + return self._complete("sms_abort_reservation", wait) + + def _complete(self, function_name: str, wait: WaitOptions) -> StoreStatus: + native_wait = _native_wait(wait) + with self._store._lock: + with self._lock: + self._release_views() + if self._handle is None: + return StoreStatus.INVALID_RESERVATION + function = getattr(self._store._lib, function_name) + return _store_status(int(function(self._handle, ctypes.byref(native_wait)))) + + def __enter__(self) -> "ValueReservation": + if not self.is_valid: + raise RuntimeError("the value reservation is no longer valid") + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.close() + + def close(self) -> None: + with self._store._lock: + self._close_from_store_locked() + self._store._children.discard(self) + + def _close_from_store_locked(self) -> None: + with self._lock: + self._release_views() + if self._handle is not None: + handle, self._handle = self._handle, None + self._store._lib.sms_destroy_reservation(handle) + + def _invalidate_views_locked(self) -> None: + with self._lock: + self._release_views() + + def __del__(self) -> None: + try: + self.close() + except BaseException: + pass + + +__all__ = [ + "WaitOptions", + "StoreOptions", + "RecoveryReport", + "DiagnosticsSnapshot", + "MemoryStore", + "ValueLease", + "ValueReservation", + "calculate_required_bytes", +] diff --git a/tests/SharedMemoryStore.IntegrationTests/MultiStoreLifecycleIntegrationTests.cs b/tests/SharedMemoryStore.IntegrationTests/MultiStoreLifecycleIntegrationTests.cs index 21106bb..26fb363 100644 --- a/tests/SharedMemoryStore.IntegrationTests/MultiStoreLifecycleIntegrationTests.cs +++ b/tests/SharedMemoryStore.IntegrationTests/MultiStoreLifecycleIntegrationTests.cs @@ -5,6 +5,42 @@ namespace SharedMemoryStore.IntegrationTests; public sealed class MultiStoreLifecycleIntegrationTests { + [Fact] + [Trait("Category", "Integration")] + public void LinuxMapsTheExistingFileBeforeReportingLayoutMismatch() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + var name = $"sms-linux-layout-mismatch-{Guid.NewGuid():N}"; + var createOptions = SharedMemoryStoreOptions.Create( + name, + slotCount: 6, + maxValueBytes: 128, + maxDescriptorBytes: 32, + maxKeyBytes: 32, + leaseRecordCount: 16, + openMode: OpenMode.CreateNew, + enableLeaseRecovery: true); + var mismatchedOptions = SharedMemoryStoreOptions.Create( + name, + slotCount: 5, + maxValueBytes: 128, + maxDescriptorBytes: 32, + maxKeyBytes: 32, + leaseRecordCount: 16, + openMode: OpenMode.OpenExisting, + enableLeaseRecovery: true); + + using var created = IntegrationStoreFactory.Create(createOptions); + Assert.Equal( + StoreOpenStatus.IncompatibleLayout, + Store.TryCreateOrOpen(mismatchedOptions, out var mismatched)); + Assert.Null(mismatched); + } + [Fact] [Trait("Category", "Integration")] public void MultipleNamedStoresRemainIsolated() diff --git a/tests/SharedMemoryStore.InteropAgent/AgentHost.cs b/tests/SharedMemoryStore.InteropAgent/AgentHost.cs new file mode 100644 index 0000000..13549a8 --- /dev/null +++ b/tests/SharedMemoryStore.InteropAgent/AgentHost.cs @@ -0,0 +1,53 @@ +using System.Text.Json; + +namespace SharedMemoryStore.InteropAgent; + +public static class AgentHost +{ + public static async Task RunAsync( + TextReader input, + TextWriter output, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + + using var session = new AgentSession(); + while (await input.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + AgentResponse response; + + try + { + response = session.Handle(AgentProtocol.ParseRequest(line)); + } + catch (JsonException exception) + { + response = Failure( + id: string.Empty, + statusCode: -1, + statusName: "ProtocolError", + errorCode: "invalid_request", + message: exception.Message); + } + + await AgentProtocol.WriteResponseAsync(output, response, cancellationToken).ConfigureAwait(false); + } + + return 0; + } + + internal static AgentResponse Failure( + string id, + int statusCode, + string statusName, + string errorCode, + string message) => + new() + { + Id = id, + Ok = false, + Status = new AgentStatus { Code = statusCode, Name = statusName }, + Error = new AgentError { Code = errorCode, Message = message } + }; +} diff --git a/tests/SharedMemoryStore.InteropAgent/AgentMessages.cs b/tests/SharedMemoryStore.InteropAgent/AgentMessages.cs new file mode 100644 index 0000000..0eaf8d7 --- /dev/null +++ b/tests/SharedMemoryStore.InteropAgent/AgentMessages.cs @@ -0,0 +1,52 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SharedMemoryStore.InteropAgent; + +public sealed record AgentRequest +{ + [JsonPropertyName("id")] + public required string Id { get; init; } + + [JsonPropertyName("command")] + public required string Command { get; init; } + + [JsonPropertyName("arguments")] + public JsonElement? Arguments { get; init; } +} + +public sealed record AgentResponse +{ + [JsonPropertyName("id")] + public required string Id { get; init; } + + [JsonPropertyName("ok")] + public required bool Ok { get; init; } + + [JsonPropertyName("status")] + public required AgentStatus Status { get; init; } + + [JsonPropertyName("result")] + public JsonElement? Result { get; init; } + + [JsonPropertyName("error")] + public AgentError? Error { get; init; } +} + +public sealed record AgentStatus +{ + [JsonPropertyName("code")] + public required int Code { get; init; } + + [JsonPropertyName("name")] + public required string Name { get; init; } +} + +public sealed record AgentError +{ + [JsonPropertyName("code")] + public required string Code { get; init; } + + [JsonPropertyName("message")] + public required string Message { get; init; } +} diff --git a/tests/SharedMemoryStore.InteropAgent/AgentProtocol.cs b/tests/SharedMemoryStore.InteropAgent/AgentProtocol.cs new file mode 100644 index 0000000..6e9b734 --- /dev/null +++ b/tests/SharedMemoryStore.InteropAgent/AgentProtocol.cs @@ -0,0 +1,146 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SharedMemoryStore.InteropAgent; + +public static class AgentProtocol +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNameCaseInsensitive = false, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false + }; + + public static string SerializeRequestLine(AgentRequest request) + { + ArgumentNullException.ThrowIfNull(request); + Validate(request); + return JsonSerializer.Serialize(request, SerializerOptions) + '\n'; + } + + public static string SerializeResponseLine(AgentResponse response) + { + ArgumentNullException.ThrowIfNull(response); + Validate(response); + return JsonSerializer.Serialize(response, SerializerOptions) + '\n'; + } + + public static AgentRequest ParseRequest(string line) + { + var request = ParseLine(line); + Validate(request); + return request; + } + + public static AgentResponse ParseResponse(string line) + { + var response = ParseLine(line); + Validate(response); + return response; + } + + public static async ValueTask ReadRequestAsync( + TextReader reader, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(reader); + + var line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false); + return line is null ? null : ParseRequest(line); + } + + public static async ValueTask WriteResponseAsync( + TextWriter writer, + AgentResponse response, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(writer); + + await writer.WriteAsync(SerializeResponseLine(response).AsMemory(), cancellationToken).ConfigureAwait(false); + await writer.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + public static JsonElement ToJsonElement(T value) => + JsonSerializer.SerializeToElement(value, SerializerOptions); + + public static string EncodeBytes(ReadOnlySpan value) => Convert.ToBase64String(value); + + public static byte[] DecodeBytes(string value) + { + ArgumentNullException.ThrowIfNull(value); + return Convert.FromBase64String(value); + } + + private static T ParseLine(string line) + { + ArgumentNullException.ThrowIfNull(line); + + if (string.IsNullOrWhiteSpace(line)) + { + throw new JsonException("An agent protocol line cannot be empty."); + } + + if (line.Contains('\r', StringComparison.Ordinal) || line.Contains('\n', StringComparison.Ordinal)) + { + throw new JsonException("An agent protocol frame must contain exactly one line."); + } + + return JsonSerializer.Deserialize(line, SerializerOptions) + ?? throw new JsonException("An agent protocol line must contain a JSON object."); + } + + private static void Validate(AgentRequest request) + { + if (string.IsNullOrWhiteSpace(request.Id)) + { + throw new JsonException("The request id is required."); + } + + if (string.IsNullOrWhiteSpace(request.Command)) + { + throw new JsonException("The request command is required."); + } + + if (request.Arguments is { ValueKind: not JsonValueKind.Object }) + { + throw new JsonException("Request arguments must be a JSON object when present."); + } + } + + private static void Validate(AgentResponse response) + { + if (response.Id is null) + { + throw new JsonException("The response id is required."); + } + + if (response.Status is null || string.IsNullOrWhiteSpace(response.Status.Name)) + { + throw new JsonException("A response status code and name are required."); + } + + if (response.Result is { ValueKind: JsonValueKind.Undefined }) + { + throw new JsonException("A response result cannot be undefined."); + } + + if (response.Ok && response.Error is not null) + { + throw new JsonException("A successful response cannot contain an error."); + } + + if (!response.Ok && response.Error is null) + { + throw new JsonException("A failed response must contain an error."); + } + + if (response.Error is not null + && (string.IsNullOrWhiteSpace(response.Error.Code) + || string.IsNullOrWhiteSpace(response.Error.Message))) + { + throw new JsonException("An error code and message are required."); + } + } +} diff --git a/tests/SharedMemoryStore.InteropAgent/AgentSession.cs b/tests/SharedMemoryStore.InteropAgent/AgentSession.cs new file mode 100644 index 0000000..eea5334 --- /dev/null +++ b/tests/SharedMemoryStore.InteropAgent/AgentSession.cs @@ -0,0 +1,412 @@ +using System.Buffers; +using System.Text.Json; + +namespace SharedMemoryStore.InteropAgent; + +internal sealed class AgentSession : IDisposable +{ + private readonly Dictionary _stores = new(StringComparer.Ordinal); + private readonly Dictionary _leases = new(StringComparer.Ordinal); + private readonly Dictionary _reservations = new(StringComparer.Ordinal); + + public AgentResponse Handle(AgentRequest request) + { + try + { + return request.Command switch + { + "ping" => Success(request.Id, 0, "Success", new { runtime = "dotnet", protocolVersion = 1 }), + "open" => Open(request), + "close" => Close(request), + "publish" => Publish(request), + "publishSegments" or "publishSegmented" => PublishSegments(request), + "acquire" => Acquire(request), + "release" => Release(request), + "remove" => Remove(request), + "reserve" => Reserve(request), + "reservationWrite" => ReservationWrite(request), + "advance" => Advance(request), + "commit" => Commit(request), + "abort" => Abort(request), + "recoverLeases" => RecoverLeases(request), + "recoverReservations" => RecoverReservations(request), + "diagnostics" => Diagnostics(request), + "crash" => Crash(), + _ => AgentHost.Failure( + request.Id, + statusCode: -2, + statusName: "UnsupportedCommand", + errorCode: "unsupported_command", + message: $"The command '{request.Command}' is not implemented by this agent.") + }; + } + catch (Exception exception) when (exception is JsonException or FormatException or KeyNotFoundException or ArgumentException) + { + return AgentHost.Failure( + request.Id, + statusCode: -1, + statusName: "ProtocolError", + errorCode: "invalid_arguments", + message: exception.Message); + } + } + + public void Dispose() + { + foreach (var lease in _leases.Values) + { + lease.Dispose(); + } + + foreach (var reservation in _reservations.Values) + { + reservation.Dispose(); + } + + foreach (var store in _stores.Values) + { + store.Dispose(); + } + + _leases.Clear(); + _reservations.Clear(); + _stores.Clear(); + } + + private AgentResponse Open(AgentRequest request) + { + var arguments = Arguments(request); + var storeId = RequiredString(arguments, "storeId"); + if (_stores.Remove(storeId, out var previous)) + { + previous.Dispose(); + } + + var slotCount = RequiredInt32(arguments, "slotCount"); + var maxValueBytes = RequiredInt32(arguments, "maxValueBytes"); + var maxDescriptorBytes = RequiredInt32(arguments, "maxDescriptorBytes"); + var maxKeyBytes = RequiredInt32(arguments, "maxKeyBytes"); + var leaseRecordCount = RequiredInt32(arguments, "leaseRecordCount"); + var totalBytes = OptionalInt64(arguments, "totalBytes") ?? SharedMemoryStoreOptions.CalculateRequiredBytes( + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount); + var options = new SharedMemoryStoreOptions + { + Name = RequiredString(arguments, "name"), + OpenMode = ParseOpenMode(arguments), + TotalBytes = totalBytes, + SlotCount = slotCount, + MaxValueBytes = maxValueBytes, + MaxDescriptorBytes = maxDescriptorBytes, + MaxKeyBytes = maxKeyBytes, + LeaseRecordCount = leaseRecordCount, + EnableLeaseRecovery = OptionalBoolean(arguments, "enableLeaseRecovery") ?? false + }; + var status = MemoryStore.TryCreateOrOpen(options, Wait(arguments), out var store); + if (status == StoreOpenStatus.Success && store is not null) + { + _stores.Add(storeId, store); + } + + return Success(request.Id, (int)status, status.ToString(), new { storeId }); + } + + private AgentResponse Close(AgentRequest request) + { + var storeId = RequiredString(Arguments(request), "storeId"); + if (_stores.Remove(storeId, out var store)) + { + store.Dispose(); + } + + return Success(request.Id, 0, StoreStatus.Success.ToString(), new { storeId }); + } + + private AgentResponse Publish(AgentRequest request) + { + var arguments = Arguments(request); + var status = Store(arguments).TryPublish( + Bytes(arguments, "key"), + Bytes(arguments, "value"), + OptionalBytes(arguments, "descriptor"), + Wait(arguments)); + return Status(request.Id, status); + } + + private AgentResponse PublishSegments(AgentRequest request) + { + var arguments = Arguments(request); + var segments = ReadSegments(arguments); + var status = Store(arguments).TryPublishSegments( + Bytes(arguments, "key"), + segments, + OptionalBytes(arguments, "descriptor"), + Wait(arguments), + out var copiedBytes); + return Success(request.Id, (int)status, status.ToString(), new { copiedBytes }); + } + + private AgentResponse Acquire(AgentRequest request) + { + var arguments = Arguments(request); + var leaseId = RequiredString(arguments, "leaseId"); + var status = Store(arguments).TryAcquire(Bytes(arguments, "key"), Wait(arguments), out var lease); + if (status != StoreStatus.Success) + { + return Status(request.Id, status); + } + + _leases[leaseId] = lease; + return Success(request.Id, (int)status, status.ToString(), new + { + leaseId, + value = AgentProtocol.EncodeBytes(lease.ValueSpan), + descriptor = AgentProtocol.EncodeBytes(lease.DescriptorSpan) + }); + } + + private AgentResponse Release(AgentRequest request) + { + var arguments = Arguments(request); + var leaseId = RequiredString(arguments, "leaseId"); + var status = _leases.TryGetValue(leaseId, out var lease) + ? lease.Release(Wait(arguments)) + : StoreStatus.InvalidLease; + return Status(request.Id, status); + } + + private AgentResponse Remove(AgentRequest request) + { + var arguments = Arguments(request); + return Status(request.Id, Store(arguments).TryRemove(Bytes(arguments, "key"), Wait(arguments))); + } + + private AgentResponse Reserve(AgentRequest request) + { + var arguments = Arguments(request); + var reservationId = RequiredString(arguments, "reservationId"); + var status = Store(arguments).TryReserve( + Bytes(arguments, "key"), + RequiredInt32(arguments, "payloadLength"), + OptionalBytes(arguments, "descriptor"), + Wait(arguments), + out var reservation); + if (status == StoreStatus.Success) + { + _reservations[reservationId] = reservation; + } + + return Success(request.Id, (int)status, status.ToString(), new { reservationId }); + } + + private AgentResponse ReservationWrite(AgentRequest request) + { + var arguments = Arguments(request); + var reservationId = RequiredString(arguments, "reservationId"); + if (!_reservations.TryGetValue(reservationId, out var reservation)) + { + return Status(request.Id, StoreStatus.InvalidReservation); + } + + var data = Bytes(arguments, "data"); + var target = reservation.GetSpan(data.Length); + if (target.Length < data.Length) + { + return Status(request.Id, StoreStatus.ReservationWriteOutOfRange); + } + + data.CopyTo(target); + return Success(request.Id, 0, StoreStatus.Success.ToString(), new { written = data.Length }); + } + + private AgentResponse Advance(AgentRequest request) + { + var arguments = Arguments(request); + var reservation = Reservation(arguments); + return Status(request.Id, reservation.Advance(RequiredInt32(arguments, "byteCount"), Wait(arguments))); + } + + private AgentResponse Commit(AgentRequest request) + { + var arguments = Arguments(request); + return Status(request.Id, Reservation(arguments).Commit(Wait(arguments))); + } + + private AgentResponse Abort(AgentRequest request) + { + var arguments = Arguments(request); + return Status(request.Id, Reservation(arguments).Abort(Wait(arguments))); + } + + private AgentResponse RecoverLeases(AgentRequest request) + { + var arguments = Arguments(request); + var status = Store(arguments).TryRecoverLeases( + new LeaseRecoveryOptions(OptionalBoolean(arguments, "recoverCurrentProcess") ?? false), + Wait(arguments), + out var report); + return Success(request.Id, (int)status, status.ToString(), report); + } + + private AgentResponse RecoverReservations(AgentRequest request) + { + var arguments = Arguments(request); + var status = Store(arguments).TryRecoverReservations( + new ReservationRecoveryOptions(OptionalBoolean(arguments, "recoverCurrentProcess") ?? false), + Wait(arguments), + out var report); + return Success(request.Id, (int)status, status.ToString(), report); + } + + private AgentResponse Diagnostics(AgentRequest request) + { + var arguments = Arguments(request); + var status = Store(arguments).TryGetDiagnostics(Wait(arguments), out var snapshot); + return Success(request.Id, (int)status, status.ToString(), snapshot); + } + + private static AgentResponse Crash() + { + Environment.Exit(97); + throw new InvalidOperationException("Process termination returned unexpectedly."); + } + + private MemoryStore Store(JsonElement arguments) + { + var id = RequiredString(arguments, "storeId"); + return _stores.TryGetValue(id, out var value) + ? value + : throw new KeyNotFoundException($"Store handle '{id}' does not exist."); + } + + private ValueReservation Reservation(JsonElement arguments) + { + var id = RequiredString(arguments, "reservationId"); + return _reservations.TryGetValue(id, out var value) + ? value + : throw new KeyNotFoundException($"Reservation handle '{id}' does not exist."); + } + + private static JsonElement Arguments(AgentRequest request) => + request.Arguments is { ValueKind: JsonValueKind.Object } arguments + ? arguments + : throw new JsonException("Command arguments are required."); + + private static string RequiredString(JsonElement arguments, string name) => + arguments.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String + ? property.GetString() ?? throw new JsonException($"'{name}' must not be null.") + : throw new JsonException($"'{name}' is required and must be a string."); + + private static int RequiredInt32(JsonElement arguments, string name) => + arguments.TryGetProperty(name, out var property) && property.TryGetInt32(out var value) + ? value + : throw new JsonException($"'{name}' is required and must be a 32-bit integer."); + + private static long? OptionalInt64(JsonElement arguments, string name) => + !arguments.TryGetProperty(name, out var property) || property.ValueKind == JsonValueKind.Null + ? null + : property.TryGetInt64(out var value) + ? value + : throw new JsonException($"'{name}' must be a 64-bit integer."); + + private static bool? OptionalBoolean(JsonElement arguments, string name) => + !arguments.TryGetProperty(name, out var property) || property.ValueKind == JsonValueKind.Null + ? null + : property.ValueKind is JsonValueKind.True or JsonValueKind.False + ? property.GetBoolean() + : throw new JsonException($"'{name}' must be a boolean."); + + private static byte[] Bytes(JsonElement arguments, string name) => + AgentProtocol.DecodeBytes(RequiredString(arguments, name)); + + private static byte[] OptionalBytes(JsonElement arguments, string name) => + arguments.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String + ? AgentProtocol.DecodeBytes(property.GetString()!) + : []; + + private static ReadOnlySequence ReadSegments(JsonElement arguments) + { + if (!arguments.TryGetProperty("segments", out var property) + || property.ValueKind != JsonValueKind.Array) + { + throw new JsonException("'segments' is required and must be an array."); + } + + BufferSegment? first = null; + BufferSegment? last = null; + foreach (var item in property.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + throw new JsonException("Every segment must be a base64 string."); + } + + var bytes = AgentProtocol.DecodeBytes(item.GetString()!); + last = last is null ? first = new BufferSegment(bytes) : last.Append(bytes); + } + + return first is null + ? ReadOnlySequence.Empty + : new ReadOnlySequence(first, 0, last!, last!.Memory.Length); + } + + private static OpenMode ParseOpenMode(JsonElement arguments) + { + if (!arguments.TryGetProperty("openMode", out var property)) + { + return OpenMode.CreateOrOpen; + } + + if (property.TryGetInt32(out var numeric)) + { + return (OpenMode)numeric; + } + + return Enum.Parse(property.GetString() ?? string.Empty, ignoreCase: true); + } + + private static StoreWaitOptions Wait(JsonElement arguments) + { + var milliseconds = OptionalInt64(arguments, "timeoutMs") ?? 1000; + return milliseconds switch + { + -1 => StoreWaitOptions.Infinite, + 0 => StoreWaitOptions.NoWait, + < -1 => new StoreWaitOptions(TimeSpan.FromMilliseconds(-2)), + _ => new StoreWaitOptions(TimeSpan.FromMilliseconds(milliseconds)) + }; + } + + private static AgentResponse Status(string id, StoreStatus status) => + Success(id, (int)status, status.ToString(), result: null); + + private static AgentResponse Success(string id, int code, string name, object? result) => + new() + { + Id = id, + Ok = true, + Status = new AgentStatus { Code = code, Name = name }, + Result = result is null ? null : AgentProtocol.ToJsonElement(result) + }; + + private sealed class BufferSegment : ReadOnlySequenceSegment + { + public BufferSegment(byte[] memory) + { + Memory = memory; + } + + public BufferSegment Append(byte[] memory) + { + var segment = new BufferSegment(memory) + { + RunningIndex = RunningIndex + Memory.Length + }; + Next = segment; + return segment; + } + } +} diff --git a/tests/SharedMemoryStore.InteropAgent/Program.cs b/tests/SharedMemoryStore.InteropAgent/Program.cs new file mode 100644 index 0000000..81e6239 --- /dev/null +++ b/tests/SharedMemoryStore.InteropAgent/Program.cs @@ -0,0 +1,3 @@ +using SharedMemoryStore.InteropAgent; + +return await AgentHost.RunAsync(Console.In, Console.Out).ConfigureAwait(false); diff --git a/tests/SharedMemoryStore.InteropAgent/SharedMemoryStore.InteropAgent.csproj b/tests/SharedMemoryStore.InteropAgent/SharedMemoryStore.InteropAgent.csproj new file mode 100644 index 0000000..86ea7a6 --- /dev/null +++ b/tests/SharedMemoryStore.InteropAgent/SharedMemoryStore.InteropAgent.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + enable + false + + + + + + + diff --git a/tests/SharedMemoryStore.InteropTests/AgentLifecycleTests.cs b/tests/SharedMemoryStore.InteropTests/AgentLifecycleTests.cs new file mode 100644 index 0000000..0908d92 --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/AgentLifecycleTests.cs @@ -0,0 +1,98 @@ +using SharedMemoryStore.InteropAgent; + +namespace SharedMemoryStore.InteropTests; + +public sealed class AgentLifecycleTests +{ + [Fact] + public async Task DotNetAgentExecutesValueAndReservationLifecycles() + { + var name = $"sms-interop-agent-{Guid.NewGuid():N}"; + var requests = new[] + { + Request("1", "open", new + { + storeId = "store", + name, + openMode = 0, + slotCount = 3, + maxValueBytes = 32, + maxDescriptorBytes = 8, + maxKeyBytes = 8, + leaseRecordCount = 4, + enableLeaseRecovery = true + }), + Request("2", "publish", new + { + storeId = "store", + key = AgentProtocol.EncodeBytes([1, 0]), + value = AgentProtocol.EncodeBytes([7, 0, 9]), + descriptor = AgentProtocol.EncodeBytes([4]) + }), + Request("3", "acquire", new + { + storeId = "store", + leaseId = "lease", + key = AgentProtocol.EncodeBytes([1, 0]) + }), + Request("4", "remove", new + { + storeId = "store", + key = AgentProtocol.EncodeBytes([1, 0]) + }), + Request("5", "release", new { leaseId = "lease" }), + Request("6", "reserve", new + { + storeId = "store", + reservationId = "reservation", + key = AgentProtocol.EncodeBytes([2]), + payloadLength = 3, + descriptor = AgentProtocol.EncodeBytes([5]) + }), + Request("7", "reservationWrite", new + { + reservationId = "reservation", + data = AgentProtocol.EncodeBytes([8, 0, 6]) + }), + Request("8", "advance", new { reservationId = "reservation", byteCount = 3 }), + Request("9", "commit", new { reservationId = "reservation" }), + Request("10", "acquire", new + { + storeId = "store", + leaseId = "reservation-lease", + key = AgentProtocol.EncodeBytes([2]) + }), + Request("11", "diagnostics", new { storeId = "store" }), + Request("12", "close", new { storeId = "store" }) + }; + using var input = new StringReader(string.Concat(requests.Select(AgentProtocol.SerializeRequestLine))); + using var output = new StringWriter(); + + Assert.Equal(0, await AgentHost.RunAsync(input, output)); + + var responses = output.ToString() + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(AgentProtocol.ParseResponse) + .ToArray(); + Assert.Equal(requests.Length, responses.Length); + Assert.All(responses, response => Assert.True(response.Ok)); + Assert.Equal("Success", responses[0].Status.Name); + Assert.Equal("Success", responses[1].Status.Name); + Assert.Equal(new byte[] { 7, 0, 9 }, DecodeResult(responses[2], "value")); + Assert.Equal("RemovePending", responses[3].Status.Name); + Assert.Equal("Success", responses[4].Status.Name); + Assert.Equal(new byte[] { 8, 0, 6 }, DecodeResult(responses[9], "value")); + Assert.Equal(1, responses[10].Result!.Value.GetProperty("publishedSlotCount").GetInt32()); + } + + private static AgentRequest Request(string id, string command, T arguments) => + new() + { + Id = id, + Command = command, + Arguments = AgentProtocol.ToJsonElement(arguments) + }; + + private static byte[] DecodeResult(AgentResponse response, string property) => + AgentProtocol.DecodeBytes(response.Result!.Value.GetProperty(property).GetString()!); +} diff --git a/tests/SharedMemoryStore.InteropTests/AgentProtocolTests.cs b/tests/SharedMemoryStore.InteropTests/AgentProtocolTests.cs new file mode 100644 index 0000000..0caf43f --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/AgentProtocolTests.cs @@ -0,0 +1,131 @@ +using System.Text.Json; +using SharedMemoryStore.InteropAgent; + +namespace SharedMemoryStore.InteropTests; + +public sealed class AgentProtocolTests +{ + [Fact] + public void RequestUsesOneLfDelimitedJsonFrameAndBase64BinaryFields() + { + var key = new byte[] { 0, 1, 2, 254, 255 }; + var value = new byte[] { 10, 13, 32, 92 }; + var request = new AgentRequest + { + Id = "request-1", + Command = "publish", + Arguments = AgentProtocol.ToJsonElement(new + { + key = AgentProtocol.EncodeBytes(key), + value = AgentProtocol.EncodeBytes(value) + }) + }; + + var frame = AgentProtocol.SerializeRequestLine(request); + + Assert.True(frame.EndsWith('\n')); + Assert.Equal(-1, frame[..^1].IndexOfAny(['\r', '\n'])); + + using var document = JsonDocument.Parse(frame); + var root = document.RootElement; + Assert.Equal("request-1", root.GetProperty("id").GetString()); + Assert.Equal("publish", root.GetProperty("command").GetString()); + Assert.Equal(key, AgentProtocol.DecodeBytes(root.GetProperty("arguments").GetProperty("key").GetString()!)); + Assert.Equal(value, AgentProtocol.DecodeBytes(root.GetProperty("arguments").GetProperty("value").GetString()!)); + } + + [Fact] + public void ResponseRoundTripPreservesNumericAndSymbolicStatus() + { + var response = new AgentResponse + { + Id = "request-2", + Ok = true, + Status = new AgentStatus { Code = 0, Name = "Success" }, + Result = AgentProtocol.ToJsonElement(new { value = AgentProtocol.EncodeBytes([1, 2, 3]) }) + }; + + var parsed = AgentProtocol.ParseResponse(AgentProtocol.SerializeResponseLine(response).TrimEnd('\n')); + + Assert.Equal("request-2", parsed.Id); + Assert.True(parsed.Ok); + Assert.Equal(0, parsed.Status.Code); + Assert.Equal("Success", parsed.Status.Name); + Assert.Equal( + new byte[] { 1, 2, 3 }, + AgentProtocol.DecodeBytes(parsed.Result!.Value.GetProperty("value").GetString()!)); + } + + [Fact] + public async Task ReaderConsumesExactlyOneRequestAtATimeAndReportsEndOfInput() + { + var first = new AgentRequest { Id = "1", Command = "ping" }; + var second = new AgentRequest { Id = "2", Command = "ping" }; + using var reader = new StringReader( + AgentProtocol.SerializeRequestLine(first) + AgentProtocol.SerializeRequestLine(second)); + + var parsedFirst = await AgentProtocol.ReadRequestAsync(reader); + var parsedSecond = await AgentProtocol.ReadRequestAsync(reader); + var end = await AgentProtocol.ReadRequestAsync(reader); + + Assert.Equal("1", parsedFirst!.Id); + Assert.Equal("2", parsedSecond!.Id); + Assert.Null(end); + } + + [Fact] + public async Task WriterFlushesOneCompleteResponseFrame() + { + var response = new AgentResponse + { + Id = "request-3", + Ok = false, + Status = new AgentStatus { Code = 7, Name = "NotFound" }, + Error = new AgentError { Code = "not_found", Message = "The key was not found." } + }; + using var writer = new StringWriter(); + + await AgentProtocol.WriteResponseAsync(writer, response); + + var output = writer.ToString(); + Assert.True(output.EndsWith('\n')); + Assert.Equal(-1, output[..^1].IndexOfAny(['\r', '\n'])); + Assert.Equal(response, AgentProtocol.ParseResponse(output.TrimEnd('\n'))); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("{}")] + [InlineData("{\"id\":\"1\",\"command\":\"ping\"}\n{\"id\":\"2\",\"command\":\"ping\"}")] + public void InvalidRequestFramesAreRejected(string frame) + { + Assert.Throws(() => AgentProtocol.ParseRequest(frame)); + } + + [Fact] + public async Task HostProducesOnlyProtocolResponsesForPingAndUnsupportedCommands() + { + var ping = new AgentRequest { Id = "ping-1", Command = "ping" }; + var unsupported = new AgentRequest { Id = "future-1", Command = "future-command" }; + using var input = new StringReader( + AgentProtocol.SerializeRequestLine(ping) + AgentProtocol.SerializeRequestLine(unsupported)); + using var output = new StringWriter(); + + var exitCode = await AgentHost.RunAsync(input, output); + + Assert.Equal(0, exitCode); + var lines = output.ToString().Split('\n', StringSplitOptions.RemoveEmptyEntries); + Assert.Equal(2, lines.Length); + + var pingResponse = AgentProtocol.ParseResponse(lines[0]); + Assert.True(pingResponse.Ok); + Assert.Equal("Success", pingResponse.Status.Name); + Assert.Equal("dotnet", pingResponse.Result!.Value.GetProperty("runtime").GetString()); + + var unsupportedResponse = AgentProtocol.ParseResponse(lines[1]); + Assert.False(unsupportedResponse.Ok); + Assert.Equal("UnsupportedCommand", unsupportedResponse.Status.Name); + Assert.Equal("unsupported_command", unsupportedResponse.Error!.Code); + } +} diff --git a/tests/SharedMemoryStore.InteropTests/CoreExchangeMatrixTests.cs b/tests/SharedMemoryStore.InteropTests/CoreExchangeMatrixTests.cs new file mode 100644 index 0000000..8ac3978 --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/CoreExchangeMatrixTests.cs @@ -0,0 +1,128 @@ +using SharedMemoryStore.InteropAgent; +using SharedMemoryStore.InteropTests.TestSupport; + +namespace SharedMemoryStore.InteropTests; + +public sealed class CoreExchangeMatrixTests +{ + public static TheoryData OrderedRuntimePairs => new() + { + { "dotnet", "dotnet" }, + { "dotnet", "cpp" }, + { "dotnet", "python" }, + { "cpp", "dotnet" }, + { "cpp", "cpp" }, + { "cpp", "python" }, + { "python", "dotnet" }, + { "python", "cpp" }, + { "python", "python" } + }; + + [Theory] + [MemberData(nameof(OrderedRuntimePairs))] + public async Task ProducerAndConsumerExchangeExactBytesBothWays(string producerRuntime, string consumerRuntime) + { + var producerDefinition = AgentDefinition.Resolve(producerRuntime); + var consumerDefinition = AgentDefinition.Resolve(consumerRuntime); + if (!producerDefinition.IsAvailable() || !consumerDefinition.IsAvailable()) + { + return; + } + + await using var producer = await AgentProcess.StartAsync(producerDefinition); + await using var consumer = await AgentProcess.StartAsync(consumerDefinition); + var name = $"sms-3x3-{producerRuntime}-{consumerRuntime}-{Guid.NewGuid():N}"; + var common = new + { + name, + slotCount = 3, + maxValueBytes = 64, + maxDescriptorBytes = 16, + maxKeyBytes = 16, + leaseRecordCount = 8, + enableLeaseRecovery = true + }; + var producerOpen = await producer.SendAsync("open", new + { + storeId = "producer", + common.name, + openMode = 0, + common.slotCount, + common.maxValueBytes, + common.maxDescriptorBytes, + common.maxKeyBytes, + common.leaseRecordCount, + common.enableLeaseRecovery + }); + AssertSuccess(producerOpen); + var consumerOpen = await consumer.SendAsync("open", new + { + storeId = "consumer", + common.name, + openMode = 1, + common.slotCount, + common.maxValueBytes, + common.maxDescriptorBytes, + common.maxKeyBytes, + common.leaseRecordCount, + common.enableLeaseRecovery + }); + AssertSuccess(consumerOpen); + + var key = new byte[] { 0, 1, 0xff }; + var firstValue = new byte[] { 9, 0, 8, 0xff }; + var firstDescriptor = new byte[] { 4, 0 }; + AssertSuccess(await producer.SendAsync("publish", new + { + storeId = "producer", + key = AgentProtocol.EncodeBytes(key), + value = AgentProtocol.EncodeBytes(firstValue), + descriptor = AgentProtocol.EncodeBytes(firstDescriptor) + })); + var acquired = await consumer.SendAsync("acquire", new + { + storeId = "consumer", + leaseId = "consumer-lease", + key = AgentProtocol.EncodeBytes(key) + }); + AssertSuccess(acquired); + Assert.Equal(firstValue, Decode(acquired, "value")); + Assert.Equal(firstDescriptor, Decode(acquired, "descriptor")); + AssertSuccess(await consumer.SendAsync("release", new { leaseId = "consumer-lease" })); + AssertSuccess(await consumer.SendAsync("remove", new + { + storeId = "consumer", + key = AgentProtocol.EncodeBytes(key) + })); + + var secondValue = new byte[] { 7, 6, 0, 5 }; + AssertSuccess(await consumer.SendAsync("publish", new + { + storeId = "consumer", + key = AgentProtocol.EncodeBytes(key), + value = AgentProtocol.EncodeBytes(secondValue), + descriptor = string.Empty + })); + var reverse = await producer.SendAsync("acquire", new + { + storeId = "producer", + leaseId = "producer-lease", + key = AgentProtocol.EncodeBytes(key) + }); + AssertSuccess(reverse); + Assert.Equal(secondValue, Decode(reverse, "value")); + AssertSuccess(await producer.SendAsync("release", new { leaseId = "producer-lease" })); + AssertSuccess(await consumer.SendAsync("close", new { storeId = "consumer" })); + AssertSuccess(await producer.SendAsync("close", new { storeId = "producer" })); + } + + private static void AssertSuccess(AgentResponse response) + { + Assert.True(response.Ok, response.Error?.Message); + Assert.Equal(0, response.Status.Code); + Assert.Equal("Success", response.Status.Name); + } + + private static byte[] Decode(AgentResponse response, string property) => + AgentProtocol.DecodeBytes(response.Result!.Value.GetProperty(property).GetString()!); +} diff --git a/tests/SharedMemoryStore.InteropTests/DiagnosticsInteropTests.cs b/tests/SharedMemoryStore.InteropTests/DiagnosticsInteropTests.cs new file mode 100644 index 0000000..aeb1c09 --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/DiagnosticsInteropTests.cs @@ -0,0 +1,104 @@ +using System.Text.Json; +using SharedMemoryStore.InteropAgent; +using SharedMemoryStore.InteropTests.TestSupport; + +namespace SharedMemoryStore.InteropTests; + +public sealed class DiagnosticsInteropTests +{ + private static readonly string[] SharedFields = + [ + "totalBytes", + "slotCount", + "freeSlotCount", + "publishedSlotCount", + "pendingRemovalCount", + "activeLeaseCount", + "activeReservationCount", + "indexEntryCount", + "occupiedIndexEntryCount", + "tombstoneIndexEntryCount", + "emptyIndexEntryCount", + "usableIndexCapacity" + ]; + + [Theory] + [MemberData( + nameof(CoreExchangeMatrixTests.OrderedRuntimePairs), + MemberType = typeof(CoreExchangeMatrixTests))] + public async Task RuntimesReportEquivalentSharedStateAndCallerLocalFailures( + string firstRuntime, + string secondRuntime) + { + var firstDefinition = AgentDefinition.Resolve(firstRuntime); + var secondDefinition = AgentDefinition.Resolve(secondRuntime); + if (!firstDefinition.IsAvailable() || !secondDefinition.IsAvailable()) + { + return; + } + + await using var first = await AgentProcess.StartAsync(firstDefinition); + await using var second = await AgentProcess.StartAsync(secondDefinition); + var name = $"sms-diagnostics-{firstRuntime}-{secondRuntime}-{Guid.NewGuid():N}"; + InteropAssertions.Success(await first.SendAsync( + "open", + InteropAssertions.OpenArguments("first", name, openMode: 0))); + InteropAssertions.Success(await second.SendAsync( + "open", + InteropAssertions.OpenArguments("second", name, openMode: 1))); + + var leasedKey = AgentProtocol.EncodeBytes(new byte[] { 1, 0, 5 }); + InteropAssertions.Success(await first.SendAsync("publish", new + { + storeId = "first", + key = leasedKey, + value = AgentProtocol.EncodeBytes(new byte[] { 7, 0, 7 }), + descriptor = string.Empty + })); + InteropAssertions.Success(await first.SendAsync("acquire", new + { + storeId = "first", + leaseId = "lease", + key = leasedKey + })); + InteropAssertions.Status( + await second.SendAsync("remove", new { storeId = "second", key = leasedKey }), + 10, + "RemovePending"); + InteropAssertions.Success(await second.SendAsync("reserve", new + { + storeId = "second", + reservationId = "reservation", + key = AgentProtocol.EncodeBytes(new byte[] { 2, 0, 6 }), + payloadLength = 5, + descriptor = string.Empty + })); + + var firstDiagnostics = await first.SendAsync("diagnostics", new { storeId = "first" }); + var secondDiagnostics = await second.SendAsync("diagnostics", new { storeId = "second" }); + InteropAssertions.Success(firstDiagnostics); + InteropAssertions.Success(secondDiagnostics); + var firstResult = firstDiagnostics.Result!.Value; + var secondResult = secondDiagnostics.Result!.Value; + foreach (var field in SharedFields) + { + Assert.Equal(Number(firstResult, field), Number(secondResult, field)); + } + + Assert.Equal(6, Number(firstResult, "slotCount")); + Assert.Equal(4, Number(firstResult, "freeSlotCount")); + Assert.Equal(0, Number(firstResult, "publishedSlotCount")); + Assert.Equal(1, Number(firstResult, "pendingRemovalCount")); + Assert.Equal(1, Number(firstResult, "activeLeaseCount")); + Assert.Equal(1, Number(firstResult, "activeReservationCount")); + Assert.Equal(10, Number(secondResult, "lastFailureStatus")); + + InteropAssertions.Success(await second.SendAsync("abort", new { reservationId = "reservation" })); + InteropAssertions.Success(await first.SendAsync("release", new { leaseId = "lease" })); + InteropAssertions.Success(await second.SendAsync("close", new { storeId = "second" })); + InteropAssertions.Success(await first.SendAsync("close", new { storeId = "first" })); + } + + private static long Number(JsonElement value, string property) => + value.GetProperty(property).GetInt64(); +} diff --git a/tests/SharedMemoryStore.InteropTests/Dockerfile b/tests/SharedMemoryStore.InteropTests/Dockerfile new file mode 100644 index 0000000..861af7e --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/Dockerfile @@ -0,0 +1,34 @@ +FROM mcr.microsoft.com/dotnet/sdk:10.0-noble + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ + cmake \ + g++ \ + ninja-build \ + python3 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace +COPY . . + +RUN cmake -S . -B artifacts/docker-interop-native -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DSMS_BUILD_TESTS=ON \ + -DSMS_BUILD_SAMPLES=OFF \ + -DSMS_INSTALL=ON \ + -DSMS_PYTHON_INSTALL_DIR=shared_memory_store \ + && cmake --build artifacts/docker-interop-native --parallel \ + && ctest --test-dir artifacts/docker-interop-native --output-on-failure \ + && cmake --install artifacts/docker-interop-native \ + --component Python \ + --prefix /workspace/src/python \ + && dotnet build tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj \ + --configuration Release + +ENV SMS_CPP_AGENT=/workspace/artifacts/docker-interop-native/tests/cpp/sms_cpp_interop_agent \ + SMS_PYTHON_EXECUTABLE=python3 \ + SMS_RUN_INTEROP_STRESS=0 \ + SMS_INTEROP_STRESS_VALUES=1000 \ + SMS_INTEROP_STRESS_LIFECYCLE_CYCLES=10000 + +CMD ["bash", "-lc", "dotnet test tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj --configuration Release --no-build --filter 'FullyQualifiedName!~StressInteropTests' && if [ \"$SMS_RUN_INTEROP_STRESS\" = 1 ]; then dotnet test tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj --configuration Release --no-build --filter 'FullyQualifiedName~StressInteropTests'; fi"] diff --git a/tests/SharedMemoryStore.InteropTests/MixedLifecycleTests.cs b/tests/SharedMemoryStore.InteropTests/MixedLifecycleTests.cs new file mode 100644 index 0000000..7e0f157 --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/MixedLifecycleTests.cs @@ -0,0 +1,215 @@ +using SharedMemoryStore.InteropAgent; +using SharedMemoryStore.InteropTests.TestSupport; + +namespace SharedMemoryStore.InteropTests; + +public sealed class MixedLifecycleTests +{ + [Theory] + [MemberData( + nameof(CoreExchangeMatrixTests.OrderedRuntimePairs), + MemberType = typeof(CoreExchangeMatrixTests))] + public async Task ConcurrentForeignPublishHasExactlyOneWinner( + string firstRuntime, + string secondRuntime) + { + var firstDefinition = AgentDefinition.Resolve(firstRuntime); + var secondDefinition = AgentDefinition.Resolve(secondRuntime); + if (!firstDefinition.IsAvailable() || !secondDefinition.IsAvailable()) + { + return; + } + + await using var first = await AgentProcess.StartAsync(firstDefinition); + await using var second = await AgentProcess.StartAsync(secondDefinition); + var name = $"sms-publish-race-{firstRuntime}-{secondRuntime}-{Guid.NewGuid():N}"; + InteropAssertions.Success(await first.SendAsync( + "open", + InteropAssertions.OpenArguments("first", name, openMode: 0))); + InteropAssertions.Success(await second.SendAsync( + "open", + InteropAssertions.OpenArguments("second", name, openMode: 1))); + var key = AgentProtocol.EncodeBytes(new byte[] { 8, 0, 8 }); + var firstPublish = first.SendAsync("publish", new + { + storeId = "first", + key, + value = AgentProtocol.EncodeBytes(new byte[] { 1 }), + descriptor = string.Empty + }); + var secondPublish = second.SendAsync("publish", new + { + storeId = "second", + key, + value = AgentProtocol.EncodeBytes(new byte[] { 2 }), + descriptor = string.Empty + }); + var responses = await Task.WhenAll(firstPublish, secondPublish); + Assert.Single(responses, response => response.Status.Name == "Success" && response.Status.Code == 0); + Assert.Single(responses, response => response.Status.Name == "DuplicateKey" && response.Status.Code == 1); + Assert.All(responses, response => Assert.True(response.Ok, response.Error?.Message)); + + InteropAssertions.Success(await first.SendAsync("remove", new { storeId = "first", key })); + InteropAssertions.Success(await second.SendAsync("close", new { storeId = "second" })); + InteropAssertions.Success(await first.SendAsync("close", new { storeId = "first" })); + } + + [Theory] + [MemberData( + nameof(CoreExchangeMatrixTests.OrderedRuntimePairs), + MemberType = typeof(CoreExchangeMatrixTests))] + public async Task ForeignLeaseReservationAndSegmentsShareOneLifecycle( + string producerRuntime, + string consumerRuntime) + { + var producerDefinition = AgentDefinition.Resolve(producerRuntime); + var consumerDefinition = AgentDefinition.Resolve(consumerRuntime); + if (!producerDefinition.IsAvailable() || !consumerDefinition.IsAvailable()) + { + return; + } + + await using var producer = await AgentProcess.StartAsync(producerDefinition); + await using var consumer = await AgentProcess.StartAsync(consumerDefinition); + var name = $"sms-lifecycle-{producerRuntime}-{consumerRuntime}-{Guid.NewGuid():N}"; + InteropAssertions.Success(await producer.SendAsync( + "open", + InteropAssertions.OpenArguments("producer", name, openMode: 0))); + InteropAssertions.Success(await consumer.SendAsync( + "open", + InteropAssertions.OpenArguments("consumer", name, openMode: 1))); + + var leasedKey = new byte[] { 1, 0, 1 }; + InteropAssertions.Success(await producer.SendAsync("publish", new + { + storeId = "producer", + key = AgentProtocol.EncodeBytes(leasedKey), + value = AgentProtocol.EncodeBytes(new byte[] { 9, 0, 8 }), + descriptor = AgentProtocol.EncodeBytes(new byte[] { 7 }) + })); + InteropAssertions.Success(await producer.SendAsync("acquire", new + { + storeId = "producer", + leaseId = "foreign-lease", + key = AgentProtocol.EncodeBytes(leasedKey) + })); + InteropAssertions.Status(await consumer.SendAsync("remove", new + { + storeId = "consumer", + key = AgentProtocol.EncodeBytes(leasedKey) + }), 10, "RemovePending"); + InteropAssertions.Success(await producer.SendAsync("release", new { leaseId = "foreign-lease" })); + InteropAssertions.Status( + await producer.SendAsync("release", new { leaseId = "foreign-lease" }), + 9, + "LeaseAlreadyReleased"); + InteropAssertions.Success(await consumer.SendAsync("publish", new + { + storeId = "consumer", + key = AgentProtocol.EncodeBytes(leasedKey), + value = AgentProtocol.EncodeBytes(new byte[] { 4, 3, 0, 2 }), + descriptor = string.Empty + })); + + var committedKey = new byte[] { 2, 0, 2 }; + var committedValue = new byte[] { 5, 0, 4, 0, 3 }; + InteropAssertions.Success(await producer.SendAsync("reserve", new + { + storeId = "producer", + reservationId = "commit-reservation", + key = AgentProtocol.EncodeBytes(committedKey), + payloadLength = committedValue.Length, + descriptor = AgentProtocol.EncodeBytes(new byte[] { 6, 0 }) + })); + InteropAssertions.Status(await consumer.SendAsync("acquire", new + { + storeId = "consumer", + leaseId = "invisible-lease", + key = AgentProtocol.EncodeBytes(committedKey) + }), 2, "NotFound"); + InteropAssertions.Status(await consumer.SendAsync("publish", new + { + storeId = "consumer", + key = AgentProtocol.EncodeBytes(committedKey), + value = AgentProtocol.EncodeBytes(new byte[] { 1 }), + descriptor = string.Empty + }), 1, "DuplicateKey"); + InteropAssertions.Success(await producer.SendAsync("reservationWrite", new + { + reservationId = "commit-reservation", + data = AgentProtocol.EncodeBytes(committedValue) + })); + InteropAssertions.Success(await producer.SendAsync( + "advance", + new { reservationId = "commit-reservation", byteCount = committedValue.Length })); + InteropAssertions.Success(await producer.SendAsync( + "commit", + new { reservationId = "commit-reservation" })); + var committed = await consumer.SendAsync("acquire", new + { + storeId = "consumer", + leaseId = "committed-lease", + key = AgentProtocol.EncodeBytes(committedKey) + }); + InteropAssertions.Success(committed); + Assert.Equal(committedValue, InteropAssertions.Decode(committed, "value")); + Assert.Equal(new byte[] { 6, 0 }, InteropAssertions.Decode(committed, "descriptor")); + InteropAssertions.Success(await consumer.SendAsync("release", new { leaseId = "committed-lease" })); + + var abortedKey = new byte[] { 3, 0, 3 }; + InteropAssertions.Success(await producer.SendAsync("reserve", new + { + storeId = "producer", + reservationId = "abort-reservation", + key = AgentProtocol.EncodeBytes(abortedKey), + payloadLength = 4, + descriptor = string.Empty + })); + InteropAssertions.Success(await producer.SendAsync("reservationWrite", new + { + reservationId = "abort-reservation", + data = AgentProtocol.EncodeBytes(new byte[] { 8, 7 }) + })); + InteropAssertions.Success(await producer.SendAsync( + "advance", + new { reservationId = "abort-reservation", byteCount = 2 })); + InteropAssertions.Success(await producer.SendAsync( + "abort", + new { reservationId = "abort-reservation" })); + InteropAssertions.Success(await consumer.SendAsync("publish", new + { + storeId = "consumer", + key = AgentProtocol.EncodeBytes(abortedKey), + value = AgentProtocol.EncodeBytes(new byte[] { 1, 2 }), + descriptor = string.Empty + })); + + var segmentedKey = new byte[] { 4, 0, 4 }; + var segmented = await producer.SendAsync("publishSegments", new + { + storeId = "producer", + key = AgentProtocol.EncodeBytes(segmentedKey), + segments = new[] + { + AgentProtocol.EncodeBytes(new byte[] { 1, 0 }), + AgentProtocol.EncodeBytes(Array.Empty()), + AgentProtocol.EncodeBytes(new byte[] { 2, 3 }) + }, + descriptor = AgentProtocol.EncodeBytes(new byte[] { 9 }) + }); + InteropAssertions.Success(segmented); + Assert.Equal(4, segmented.Result!.Value.GetProperty("copiedBytes").GetInt64()); + var segmentedLease = await consumer.SendAsync("acquire", new + { + storeId = "consumer", + leaseId = "segmented-lease", + key = AgentProtocol.EncodeBytes(segmentedKey) + }); + InteropAssertions.Success(segmentedLease); + Assert.Equal(new byte[] { 1, 0, 2, 3 }, InteropAssertions.Decode(segmentedLease, "value")); + InteropAssertions.Success(await consumer.SendAsync("release", new { leaseId = "segmented-lease" })); + + InteropAssertions.Success(await consumer.SendAsync("close", new { storeId = "consumer" })); + InteropAssertions.Success(await producer.SendAsync("close", new { storeId = "producer" })); + } +} diff --git a/tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs b/tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs new file mode 100644 index 0000000..8accae2 --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/RecoveryAndOwnershipTests.cs @@ -0,0 +1,243 @@ +using System.Diagnostics; +using SharedMemoryStore.InteropAgent; +using SharedMemoryStore.InteropTests.TestSupport; + +namespace SharedMemoryStore.InteropTests; + +public sealed class RecoveryAndOwnershipTests +{ + public static TheoryData Runtimes => new() + { + "dotnet", + "cpp", + "python" + }; + + [Theory] + [MemberData( + nameof(CoreExchangeMatrixTests.OrderedRuntimePairs), + MemberType = typeof(CoreExchangeMatrixTests))] + public async Task SurvivorRecoversForeignCrashedLeaseAndReservation( + string survivorRuntime, + string crashedRuntime) + { + var survivorDefinition = AgentDefinition.Resolve(survivorRuntime); + var crashedDefinition = AgentDefinition.Resolve(crashedRuntime); + if (!survivorDefinition.IsAvailable() || !crashedDefinition.IsAvailable()) + { + return; + } + + var name = $"sms-recovery-{survivorRuntime}-{crashedRuntime}-{Guid.NewGuid():N}"; + await using var survivor = await AgentProcess.StartAsync(survivorDefinition); + InteropAssertions.Success(await survivor.SendAsync( + "open", + InteropAssertions.OpenArguments("survivor", name, openMode: 0))); + + var leasedKey = new byte[] { 1, 0, 8 }; + await using (var crashedLeaseOwner = await AgentProcess.StartAsync(crashedDefinition)) + { + InteropAssertions.Success(await crashedLeaseOwner.SendAsync( + "open", + InteropAssertions.OpenArguments("crashed-lease-owner", name, openMode: 1))); + InteropAssertions.Success(await crashedLeaseOwner.SendAsync("publish", new + { + storeId = "crashed-lease-owner", + key = AgentProtocol.EncodeBytes(leasedKey), + value = AgentProtocol.EncodeBytes(new byte[] { 7, 0, 6 }), + descriptor = string.Empty + })); + InteropAssertions.Success(await crashedLeaseOwner.SendAsync("acquire", new + { + storeId = "crashed-lease-owner", + leaseId = "abandoned-lease", + key = AgentProtocol.EncodeBytes(leasedKey) + })); + await crashedLeaseOwner.CrashAsync(); + } + + var recoveredLeases = await survivor.SendAsync("recoverLeases", new + { + storeId = "survivor", + recoverCurrentProcess = false + }); + InteropAssertions.Success(recoveredLeases); + Assert.Equal(1, recoveredLeases.Result!.Value.GetProperty("recoveredLeaseCount").GetInt32()); + InteropAssertions.Success(await survivor.SendAsync("remove", new + { + storeId = "survivor", + key = AgentProtocol.EncodeBytes(leasedKey) + })); + + var reservedKey = new byte[] { 2, 0, 9 }; + await using (var crashedReservationOwner = await AgentProcess.StartAsync(crashedDefinition)) + { + InteropAssertions.Success(await crashedReservationOwner.SendAsync( + "open", + InteropAssertions.OpenArguments("crashed-reservation-owner", name, openMode: 1))); + InteropAssertions.Success(await crashedReservationOwner.SendAsync("reserve", new + { + storeId = "crashed-reservation-owner", + reservationId = "abandoned-reservation", + key = AgentProtocol.EncodeBytes(reservedKey), + payloadLength = 6, + descriptor = AgentProtocol.EncodeBytes(new byte[] { 3 }) + })); + InteropAssertions.Success(await crashedReservationOwner.SendAsync("reservationWrite", new + { + reservationId = "abandoned-reservation", + data = AgentProtocol.EncodeBytes(new byte[] { 5, 0, 4 }) + })); + InteropAssertions.Success(await crashedReservationOwner.SendAsync( + "advance", + new { reservationId = "abandoned-reservation", byteCount = 3 })); + await crashedReservationOwner.CrashAsync(); + } + + var recoveredReservations = await survivor.SendAsync("recoverReservations", new + { + storeId = "survivor", + recoverCurrentProcess = false + }); + InteropAssertions.Success(recoveredReservations); + Assert.Equal( + 1, + recoveredReservations.Result!.Value.GetProperty("recoveredReservationCount").GetInt32()); + InteropAssertions.Success(await survivor.SendAsync("publish", new + { + storeId = "survivor", + key = AgentProtocol.EncodeBytes(reservedKey), + value = AgentProtocol.EncodeBytes(new byte[] { 1, 0, 2 }), + descriptor = string.Empty + })); + InteropAssertions.Success(await survivor.SendAsync("close", new { storeId = "survivor" })); + } + + [Theory] + [MemberData(nameof(Runtimes))] + public async Task NoWaitAndBoundedWaitObserveForeignStoreLock(string runtime) + { + var definition = AgentDefinition.Resolve(runtime); + if (!definition.IsAvailable()) + { + return; + } + + await using var agent = await AgentProcess.StartAsync(definition); + var name = $"sms-contention-{runtime}-{Guid.NewGuid():N}"; + InteropAssertions.Success(await agent.SendAsync( + "open", + InteropAssertions.OpenArguments("store", name, openMode: 0))); + var key = AgentProtocol.EncodeBytes(new byte[] { 4, 0, 4 }); + var value = AgentProtocol.EncodeBytes(new byte[] { 8, 0, 8 }); + + using (await ForeignStoreLock.AcquireAsync(name)) + { + InteropAssertions.Status(await agent.SendAsync("publish", new + { + storeId = "store", + key, + value, + descriptor = string.Empty, + timeoutMs = 0 + }), 21, "StoreBusy"); + + var stopwatch = Stopwatch.StartNew(); + var bounded = await agent.SendAsync("publish", new + { + storeId = "store", + key, + value, + descriptor = string.Empty, + timeoutMs = 40 + }); + stopwatch.Stop(); + InteropAssertions.Status(bounded, 21, "StoreBusy"); + Assert.InRange(stopwatch.ElapsedMilliseconds, 10, 500); + } + + InteropAssertions.Success(await agent.SendAsync("publish", new + { + storeId = "store", + key, + value, + descriptor = string.Empty, + timeoutMs = 1000 + })); + InteropAssertions.Success(await agent.SendAsync("close", new { storeId = "store" })); + } + + [Theory] + [MemberData(nameof(Runtimes))] + public async Task EveryRuntimeRejectsMismatchedExistingLayout(string runtime) + { + var creatorDefinition = AgentDefinition.Resolve("dotnet"); + var openerDefinition = AgentDefinition.Resolve(runtime); + if (!creatorDefinition.IsAvailable() || !openerDefinition.IsAvailable()) + { + return; + } + + await using var creator = await AgentProcess.StartAsync(creatorDefinition); + await using var opener = await AgentProcess.StartAsync(openerDefinition); + var name = $"sms-layout-mismatch-{runtime}-{Guid.NewGuid():N}"; + InteropAssertions.Success(await creator.SendAsync( + "open", + InteropAssertions.OpenArguments("creator", name, openMode: 0, slotCount: 6))); + var mismatch = await opener.SendAsync( + "open", + InteropAssertions.OpenArguments("mismatch", name, openMode: 1, slotCount: 5)); + InteropAssertions.Status(mismatch, 4, "IncompatibleLayout"); + InteropAssertions.Success(await creator.SendAsync("close", new { storeId = "creator" })); + } + + [Fact] + public async Task ThreeLinuxOwnersCleanOnlyAfterFinalClose() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + var definitions = InteropAssertions.Runtimes.Select(AgentDefinition.Resolve).ToArray(); + if (definitions.Any(definition => !definition.IsAvailable())) + { + return; + } + + await using var dotnet = await AgentProcess.StartAsync(definitions[0]); + await using var cpp = await AgentProcess.StartAsync(definitions[1]); + await using var python = await AgentProcess.StartAsync(definitions[2]); + var name = $"sms-three-owners-{Guid.NewGuid():N}"; + InteropAssertions.Success(await dotnet.SendAsync( + "open", + InteropAssertions.OpenArguments("dotnet", name, openMode: 0))); + InteropAssertions.Success(await cpp.SendAsync( + "open", + InteropAssertions.OpenArguments("cpp", name, openMode: 1))); + InteropAssertions.Success(await python.SendAsync( + "open", + InteropAssertions.OpenArguments("python", name, openMode: 1))); + + var regionPath = ForeignStoreLock.LinuxRegionPath(name); + var synchronizationPath = ForeignStoreLock.LinuxSynchronizationPath(name); + var ownersPath = ForeignStoreLock.LinuxOwnersPath(name); + var lifecyclePath = ForeignStoreLock.LinuxLifecyclePath(name); + Assert.True(File.Exists(regionPath)); + Assert.True(File.Exists(synchronizationPath)); + Assert.Equal(3, File.ReadAllLines(ownersPath).Length); + + InteropAssertions.Success(await dotnet.SendAsync("close", new { storeId = "dotnet" })); + Assert.True(File.Exists(regionPath)); + Assert.Equal(2, File.ReadAllLines(ownersPath).Length); + InteropAssertions.Success(await cpp.SendAsync("close", new { storeId = "cpp" })); + Assert.True(File.Exists(regionPath)); + Assert.Single(File.ReadAllLines(ownersPath)); + InteropAssertions.Success(await python.SendAsync("close", new { storeId = "python" })); + + Assert.False(File.Exists(regionPath)); + Assert.False(File.Exists(synchronizationPath)); + Assert.False(File.Exists(ownersPath)); + Assert.True(File.Exists(lifecyclePath)); + } +} diff --git a/tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj b/tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj new file mode 100644 index 0000000..ba4f772 --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/SharedMemoryStore.InteropTests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + diff --git a/tests/SharedMemoryStore.InteropTests/StressInteropTests.cs b/tests/SharedMemoryStore.InteropTests/StressInteropTests.cs new file mode 100644 index 0000000..827c7cd --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/StressInteropTests.cs @@ -0,0 +1,365 @@ +using SharedMemoryStore.InteropAgent; +using SharedMemoryStore.InteropTests.TestSupport; + +namespace SharedMemoryStore.InteropTests; + +public sealed class StressInteropTests +{ + private const string StressOptInVariable = "SMS_RUN_INTEROP_STRESS"; + + public static TheoryData OrderedRuntimePairs => new() + { + { "dotnet", "dotnet" }, + { "dotnet", "cpp" }, + { "dotnet", "python" }, + { "cpp", "dotnet" }, + { "cpp", "cpp" }, + { "cpp", "python" }, + { "python", "dotnet" }, + { "python", "cpp" }, + { "python", "python" } + }; + + [Theory] + [MemberData(nameof(OrderedRuntimePairs))] + public async Task OrderedPairsExchangeTheConfiguredValueCount( + string producerRuntime, + string consumerRuntime) + { + if (!StressIsEnabled()) + { + return; + } + + var valueCount = ReadBoundedCount("SMS_INTEROP_STRESS_VALUES", 1_000, 100_000); + var producerDefinition = AgentDefinition.Resolve(producerRuntime); + var consumerDefinition = AgentDefinition.Resolve(consumerRuntime); + if (!AgentsAreAvailable(producerDefinition, consumerDefinition)) + { + return; + } + + await using var producer = await AgentProcess.StartAsync(producerDefinition); + await using var consumer = await AgentProcess.StartAsync(consumerDefinition); + var name = $"sms-stress-values-{producerRuntime}-{consumerRuntime}-{Environment.ProcessId}-{Guid.NewGuid():N}"; + var options = new + { + name, + slotCount = 4, + maxValueBytes = 64, + maxDescriptorBytes = 16, + maxKeyBytes = 16, + leaseRecordCount = 16, + enableLeaseRecovery = true + }; + + AssertSuccess(await producer.SendAsync("open", new + { + storeId = "producer", + options.name, + openMode = 0, + options.slotCount, + options.maxValueBytes, + options.maxDescriptorBytes, + options.maxKeyBytes, + options.leaseRecordCount, + options.enableLeaseRecovery + })); + AssertSuccess(await consumer.SendAsync("open", new + { + storeId = "consumer", + options.name, + openMode = 1, + options.slotCount, + options.maxValueBytes, + options.maxDescriptorBytes, + options.maxKeyBytes, + options.leaseRecordCount, + options.enableLeaseRecovery + })); + + var seed = StableSeed(producerRuntime, consumerRuntime); + for (var iteration = 0; iteration < valueCount; iteration++) + { + var key = IterationBytes(0x4b, iteration, seed, 9); + var value = IterationBytes(0x56, iteration, seed, 37); + var descriptor = IterationBytes(0x44, iteration, seed, 7); + AssertSuccess(await producer.SendAsync("publish", new + { + storeId = "producer", + key = AgentProtocol.EncodeBytes(key), + value = AgentProtocol.EncodeBytes(value), + descriptor = AgentProtocol.EncodeBytes(descriptor) + })); + + var acquired = await consumer.SendAsync("acquire", new + { + storeId = "consumer", + leaseId = "stress-lease", + key = AgentProtocol.EncodeBytes(key) + }); + AssertSuccess(acquired); + Assert.Equal(value, Decode(acquired, "value")); + Assert.Equal(descriptor, Decode(acquired, "descriptor")); + AssertSuccess(await consumer.SendAsync("release", new { leaseId = "stress-lease" })); + AssertSuccess(await consumer.SendAsync("remove", new + { + storeId = "consumer", + key = AgentProtocol.EncodeBytes(key) + })); + } + + AssertSuccess(await consumer.SendAsync("close", new { storeId = "consumer" })); + AssertSuccess(await producer.SendAsync("close", new { storeId = "producer" })); + } + + [Fact] + public async Task AllRuntimesCompleteConfiguredMixedLifecycleCycles() + { + if (!StressIsEnabled()) + { + return; + } + + var cycleCount = ReadBoundedCount("SMS_INTEROP_STRESS_LIFECYCLE_CYCLES", 10_000, 100_000); + var definitions = new[] + { + AgentDefinition.Resolve("dotnet"), + AgentDefinition.Resolve("cpp"), + AgentDefinition.Resolve("python") + }; + if (!AgentsAreAvailable(definitions)) + { + return; + } + + await using var dotnet = await AgentProcess.StartAsync(definitions[0]); + await using var cpp = await AgentProcess.StartAsync(definitions[1]); + await using var python = await AgentProcess.StartAsync(definitions[2]); + var participants = new[] + { + new Participant("dotnet-store", dotnet), + new Participant("cpp-store", cpp), + new Participant("python-store", python) + }; + var name = $"sms-stress-lifecycle-{Environment.ProcessId}-{Guid.NewGuid():N}"; + var options = new + { + name, + slotCount = 4, + maxValueBytes = 32, + maxDescriptorBytes = 8, + maxKeyBytes = 8, + leaseRecordCount = 16, + enableLeaseRecovery = true + }; + + for (var index = 0; index < participants.Length; index++) + { + var participant = participants[index]; + AssertSuccess(await participant.Agent.SendAsync("open", new + { + storeId = participant.StoreId, + options.name, + openMode = index == 0 ? 0 : 1, + options.slotCount, + options.maxValueBytes, + options.maxDescriptorBytes, + options.maxKeyBytes, + options.leaseRecordCount, + options.enableLeaseRecovery + })); + } + + for (var cycle = 0; cycle < cycleCount; cycle++) + { + var publisher = participants[cycle % participants.Length]; + var reader = participants[(cycle + 1) % participants.Length]; + var reservationOwner = participants[(cycle + 2) % participants.Length]; + var leaseKey = IterationKey(0x4c, cycle); + var reservationKey = IterationKey(0x52, cycle); + var value = IterationBytes(0x50, cycle, 0x13579bdfu, 12); + var descriptor = IterationBytes(0x44, cycle, 0x2468ace0u, 3); + + AssertSuccess(await publisher.Agent.SendAsync("publish", new + { + storeId = publisher.StoreId, + key = AgentProtocol.EncodeBytes(leaseKey), + value = AgentProtocol.EncodeBytes(value), + descriptor = AgentProtocol.EncodeBytes(descriptor) + })); + AssertSuccess(await reader.Agent.SendAsync("acquire", new + { + storeId = reader.StoreId, + leaseId = "stress-lifecycle-lease", + key = AgentProtocol.EncodeBytes(leaseKey) + })); + AssertStatus(await publisher.Agent.SendAsync("remove", new + { + storeId = publisher.StoreId, + key = AgentProtocol.EncodeBytes(leaseKey) + }), 10, "RemovePending"); + AssertSuccess(await reader.Agent.SendAsync("release", new { leaseId = "stress-lifecycle-lease" })); + + AssertSuccess(await reservationOwner.Agent.SendAsync("reserve", new + { + storeId = reservationOwner.StoreId, + reservationId = "stress-reservation", + key = AgentProtocol.EncodeBytes(reservationKey), + payloadLength = value.Length, + descriptor = AgentProtocol.EncodeBytes(descriptor) + })); + AssertSuccess(await reservationOwner.Agent.SendAsync("reservationWrite", new + { + reservationId = "stress-reservation", + data = AgentProtocol.EncodeBytes(value) + })); + AssertSuccess(await reservationOwner.Agent.SendAsync("advance", new + { + reservationId = "stress-reservation", + byteCount = value.Length + })); + + if ((cycle & 1) == 0) + { + AssertSuccess(await reservationOwner.Agent.SendAsync("abort", new + { + reservationId = "stress-reservation" + })); + } + else + { + AssertSuccess(await reservationOwner.Agent.SendAsync("commit", new + { + reservationId = "stress-reservation" + })); + var committed = await publisher.Agent.SendAsync("acquire", new + { + storeId = publisher.StoreId, + leaseId = "stress-reservation-lease", + key = AgentProtocol.EncodeBytes(reservationKey) + }); + AssertSuccess(committed); + Assert.Equal(value, Decode(committed, "value")); + Assert.Equal(descriptor, Decode(committed, "descriptor")); + AssertSuccess(await publisher.Agent.SendAsync("release", new + { + leaseId = "stress-reservation-lease" + })); + AssertSuccess(await publisher.Agent.SendAsync("remove", new + { + storeId = publisher.StoreId, + key = AgentProtocol.EncodeBytes(reservationKey) + })); + } + } + + for (var index = participants.Length - 1; index >= 0; index--) + { + var participant = participants[index]; + AssertSuccess(await participant.Agent.SendAsync("close", new { storeId = participant.StoreId })); + } + } + + private static bool StressIsEnabled() => + string.Equals(Environment.GetEnvironmentVariable(StressOptInVariable), "1", StringComparison.Ordinal); + + private static int ReadBoundedCount(string variable, int defaultValue, int maximum) + { + var raw = Environment.GetEnvironmentVariable(variable); + if (string.IsNullOrWhiteSpace(raw)) + { + return defaultValue; + } + + if (!int.TryParse(raw, out var parsed) || parsed < 1 || parsed > maximum) + { + throw new InvalidOperationException($"{variable} must be between 1 and {maximum}."); + } + + return parsed; + } + + private static bool AgentsAreAvailable(params AgentDefinition[] definitions) => + definitions.All(definition => definition.IsAvailable() && PythonNativeLibraryIsAvailable(definition)); + + private static bool PythonNativeLibraryIsAvailable(AgentDefinition definition) + { + if (definition.Runtime != "python") + { + return true; + } + + var agentScript = definition.Arguments.FirstOrDefault(); + var testsDirectory = agentScript is null ? null : Directory.GetParent(Path.GetDirectoryName(agentScript)!); + var repository = testsDirectory?.Parent; + if (repository is null) + { + return false; + } + + var libraryName = OperatingSystem.IsWindows() ? "shared_memory_store.dll" : "libshared_memory_store.so"; + return File.Exists(Path.Combine( + repository.FullName, + "src", + "python", + "shared_memory_store", + libraryName)); + } + + private static uint StableSeed(string first, string second) + { + var seed = 2166136261u; + foreach (var current in $"{first}>{second}") + { + seed = unchecked((seed ^ current) * 16777619u); + } + + return seed; + } + + private static byte[] IterationKey(byte prefix, int iteration) + { + return + [ + prefix, + (byte)iteration, + (byte)(iteration >> 8), + (byte)(iteration >> 16), + (byte)(iteration >> 24) + ]; + } + + private static byte[] IterationBytes(byte prefix, int iteration, uint seed, int length) + { + var bytes = new byte[length]; + var state = unchecked(seed ^ ((uint)iteration * 0x9e3779b9u) ^ prefix); + for (var index = 0; index < bytes.Length; index++) + { + state = unchecked((state * 1664525u) + 1013904223u); + bytes[index] = (byte)(state >> 24); + } + + bytes[0] = prefix; + if (bytes.Length > 1) + { + bytes[1] = 0; + } + + return bytes; + } + + private static void AssertSuccess(AgentResponse response) => AssertStatus(response, 0, "Success"); + + private static void AssertStatus(AgentResponse response, int code, string name) + { + Assert.True(response.Ok, response.Error?.Message); + Assert.Equal(code, response.Status.Code); + Assert.Equal(name, response.Status.Name); + } + + private static byte[] Decode(AgentResponse response, string property) => + AgentProtocol.DecodeBytes(response.Result!.Value.GetProperty(property).GetString()!); + + private sealed record Participant(string StoreId, AgentProcess Agent); +} diff --git a/tests/SharedMemoryStore.InteropTests/TestSupport/AgentProcess.cs b/tests/SharedMemoryStore.InteropTests/TestSupport/AgentProcess.cs new file mode 100644 index 0000000..f7b4ced --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/TestSupport/AgentProcess.cs @@ -0,0 +1,214 @@ +using System.Diagnostics; +using SharedMemoryStore.InteropAgent; + +namespace SharedMemoryStore.InteropTests.TestSupport; + +internal sealed record AgentDefinition( + string Runtime, + string Executable, + IReadOnlyList Arguments, + IReadOnlyDictionary? Environment = null) +{ + public static AgentDefinition Resolve(string runtime) + { + var repository = RepositoryRoot(); + return runtime switch + { + "dotnet" => new AgentDefinition( + runtime, + "dotnet", + [typeof(AgentHost).Assembly.Location]), + "cpp" => new AgentDefinition( + runtime, + System.Environment.GetEnvironmentVariable("SMS_CPP_AGENT") ?? DefaultCppAgent(repository), + []), + "python" => new AgentDefinition( + runtime, + System.Environment.GetEnvironmentVariable("SMS_PYTHON_EXECUTABLE") ?? (OperatingSystem.IsWindows() ? "python" : "python3"), + [Path.Combine(repository, "tests", "python", "interop_agent.py")], + new Dictionary + { + ["PYTHONPATH"] = Path.Combine(repository, "src", "python") + }), + _ => throw new ArgumentOutOfRangeException(nameof(runtime), runtime, "Unknown agent runtime.") + }; + } + + public bool IsAvailable() + { + if (Path.IsPathFullyQualified(Executable) && !File.Exists(Executable)) + { + return false; + } + + if (Runtime == "python" && (Arguments.Count == 0 || !File.Exists(Arguments[0]))) + { + return false; + } + + if (Runtime == "python") + { + var nativeLibrary = OperatingSystem.IsWindows() + ? "shared_memory_store.dll" + : "libshared_memory_store.so"; + if (!File.Exists(Path.Combine( + RepositoryRoot(), + "src", + "python", + "shared_memory_store", + nativeLibrary))) + { + return false; + } + } + + return true; + } + + private static string DefaultCppAgent(string repository) => OperatingSystem.IsWindows() + ? Path.Combine(repository, "artifacts", "native-win", "sms_cpp_interop_agent.exe") + : Path.Combine(repository, "artifacts", "cmake-wsl", "tests", "cpp", "sms_cpp_interop_agent"); + + private static string RepositoryRoot() + { + var current = new DirectoryInfo(AppContext.BaseDirectory); + while (current is not null && !File.Exists(Path.Combine(current.FullName, "SharedMemoryStore.slnx"))) + { + current = current.Parent; + } + + return current?.FullName ?? throw new DirectoryNotFoundException("Could not locate the SharedMemoryStore repository root."); + } +} + +internal sealed class AgentProcess : IAsyncDisposable +{ + private readonly AgentDefinition _definition; + private readonly Process _process; + private readonly Task _stderr; + private int _requestSequence; + + private AgentProcess(AgentDefinition definition, Process process) + { + _definition = definition; + _process = process; + _stderr = process.StandardError.ReadToEndAsync(); + } + + public static async Task StartAsync(AgentDefinition definition) + { + var startInfo = new ProcessStartInfo + { + FileName = definition.Executable, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WorkingDirectory = Directory.GetCurrentDirectory() + }; + foreach (var argument in definition.Arguments) + { + startInfo.ArgumentList.Add(argument); + } + + if (definition.Environment is not null) + { + foreach (var pair in definition.Environment) + { + startInfo.Environment[pair.Key] = pair.Value; + } + } + + var process = Process.Start(startInfo) + ?? throw new InvalidOperationException($"Could not start the {definition.Runtime} interoperability agent."); + var result = new AgentProcess(definition, process); + var ping = await result.SendAsync("ping", arguments: null).ConfigureAwait(false); + if (!ping.Ok || ping.Status.Name != "Success") + { + await result.DisposeAsync().ConfigureAwait(false); + throw new InvalidOperationException($"The {definition.Runtime} agent did not answer ping successfully."); + } + + return result; + } + + public async Task SendAsync(string command, T? arguments, TimeSpan? timeout = null) + { + if (_process.HasExited) + { + throw new InvalidOperationException( + $"The {_definition.Runtime} agent exited with code {_process.ExitCode}. stderr: {await _stderr.ConfigureAwait(false)}"); + } + + var request = new AgentRequest + { + Id = Interlocked.Increment(ref _requestSequence).ToString(System.Globalization.CultureInfo.InvariantCulture), + Command = command, + Arguments = arguments is null ? null : AgentProtocol.ToJsonElement(arguments) + }; + await _process.StandardInput.WriteAsync(AgentProtocol.SerializeRequestLine(request)).ConfigureAwait(false); + await _process.StandardInput.FlushAsync().ConfigureAwait(false); + using var cancellation = new CancellationTokenSource(timeout ?? TimeSpan.FromSeconds(10)); + var line = await _process.StandardOutput.ReadLineAsync(cancellation.Token).ConfigureAwait(false); + if (line is null) + { + throw new InvalidOperationException( + $"The {_definition.Runtime} agent closed stdout. stderr: {await _stderr.ConfigureAwait(false)}"); + } + + var response = AgentProtocol.ParseResponse(line); + Assert.Equal(request.Id, response.Id); + return response; + } + + public async Task CrashAsync(TimeSpan? timeout = null) + { + if (_process.HasExited) + { + throw new InvalidOperationException( + $"The {_definition.Runtime} agent already exited with code {_process.ExitCode}."); + } + + var request = new AgentRequest + { + Id = Interlocked.Increment(ref _requestSequence).ToString(System.Globalization.CultureInfo.InvariantCulture), + Command = "crash", + Arguments = null + }; + await _process.StandardInput.WriteAsync(AgentProtocol.SerializeRequestLine(request)).ConfigureAwait(false); + await _process.StandardInput.FlushAsync().ConfigureAwait(false); + + using var cancellation = new CancellationTokenSource(timeout ?? TimeSpan.FromSeconds(10)); + await _process.WaitForExitAsync(cancellation.Token).ConfigureAwait(false); + var stderr = await _stderr.ConfigureAwait(false); + Assert.True( + _process.ExitCode == 97, + $"The {_definition.Runtime} crash command exited with code {_process.ExitCode}; expected 97. stderr: {stderr}"); + } + + public async ValueTask DisposeAsync() + { + try + { + if (!_process.HasExited) + { + _process.StandardInput.Close(); + } + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await _process.WaitForExitAsync(timeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + if (!_process.HasExited) + { + _process.Kill(entireProcessTree: true); + await _process.WaitForExitAsync().ConfigureAwait(false); + } + } + finally + { + _process.Dispose(); + } + } +} diff --git a/tests/SharedMemoryStore.InteropTests/TestSupport/ForeignStoreLock.cs b/tests/SharedMemoryStore.InteropTests/TestSupport/ForeignStoreLock.cs new file mode 100644 index 0000000..6516426 --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/TestSupport/ForeignStoreLock.cs @@ -0,0 +1,180 @@ +using System.Security.Cryptography; +using System.Text; +using System.Runtime.Versioning; + +namespace SharedMemoryStore.InteropTests.TestSupport; + +internal sealed class ForeignStoreLock : IDisposable +{ + private readonly string _publicName; + private readonly ManualResetEventSlim _release = new(initialState: false); + private readonly TaskCompletionSource _acquired = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly Thread _holder; + private int _disposed; + + private ForeignStoreLock(string publicName) + { + _publicName = publicName; + _holder = new Thread(Hold) + { + IsBackground = true, + Name = "SharedMemoryStore foreign-lock test holder" + }; + } + + public static async Task AcquireAsync(string publicName) + { + var result = new ForeignStoreLock(publicName); + result._holder.Start(); + try + { + await result._acquired.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); + return result; + } + catch + { + result.Dispose(); + throw; + } + } + + public static string LinuxRegionPath(string publicName) => + LinuxPath(publicName, ".region"); + + public static string LinuxSynchronizationPath(string publicName) => + LinuxPath(publicName, ".lock"); + + public static string LinuxOwnersPath(string publicName) => + LinuxPath(publicName, ".owners"); + + public static string LinuxLifecyclePath(string publicName) => + LinuxPath(publicName, ".lifecycle"); + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _release.Set(); + if (_holder.IsAlive && !_holder.Join(TimeSpan.FromSeconds(5))) + { + throw new TimeoutException("The foreign store lock holder did not stop."); + } + + _release.Dispose(); + } + + private void Hold() + { + try + { + if (OperatingSystem.IsWindows()) + { + HoldWindows(); + } + else if (OperatingSystem.IsLinux()) + { + HoldLinux(); + } + else + { + throw new PlatformNotSupportedException("Interop contention tests support Windows and Linux."); + } + } + catch (Exception exception) + { + _acquired.TrySetException(exception); + } + } + + private void HoldWindows() + { + using var mutex = new Mutex(initiallyOwned: false, BuildWindowsSynchronizationName(_publicName)); + var ownsMutex = false; + try + { + try + { + ownsMutex = mutex.WaitOne(TimeSpan.FromSeconds(5)); + } + catch (AbandonedMutexException) + { + ownsMutex = true; + } + + if (!ownsMutex) + { + throw new TimeoutException("Could not acquire the Windows interoperability mutex."); + } + + _acquired.TrySetResult(); + _release.Wait(); + } + finally + { + if (ownsMutex) + { + mutex.ReleaseMutex(); + } + } + } + + [SupportedOSPlatform("linux")] + private void HoldLinux() + { + var path = LinuxSynchronizationPath(_publicName); + using var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); + stream.Lock(0, 1); + try + { + _acquired.TrySetResult(); + _release.Wait(); + } + finally + { + stream.Unlock(0, 1); + } + } + + private static string BuildWindowsSynchronizationName(string publicName) + { + var scope = publicName.StartsWith(@"Global\", StringComparison.OrdinalIgnoreCase) + ? @"Global\" + : @"Local\"; + var sanitized = string.Create(publicName.Length, publicName, static (destination, source) => + { + for (var index = 0; index < source.Length; index++) + { + var value = source[index]; + destination[index] = char.IsLetterOrDigit(value) || value is '-' or '_' ? value : '_'; + } + }); + return scope + "SharedMemoryStore-" + sanitized; + } + + private static string LinuxPath(string publicName, string suffix) + { + var sanitized = new StringBuilder(publicName.Length); + foreach (var value in publicName) + { + sanitized.Append(char.IsAsciiLetterOrDigit(value) || value is '-' or '_' or '.' ? value : '_'); + } + + var readable = sanitized.ToString().Trim('_', '.'); + if (readable.Length == 0) + { + readable = "store"; + } + else if (readable.Length > 80) + { + readable = readable[..80]; + } + + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(publicName)); + var digest = Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); + var directory = Path.Combine(Directory.Exists("/dev/shm") ? "/dev/shm" : Path.GetTempPath(), "SharedMemoryStore"); + return Path.Combine(directory, $"sms-{readable}-{digest}{suffix}"); + } +} diff --git a/tests/SharedMemoryStore.InteropTests/TestSupport/InteropAssertions.cs b/tests/SharedMemoryStore.InteropTests/TestSupport/InteropAssertions.cs new file mode 100644 index 0000000..06468cc --- /dev/null +++ b/tests/SharedMemoryStore.InteropTests/TestSupport/InteropAssertions.cs @@ -0,0 +1,47 @@ +using SharedMemoryStore.InteropAgent; + +namespace SharedMemoryStore.InteropTests.TestSupport; + +internal static class InteropAssertions +{ + public static readonly string[] Runtimes = ["dotnet", "cpp", "python"]; + + public static object OpenArguments( + string storeId, + string name, + int openMode, + bool enableLeaseRecovery = true, + int slotCount = 6, + int maxValueBytes = 128, + int maxDescriptorBytes = 32, + int maxKeyBytes = 32, + int leaseRecordCount = 16) => new + { + storeId, + name, + openMode, + slotCount, + maxValueBytes, + maxDescriptorBytes, + maxKeyBytes, + leaseRecordCount, + enableLeaseRecovery + }; + + public static void Success(AgentResponse response) + { + Assert.True(response.Ok, response.Error?.Message); + Assert.Equal(0, response.Status.Code); + Assert.Equal("Success", response.Status.Name); + } + + public static void Status(AgentResponse response, int code, string name) + { + Assert.True(response.Ok, response.Error?.Message); + Assert.Equal(code, response.Status.Code); + Assert.Equal(name, response.Status.Name); + } + + public static byte[] Decode(AgentResponse response, string property) => + AgentProtocol.DecodeBytes(response.Result!.Value.GetProperty(property).GetString()!); +} diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt new file mode 100644 index 0000000..3b35f3c --- /dev/null +++ b/tests/cpp/CMakeLists.txt @@ -0,0 +1,78 @@ +set( + _sms_native_test_sources + protocol_tests.cpp + store_tests.cpp + c_api_tests.cpp + lifecycle_tests.cpp + diagnostics_tests.cpp) + +function(_sms_add_native_test source_name) + set(_source "${CMAKE_CURRENT_SOURCE_DIR}/${source_name}") + if(NOT EXISTS "${_source}") + return() + endif() + + get_filename_component(_test_stem "${source_name}" NAME_WE) + set(_target "sms_${_test_stem}") + + add_executable(${_target} "${_source}") + if(_test_stem STREQUAL "protocol_tests") + target_sources( + ${_target} + PRIVATE "${PROJECT_SOURCE_DIR}/src/cpp/src/protocol.cpp") + endif() + target_compile_features(${_target} PRIVATE cxx_std_20) + set_target_properties(${_target} PROPERTIES CXX_EXTENSIONS OFF) + target_include_directories( + ${_target} PRIVATE "${PROJECT_SOURCE_DIR}/src/cpp/src") + target_compile_definitions( + ${_target} PRIVATE SMS_REPOSITORY_ROOT="${PROJECT_SOURCE_DIR}") + target_link_libraries( + ${_target} PRIVATE SharedMemoryStore::SharedMemoryStore Threads::Threads) + + if(MSVC) + target_compile_options(${_target} PRIVATE /W4 /permissive- /EHsc) + else() + target_compile_options(${_target} PRIVATE -Wall -Wextra -Wpedantic) + endif() + + if(WIN32) + add_custom_command( + TARGET ${_target} + POST_BUILD + COMMAND + "${CMAKE_COMMAND}" -E copy_if_different + "$" "$" + VERBATIM) + endif() + + add_test(NAME "SharedMemoryStore.${_test_stem}" COMMAND ${_target}) + set_tests_properties( + "SharedMemoryStore.${_test_stem}" + PROPERTIES WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}") +endfunction() + +foreach(_sms_test_source IN LISTS _sms_native_test_sources) + _sms_add_native_test("${_sms_test_source}") +endforeach() + +set(_sms_interop_agent_source "${CMAKE_CURRENT_SOURCE_DIR}/interop_agent.cpp") +if(EXISTS "${_sms_interop_agent_source}") + add_executable(sms_cpp_interop_agent "${_sms_interop_agent_source}") + target_compile_features(sms_cpp_interop_agent PRIVATE cxx_std_20) + set_target_properties(sms_cpp_interop_agent PROPERTIES CXX_EXTENSIONS OFF) + target_link_libraries( + sms_cpp_interop_agent + PRIVATE SharedMemoryStore::SharedMemoryStore Threads::Threads) + + if(WIN32) + add_custom_command( + TARGET sms_cpp_interop_agent + POST_BUILD + COMMAND + "${CMAKE_COMMAND}" -E copy_if_different + "$" + "$" + VERBATIM) + endif() +endif() diff --git a/tests/cpp/c_api_tests.cpp b/tests/cpp/c_api_tests.cpp new file mode 100644 index 0000000..403943e --- /dev/null +++ b/tests/cpp/c_api_tests.cpp @@ -0,0 +1,64 @@ +#include "shared_memory_store/c_api.h" +#include "test_support.hpp" + +#include +#include + +int main() { + SMS_CHECK(sms_abi_version() == SMS_C_ABI_VERSION); + sms_protocol_info protocol{}; + protocol.struct_size = sizeof(protocol); + protocol.abi_version = SMS_C_ABI_VERSION; + SMS_CHECK(sms_get_protocol_info(&protocol) == SMS_STATUS_SUCCESS); + SMS_CHECK(protocol.store_header_size == 160); + SMS_CHECK(protocol.slot_metadata_size == 72); + SMS_CHECK(sms_get_protocol_info(nullptr) == SMS_STATUS_UNKNOWN_FAILURE); + std::uint32_t field_offset{}; + SMS_CHECK(sms_get_layout_field_offset(SMS_LAYOUT_FIELD_HEADER_SEQUENCE, &field_offset) == SMS_STATUS_SUCCESS); + SMS_CHECK(field_offset == 152); + SMS_CHECK(sms_get_layout_field_offset(SMS_LAYOUT_FIELD_SLOT_KEY_HASH, &field_offset) == SMS_STATUS_SUCCESS); + SMS_CHECK(field_offset == 40); + + std::int64_t required{}; + SMS_CHECK(sms_calculate_required_bytes(2, 64, 8, 16, 4, &required) == SMS_OPEN_SUCCESS); + SMS_CHECK(required > 0); + const auto name = sms_test_name("c-api"); + sms_store_options options{}; + options.struct_size = sizeof(options); + options.abi_version = SMS_C_ABI_VERSION; + options.name_utf8 = name.data(); + options.name_length = name.size(); + options.open_mode = SMS_OPEN_MODE_CREATE_NEW; + options.total_bytes = required; + options.slot_count = 2; + options.max_value_bytes = 64; + options.max_descriptor_bytes = 8; + options.max_key_bytes = 16; + options.lease_record_count = 4; + options.enable_lease_recovery = 1; + sms_wait_options wait{sizeof(wait), SMS_C_ABI_VERSION, 1000}; + sms_store* store{}; + SMS_CHECK(sms_open_store(&options, &wait, &store) == SMS_OPEN_SUCCESS); + SMS_CHECK(store != nullptr); + sms_store_layout layout{}; + layout.struct_size = sizeof(layout); + layout.abi_version = SMS_C_ABI_VERSION; + SMS_CHECK(sms_get_store_layout(store, &wait, &layout) == SMS_STATUS_SUCCESS); + SMS_CHECK(layout.total_bytes == required); + SMS_CHECK(layout.slot_count == 2); + const std::array key{1, 0}; + const std::array value{2, 0, 3}; + SMS_CHECK(sms_publish(store, {key.data(), key.size()}, {value.data(), value.size()}, {}, &wait) == SMS_STATUS_SUCCESS); + sms_lease* lease{}; + SMS_CHECK(sms_acquire(store, {key.data(), key.size()}, &wait, &lease) == SMS_STATUS_SUCCESS); + SMS_CHECK(sms_lease_is_valid(lease) == 1); + const auto view = sms_lease_value(lease); + SMS_CHECK(view.length == value.size()); + SMS_CHECK(std::memcmp(view.data, value.data(), value.size()) == 0); + SMS_CHECK(sms_release_lease(lease, &wait) == SMS_STATUS_SUCCESS); + SMS_CHECK(sms_release_lease(lease, &wait) == SMS_STATUS_LEASE_ALREADY_RELEASED); + sms_destroy_lease(lease); + sms_close_store(store); + SMS_CHECK(sms_publish(nullptr, {}, {}, {}, &wait) == SMS_STATUS_STORE_DISPOSED); + return 0; +} diff --git a/tests/cpp/diagnostics_tests.cpp b/tests/cpp/diagnostics_tests.cpp new file mode 100644 index 0000000..93baddd --- /dev/null +++ b/tests/cpp/diagnostics_tests.cpp @@ -0,0 +1,25 @@ +#include "test_support.hpp" + +#include + +int main() { + using namespace shared_memory_store; + auto options = sms_test_options("diagnostics", 2, 4); + memory_store store; + SMS_CHECK(memory_store::try_create_or_open(options, store) == open_status::success); + const std::array missing{9}; + value_lease lease; + SMS_CHECK(store.try_acquire(sms_test_bytes(missing), lease) == status::not_found); + const std::array key{1}; + const std::array value{2}; + SMS_CHECK(store.try_publish(sms_test_bytes(key), sms_test_bytes(value)) == status::success); + diagnostics_snapshot diagnostics; + SMS_CHECK(store.try_get_diagnostics(diagnostics) == status::success); + SMS_CHECK(diagnostics.slot_count() == 2); + SMS_CHECK(diagnostics.published_slot_count() == 1); + SMS_CHECK(diagnostics.free_slot_count() == 1); + SMS_CHECK(diagnostics.occupied_index_entry_count() == 1); + SMS_CHECK(diagnostics.failure_count(status::not_found) == 1); + SMS_CHECK(diagnostics.last_failure_status() == status::not_found); + return 0; +} diff --git a/tests/cpp/interop_agent.cpp b/tests/cpp/interop_agent.cpp new file mode 100644 index 0000000..c511a89 --- /dev/null +++ b/tests/cpp/interop_agent.cpp @@ -0,0 +1,987 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace json { + +struct value { + using array = std::vector; + using object = std::map>; + using storage = std::variant; + + value() : data(nullptr) {} + value(std::nullptr_t) : data(nullptr) {} + value(bool input) : data(input) {} + value(int input) : data(static_cast(input)) {} + value(std::int64_t input) : data(input) {} + value(double input) : data(input) {} + value(const char* input) : data(std::string(input)) {} + value(std::string input) : data(std::move(input)) {} + value(array input) : data(std::move(input)) {} + value(object input) : data(std::move(input)) {} + + storage data; +}; + +void append_utf8(std::string& destination, std::uint32_t code_point) { + if (code_point <= 0x7fu) { + destination.push_back(static_cast(code_point)); + } else if (code_point <= 0x7ffu) { + destination.push_back(static_cast(0xc0u | (code_point >> 6u))); + destination.push_back(static_cast(0x80u | (code_point & 0x3fu))); + } else if (code_point <= 0xffffu) { + destination.push_back(static_cast(0xe0u | (code_point >> 12u))); + destination.push_back(static_cast(0x80u | ((code_point >> 6u) & 0x3fu))); + destination.push_back(static_cast(0x80u | (code_point & 0x3fu))); + } else if (code_point <= 0x10ffffu) { + destination.push_back(static_cast(0xf0u | (code_point >> 18u))); + destination.push_back(static_cast(0x80u | ((code_point >> 12u) & 0x3fu))); + destination.push_back(static_cast(0x80u | ((code_point >> 6u) & 0x3fu))); + destination.push_back(static_cast(0x80u | (code_point & 0x3fu))); + } else { + throw std::runtime_error("JSON string contains an invalid Unicode scalar value."); + } +} + +class parser { +public: + explicit parser(std::string_view input) : input_(input) {} + + value parse() { + skip_space(); + auto result = parse_value(); + skip_space(); + if (position_ != input_.size()) fail("Unexpected characters after the JSON value."); + return result; + } + +private: + [[noreturn]] void fail(const char* message) const { + throw std::runtime_error(std::string(message) + " At byte " + std::to_string(position_) + '.'); + } + + void skip_space() { + while (position_ < input_.size()) { + const auto current = input_[position_]; + if (current != ' ' && current != '\t' && current != '\r' && current != '\n') break; + ++position_; + } + } + + char take() { + if (position_ == input_.size()) fail("Unexpected end of JSON input."); + return input_[position_++]; + } + + bool consume(char expected) { + if (position_ < input_.size() && input_[position_] == expected) { + ++position_; + return true; + } + return false; + } + + void expect_literal(std::string_view expected) { + if (input_.substr(position_, expected.size()) != expected) fail("Invalid JSON literal."); + position_ += expected.size(); + } + + value parse_value() { + if (position_ == input_.size()) fail("A JSON value is required."); + switch (input_[position_]) { + case 'n': expect_literal("null"); return nullptr; + case 't': expect_literal("true"); return true; + case 'f': expect_literal("false"); return false; + case '"': return parse_string(); + case '[': return parse_array(); + case '{': return parse_object(); + default: + if (input_[position_] == '-' || (input_[position_] >= '0' && input_[position_] <= '9')) + return parse_number(); + fail("Invalid JSON value."); + } + } + + std::uint32_t parse_hex_quad() { + std::uint32_t result{}; + for (int index = 0; index < 4; ++index) { + const auto current = take(); + result <<= 4u; + if (current >= '0' && current <= '9') result |= static_cast(current - '0'); + else if (current >= 'a' && current <= 'f') result |= static_cast(current - 'a' + 10); + else if (current >= 'A' && current <= 'F') result |= static_cast(current - 'A' + 10); + else fail("Invalid hexadecimal JSON escape."); + } + return result; + } + + std::string parse_string() { + if (take() != '"') fail("A JSON string was expected."); + std::string result; + while (true) { + const auto current = static_cast(take()); + if (current == '"') return result; + if (current < 0x20u) fail("Unescaped control character in JSON string."); + if (current != '\\') { + result.push_back(static_cast(current)); + continue; + } + + switch (take()) { + case '"': result.push_back('"'); break; + case '\\': result.push_back('\\'); break; + case '/': result.push_back('/'); break; + case 'b': result.push_back('\b'); break; + case 'f': result.push_back('\f'); break; + case 'n': result.push_back('\n'); break; + case 'r': result.push_back('\r'); break; + case 't': result.push_back('\t'); break; + case 'u': { + auto scalar = parse_hex_quad(); + if (scalar >= 0xd800u && scalar <= 0xdbffu) { + if (take() != '\\' || take() != 'u') fail("A high surrogate must be followed by a low surrogate."); + const auto low = parse_hex_quad(); + if (low < 0xdc00u || low > 0xdfffu) fail("Invalid low surrogate in JSON string."); + scalar = 0x10000u + ((scalar - 0xd800u) << 10u) + (low - 0xdc00u); + } else if (scalar >= 0xdc00u && scalar <= 0xdfffu) { + fail("Unexpected low surrogate in JSON string."); + } + append_utf8(result, scalar); + break; + } + default: fail("Invalid JSON string escape."); + } + } + } + + value parse_number() { + const auto start = position_; + consume('-'); + if (consume('0')) { + if (position_ < input_.size() && input_[position_] >= '0' && input_[position_] <= '9') + fail("A JSON number cannot contain a leading zero."); + } else { + if (position_ == input_.size() || input_[position_] < '1' || input_[position_] > '9') + fail("Invalid JSON number."); + while (position_ < input_.size() && input_[position_] >= '0' && input_[position_] <= '9') ++position_; + } + bool floating = false; + if (consume('.')) { + floating = true; + if (position_ == input_.size() || input_[position_] < '0' || input_[position_] > '9') + fail("A JSON fraction requires at least one digit."); + while (position_ < input_.size() && input_[position_] >= '0' && input_[position_] <= '9') ++position_; + } + if (position_ < input_.size() && (input_[position_] == 'e' || input_[position_] == 'E')) { + floating = true; + ++position_; + if (position_ < input_.size() && (input_[position_] == '+' || input_[position_] == '-')) ++position_; + if (position_ == input_.size() || input_[position_] < '0' || input_[position_] > '9') + fail("A JSON exponent requires at least one digit."); + while (position_ < input_.size() && input_[position_] >= '0' && input_[position_] <= '9') ++position_; + } + const auto token = input_.substr(start, position_ - start); + if (!floating) { + std::int64_t result{}; + const auto parsed = std::from_chars(token.data(), token.data() + token.size(), result); + if (parsed.ec != std::errc{} || parsed.ptr != token.data() + token.size()) fail("JSON integer is out of range."); + return result; + } + std::string owned(token); + char* end{}; + const auto result = std::strtod(owned.c_str(), &end); + if (end != owned.c_str() + owned.size()) fail("Invalid JSON floating-point number."); + return result; + } + + value::array parse_array() { + take(); + skip_space(); + value::array result; + if (consume(']')) return result; + while (true) { + skip_space(); + result.push_back(parse_value()); + skip_space(); + if (consume(']')) return result; + if (!consume(',')) fail("A comma was expected in the JSON array."); + } + } + + value::object parse_object() { + take(); + skip_space(); + value::object result; + if (consume('}')) return result; + while (true) { + skip_space(); + if (position_ == input_.size() || input_[position_] != '"') fail("A JSON object key must be a string."); + auto key = parse_string(); + skip_space(); + if (!consume(':')) fail("A colon was expected after the JSON object key."); + skip_space(); + auto [iterator, inserted] = result.emplace(std::move(key), parse_value()); + if (!inserted) fail("Duplicate JSON object keys are not supported."); + skip_space(); + if (consume('}')) return result; + if (!consume(',')) fail("A comma was expected in the JSON object."); + } + } + + std::string_view input_; + std::size_t position_{}; +}; + +void append_escaped(std::string& output, std::string_view input) { + static constexpr char hex[] = "0123456789abcdef"; + output.push_back('"'); + for (const auto current : input) { + const auto byte = static_cast(current); + switch (byte) { + case '"': output += "\\\""; break; + case '\\': output += "\\\\"; break; + case '\b': output += "\\b"; break; + case '\f': output += "\\f"; break; + case '\n': output += "\\n"; break; + case '\r': output += "\\r"; break; + case '\t': output += "\\t"; break; + default: + if (byte < 0x20u) { + output += "\\u00"; + output.push_back(hex[byte >> 4u]); + output.push_back(hex[byte & 0x0fu]); + } else { + output.push_back(current); + } + } + } + output.push_back('"'); +} + +void append_json(std::string& output, const value& input) { + std::visit([&output](const auto& current) { + using type = std::decay_t; + if constexpr (std::is_same_v) output += "null"; + else if constexpr (std::is_same_v) output += current ? "true" : "false"; + else if constexpr (std::is_same_v) output += std::to_string(current); + else if constexpr (std::is_same_v) output += std::to_string(current); + else if constexpr (std::is_same_v) append_escaped(output, current); + else if constexpr (std::is_same_v) { + output.push_back('['); + bool first = true; + for (const auto& item : current) { + if (!first) output.push_back(','); + first = false; + append_json(output, item); + } + output.push_back(']'); + } else { + output.push_back('{'); + bool first = true; + for (const auto& [key, item] : current) { + if (!first) output.push_back(','); + first = false; + append_escaped(output, key); + output.push_back(':'); + append_json(output, item); + } + output.push_back('}'); + } + }, input.data); +} + +std::string dump(const value& input) { + std::string result; + result.reserve(256); + append_json(result, input); + return result; +} + +const value::object& object_value(const value& input, std::string_view description) { + const auto* result = std::get_if(&input.data); + if (!result) throw std::runtime_error(std::string(description) + " must be a JSON object."); + return *result; +} + +const value::array& array_value(const value& input, std::string_view description) { + const auto* result = std::get_if(&input.data); + if (!result) throw std::runtime_error(std::string(description) + " must be a JSON array."); + return *result; +} + +const std::string& string_value(const value& input, std::string_view description) { + const auto* result = std::get_if(&input.data); + if (!result) throw std::runtime_error(std::string(description) + " must be a JSON string."); + return *result; +} + +} // namespace json + +class protocol_error : public std::runtime_error { +public: + protocol_error(std::string code, std::string message) + : std::runtime_error(std::move(message)), code_(std::move(code)) {} + const std::string& code() const noexcept { return code_; } +private: + std::string code_; +}; + +const json::value* find(const json::value::object& object, std::string_view key) { + const auto iterator = object.find(key); + return iterator == object.end() ? nullptr : &iterator->second; +} + +const json::value& require(const json::value::object& object, std::string_view key) { + const auto* result = find(object, key); + if (!result) throw protocol_error("invalid_arguments", "The '" + std::string(key) + "' argument is required."); + return *result; +} + +std::string string_argument(const json::value::object& object, std::string_view key, + std::string default_value = {}) { + const auto* input = find(object, key); + if (!input) return default_value; + try { return json::string_value(*input, std::string("The '") + std::string(key) + "' argument"); } + catch (const std::exception& exception) { throw protocol_error("invalid_arguments", exception.what()); } +} + +std::string required_string(const json::value::object& object, std::string_view key) { + const auto result = string_argument(object, key); + if (result.empty()) throw protocol_error("invalid_arguments", "The '" + std::string(key) + "' argument is required."); + return result; +} + +std::int64_t integer_argument(const json::value::object& object, std::string_view key, + std::int64_t default_value) { + const auto* input = find(object, key); + if (!input || std::holds_alternative(input->data)) return default_value; + if (const auto* result = std::get_if(&input->data)) return *result; + throw protocol_error("invalid_arguments", "The '" + std::string(key) + "' argument must be an integer."); +} + +std::int32_t int32_argument(const json::value::object& object, std::string_view key, + std::int32_t default_value) { + const auto result = integer_argument(object, key, default_value); + if (result < std::numeric_limits::min() || result > std::numeric_limits::max()) + throw protocol_error("invalid_arguments", "The '" + std::string(key) + "' argument is outside the Int32 range."); + return static_cast(result); +} + +std::int32_t required_int32(const json::value::object& object, std::string_view key) { + const auto* input = find(object, key); + if (!input || std::holds_alternative(input->data)) + throw protocol_error("invalid_arguments", "The '" + std::string(key) + "' argument is required."); + return int32_argument(object, key, 0); +} + +bool bool_argument(const json::value::object& object, std::string_view key, bool default_value) { + const auto* input = find(object, key); + if (!input || std::holds_alternative(input->data)) return default_value; + if (const auto* result = std::get_if(&input->data)) return *result; + throw protocol_error("invalid_arguments", "The '" + std::string(key) + "' argument must be a Boolean."); +} + +std::vector decode_base64(std::string_view input) { + if (input.empty()) return {}; + if (input.size() % 4u != 0u) + throw protocol_error("invalid_base64", "A base64 field has an invalid length."); + + const auto decode_character = [](char current) -> int { + if (current >= 'A' && current <= 'Z') return current - 'A'; + if (current >= 'a' && current <= 'z') return current - 'a' + 26; + if (current >= '0' && current <= '9') return current - '0' + 52; + if (current == '+') return 62; + if (current == '/') return 63; + return -1; + }; + + std::vector result; + result.reserve((input.size() / 4u) * 3u); + for (std::size_t offset = 0; offset < input.size(); offset += 4u) { + const bool final_group = offset + 4u == input.size(); + const bool pad2 = input[offset + 2u] == '='; + const bool pad3 = input[offset + 3u] == '='; + if ((!final_group && (pad2 || pad3)) || (pad2 && !pad3)) + throw protocol_error("invalid_base64", "A base64 field has invalid padding."); + const auto a = decode_character(input[offset]); + const auto b = decode_character(input[offset + 1u]); + const auto c = pad2 ? 0 : decode_character(input[offset + 2u]); + const auto d = pad3 ? 0 : decode_character(input[offset + 3u]); + if (a < 0 || b < 0 || c < 0 || d < 0) + throw protocol_error("invalid_base64", "A base64 field contains an invalid character."); + const auto bits = (static_cast(a) << 18u) | + (static_cast(b) << 12u) | + (static_cast(c) << 6u) | + static_cast(d); + result.push_back(static_cast((bits >> 16u) & 0xffu)); + if (!pad2) result.push_back(static_cast((bits >> 8u) & 0xffu)); + if (!pad3) result.push_back(static_cast(bits & 0xffu)); + } + return result; +} + +std::string encode_base64(std::span input) { + static constexpr char alphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string result; + result.reserve(((input.size() + 2u) / 3u) * 4u); + for (std::size_t offset = 0; offset < input.size(); offset += 3u) { + const auto remaining = input.size() - offset; + const auto a = std::to_integer(input[offset]); + const auto b = remaining > 1u ? std::to_integer(input[offset + 1u]) : 0u; + const auto c = remaining > 2u ? std::to_integer(input[offset + 2u]) : 0u; + const auto bits = (a << 16u) | (b << 8u) | c; + result.push_back(alphabet[(bits >> 18u) & 0x3fu]); + result.push_back(alphabet[(bits >> 12u) & 0x3fu]); + result.push_back(remaining > 1u ? alphabet[(bits >> 6u) & 0x3fu] : '='); + result.push_back(remaining > 2u ? alphabet[bits & 0x3fu] : '='); + } + return result; +} + +std::vector bytes_argument(const json::value::object& object, std::string_view key, + bool required_field = false) { + const auto* input = find(object, key); + if (!input) { + if (required_field) throw protocol_error("invalid_arguments", "The '" + std::string(key) + "' argument is required."); + return {}; + } + if (!required_field && std::holds_alternative(input->data)) return {}; + try { return decode_base64(json::string_value(*input, std::string("The '") + std::string(key) + "' argument")); } + catch (const protocol_error&) { throw; } + catch (const std::exception& exception) { throw protocol_error("invalid_arguments", exception.what()); } +} + +shared_memory_store::wait_options wait_argument(const json::value::object& arguments) { + return {integer_argument(arguments, "timeoutMs", 1000)}; +} + +shared_memory_store::open_mode open_mode_argument(const json::value::object& arguments) { + const auto* input = find(arguments, "openMode"); + if (!input) return shared_memory_store::open_mode::create_or_open; + if (const auto* number = std::get_if(&input->data)) { + if (*number >= std::numeric_limits::min() && + *number <= std::numeric_limits::max()) + return static_cast(static_cast(*number)); + } else if (const auto* text = std::get_if(&input->data)) { + std::string normalized; + normalized.reserve(text->size()); + for (const auto current : *text) { + if (current != '_' && current != '-' && current != '/') + normalized.push_back(static_cast(std::tolower(static_cast(current)))); + } + if (normalized == "createnew" || normalized == "create") return shared_memory_store::open_mode::create_new; + if (normalized == "openexisting" || normalized == "open") return shared_memory_store::open_mode::open_existing; + if (normalized == "createoropen") return shared_memory_store::open_mode::create_or_open; + } + throw protocol_error("invalid_arguments", "The 'openMode' argument is invalid."); +} + +const char* open_status_name(shared_memory_store::open_status input) noexcept { + using enum shared_memory_store::open_status; + switch (input) { + case success: return "Success"; + case already_exists: return "AlreadyExists"; + case not_found: return "NotFound"; + case invalid_options: return "InvalidOptions"; + case incompatible_layout: return "IncompatibleLayout"; + case unsupported_platform: return "UnsupportedPlatform"; + case insufficient_capacity: return "InsufficientCapacity"; + case access_denied: return "AccessDenied"; + case mapping_failed: return "MappingFailed"; + case store_busy: return "StoreBusy"; + case operation_canceled: return "OperationCanceled"; + } + return "UnknownOpenStatus"; +} + +const char* status_name(shared_memory_store::status input) noexcept { + using enum shared_memory_store::status; + switch (input) { + case success: return "Success"; + case duplicate_key: return "DuplicateKey"; + case not_found: return "NotFound"; + case key_too_large: return "KeyTooLarge"; + case value_too_large: return "ValueTooLarge"; + case descriptor_too_large: return "DescriptorTooLarge"; + case store_full: return "StoreFull"; + case lease_table_full: return "LeaseTableFull"; + case invalid_lease: return "InvalidLease"; + case lease_already_released: return "LeaseAlreadyReleased"; + case remove_pending: return "RemovePending"; + case unsupported_platform: return "UnsupportedPlatform"; + case store_disposed: return "StoreDisposed"; + case corrupt_store: return "CorruptStore"; + case access_denied: return "AccessDenied"; + case unknown_failure: return "UnknownFailure"; + case invalid_reservation: return "InvalidReservation"; + case reservation_incomplete: return "ReservationIncomplete"; + case reservation_already_completed: return "ReservationAlreadyCompleted"; + case reservation_write_out_of_range: return "ReservationWriteOutOfRange"; + case invalid_key: return "InvalidKey"; + case store_busy: return "StoreBusy"; + case operation_canceled: return "OperationCanceled"; + } + return "UnknownStatus"; +} + +json::value status_json(std::int32_t code, const char* name) { + return json::value::object{{"code", code}, {"name", name}}; +} + +json::value response(std::string id, std::int32_t code, const char* name, json::value result = nullptr) { + json::value::object output{{"id", std::move(id)}, {"ok", true}, {"status", status_json(code, name)}}; + if (!std::holds_alternative(result.data)) output.emplace("result", std::move(result)); + return output; +} + +json::value response(std::string id, shared_memory_store::status operation_status, + json::value result = nullptr) { + return response(std::move(id), static_cast(operation_status), + status_name(operation_status), std::move(result)); +} + +json::value response(std::string id, shared_memory_store::open_status operation_status, + json::value result = nullptr) { + return response(std::move(id), static_cast(operation_status), + open_status_name(operation_status), std::move(result)); +} + +json::value failure(std::string id, std::int32_t status_code, const char* status_name_value, + std::string error_code, std::string message) { + return json::value::object{ + {"error", json::value::object{{"code", std::move(error_code)}, {"message", std::move(message)}}}, + {"id", std::move(id)}, + {"ok", false}, + {"status", status_json(status_code, status_name_value)}}; +} + +struct lease_entry { + std::string store_id; + shared_memory_store::value_lease lease; +}; + +struct reservation_entry { + std::string store_id; + shared_memory_store::value_reservation reservation; +}; + +class agent { +public: + json::value handle(const json::value& request) { + const auto& root = json::object_value(request, "The request"); + const auto id = required_request_string(root, "id"); + const auto command = required_request_string(root, "command"); + static const json::value::object empty_arguments; + const auto* arguments_value = find(root, "arguments"); + const auto& arguments = arguments_value + ? checked_object(*arguments_value, "Request arguments") + : empty_arguments; + + if (command == "ping") { + return response(id, shared_memory_store::status::success, + json::value::object{{"protocolVersion", 1}, {"runtime", "cpp"}}); + } + if (command == "open" || command == "create" || command == "open/create") + return open(id, arguments, command); + if (command == "close") return close(id, arguments); + if (command == "publish") return publish(id, arguments); + if (command == "publishSegments" || command == "publishSegmented") return publish_segments(id, arguments); + if (command == "acquire") return acquire(id, arguments); + if (command == "read") return read(id, arguments); + if (command == "release") return release(id, arguments); + if (command == "remove") return remove(id, arguments); + if (command == "reserve") return reserve(id, arguments); + if (command == "reservationWrite" || command == "write") return reservation_write(id, arguments); + if (command == "advance") return advance(id, arguments); + if (command == "commit") return commit(id, arguments); + if (command == "abort") return abort(id, arguments); + if (command == "recoverLeases") return recover(id, arguments, false); + if (command == "recoverReservations") return recover(id, arguments, true); + if (command == "diagnostics") return diagnostics(id, arguments); + if (command == "crash") { + const auto exit_code = int32_argument(arguments, "exitCode", 97); + std::_Exit(exit_code); + } + throw protocol_error("unsupported_command", "The command '" + command + "' is not implemented by this agent."); + } + +private: + static const json::value::object& checked_object(const json::value& input, std::string_view description) { + try { return json::object_value(input, description); } + catch (const std::exception& exception) { throw protocol_error("invalid_request", exception.what()); } + } + + static std::string required_request_string(const json::value::object& root, std::string_view key) { + const auto* input = find(root, key); + if (!input) throw protocol_error("invalid_request", "The request '" + std::string(key) + "' is required."); + try { + const auto& result = json::string_value(*input, std::string("The request '") + std::string(key) + "'"); + if (result.empty()) throw std::runtime_error("The request '" + std::string(key) + "' cannot be empty."); + return result; + } catch (const std::exception& exception) { + throw protocol_error("invalid_request", exception.what()); + } + } + + shared_memory_store::memory_store& store(const json::value::object& arguments, std::string& id) { + id = required_string(arguments, "storeId"); + const auto iterator = stores_.find(id); + if (iterator == stores_.end()) throw protocol_error("invalid_arguments", "The store handle '" + id + "' is unknown."); + return iterator->second; + } + + lease_entry& lease(const json::value::object& arguments, std::string& id) { + id = required_string(arguments, "leaseId"); + const auto iterator = leases_.find(id); + if (iterator == leases_.end()) throw protocol_error("invalid_arguments", "The lease handle '" + id + "' is unknown."); + return iterator->second; + } + + reservation_entry& reservation(const json::value::object& arguments, std::string& id) { + id = required_string(arguments, "reservationId"); + const auto iterator = reservations_.find(id); + if (iterator == reservations_.end()) + throw protocol_error("invalid_arguments", "The reservation handle '" + id + "' is unknown."); + return iterator->second; + } + + json::value open(const std::string& request_id, const json::value::object& arguments, + std::string_view command) { + const auto handle_id = required_string(arguments, "storeId"); + stores_.erase(handle_id); + + shared_memory_store::store_options options; + options.name = required_string(arguments, "name"); + options.mode = command == "create" ? shared_memory_store::open_mode::create_new : open_mode_argument(arguments); + options.slot_count = required_int32(arguments, "slotCount"); + options.max_value_bytes = required_int32(arguments, "maxValueBytes"); + options.max_descriptor_bytes = required_int32(arguments, "maxDescriptorBytes"); + options.max_key_bytes = required_int32(arguments, "maxKeyBytes"); + options.lease_record_count = required_int32(arguments, "leaseRecordCount"); + options.enable_lease_recovery = bool_argument(arguments, "enableLeaseRecovery", false); + const auto* total_bytes = find(arguments, "totalBytes"); + if (!total_bytes || std::holds_alternative(total_bytes->data)) { + try { + options.total_bytes = shared_memory_store::store_options::calculate_required_bytes( + options.slot_count, options.max_value_bytes, options.max_descriptor_bytes, + options.max_key_bytes, options.lease_record_count); + } catch (const std::exception&) { + return response(request_id, shared_memory_store::open_status::invalid_options, + json::value::object{{"storeId", handle_id}, {"totalBytes", 0}}); + } + } else { + options.total_bytes = integer_argument(arguments, "totalBytes", 0); + } + + shared_memory_store::memory_store opened; + const auto result = shared_memory_store::memory_store::try_create_or_open(options, opened, wait_argument(arguments)); + if (result == shared_memory_store::open_status::success) + stores_.emplace(handle_id, std::move(opened)); + return response(request_id, result, + json::value::object{{"storeId", handle_id}, {"totalBytes", options.total_bytes}}); + } + + json::value close(const std::string& request_id, const json::value::object& arguments) { + const auto handle_id = required_string(arguments, "storeId"); + const auto iterator = stores_.find(handle_id); + if (iterator != stores_.end()) stores_.erase(iterator); + return response(request_id, shared_memory_store::status::success, + json::value::object{{"closed", true}, {"storeId", handle_id}}); + } + + json::value publish(const std::string& request_id, const json::value::object& arguments) { + std::string handle_id; + auto& target = store(arguments, handle_id); + const auto key = bytes_argument(arguments, "key", true); + const auto value = bytes_argument(arguments, "value", true); + const auto descriptor = bytes_argument(arguments, "descriptor"); + const auto result = target.try_publish(key, value, descriptor, wait_argument(arguments)); + return response(request_id, result, + json::value::object{{"storeId", handle_id}, {"valueLength", static_cast(value.size())}}); + } + + json::value publish_segments(const std::string& request_id, const json::value::object& arguments) { + std::string handle_id; + auto& target = store(arguments, handle_id); + const auto key = bytes_argument(arguments, "key", true); + const auto descriptor = bytes_argument(arguments, "descriptor"); + const auto& inputs = json::array_value(require(arguments, "segments"), "The 'segments' argument"); + std::vector> owned; + std::vector> segments; + owned.reserve(inputs.size()); + segments.reserve(inputs.size()); + for (const auto& item : inputs) { + owned.push_back(decode_base64(json::string_value(item, "A segment"))); + segments.emplace_back(owned.back()); + } + std::int64_t copied{}; + const auto result = target.try_publish_segments(key, segments, descriptor, copied, wait_argument(arguments)); + return response(request_id, result, + json::value::object{{"copiedBytes", copied}, {"storeId", handle_id}}); + } + + json::value acquire(const std::string& request_id, const json::value::object& arguments) { + std::string store_id; + auto& target = store(arguments, store_id); + const auto lease_id = required_string(arguments, "leaseId"); + leases_.erase(lease_id); + const auto key = bytes_argument(arguments, "key", true); + shared_memory_store::value_lease acquired; + const auto result = target.try_acquire(key, acquired, wait_argument(arguments)); + json::value::object output{{"leaseId", lease_id}, {"storeId", store_id}}; + if (result == shared_memory_store::status::success) { + output.emplace("descriptor", encode_base64(acquired.descriptor())); + output.emplace("value", encode_base64(acquired.value())); + leases_.emplace(lease_id, lease_entry{store_id, std::move(acquired)}); + } + return response(request_id, result, std::move(output)); + } + + json::value read(const std::string& request_id, const json::value::object& arguments) { + std::string lease_id; + auto& entry = lease(arguments, lease_id); + if (!entry.lease.valid()) + return response(request_id, shared_memory_store::status::invalid_lease, + json::value::object{{"leaseId", lease_id}}); + return response(request_id, shared_memory_store::status::success, + json::value::object{{"descriptor", encode_base64(entry.lease.descriptor())}, + {"leaseId", lease_id}, + {"value", encode_base64(entry.lease.value())}}); + } + + json::value release(const std::string& request_id, const json::value::object& arguments) { + const auto lease_id = required_string(arguments, "leaseId"); + const auto iterator = leases_.find(lease_id); + if (iterator == leases_.end()) + return response(request_id, shared_memory_store::status::invalid_lease); + auto& entry = iterator->second; + const auto result = entry.lease.release(wait_argument(arguments)); + return response(request_id, result, + json::value::object{{"leaseId", lease_id}, {"valid", entry.lease.valid()}}); + } + + json::value remove(const std::string& request_id, const json::value::object& arguments) { + std::string store_id; + auto& target = store(arguments, store_id); + const auto key = bytes_argument(arguments, "key", true); + const auto result = target.try_remove(key, wait_argument(arguments)); + return response(request_id, result, json::value::object{{"storeId", store_id}}); + } + + json::value reserve(const std::string& request_id, const json::value::object& arguments) { + std::string store_id; + auto& target = store(arguments, store_id); + const auto reservation_id = required_string(arguments, "reservationId"); + reservations_.erase(reservation_id); + const auto key = bytes_argument(arguments, "key", true); + const auto descriptor = bytes_argument(arguments, "descriptor"); + const auto payload_length = required_int32(arguments, "payloadLength"); + shared_memory_store::value_reservation reserved; + const auto result = target.try_reserve(key, payload_length, descriptor, reserved, wait_argument(arguments)); + json::value::object output{{"payloadLength", payload_length}, + {"reservationId", reservation_id}, + {"storeId", store_id}}; + if (result == shared_memory_store::status::success) { + output.emplace("bytesWritten", reserved.bytes_written()); + output.emplace("remainingBytes", reserved.remaining_bytes()); + reservations_.emplace(reservation_id, reservation_entry{store_id, std::move(reserved)}); + } + return response(request_id, result, std::move(output)); + } + + json::value reservation_write(const std::string& request_id, const json::value::object& arguments) { + const auto reservation_id = required_string(arguments, "reservationId"); + const auto iterator = reservations_.find(reservation_id); + if (iterator == reservations_.end()) + return response(request_id, shared_memory_store::status::invalid_reservation); + auto& entry = iterator->second; + const auto data = bytes_argument(arguments, "data", true); + if (!entry.reservation.valid()) + return response(request_id, shared_memory_store::status::invalid_reservation, + reservation_result(reservation_id, entry.reservation, 0)); + if (data.size() > static_cast(std::max(0, entry.reservation.remaining_bytes()))) + return response(request_id, shared_memory_store::status::reservation_write_out_of_range, + reservation_result(reservation_id, entry.reservation, 0)); + if (!data.empty()) { + const auto buffer = entry.reservation.buffer(static_cast(data.size())); + if (buffer.size() < data.size()) + return response(request_id, shared_memory_store::status::invalid_reservation, + reservation_result(reservation_id, entry.reservation, 0)); + std::memcpy(buffer.data(), data.data(), data.size()); + } + return response(request_id, shared_memory_store::status::success, + reservation_result(reservation_id, entry.reservation, + static_cast(data.size()))); + } + + json::value advance(const std::string& request_id, const json::value::object& arguments) { + std::string reservation_id; + auto& entry = reservation(arguments, reservation_id); + const auto result = entry.reservation.advance(required_int32(arguments, "byteCount"), wait_argument(arguments)); + return response(request_id, result, reservation_result(reservation_id, entry.reservation, 0)); + } + + json::value commit(const std::string& request_id, const json::value::object& arguments) { + std::string reservation_id; + auto& entry = reservation(arguments, reservation_id); + const auto result = entry.reservation.commit(wait_argument(arguments)); + return response(request_id, result, reservation_result(reservation_id, entry.reservation, 0)); + } + + json::value abort(const std::string& request_id, const json::value::object& arguments) { + std::string reservation_id; + auto& entry = reservation(arguments, reservation_id); + const auto result = entry.reservation.abort(wait_argument(arguments)); + return response(request_id, result, reservation_result(reservation_id, entry.reservation, 0)); + } + + static json::value reservation_result(const std::string& reservation_id, + const shared_memory_store::value_reservation& reservation, + std::int64_t bytes_copied) { + return json::value::object{{"bytesCopied", bytes_copied}, + {"bytesWritten", reservation.bytes_written()}, + {"payloadLength", reservation.payload_length()}, + {"remainingBytes", reservation.remaining_bytes()}, + {"reservationId", reservation_id}, + {"written", bytes_copied}, + {"valid", reservation.valid()}}; + } + + json::value recover(const std::string& request_id, const json::value::object& arguments, + bool reservations) { + std::string store_id; + auto& target = store(arguments, store_id); + shared_memory_store::recovery_report report{}; + const auto current = bool_argument(arguments, "recoverCurrentProcess", false); + const auto result = reservations + ? target.try_recover_reservations(current, report, wait_argument(arguments)) + : target.try_recover_leases(current, report, wait_argument(arguments)); + json::value::object output{{"activeCount", report.active_count}, + {"failedCount", report.failed_count}, + {"failedRecoveryCount", report.failed_count}, + {"recoveredCount", report.recovered_count}, + {"scannedCount", report.scanned_count}, + {"storeId", store_id}, + {"unsupportedCount", report.unsupported_count}}; + if (reservations) { + output.emplace("activeReservationCount", report.active_count); + output.emplace("recoveredReservationCount", report.recovered_count); + output.emplace("scannedReservationCount", report.scanned_count); + output.emplace("unsupportedReservationCount", report.unsupported_count); + } else { + output.emplace("activeLeaseCount", report.active_count); + output.emplace("recoveredLeaseCount", report.recovered_count); + output.emplace("scannedRecordCount", report.scanned_count); + output.emplace("unsupportedLeaseCount", report.unsupported_count); + } + return response(request_id, result, std::move(output)); + } + + json::value diagnostics(const std::string& request_id, const json::value::object& arguments) { + std::string store_id; + auto& target = store(arguments, store_id); + shared_memory_store::diagnostics_snapshot snapshot; + const auto result = target.try_get_diagnostics(snapshot, wait_argument(arguments)); + json::value::object output{{"storeId", store_id}}; + if (result == shared_memory_store::status::success) { + const auto& native = snapshot.native(); + json::value::array failures; + failures.reserve(23); + for (int index = 0; index < 23; ++index) failures.emplace_back(native.failure_counts[index]); + output.insert({ + {"abortedReservationCount", native.aborted_reservation_count}, + {"activeLeaseCount", native.active_lease_count}, + {"activeLeaseRecoveryCount", native.active_lease_recovery_count}, + {"activeReservationCount", native.active_reservation_count}, + {"activeReservationRecoveryCount", native.active_reservation_recovery_count}, + {"capacityPressureCount", native.capacity_pressure_count}, + {"emptyIndexEntryCount", native.empty_index_entry_count}, + {"failedLeaseRecoveryCount", native.failed_lease_recovery_count}, + {"failedReservationRecoveryCount", native.failed_reservation_recovery_count}, + {"failureCounts", std::move(failures)}, + {"freeSlotCount", native.free_slot_count}, + {"indexCompactionCount", native.index_compaction_count}, + {"indexEntryCount", native.index_entry_count}, + {"lastFailureStatus", native.last_failure_status}, + {"lastObservedProbeLength", native.last_observed_probe_length}, + {"maxObservedProbeLength", native.max_observed_probe_length}, + {"occupiedIndexEntryCount", native.occupied_index_entry_count}, + {"pendingRemovalCount", native.pending_removal_count}, + {"publishedSlotCount", native.published_slot_count}, + {"recoveredLeaseCount", native.recovered_lease_count}, + {"recoveredReservationCount", native.recovered_reservation_count}, + {"slotCount", native.slot_count}, + {"tombstoneIndexEntryCount", native.tombstone_index_entry_count}, + {"tombstonePressureRatio", native.index_entry_count == 0 + ? 0.0 + : static_cast(native.tombstone_index_entry_count) / + static_cast(native.index_entry_count)}, + {"totalBytes", native.total_bytes}, + {"unsupportedLeaseRecoveryCount", native.unsupported_lease_recovery_count}, + {"unsupportedReservationRecoveryCount", native.unsupported_reservation_recovery_count}, + {"usableIndexCapacity", native.usable_index_capacity} + }); + } + return response(request_id, result, std::move(output)); + } + + std::unordered_map stores_; + std::unordered_map leases_; + std::unordered_map reservations_; +}; + +} // namespace + +int main() { + std::ios::sync_with_stdio(false); + std::cin.tie(nullptr); + + agent participant; + std::string line; + while (std::getline(std::cin, line)) { + std::string request_id; + json::value output; + try { + const auto request = json::parser(line).parse(); + if (const auto* root = std::get_if(&request.data)) { + if (const auto* id = find(*root, "id")) { + if (const auto* text = std::get_if(&id->data)) request_id = *text; + } + } + output = participant.handle(request); + } catch (const protocol_error& exception) { + const bool unsupported = exception.code() == "unsupported_command"; + output = failure(request_id, unsupported ? -2 : -1, + unsupported ? "UnsupportedCommand" : "ProtocolError", + exception.code(), exception.what()); + } catch (const std::exception& exception) { + output = failure(request_id, -1, "ProtocolError", "invalid_request", exception.what()); + } catch (...) { + output = failure(request_id, -1, "ProtocolError", "invalid_request", "Unknown request-processing failure."); + } + std::cout << json::dump(output) << '\n' << std::flush; + } + return 0; +} diff --git a/tests/cpp/lifecycle_tests.cpp b/tests/cpp/lifecycle_tests.cpp new file mode 100644 index 0000000..b1460cb --- /dev/null +++ b/tests/cpp/lifecycle_tests.cpp @@ -0,0 +1,49 @@ +#include "test_support.hpp" + +#include +#include + +int main() { + using namespace shared_memory_store; + auto options = sms_test_options("lifecycle", 3, 6); + memory_store store; + SMS_CHECK(memory_store::try_create_or_open(options, store) == open_status::success); + const std::array key{1}; + const std::array descriptor{7, 8}; + value_reservation reservation; + SMS_CHECK(store.try_reserve(sms_test_bytes(key), 5, sms_test_bytes(descriptor), reservation) == status::success); + SMS_CHECK(reservation.valid()); + SMS_CHECK(reservation.remaining_bytes() == 5); + SMS_CHECK(reservation.commit() == status::reservation_incomplete); + auto buffer = reservation.buffer(5); + SMS_CHECK(buffer.size() == 5); + const std::array payload{10, 11, 0, 12, 13}; + std::memcpy(buffer.data(), payload.data(), payload.size()); + SMS_CHECK(reservation.advance(6) == status::reservation_write_out_of_range); + SMS_CHECK(reservation.advance(5) == status::success); + SMS_CHECK(reservation.commit() == status::success); + SMS_CHECK(!reservation.valid()); + + value_lease lease; + SMS_CHECK(store.try_acquire(sms_test_bytes(key), lease) == status::success); + SMS_CHECK(std::memcmp(lease.value().data(), payload.data(), payload.size()) == 0); + SMS_CHECK(lease.release() == status::success); + SMS_CHECK(store.try_remove(sms_test_bytes(key)) == status::success); + + const std::array segment_key{2}; + const std::array first{std::byte{1}, std::byte{2}}; + const std::array second{std::byte{3}, std::byte{0}, std::byte{4}}; + const std::array, 2> segments{first, second}; + std::int64_t copied{}; + SMS_CHECK(store.try_publish_segments(sms_test_bytes(segment_key), segments, {}, copied) == status::success); + SMS_CHECK(copied == 5); + + const std::array recover_key{3}; + value_reservation abandoned; + SMS_CHECK(store.try_reserve(sms_test_bytes(recover_key), 1, {}, abandoned) == status::success); + recovery_report report{}; + SMS_CHECK(store.try_recover_reservations(true, report) == status::success); + SMS_CHECK(report.recovered_count == 1); + SMS_CHECK(!abandoned.valid()); + return 0; +} diff --git a/tests/cpp/package_consumer/CMakeLists.txt b/tests/cpp/package_consumer/CMakeLists.txt new file mode 100644 index 0000000..7b4206d --- /dev/null +++ b/tests/cpp/package_consumer/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.20) +project(SharedMemoryStorePackageConsumer LANGUAGES CXX) + +find_package(SharedMemoryStore CONFIG REQUIRED) +add_executable(shared_memory_store_package_consumer main.cpp) +target_compile_features(shared_memory_store_package_consumer PRIVATE cxx_std_20) +target_link_libraries( + shared_memory_store_package_consumer + PRIVATE SharedMemoryStore::SharedMemoryStore) + +if(WIN32) + add_custom_command( + TARGET shared_memory_store_package_consumer + POST_BUILD + COMMAND + "${CMAKE_COMMAND}" -E copy_if_different + "$" + "$" + VERBATIM) +endif() diff --git a/tests/cpp/package_consumer/main.cpp b/tests/cpp/package_consumer/main.cpp new file mode 100644 index 0000000..6e27313 --- /dev/null +++ b/tests/cpp/package_consumer/main.cpp @@ -0,0 +1,35 @@ +#include + +#include +#include +#include +#include + +#if defined(_WIN32) +# define NOMINMAX +# include +#else +# include +#endif + +int main() { +#if defined(_WIN32) + const auto pid = GetCurrentProcessId(); +#else + const auto pid = getpid(); +#endif + using namespace shared_memory_store; + auto options = store_options::create( + "sms-installed-consumer-" + std::to_string(pid), 2, 32, 8, 8, 4, + open_mode::create_new); + memory_store store; + if (memory_store::try_create_or_open(options, store) != open_status::success) return 1; + const std::array key{std::byte{1}}; + const std::array payload{std::byte{2}, std::byte{0}, std::byte{3}}; + if (store.try_publish(key, payload) != status::success) return 2; + value_lease lease; + if (store.try_acquire(key, lease) != status::success) return 3; + if (lease.value().size() != payload.size() || + std::memcmp(lease.value().data(), payload.data(), payload.size()) != 0) return 4; + return lease.release() == status::success ? 0 : 5; +} diff --git a/tests/cpp/protocol_tests.cpp b/tests/cpp/protocol_tests.cpp new file mode 100644 index 0000000..004c938 --- /dev/null +++ b/tests/cpp/protocol_tests.cpp @@ -0,0 +1,51 @@ +#include "internal.hpp" +#include "test_support.hpp" + +#include +#include + +int main() { + using namespace sms::detail; + Layout layout{}; + SMS_CHECK(Layout::calculate(0, 3, 17, 5, 9, 4, layout)); + SMS_CHECK(layout.header_length == 160); + SMS_CHECK(layout.index_entry_count == 8); + SMS_CHECK(layout.index_entry_size == 48); + SMS_CHECK(layout.index_offset == 160); + SMS_CHECK(layout.index_length == 384); + SMS_CHECK(layout.lease_registry_offset == 544); + SMS_CHECK(layout.slot_metadata_offset == 704); + SMS_CHECK(layout.descriptor_storage_offset == 920); + SMS_CHECK(layout.payload_storage_offset == 944); + SMS_CHECK(layout.required_bytes == 1016); + SMS_CHECK(!Layout::calculate(0, 0, 1, 0, 1, 1, layout)); + SMS_CHECK(!Layout::calculate(0, 1, 1, 0, INT32_MAX, 1, layout)); + + constexpr std::array hello{'h', 'e', 'l', 'l', 'o'}; + SMS_CHECK(hash_key(hello) == 0xa430d84680aabd0bULL); + constexpr std::array binary{0x00, 0x01, 0xff, 0x80}; + SMS_CHECK(hash_key(binary) == 0x4653dd7f9a76930dULL); + + ResourceName simple{}; + SMS_CHECK(make_resource_name("sms.compatibility", simple)); + SMS_CHECK(simple.fragment == "sms-sms.compatibility-251220bcba0b63e6"); +#if defined(_WIN32) + SMS_CHECK(simple.windows_region_name == L"sms.compatibility"); + SMS_CHECK(simple.windows_lock_name == L"Local\\SharedMemoryStore-sms_compatibility"); +#endif + ResourceName separator{}; + SMS_CHECK(make_resource_name("store/name", separator)); + SMS_CHECK(separator.fragment == "sms-store_name-549a0c43d4d76e02"); + ResourceName collision{}; + SMS_CHECK(make_resource_name("store?name", collision)); + SMS_CHECK(collision.fragment == "sms-store_name-0f08216b745495cd"); + ResourceName unicode{}; + SMS_CHECK(make_resource_name("caf\xC3\xA9/\xE5\x85\xB1\xE4\xBA\xAB/\xF0\x9F\x98\x80", unicode)); + SMS_CHECK(unicode.fragment == "sms-caf-0f903cf0f516f93b"); +#if defined(_WIN32) + SMS_CHECK(unicode.windows_lock_name == L"Local\\SharedMemoryStore-caf\u00e9_\u5171\u4eab___"); +#endif + SMS_CHECK(utf16_length("\xF0\x9F\x98\x80") == 2); + SMS_CHECK(!valid_utf8("\xC0\x80")); + return 0; +} diff --git a/tests/cpp/store_tests.cpp b/tests/cpp/store_tests.cpp new file mode 100644 index 0000000..f07a5e6 --- /dev/null +++ b/tests/cpp/store_tests.cpp @@ -0,0 +1,38 @@ +#include "test_support.hpp" + +#include +#include + +int main() { + using namespace shared_memory_store; + auto options = sms_test_options("store"); + memory_store store; + SMS_CHECK(memory_store::try_create_or_open(options, store) == open_status::success); + + const std::array key{0, 1, 255}; + const std::array value{9, 0, 8, 7, 255}; + const std::array descriptor{4, 0}; + SMS_CHECK(store.try_publish(sms_test_bytes(key), sms_test_bytes(value), sms_test_bytes(descriptor)) == status::success); + SMS_CHECK(store.try_publish(sms_test_bytes(key), sms_test_bytes(value)) == status::duplicate_key); + + value_lease lease; + SMS_CHECK(store.try_acquire(sms_test_bytes(key), lease) == status::success); + SMS_CHECK(lease.valid()); + SMS_CHECK(lease.value().size() == value.size()); + SMS_CHECK(std::memcmp(lease.value().data(), value.data(), value.size()) == 0); + SMS_CHECK(std::memcmp(lease.descriptor().data(), descriptor.data(), descriptor.size()) == 0); + SMS_CHECK(store.try_remove(sms_test_bytes(key)) == status::remove_pending); + SMS_CHECK(lease.value().size() == value.size()); + SMS_CHECK(lease.release() == status::success); + SMS_CHECK(!lease.valid()); + SMS_CHECK(lease.release() == status::lease_already_released); + SMS_CHECK(store.try_acquire(sms_test_bytes(key), lease) == status::not_found); + + const std::array replacement{42}; + SMS_CHECK(store.try_publish(sms_test_bytes(key), sms_test_bytes(replacement)) == status::success); + SMS_CHECK(store.try_remove(sms_test_bytes(key)) == status::success); + SMS_CHECK(store.try_remove(sms_test_bytes(key)) == status::not_found); + store.close(); + SMS_CHECK(store.try_publish(sms_test_bytes(key), sms_test_bytes(value)) == status::store_disposed); + return 0; +} diff --git a/tests/cpp/test_support.hpp b/tests/cpp/test_support.hpp new file mode 100644 index 0000000..a8073f0 --- /dev/null +++ b/tests/cpp/test_support.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include "shared_memory_store/store.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# define NOMINMAX +# include +#else +# include +#endif + +#define SMS_CHECK(expression) do { \ + if (!(expression)) { \ + std::cerr << __FILE__ << ':' << __LINE__ << ": check failed: " #expression << '\n'; \ + return 1; \ + } \ +} while (false) + +inline std::span sms_test_bytes(std::string_view value) noexcept { + return {reinterpret_cast(value.data()), value.size()}; +} + +template +inline std::span sms_test_bytes(const std::array& value) noexcept { + return {reinterpret_cast(value.data()), value.size()}; +} + +inline std::string sms_test_name(std::string_view suffix) { +#if defined(_WIN32) + const auto pid = GetCurrentProcessId(); +#else + const auto pid = getpid(); +#endif + return "sms-native-" + std::to_string(pid) + "-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + "-" + std::string(suffix); +} + +inline shared_memory_store::store_options sms_test_options(std::string suffix, std::int32_t slots = 4, + std::int32_t leases = 8) { + return shared_memory_store::store_options::create( + sms_test_name(suffix), slots, 128, 32, 32, leases, + shared_memory_store::open_mode::create_new, true); +} diff --git a/tests/python/_support.py b/tests/python/_support.py new file mode 100644 index 0000000..4604878 --- /dev/null +++ b/tests/python/_support.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import os +import unittest +import uuid + + +def unique_store_name(label: str) -> str: + return f"sms-python-{label}-{os.getpid()}-{uuid.uuid4().hex}" + + +def require_native() -> None: + from shared_memory_store import calculate_required_bytes + + try: + calculate_required_bytes( + slot_count=1, + max_value_bytes=1, + max_descriptor_bytes=0, + max_key_bytes=1, + lease_record_count=1, + ) + except (ImportError, OSError) as error: + raise unittest.SkipTest(f"native SharedMemoryStore artifact is unavailable: {error}") from error + + +def create_options(label: str, *, slots: int = 3, recovery: bool = True): + from shared_memory_store import StoreOptions + + return StoreOptions.create( + unique_store_name(label), + slot_count=slots, + max_value_bytes=128, + max_descriptor_bytes=32, + max_key_bytes=32, + lease_record_count=8, + enable_lease_recovery=recovery, + ) diff --git a/tests/python/interop_agent.py b/tests/python/interop_agent.py new file mode 100644 index 0000000..27126c2 --- /dev/null +++ b/tests/python/interop_agent.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Line-delimited JSON interoperability participant for the Python runtime.""" + +from __future__ import annotations + +import base64 +import json +import os +import sys +from typing import Any, Optional + +from shared_memory_store import ( + MemoryStore, + OpenMode, + StoreOpenStatus, + StoreOptions, + StoreStatus, + ValueLease, + ValueReservation, + WaitOptions, +) + + +def _symbolic_name(value: Any) -> str: + return "".join(part.title() for part in value.name.split("_")) + + +def _bytes(arguments: dict[str, Any], name: str, default: Optional[bytes] = None) -> bytes: + value = arguments.get(name) + if value is None and default is not None: + return default + if not isinstance(value, str): + raise ValueError(f"argument {name!r} must be a base64 string") + return base64.b64decode(value, validate=True) + + +def _encoded(value: bytes) -> str: + return base64.b64encode(value).decode("ascii") + + +def _wait(arguments: dict[str, Any]) -> WaitOptions: + timeout = arguments.get("timeoutMs", arguments.get("timeoutMilliseconds", 1000)) + return WaitOptions(timeout) + + +class Agent: + def __init__(self) -> None: + self.stores: dict[str, MemoryStore] = {} + self.leases: dict[str, ValueLease] = {} + self.reservations: dict[str, ValueReservation] = {} + + def close(self) -> None: + for lease in list(self.leases.values()): + lease.close() + for reservation in list(self.reservations.values()): + reservation.close() + for store in list(self.stores.values()): + store.close() + self.leases.clear() + self.reservations.clear() + self.stores.clear() + + def handle(self, request: dict[str, Any]) -> dict[str, Any]: + request_id = request.get("id") + command = request.get("command") + arguments = request.get("arguments") or {} + if not isinstance(request_id, str) or not request_id.strip(): + raise ValueError("the request id is required") + if not isinstance(command, str) or not command.strip(): + raise ValueError("the request command is required") + if not isinstance(arguments, dict): + raise ValueError("request arguments must be an object") + + method = getattr(self, f"command_{command}", None) + if method is None: + return { + "id": request_id, + "ok": False, + "status": {"code": -2, "name": "UnsupportedCommand"}, + "error": { + "code": "unsupported_command", + "message": f"The command {command!r} is not implemented by this agent.", + }, + } + status, result = method(arguments) + response: dict[str, Any] = { + "id": request_id, + "ok": True, + "status": {"code": int(status), "name": _symbolic_name(status)}, + } + if result is not None: + response["result"] = result + return response + + def _store(self, arguments: dict[str, Any]) -> MemoryStore: + store_id = arguments.get("storeId") + if not isinstance(store_id, str) or store_id not in self.stores: + raise ValueError(f"unknown storeId: {store_id!r}") + return self.stores[store_id] + + def _lease(self, arguments: dict[str, Any]) -> ValueLease: + lease_id = arguments.get("leaseId") + if not isinstance(lease_id, str) or lease_id not in self.leases: + raise ValueError(f"unknown leaseId: {lease_id!r}") + return self.leases[lease_id] + + def _reservation(self, arguments: dict[str, Any]) -> ValueReservation: + reservation_id = arguments.get("reservationId") + if not isinstance(reservation_id, str) or reservation_id not in self.reservations: + raise ValueError(f"unknown reservationId: {reservation_id!r}") + return self.reservations[reservation_id] + + def command_ping(self, arguments: dict[str, Any]) -> tuple[StoreStatus, dict[str, Any]]: + del arguments + return StoreStatus.SUCCESS, {"runtime": "python", "protocolVersion": 1} + + def command_open(self, arguments: dict[str, Any]) -> tuple[StoreOpenStatus, None]: + try: + options = StoreOptions.create( + arguments["name"], + slot_count=arguments["slotCount"], + max_value_bytes=arguments["maxValueBytes"], + max_descriptor_bytes=arguments["maxDescriptorBytes"], + max_key_bytes=arguments["maxKeyBytes"], + lease_record_count=arguments["leaseRecordCount"], + open_mode=OpenMode(arguments.get("openMode", int(OpenMode.CREATE_OR_OPEN))), + enable_lease_recovery=arguments.get("enableLeaseRecovery", False), + ) + except (KeyError, TypeError, ValueError): + return StoreOpenStatus.INVALID_OPTIONS, None + status, store = MemoryStore.open(options, wait=_wait(arguments)) + if status is StoreOpenStatus.SUCCESS: + store_id = arguments.get("storeId") + if not isinstance(store_id, str) or not store_id: + assert store is not None + store.close() + return StoreOpenStatus.INVALID_OPTIONS, None + prior = self.stores.pop(store_id, None) + if prior is not None: + prior.close() + assert store is not None + self.stores[store_id] = store + return status, None + + def command_close(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + store_id = arguments.get("storeId") + store = self.stores.pop(store_id, None) if isinstance(store_id, str) else None + if store is None: + return StoreStatus.STORE_DISPOSED, None + store.close() + return StoreStatus.SUCCESS, None + + def command_publish(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + status = self._store(arguments).publish( + _bytes(arguments, "key"), + _bytes(arguments, "value"), + _bytes(arguments, "descriptor", b""), + wait=_wait(arguments), + ) + return status, None + + def command_publishSegments(self, arguments: dict[str, Any]) -> tuple[StoreStatus, dict[str, int]]: + encoded_segments = arguments.get("segments") + if not isinstance(encoded_segments, list): + raise ValueError("segments must be an array") + segments = [base64.b64decode(value, validate=True) for value in encoded_segments] + status, copied = self._store(arguments).publish_segments( + _bytes(arguments, "key"), + segments, + _bytes(arguments, "descriptor", b""), + wait=_wait(arguments), + ) + return status, {"copiedBytes": copied} + + def command_acquire(self, arguments: dict[str, Any]) -> tuple[StoreStatus, Optional[dict[str, str]]]: + status, lease = self._store(arguments).acquire(_bytes(arguments, "key"), wait=_wait(arguments)) + if status is not StoreStatus.SUCCESS: + return status, None + lease_id = arguments.get("leaseId") + if not isinstance(lease_id, str) or not lease_id: + assert lease is not None + lease.close() + return StoreStatus.INVALID_LEASE, None + prior = self.leases.pop(lease_id, None) + if prior is not None: + prior.close() + assert lease is not None + self.leases[lease_id] = lease + return status, {"value": _encoded(bytes(lease.value)), "descriptor": _encoded(bytes(lease.descriptor))} + + def command_read(self, arguments: dict[str, Any]) -> tuple[StoreStatus, Optional[dict[str, str]]]: + lease = self._lease(arguments) + if not lease.is_valid: + return StoreStatus.INVALID_LEASE, None + return StoreStatus.SUCCESS, { + "value": _encoded(bytes(lease.value)), + "descriptor": _encoded(bytes(lease.descriptor)), + } + + def command_release(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + return self._lease(arguments).release(wait=_wait(arguments)), None + + def command_remove(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + return self._store(arguments).remove(_bytes(arguments, "key"), wait=_wait(arguments)), None + + def command_reserve(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + status, reservation = self._store(arguments).reserve( + _bytes(arguments, "key"), + arguments.get("payloadLength"), + _bytes(arguments, "descriptor", b""), + wait=_wait(arguments), + ) + if status is not StoreStatus.SUCCESS: + return status, None + reservation_id = arguments.get("reservationId") + if not isinstance(reservation_id, str) or not reservation_id: + assert reservation is not None + reservation.close() + return StoreStatus.INVALID_RESERVATION, None + prior = self.reservations.pop(reservation_id, None) + if prior is not None: + prior.close() + assert reservation is not None + self.reservations[reservation_id] = reservation + return status, None + + def command_reservationWrite(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + reservation = self._reservation(arguments) + data = _bytes(arguments, "data") + view = reservation.buffer(len(data)) + try: + if len(view) < len(data): + return StoreStatus.RESERVATION_WRITE_OUT_OF_RANGE, None + view[: len(data)] = data + return StoreStatus.SUCCESS, None + finally: + view.release() + + def command_advance(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + return self._reservation(arguments).advance(arguments.get("byteCount"), wait=_wait(arguments)), None + + def command_commit(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + return self._reservation(arguments).commit(wait=_wait(arguments)), None + + def command_abort(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + return self._reservation(arguments).abort(wait=_wait(arguments)), None + + def command_recoverLeases(self, arguments: dict[str, Any]) -> tuple[StoreStatus, dict[str, Any]]: + status, report = self._store(arguments).recover_leases( + arguments.get("recoverCurrentProcess", False), wait=_wait(arguments) + ) + return status, _report(report, reservations=False, store_id=arguments.get("storeId")) + + def command_recoverReservations(self, arguments: dict[str, Any]) -> tuple[StoreStatus, dict[str, Any]]: + status, report = self._store(arguments).recover_reservations( + arguments.get("recoverCurrentProcess", False), wait=_wait(arguments) + ) + return status, _report(report, reservations=True, store_id=arguments.get("storeId")) + + def command_diagnostics(self, arguments: dict[str, Any]) -> tuple[StoreStatus, Optional[dict[str, Any]]]: + status, snapshot = self._store(arguments).diagnostics(wait=_wait(arguments)) + if snapshot is None: + return status, None + return status, { + "storeId": arguments.get("storeId"), + "totalBytes": snapshot.total_bytes, + "slotCount": snapshot.slot_count, + "freeSlotCount": snapshot.free_slot_count, + "publishedSlotCount": snapshot.published_slot_count, + "pendingRemovalCount": snapshot.pending_removal_count, + "activeLeaseCount": snapshot.active_lease_count, + "activeReservationCount": snapshot.active_reservation_count, + "indexEntryCount": snapshot.index_entry_count, + "occupiedIndexEntryCount": snapshot.occupied_index_entry_count, + "tombstoneIndexEntryCount": snapshot.tombstone_index_entry_count, + "emptyIndexEntryCount": snapshot.empty_index_entry_count, + "usableIndexCapacity": snapshot.usable_index_capacity, + "lastObservedProbeLength": snapshot.last_observed_probe_length, + "maxObservedProbeLength": snapshot.max_observed_probe_length, + "lastFailureStatus": int(snapshot.last_failure_status), + "abortedReservationCount": snapshot.aborted_reservation_count, + "recoveredLeaseCount": snapshot.recovered_lease_count, + "activeLeaseRecoveryCount": snapshot.active_lease_recovery_count, + "unsupportedLeaseRecoveryCount": snapshot.unsupported_lease_recovery_count, + "failedLeaseRecoveryCount": snapshot.failed_lease_recovery_count, + "recoveredReservationCount": snapshot.recovered_reservation_count, + "activeReservationRecoveryCount": snapshot.active_reservation_recovery_count, + "unsupportedReservationRecoveryCount": snapshot.unsupported_reservation_recovery_count, + "failedReservationRecoveryCount": snapshot.failed_reservation_recovery_count, + "capacityPressureCount": snapshot.capacity_pressure_count, + "indexCompactionCount": snapshot.index_compaction_count, + "failureCounts": list(snapshot.failure_counts), + } + + def command_crash(self, arguments: dict[str, Any]) -> tuple[StoreStatus, None]: + del arguments + os._exit(97) + + command_publishSegmented = command_publishSegments + command_segmentedPublish = command_publishSegments + + +def _report(report: Any, *, reservations: bool, store_id: Any) -> dict[str, Any]: + result: dict[str, Any] = { + "storeId": store_id, + "scannedCount": report.scanned_count, + "recoveredCount": report.recovered_count, + "activeCount": report.active_count, + "unsupportedCount": report.unsupported_count, + "failedCount": report.failed_count, + "failedRecoveryCount": report.failed_count, + } + if reservations: + result.update( + scannedReservationCount=report.scanned_count, + recoveredReservationCount=report.recovered_count, + activeReservationCount=report.active_count, + unsupportedReservationCount=report.unsupported_count, + ) + else: + result.update( + scannedRecordCount=report.scanned_count, + recoveredLeaseCount=report.recovered_count, + activeLeaseCount=report.active_count, + unsupportedLeaseCount=report.unsupported_count, + ) + return result + + +def main() -> int: + agent = Agent() + try: + for line in sys.stdin: + request_id = "" + try: + if "\n" in line[:-1] or "\r" in line: + raise ValueError("an agent protocol frame must contain exactly one LF-delimited line") + request = json.loads(line) + if not isinstance(request, dict): + raise ValueError("an agent protocol frame must be a JSON object") + request_id = request.get("id") if isinstance(request.get("id"), str) else "" + response = agent.handle(request) + except Exception as error: + response = { + "id": request_id, + "ok": False, + "status": {"code": -1, "name": "ProtocolError"}, + "error": {"code": "invalid_request", "message": str(error) or type(error).__name__}, + } + sys.stdout.write(json.dumps(response, separators=(",", ":")) + "\n") + sys.stdout.flush() + finally: + agent.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/python/test_diagnostics.py b/tests/python/test_diagnostics.py new file mode 100644 index 0000000..eb7eaf6 --- /dev/null +++ b/tests/python/test_diagnostics.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError +import unittest + +from shared_memory_store import MemoryStore, StoreOpenStatus, StoreStatus + +from _support import create_options, require_native + + +class DiagnosticsTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + require_native() + + def test_snapshot_reports_shared_state_and_local_failures(self) -> None: + status, store = MemoryStore.open(create_options("diagnostics", slots=3)) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store: + self.assertEqual(StoreStatus.NOT_FOUND, store.acquire(b"missing")[0]) + self.assertEqual(StoreStatus.SUCCESS, store.publish(b"one", b"1")) + acquired, lease = store.acquire(b"one") + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert lease is not None + self.assertEqual(StoreStatus.REMOVE_PENDING, store.remove(b"one")) + reserved, reservation = store.reserve(b"two", 2) + self.assertEqual(StoreStatus.SUCCESS, reserved) + assert reservation is not None + + diagnostic_status, snapshot = store.get_diagnostics() + self.assertEqual(StoreStatus.SUCCESS, diagnostic_status) + assert snapshot is not None + self.assertEqual(3, snapshot.slot_count) + self.assertEqual(1, snapshot.free_slot_count) + self.assertEqual(0, snapshot.published_slot_count) + self.assertEqual(1, snapshot.pending_removal_count) + self.assertEqual(1, snapshot.active_lease_count) + self.assertEqual(1, snapshot.active_reservation_count) + self.assertEqual(StoreStatus.REMOVE_PENDING, snapshot.last_failure_status) + self.assertEqual(1, snapshot.failure_count(StoreStatus.NOT_FOUND)) + self.assertEqual(1, snapshot.get_failure_count(StoreStatus.REMOVE_PENDING)) + self.assertEqual(23, len(snapshot.failure_counts)) + with self.assertRaises(FrozenInstanceError): + snapshot.slot_count = 4 # type: ignore[misc] + + lease.close() + reservation.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_installed_package.py b/tests/python/test_installed_package.py new file mode 100644 index 0000000..791e866 --- /dev/null +++ b/tests/python/test_installed_package.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from importlib.resources import files +import os +from pathlib import Path +import sys +import unittest + +import shared_memory_store +from shared_memory_store import MemoryStore, StoreOpenStatus, StoreStatus + +from _support import create_options + + +class InstalledPackageTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + if os.environ.get("SMS_TEST_INSTALLED_PACKAGE") != "1": + raise unittest.SkipTest("set SMS_TEST_INSTALLED_PACKAGE=1 in the clean wheel environment") + + def test_import_and_native_load_do_not_resolve_from_repository_source(self) -> None: + package_file = Path(shared_memory_store.__file__).resolve() + repository_source = Path(__file__).resolve().parents[2] / "src" / "python" + self.assertFalse(package_file.is_relative_to(repository_source)) + filename = "shared_memory_store.dll" if sys.platform == "win32" else "libshared_memory_store.so" + self.assertTrue(files("shared_memory_store").joinpath(filename).is_file()) + loaded = shared_memory_store.native_library_path().resolve() + self.assertEqual(filename, loaded.name) + self.assertEqual(package_file.parent, loaded.parent) + + def test_clean_wheel_executes_store_lifecycle(self) -> None: + status, store = MemoryStore.open(create_options("installed")) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store: + self.assertEqual(StoreStatus.SUCCESS, store.publish(b"key", b"value")) + acquired, lease = store.acquire(b"key") + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert lease is not None + with lease: + self.assertEqual(b"value", bytes(lease.value)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_interop_agent.py b/tests/python/test_interop_agent.py new file mode 100644 index 0000000..7457b13 --- /dev/null +++ b/tests/python/test_interop_agent.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import base64 +import unittest + +from interop_agent import Agent + +from _support import require_native, unique_store_name + + +def encoded(value: bytes) -> str: + return base64.b64encode(value).decode("ascii") + + +class InteropAgentTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + require_native() + + def test_agent_executes_binary_value_and_reservation_lifecycles(self) -> None: + agent = Agent() + try: + common = { + "storeId": "store", + "name": unique_store_name("agent"), + "openMode": 0, + "slotCount": 3, + "maxValueBytes": 32, + "maxDescriptorBytes": 8, + "maxKeyBytes": 8, + "leaseRecordCount": 4, + "enableLeaseRecovery": True, + } + self.assert_success(agent.handle({"id": "1", "command": "ping"})) + self.assert_success(agent.handle({"id": "2", "command": "open", "arguments": common})) + self.assert_success( + agent.handle( + { + "id": "3", + "command": "publish", + "arguments": { + "storeId": "store", + "key": encoded(b"k\x00"), + "value": encoded(b"v\x00\xff"), + "descriptor": encoded(b"d"), + }, + } + ) + ) + acquired = agent.handle( + { + "id": "4", + "command": "acquire", + "arguments": {"storeId": "store", "leaseId": "lease", "key": encoded(b"k\x00")}, + } + ) + self.assert_success(acquired) + self.assertEqual(encoded(b"v\x00\xff"), acquired["result"]["value"]) + self.assert_success( + agent.handle({"id": "5", "command": "release", "arguments": {"leaseId": "lease"}}) + ) + + self.assert_success( + agent.handle( + { + "id": "6", + "command": "reserve", + "arguments": { + "storeId": "store", + "reservationId": "reservation", + "key": encoded(b"r"), + "payloadLength": 3, + "descriptor": "", + }, + } + ) + ) + self.assert_success( + agent.handle( + { + "id": "7", + "command": "reservationWrite", + "arguments": {"reservationId": "reservation", "data": encoded(b"a\x00b")}, + } + ) + ) + self.assert_success( + agent.handle( + { + "id": "8", + "command": "advance", + "arguments": {"reservationId": "reservation", "byteCount": 3}, + } + ) + ) + self.assert_success( + agent.handle({"id": "9", "command": "commit", "arguments": {"reservationId": "reservation"}}) + ) + diagnostics = agent.handle( + {"id": "10", "command": "diagnostics", "arguments": {"storeId": "store"}} + ) + self.assert_success(diagnostics) + self.assertEqual(2, diagnostics["result"]["publishedSlotCount"]) + finally: + agent.close() + + def test_unknown_command_is_a_protocol_failure(self) -> None: + agent = Agent() + try: + response = agent.handle({"id": "1", "command": "unknown"}) + self.assertFalse(response["ok"]) + self.assertEqual("unsupported_command", response["error"]["code"]) + finally: + agent.close() + + def assert_success(self, response: dict) -> None: + self.assertTrue(response["ok"], response.get("error")) + self.assertEqual(0, response["status"]["code"]) + self.assertEqual("Success", response["status"]["name"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_lifecycle.py b/tests/python/test_lifecycle.py new file mode 100644 index 0000000..abea576 --- /dev/null +++ b/tests/python/test_lifecycle.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import unittest + +from shared_memory_store import MemoryStore, StoreOpenStatus, StoreStatus + +from _support import create_options, require_native + + +class LifecycleTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + require_native() + + def test_segmented_publish_preserves_logical_bytes(self) -> None: + status, store = MemoryStore.open(create_options("segments")) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store: + source = memoryview(bytearray(b"abcdef"))[::2] + published, copied = store.publish_segments( + b"key", + [b"\x00\x01", bytearray(b"\xfe"), source, memoryview(b"\xff")], + b"meta", + ) + source.release() + self.assertEqual(StoreStatus.SUCCESS, published) + self.assertEqual(7, copied) + acquired, lease = store.acquire(b"key") + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert lease is not None + with lease: + self.assertEqual(b"\x00\x01\xfeace\xff", bytes(lease.value)) + self.assertEqual(b"meta", bytes(lease.descriptor)) + + empty_status, empty_copied = store.publish_segments(b"empty", []) + self.assertEqual(StoreStatus.SUCCESS, empty_status) + self.assertEqual(0, empty_copied) + acquired, empty_lease = store.acquire(b"empty") + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert empty_lease is not None + with empty_lease: + self.assertEqual(b"", bytes(empty_lease.value)) + + def test_reservation_progress_commit_and_view_invalidation(self) -> None: + status, store = MemoryStore.open(create_options("reservation")) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store: + reserved, reservation = store.reserve(b"key", 5, b"desc") + self.assertEqual(StoreStatus.SUCCESS, reserved) + assert reservation is not None + self.assertEqual(5, reservation.payload_length) + self.assertEqual(0, reservation.bytes_written) + self.assertEqual(5, reservation.remaining_bytes) + self.assertEqual(StoreStatus.RESERVATION_INCOMPLETE, reservation.commit()) + + first = reservation.buffer(2) + self.assertFalse(first.readonly) + first[:2] = b"ab" + self.assertEqual(StoreStatus.SUCCESS, reservation.advance(2)) + with self.assertRaises(ValueError): + bytes(first) + self.assertEqual(2, reservation.bytes_written) + self.assertEqual(3, reservation.remaining_bytes) + second = reservation.buffer(3) + self.assertEqual(3, len(second)) + second[:] = b"c\x00d" + self.assertEqual(StoreStatus.RESERVATION_WRITE_OUT_OF_RANGE, reservation.advance(4)) + with self.assertRaises(ValueError): + bytes(second) + # A failed advance invalidates the immediate view, but not progress. + second = reservation.buffer(3) + second[:] = b"c\x00d" + self.assertEqual(StoreStatus.SUCCESS, reservation.advance(3)) + self.assertEqual(StoreStatus.SUCCESS, reservation.commit()) + self.assertFalse(reservation.is_valid) + self.assertEqual(StoreStatus.RESERVATION_ALREADY_COMPLETED, reservation.commit()) + + acquired, lease = store.acquire(b"key") + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert lease is not None + with lease: + self.assertEqual(b"abc\x00d", bytes(lease.value)) + self.assertEqual(b"desc", bytes(lease.descriptor)) + + def test_context_exit_aborts_and_allows_key_reuse(self) -> None: + status, store = MemoryStore.open(create_options("abort")) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store: + reserved, reservation = store.reserve(b"key", 3) + self.assertEqual(StoreStatus.SUCCESS, reserved) + assert reservation is not None + with reservation: + view = reservation.buffer() + view[:] = b"abc" + with self.assertRaises(ValueError): + bytes(view) + self.assertEqual(StoreStatus.NOT_FOUND, store.acquire(b"key")[0]) + self.assertEqual(StoreStatus.SUCCESS, store.publish(b"key", b"replacement")) + + def test_current_process_recovery_invalidates_owned_tokens(self) -> None: + status, store = MemoryStore.open(create_options("recovery", recovery=True)) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store: + self.assertEqual(StoreStatus.SUCCESS, store.publish(b"lease", b"value")) + acquired, lease = store.acquire(b"lease") + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert lease is not None + lease_view = lease.value + recovery_status, lease_report = store.recover_leases(True) + self.assertEqual(StoreStatus.SUCCESS, recovery_status) + self.assertEqual(1, lease_report.recovered_count) + self.assertFalse(lease.is_valid) + with self.assertRaises(ValueError): + bytes(lease_view) + + reserved, reservation = store.reserve(b"reservation", 2) + self.assertEqual(StoreStatus.SUCCESS, reserved) + assert reservation is not None + reservation_view = reservation.buffer() + reservation_view[:] = b"no" + recovery_status, reservation_report = store.recover_reservations(True) + self.assertEqual(StoreStatus.SUCCESS, recovery_status) + self.assertEqual(1, reservation_report.recovered_count) + self.assertFalse(reservation.is_valid) + with self.assertRaises(ValueError): + bytes(reservation_view) + self.assertEqual(StoreStatus.NOT_FOUND, store.acquire(b"reservation")[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_native_abi.py b/tests/python/test_native_abi.py new file mode 100644 index 0000000..e9170ab --- /dev/null +++ b/tests/python/test_native_abi.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import ctypes +from dataclasses import FrozenInstanceError +import unittest + +from shared_memory_store import ( + ABI_VERSION, + LAYOUT_MAJOR_VERSION, + LAYOUT_MINOR_VERSION, + RESOURCE_NAMING_VERSION, + OpenMode, + StoreOpenStatus, + StoreStatus, + WaitOptions, +) +from shared_memory_store import _native + + +class NativeAbiTests(unittest.TestCase): + def test_protocol_and_status_numbers_are_exact(self) -> None: + self.assertEqual(0x00010000, ABI_VERSION) + self.assertEqual((1, 2, 1), (LAYOUT_MAJOR_VERSION, LAYOUT_MINOR_VERSION, RESOURCE_NAMING_VERSION)) + self.assertEqual([0, 1, 2], [int(value) for value in OpenMode]) + self.assertEqual(list(range(11)), [int(value) for value in StoreOpenStatus]) + self.assertEqual(list(range(23)), [int(value) for value in StoreStatus]) + + def test_every_ctypes_structure_size_and_critical_offset(self) -> None: + expected = { + _native.Bytes: (16, {"data": 0, "length": 8}), + _native.MutableBytes: (16, {"data": 0, "length": 8}), + _native.Segment: (16, {"data": 0, "length": 8}), + _native.WaitOptions: (16, {"struct_size": 0, "abi_version": 4, "timeout_milliseconds": 8}), + _native.StoreOptions: ( + 72, + { + "struct_size": 0, + "abi_version": 4, + "name_utf8": 8, + "name_length": 16, + "open_mode": 24, + "total_bytes": 32, + "slot_count": 40, + "lease_record_count": 56, + "enable_lease_recovery": 60, + }, + ), + _native.RecoveryReport: (32, {"scanned_count": 8, "failed_count": 24}), + _native.Diagnostics: ( + 344, + { + "total_bytes": 8, + "slot_count": 16, + "last_failure_status": 68, + "aborted_reservation_count": 72, + "failure_counts": 160, + }, + ), + _native.ProtocolInfo: (36, {"layout_major": 8, "lease_record_size": 32}), + _native.StoreLayout: ( + 144, + { + "total_bytes": 8, + "slot_count": 16, + "index_offset": 48, + "descriptor_stride": 96, + "descriptor_storage_offset": 104, + "required_bytes": 136, + }, + ), + } + for structure, (size, offsets) in expected.items(): + with self.subTest(structure=structure.__name__): + self.assertEqual(size, ctypes.sizeof(structure)) + for field, offset in offsets.items(): + self.assertEqual(offset, getattr(structure, field).offset, field) + + self.assertIs(ctypes.c_uint64, dict(_native.Bytes._fields_)["length"]) + self.assertIs(ctypes.c_uint64, dict(_native.StoreOptions._fields_)["name_length"]) + self.assertIs(ctypes.c_uint64, dict(_native.Segment._fields_)["length"]) + + def test_wait_options_are_validated_and_immutable(self) -> None: + self.assertEqual(1000, WaitOptions.default().timeout_milliseconds) + self.assertEqual(0, WaitOptions.no_wait().timeout_milliseconds) + self.assertEqual(-1, WaitOptions.infinite().timeout_milliseconds) + with self.assertRaises(ValueError): + WaitOptions(-2) + with self.assertRaises(TypeError): + WaitOptions(True) + with self.assertRaises(FrozenInstanceError): + WaitOptions.DEFAULT.timeout_milliseconds = 1 # type: ignore[misc] + + def test_loader_has_one_platform_specific_package_artifact_name(self) -> None: + filename = _native._library_filename() + self.assertIn(filename, {"shared_memory_store.dll", "libshared_memory_store.so"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_protocol_manifest.py b/tests/python/test_protocol_manifest.py new file mode 100644 index 0000000..db569b0 --- /dev/null +++ b/tests/python/test_protocol_manifest.py @@ -0,0 +1,628 @@ +from __future__ import annotations + +import ctypes +import hashlib +import json +from pathlib import Path +import struct +import unittest +import unicodedata + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MANIFEST_PATH = REPOSITORY_ROOT / "protocol" / "fixtures" / "v1.2" / "manifest.json" + +INT32_MIN = -(1 << 31) +INT32_MAX = (1 << 31) - 1 +INT64_MIN = -(1 << 63) +INT64_MAX = (1 << 63) - 1 + + +class LayoutInvalidArgument(ValueError): + pass + + +class LayoutArithmeticOverflow(OverflowError): + pass + + +def _checked_int32(value: int) -> int: + if value < INT32_MIN or value > INT32_MAX: + raise LayoutArithmeticOverflow + return value + + +def _checked_int64(value: int) -> int: + if value < INT64_MIN or value > INT64_MAX: + raise LayoutArithmeticOverflow + return value + + +def _align_int32(value: int) -> int: + return _checked_int32(_checked_int32(value + 7) & ~7) + + +def _align_int64(value: int) -> int: + return _checked_int64(_checked_int64(value + 7) & ~7) + + +def _next_power_of_two(value: int) -> int: + if value <= 0 or value > 1 << 30: + raise LayoutArithmeticOverflow + + result = 1 + while result < value: + result = _checked_int32(result << 1) + return result + + +def _calculate_layout( + *, + slot_count: int, + max_value_bytes: int, + max_descriptor_bytes: int, + max_key_bytes: int, + lease_record_count: int, +) -> dict[str, int]: + if slot_count < 1: + raise LayoutInvalidArgument + if max_value_bytes < 1: + raise LayoutInvalidArgument + if max_descriptor_bytes < 0: + raise LayoutInvalidArgument + if max_key_bytes < 1: + raise LayoutInvalidArgument + if lease_record_count < 1: + raise LayoutInvalidArgument + + for value in ( + slot_count, + max_value_bytes, + max_descriptor_bytes, + max_key_bytes, + lease_record_count, + ): + _checked_int32(value) + + header_length = _align_int32(160) + doubled_slots = _checked_int32(slot_count * 2) + index_entry_count = _next_power_of_two(max(4, doubled_slots)) + index_entry_size = _align_int32(_checked_int32(32 + max_key_bytes)) + index_offset = header_length + index_length = _checked_int64(index_entry_count * index_entry_size) + lease_registry_offset = _align_int64(_checked_int64(index_offset + index_length)) + lease_registry_length = _checked_int64(lease_record_count * 40) + slot_metadata_offset = _align_int64( + _checked_int64(lease_registry_offset + lease_registry_length) + ) + slot_metadata_length = _checked_int64(slot_count * 72) + descriptor_stride = _align_int32(max(1, max_descriptor_bytes)) + descriptor_storage_offset = _align_int64( + _checked_int64(slot_metadata_offset + slot_metadata_length) + ) + descriptor_storage_length = _checked_int64(slot_count * descriptor_stride) + payload_stride = _align_int32(max(1, max_value_bytes)) + payload_storage_offset = _align_int64( + _checked_int64(descriptor_storage_offset + descriptor_storage_length) + ) + payload_storage_length = _checked_int64(slot_count * payload_stride) + required_bytes = _align_int64( + _checked_int64(payload_storage_offset + payload_storage_length) + ) + + return { + "header_length": header_length, + "index_entry_count": index_entry_count, + "index_entry_size": index_entry_size, + "index_offset": index_offset, + "index_length": index_length, + "lease_registry_offset": lease_registry_offset, + "lease_registry_length": lease_registry_length, + "slot_metadata_offset": slot_metadata_offset, + "slot_metadata_length": slot_metadata_length, + "descriptor_stride": descriptor_stride, + "descriptor_storage_offset": descriptor_storage_offset, + "descriptor_storage_length": descriptor_storage_length, + "payload_stride": payload_stride, + "payload_storage_offset": payload_storage_offset, + "payload_storage_length": payload_storage_length, + "required_bytes": required_bytes, + } + + +def _fnv1a_64(value: bytes) -> int: + result = 0xCBF29CE484222325 + for current in value: + result ^= current + result = (result * 0x00000100000001B3) & 0xFFFFFFFFFFFFFFFF + return result + + +def _utf16_code_units(value: str) -> list[int]: + encoded = value.encode("utf-16-le") + return [int.from_bytes(encoded[index : index + 2], "little") for index in range(0, len(encoded), 2)] + + +def _is_dotnet_letter_or_digit(code_unit: int) -> bool: + category = unicodedata.category(chr(code_unit)) + return category.startswith("L") or category == "Nd" + + +def _derive_resource_names(public_name: str) -> dict[str, object]: + code_units = _utf16_code_units(public_name) + + windows_readable = "".join( + chr(code_unit) + if _is_dotnet_letter_or_digit(code_unit) or code_unit in (ord("-"), ord("_")) + else "_" + for code_unit in code_units + ) + scope = "Global\\" if public_name[:7].lower() == "global\\" else "Local\\" + + linux_readable = "".join( + chr(code_unit) + if code_unit < 128 + and (chr(code_unit).isalnum() or code_unit in (ord("-"), ord("_"), ord("."))) + else "_" + for code_unit in code_units + ).strip("_.") + linux_readable = (linux_readable or "store")[:80] + digest = hashlib.sha256(public_name.encode("utf-8")).hexdigest()[:16] + fragment = f"sms-{linux_readable}-{digest}" + + return { + "utf16_code_units": len(code_units), + "windows_region_name": public_name, + "windows_synchronization_name": f"{scope}SharedMemoryStore-{windows_readable}", + "linux_sha256_prefix_hex": digest, + "linux_fragment": fragment, + "linux_files": { + "region": f"{fragment}.region", + "synchronization": f"{fragment}.lock", + "owners": f"{fragment}.owners", + "lifecycle": f"{fragment}.lifecycle", + }, + } + + +def _read_int32(region: bytes, offset: int) -> int: + return struct.unpack_from(" int: + return struct.unpack_from(" int: + return struct.unpack_from(" dict[str, object]: + index_state_names = ("Empty", "Occupied", "Tombstone") + slot_state_names = ("Free", "Publishing", "Published", "RemoveRequested", "Reclaiming") + lease_state_names = ("Free", "Active", "Released", "Abandoned") + + index_entries: list[dict[str, object]] = [] + index_count = _read_int32(region, 44) + index_stride = _read_int32(region, 48) + index_offset = _read_int64(region, 56) + for entry_index in range(index_count): + offset = index_offset + (entry_index * index_stride) + state = _read_int32(region, offset) + if state == 0: + continue + key_length = _read_int32(region, offset + 4) + index_entries.append( + { + "entry_index": entry_index, + "state": state, + "state_name": index_state_names[state], + "key_hex": region[offset + 32 : offset + 32 + key_length].hex(), + "key_hash_hex": f"{_read_uint64(region, offset + 8):016x}", + "slot_index": _read_int32(region, offset + 16), + "slot_generation": _read_int32(region, offset + 20), + "slot_reuse_epoch": _read_int64(region, offset + 24), + } + ) + + lease_records: list[dict[str, object]] = [] + lease_count = _read_int32(region, 28) + lease_offset = _read_int64(region, 72) + for record_index in range(lease_count): + offset = lease_offset + (record_index * 40) + state = _read_int32(region, offset) + if state == 0: + continue + lease_records.append( + { + "record_id": _read_int32(region, offset + 4), + "state": state, + "state_name": lease_state_names[state], + "slot_index": _read_int32(region, offset + 8), + "slot_generation": _read_int32(region, offset + 12), + "slot_reuse_epoch": _read_int64(region, offset + 16), + "owner_process_id": _read_int32(region, offset + 24), + "acquire_sequence": _read_int64(region, offset + 32), + } + ) + + slots: list[dict[str, object]] = [] + slot_count = _read_int32(region, 24) + slot_offset = _read_int64(region, 88) + for slot_index in range(slot_count): + offset = slot_offset + (slot_index * 72) + state = _read_int32(region, offset) + descriptor_length = _read_int32(region, offset + 24) + payload_length = _read_int32(region, offset + 28) + descriptor_offset = _read_int64(region, offset + 48) + payload_offset = _read_int64(region, offset + 56) + slots.append( + { + "slot_index": slot_index, + "state": state, + "state_name": slot_state_names[state], + "generation": _read_int32(region, offset + 4), + "reuse_epoch": _read_int64(region, offset + 8), + "usage_count": _read_int32(region, offset + 16), + "key_length": _read_int32(region, offset + 20), + "descriptor_hex": region[ + descriptor_offset : descriptor_offset + descriptor_length + ].hex(), + "payload_hex": region[payload_offset : payload_offset + payload_length].hex(), + "publisher_process_id": _read_int32(region, offset + 32), + "reservation_bytes_written": _read_int32(region, offset + 36), + "key_hash_hex": f"{_read_uint64(region, offset + 40):016x}", + "descriptor_offset": descriptor_offset, + "payload_offset": payload_offset, + "committed_sequence": _read_int64(region, offset + 64), + } + ) + + return { + "format_version": 1, + "fixture": fixture_name, + "offline_only": True, + "protocol": { + "layout_major": _read_int32(region, 4), + "layout_minor": _read_int32(region, 8), + "byte_order": "little", + }, + "header": { + "magic_hex": f"{_read_int32(region, 0) & 0xFFFFFFFF:08x}", + "header_length": _read_int32(region, 12), + "total_bytes": _read_int64(region, 16), + "slot_count": slot_count, + "lease_record_count": lease_count, + "max_key_bytes": _read_int32(region, 32), + "max_descriptor_bytes": _read_int32(region, 36), + "max_value_bytes": _read_int32(region, 40), + "index_entry_count": index_count, + "index_entry_size": index_stride, + "index_offset": index_offset, + "index_length": _read_int64(region, 64), + "lease_registry_offset": lease_offset, + "lease_registry_length": _read_int64(region, 80), + "slot_metadata_offset": slot_offset, + "slot_metadata_length": _read_int64(region, 96), + "descriptor_storage_offset": _read_int64(region, 104), + "descriptor_storage_length": _read_int64(region, 112), + "payload_storage_offset": _read_int64(region, 120), + "payload_storage_length": _read_int64(region, 128), + "store_id_hex": f"{_read_uint64(region, 136):016x}", + "store_state": _read_int32(region, 144), + "sequence": _read_int64(region, 152), + }, + "index_entries": index_entries, + "lease_records": lease_records, + "slots": slots, + } + + +class StoreHeader(ctypes.LittleEndianStructure): + _pack_ = 8 + _fields_ = [ + ("magic", ctypes.c_int32), + ("layout_major_version", ctypes.c_int32), + ("layout_minor_version", ctypes.c_int32), + ("header_length", ctypes.c_int32), + ("total_bytes", ctypes.c_int64), + ("slot_count", ctypes.c_int32), + ("lease_record_count", ctypes.c_int32), + ("max_key_bytes", ctypes.c_int32), + ("max_descriptor_bytes", ctypes.c_int32), + ("max_value_bytes", ctypes.c_int32), + ("index_entry_count", ctypes.c_int32), + ("index_entry_size", ctypes.c_int32), + ("index_offset", ctypes.c_int64), + ("index_length", ctypes.c_int64), + ("lease_registry_offset", ctypes.c_int64), + ("lease_registry_length", ctypes.c_int64), + ("slot_metadata_offset", ctypes.c_int64), + ("slot_metadata_length", ctypes.c_int64), + ("descriptor_storage_offset", ctypes.c_int64), + ("descriptor_storage_length", ctypes.c_int64), + ("payload_storage_offset", ctypes.c_int64), + ("payload_storage_length", ctypes.c_int64), + ("store_id", ctypes.c_int64), + ("store_state", ctypes.c_int32), + ("reserved", ctypes.c_int32), + ("sequence", ctypes.c_int64), + ] + + +class IndexEntryHeader(ctypes.LittleEndianStructure): + _pack_ = 8 + _fields_ = [ + ("state", ctypes.c_int32), + ("key_length", ctypes.c_int32), + ("key_hash", ctypes.c_uint64), + ("slot_index", ctypes.c_int32), + ("slot_generation", ctypes.c_int32), + ("slot_reuse_epoch", ctypes.c_int64), + ] + + +class SlotMetadata(ctypes.LittleEndianStructure): + _pack_ = 8 + _fields_ = [ + ("state", ctypes.c_int32), + ("generation", ctypes.c_int32), + ("reuse_epoch", ctypes.c_int64), + ("usage_count", ctypes.c_int32), + ("key_length", ctypes.c_int32), + ("descriptor_length", ctypes.c_int32), + ("value_length", ctypes.c_int32), + ("publisher_process_id", ctypes.c_int32), + ("reserved", ctypes.c_int32), + ("key_hash", ctypes.c_uint64), + ("descriptor_offset", ctypes.c_int64), + ("payload_offset", ctypes.c_int64), + ("committed_sequence", ctypes.c_int64), + ] + + +class LeaseRecord(ctypes.LittleEndianStructure): + _pack_ = 8 + _fields_ = [ + ("state", ctypes.c_int32), + ("lease_record_id", ctypes.c_int32), + ("slot_index", ctypes.c_int32), + ("slot_generation", ctypes.c_int32), + ("slot_reuse_epoch", ctypes.c_int64), + ("owner_process_id", ctypes.c_int32), + ("reserved", ctypes.c_int32), + ("acquire_sequence", ctypes.c_int64), + ] + + +class ProtocolManifestTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + + def test_protocol_identity_and_magic_are_exact(self) -> None: + protocol = self.manifest["protocol"] + self.assertEqual(1, self.manifest["format_version"]) + self.assertEqual(1, protocol["layout_major"]) + self.assertEqual(2, protocol["layout_minor"]) + self.assertEqual(1, protocol["resource_naming_version"]) + self.assertEqual("little", protocol["byte_order"]) + self.assertEqual(8, protocol["max_field_alignment"]) + + magic = protocol["magic"] + self.assertEqual("SMS1", magic["ascii"]) + self.assertEqual(0x31534D53, magic["integer_value"]) + self.assertEqual(f"{magic['integer_value']:08x}", magic["integer_hex"]) + self.assertEqual( + magic["ascii"].encode("ascii"), + bytes.fromhex(magic["little_endian_bytes_hex"]), + ) + self.assertEqual( + magic["integer_value"], + int.from_bytes(bytes.fromhex(magic["little_endian_bytes_hex"]), "little"), + ) + + def test_every_record_size_type_offset_and_padding(self) -> None: + structures = { + "store_header": StoreHeader, + "index_entry_header": IndexEntryHeader, + "slot_metadata": SlotMetadata, + "lease_record": LeaseRecord, + } + type_names = { + ctypes.c_int32: "int32", + ctypes.c_int64: "int64", + ctypes.c_uint64: "uint64", + } + + self.assertEqual(set(structures), set(self.manifest["records"])) + for record_name, structure in structures.items(): + with self.subTest(record=record_name): + record = self.manifest["records"][record_name] + self.assertEqual(record["size"], ctypes.sizeof(structure)) + declared_fields = dict(structure._fields_) + self.assertEqual(set(record["fields"]), set(declared_fields)) + occupied: set[int] = set() + for field_name, field_type in structure._fields_: + expected = record["fields"][field_name] + descriptor = getattr(structure, field_name) + self.assertEqual(expected["offset"], descriptor.offset) + self.assertEqual(expected["type"], type_names[field_type]) + occupied.update(range(descriptor.offset, descriptor.offset + ctypes.sizeof(field_type))) + + actual_padding = sorted(set(range(ctypes.sizeof(structure))) - occupied) + expected_padding = sorted( + byte + for padding in record["padding"] + for byte in range(padding["offset"], padding["offset"] + padding["length"]) + ) + self.assertEqual(expected_padding, actual_padding) + + self.assertEqual( + self.manifest["records"]["index_entry_header"]["size"], + self.manifest["records"]["index_entry_header"]["inline_key_offset"], + ) + + def test_state_open_mode_and_status_assignments_are_exact(self) -> None: + self.assertEqual( + { + "store": {"Initializing": 0, "Ready": 1, "Disposing": 2, "Corrupt": 3, "Unsupported": 4}, + "index": {"Empty": 0, "Occupied": 1, "Tombstone": 2}, + "slot": {"Free": 0, "Publishing": 1, "Published": 2, "RemoveRequested": 3, "Reclaiming": 4}, + "lease": {"Free": 0, "Active": 1, "Released": 2, "Abandoned": 3}, + }, + self.manifest["states"], + ) + self.assertEqual( + {"CreateNew": 0, "OpenExisting": 1, "CreateOrOpen": 2}, + self.manifest["open_modes"], + ) + self.assertEqual( + { + "Success": 0, + "AlreadyExists": 1, + "NotFound": 2, + "InvalidOptions": 3, + "IncompatibleLayout": 4, + "UnsupportedPlatform": 5, + "InsufficientCapacity": 6, + "AccessDenied": 7, + "MappingFailed": 8, + "StoreBusy": 9, + "OperationCanceled": 10, + }, + self.manifest["status_values"]["store_open_status"], + ) + self.assertEqual( + { + "Success": 0, + "DuplicateKey": 1, + "NotFound": 2, + "KeyTooLarge": 3, + "ValueTooLarge": 4, + "DescriptorTooLarge": 5, + "StoreFull": 6, + "LeaseTableFull": 7, + "InvalidLease": 8, + "LeaseAlreadyReleased": 9, + "RemovePending": 10, + "UnsupportedPlatform": 11, + "StoreDisposed": 12, + "CorruptStore": 13, + "AccessDenied": 14, + "UnknownFailure": 15, + "InvalidReservation": 16, + "ReservationIncomplete": 17, + "ReservationAlreadyCompleted": 18, + "ReservationWriteOutOfRange": 19, + "InvalidKey": 20, + "StoreBusy": 21, + "OperationCanceled": 22, + }, + self.manifest["status_values"]["store_status"], + ) + + def test_every_fnv1a_vector(self) -> None: + specification = self.manifest["fnv1a_64"] + self.assertEqual(0xCBF29CE484222325, int(specification["offset_basis_hex"], 16)) + self.assertEqual(0x00000100000001B3, int(specification["prime_hex"], 16)) + names: set[str] = set() + for vector in specification["vectors"]: + with self.subTest(vector=vector["name"]): + self.assertNotIn(vector["name"], names) + names.add(vector["name"]) + value = bytes.fromhex(vector["bytes_hex"]) + self.assertEqual(bool(value), vector["valid_store_key"]) + self.assertEqual(16, len(vector["expected_hash_hex"])) + self.assertEqual( + int(vector["expected_hash_hex"], 16), + _fnv1a_64(value), + ) + + def test_every_successful_layout_vector(self) -> None: + names: set[str] = set() + for vector in self.manifest["layout_calculation"]["vectors"]: + with self.subTest(vector=vector["name"]): + self.assertNotIn(vector["name"], names) + names.add(vector["name"]) + self.assertEqual(vector["expected"], _calculate_layout(**vector["input"])) + + def test_every_layout_error_vector(self) -> None: + error_types = { + "invalid_argument": LayoutInvalidArgument, + "arithmetic_overflow": LayoutArithmeticOverflow, + } + names: set[str] = set() + for vector in self.manifest["layout_calculation"]["error_vectors"]: + with self.subTest(vector=vector["name"]): + self.assertNotIn(vector["name"], names) + names.add(vector["name"]) + expected_error = vector["expected_error"] + self.assertIn(expected_error, error_types) + with self.assertRaises(error_types[expected_error]): + _calculate_layout(**vector["input"]) + + def test_every_offline_mapped_region_fixture_and_snapshot(self) -> None: + expected_names = { + "empty", + "published", + "pending-reservation", + "pending-removal", + "reused-slot", + } + fixtures = self.manifest["mapped_region_fixtures"] + self.assertEqual(expected_names, {fixture["name"] for fixture in fixtures}) + + for fixture in fixtures: + with self.subTest(fixture=fixture["name"]): + self.assertTrue(fixture["offline_only"]) + binary_path = MANIFEST_PATH.parent / fixture["binary_file"] + snapshot_path = MANIFEST_PATH.parent / fixture["snapshot_file"] + region = binary_path.read_bytes() + snapshot_bytes = snapshot_path.read_bytes() + + self.assertEqual(fixture["byte_length"], len(region)) + self.assertEqual( + fixture["binary_sha256_hex"], hashlib.sha256(region).hexdigest() + ) + self.assertEqual( + fixture["snapshot_sha256_hex"], + hashlib.sha256(snapshot_bytes).hexdigest(), + ) + + snapshot = json.loads(snapshot_bytes) + self.assertEqual( + snapshot, + _normalize_mapped_region(fixture["name"], region), + ) + + def test_every_resource_name_vector(self) -> None: + specification = self.manifest["resource_naming"] + self.assertEqual(1, specification["version"]) + self.assertEqual("0700", specification["linux_directory_mode_octal"]) + self.assertEqual("0600", specification["linux_file_mode_octal"]) + names: set[str] = set() + linux_fragments: set[str] = set() + for vector in specification["vectors"]: + with self.subTest(vector=vector["name"]): + self.assertNotIn(vector["name"], names) + names.add(vector["name"]) + derived = _derive_resource_names(vector["public_name"]) + self.assertEqual(vector["utf16_code_units"], derived.pop("utf16_code_units")) + self.assertLessEqual( + vector["utf16_code_units"], + specification["maximum_public_name_utf16_code_units"], + ) + self.assertEqual(vector["expected"], derived) + fragment = vector["expected"]["linux_fragment"] + self.assertNotIn(fragment, linux_fragments) + linux_fragments.add(fragment) + readable = fragment.removeprefix("sms-").rsplit("-", 1)[0] + self.assertLessEqual( + len(_utf16_code_units(readable)), + specification["maximum_linux_readable_utf16_code_units"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_store.py b/tests/python/test_store.py new file mode 100644 index 0000000..97f37e2 --- /dev/null +++ b/tests/python/test_store.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from dataclasses import replace +from concurrent.futures import ThreadPoolExecutor +import gc +import unittest +import weakref + +from shared_memory_store import ( + MemoryStore, + OpenMode, + StoreOpenStatus, + StoreStatus, + calculate_required_bytes, +) + +from _support import create_options, require_native + + +class MemoryStoreTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + require_native() + + def test_required_bytes_and_all_open_modes(self) -> None: + options = create_options("open") + self.assertEqual( + options.total_bytes, + calculate_required_bytes( + slot_count=options.slot_count, + max_value_bytes=options.max_value_bytes, + max_descriptor_bytes=options.max_descriptor_bytes, + max_key_bytes=options.max_key_bytes, + lease_record_count=options.lease_record_count, + ), + ) + with self.assertRaises(ValueError): + calculate_required_bytes( + slot_count=0, + max_value_bytes=1, + max_descriptor_bytes=0, + max_key_bytes=1, + lease_record_count=1, + ) + + create_new = replace(options, open_mode=OpenMode.CREATE_NEW) + status, creator = MemoryStore.open(create_new) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + self.assertIsNotNone(creator) + assert creator is not None + try: + duplicate, absent = MemoryStore.open(create_new) + self.assertEqual(StoreOpenStatus.ALREADY_EXISTS, duplicate) + self.assertIsNone(absent) + opened, peer = MemoryStore.open(replace(options, open_mode=OpenMode.OPEN_EXISTING)) + self.assertEqual(StoreOpenStatus.SUCCESS, opened) + self.assertIsNotNone(peer) + assert peer is not None + peer.close() + peer.close() + + incompatible, incompatible_store = MemoryStore.open( + replace(options, open_mode=OpenMode.OPEN_EXISTING, max_value_bytes=64) + ) + self.assertEqual(StoreOpenStatus.INCOMPATIBLE_LAYOUT, incompatible) + self.assertIsNone(incompatible_store) + finally: + creator.close() + + insufficient, no_store = MemoryStore.open(replace(options, total_bytes=options.total_bytes - 1)) + self.assertEqual(StoreOpenStatus.INSUFFICIENT_CAPACITY, insufficient) + self.assertIsNone(no_store) + for invalid_name in ("", " ", "embedded\x00nul", "\ud800"): + invalid, no_store = MemoryStore.open(replace(options, name=invalid_name)) + self.assertEqual(StoreOpenStatus.INVALID_OPTIONS, invalid) + self.assertIsNone(no_store) + + def test_binary_publish_acquire_release_remove_and_reuse(self) -> None: + status, store = MemoryStore.open(create_options("basic")) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store: + key = bytearray(b"key\x00\xff") + value = bytearray(b"payload\x00\xfe") + descriptor = bytearray(b"descriptor\x00") + self.assertEqual(StoreStatus.SUCCESS, store.publish(key, value, descriptor)) + value[:] = b"X" * len(value) + self.assertEqual(StoreStatus.DUPLICATE_KEY, store.publish(key, b"other")) + + acquired, lease = store.acquire(memoryview(key)) + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert lease is not None + value_view = lease.value + descriptor_view = lease.descriptor + self.assertTrue(value_view.readonly) + self.assertTrue(descriptor_view.readonly) + self.assertEqual(b"payload\x00\xfe", bytes(value_view)) + self.assertEqual(b"descriptor\x00", bytes(descriptor_view)) + with self.assertRaises(TypeError): + value_view[0] = 1 + + self.assertEqual(StoreStatus.REMOVE_PENDING, store.remove(key)) + self.assertEqual(StoreStatus.NOT_FOUND, store.acquire(key)[0]) + self.assertEqual(StoreStatus.SUCCESS, lease.release()) + self.assertEqual(StoreStatus.LEASE_ALREADY_RELEASED, lease.release()) + with self.assertRaises(ValueError): + bytes(value_view) + lease.close() + + self.assertEqual(StoreStatus.SUCCESS, store.publish(key, b"replacement")) + self.assertEqual(StoreStatus.SUCCESS, store.remove(key)) + + def test_view_retains_lease_and_store_ownership(self) -> None: + status, store = MemoryStore.open(create_options("ownership")) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + self.assertEqual(StoreStatus.SUCCESS, store.publish(b"k", b"owned")) + acquired, lease = store.acquire(b"k") + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert lease is not None + reference = weakref.ref(lease) + view = lease.value + del lease + gc.collect() + self.assertIsNotNone(reference()) + self.assertEqual(b"owned", bytes(view)) + view.release() + del view + gc.collect() + self.assertIsNone(reference()) + diagnostic_status, snapshot = store.diagnostics() + self.assertEqual(StoreStatus.SUCCESS, diagnostic_status) + assert snapshot is not None + self.assertEqual(0, snapshot.active_lease_count) + store.close() + + def test_store_close_invalidates_children_and_views(self) -> None: + status, store = MemoryStore.open(create_options("close")) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + self.assertEqual(StoreStatus.SUCCESS, store.publish(b"k", b"value")) + acquired, lease = store.acquire(b"k") + self.assertEqual(StoreStatus.SUCCESS, acquired) + assert lease is not None + view = lease.value + store.close() + store.close() + self.assertFalse(store.is_open) + self.assertFalse(lease.is_valid) + self.assertEqual(StoreStatus.STORE_DISPOSED, store.publish(b"x", b"y")) + self.assertEqual(StoreStatus.STORE_DISPOSED, store.remove(b"x")) + self.assertEqual(StoreStatus.STORE_DISPOSED, store.acquire(b"x")[0]) + with self.assertRaises(ValueError): + bytes(view) + + def test_invalid_and_oversized_inputs_return_contract_statuses(self) -> None: + options = create_options("validation") + status, store = MemoryStore.open(options) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store: + self.assertEqual(StoreStatus.INVALID_KEY, store.publish(b"", b"value")) + self.assertEqual(StoreStatus.KEY_TOO_LARGE, store.publish(b"k" * 33, b"value")) + self.assertEqual(StoreStatus.VALUE_TOO_LARGE, store.publish(b"k", b"v" * 129)) + self.assertEqual(StoreStatus.DESCRIPTOR_TOO_LARGE, store.publish(b"k", b"v", b"d" * 33)) + with self.assertRaises(TypeError): + store.publish("not bytes", b"value") + + def test_concurrent_duplicate_publish_has_one_winner(self) -> None: + status, store = MemoryStore.open(create_options("duplicate-race")) + self.assertEqual(StoreOpenStatus.SUCCESS, status) + assert store is not None + with store, ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(lambda value: store.publish(b"same", bytes([value])), range(8))) + self.assertEqual(1, results.count(StoreStatus.SUCCESS)) + self.assertEqual(7, results.count(StoreStatus.DUPLICATE_KEY)) + + +if __name__ == "__main__": + unittest.main() From c4211aba1036c5b67b6acd649874c765130f3f2c Mon Sep 17 00:00:00 2001 From: Ran Trifon Date: Fri, 10 Jul 2026 15:17:20 +0300 Subject: [PATCH 2/3] Preserve protocol snapshot line endings --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b76b1ec --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +protocol/fixtures/v1.2/*.snapshot.json text eol=lf From 210b8a21078034d7aba62f65d4afc6fc3dc094c2 Mon Sep 17 00:00:00 2001 From: Ran Trifon Date: Fri, 10 Jul 2026 15:36:26 +0300 Subject: [PATCH 3/3] Upgrade CI actions to Node 24 --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 106c856..0ae6c8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,10 +25,10 @@ jobs: steps: - name: Check out source - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up .NET 10 - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 with: dotnet-version: 10.0.x @@ -62,10 +62,10 @@ jobs: steps: - name: Check out source - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up .NET 10 - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 with: dotnet-version: 10.0.x @@ -84,7 +84,7 @@ jobs: steps: - name: Check out source - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Validate CMake build, tests, install, and clean consumer shell: pwsh @@ -101,7 +101,7 @@ jobs: steps: - name: Check out source - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Python 3.12 uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -127,10 +127,10 @@ jobs: steps: - name: Check out source - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up .NET 10 - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 with: dotnet-version: 10.0.x