Cbones

.github/workflows/ci.yml

@@ -6,52 +6,95 @@ on:
     tags:
       - "v*" # Semantic version tags
   pull_request:
+permissions:
+  contents: read
 jobs:
   test:
     strategy:
       matrix:
-        py: ["3.9", "3.10", "3.11", "3.12", "3.13"]
-    runs-on: ubuntu-latest
+        py: ["3.11", "3.12", "3.13", "3.14"]
+        os: [ubuntu-latest, macos-latest]
+        exclude:
+          # Only test the latest stable Python on macOS to keep CI fast
+          - os: macos-latest
+            py: "3.11"
+          - os: macos-latest
+            py: "3.12"
+          - os: macos-latest
+            py: "3.14"
+    runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0 # full history
           fetch-tags: true # redundant in v4, but explicit
-          submodules: "recursive"
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.py }}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install submodules/daggerml_cli/
-          pip install .[dev]
+      - name: Install uv
+        run: pip install uv==0.11.7
+      - name: Sync dependencies
+        run: uv sync --group dev
       - name: pytest
-        run: python -m pytest -m "not slow" .
+        run: uv run pytest .
+  sanitize:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+      - name: Install uv
+        run: pip install uv==0.11.7
+      - name: Build with ASAN + UBSAN
+        env:
+          CMAKE_ARGS: "-DDML_ENABLE_ASAN=ON -DDML_ENABLE_UBSAN=ON"
+        run: uv sync --group dev
+      - name: Run tests with sanitizers
+        env:
+          ASAN_OPTIONS: "detect_leaks=0:abort_on_error=1"
+          UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+        run: |
+          LIBASAN=$(gcc -print-file-name=libasan.so)
+          LD_PRELOAD="$LIBASAN" uv run pytest tests/contracts tests/contrib src/daggerml
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
-      - name: Install ruff
-        run: pip install ruff
+      - name: Install uv
+        run: pip install uv==0.11.7
+      - name: Sync dependencies
+        run: uv sync --group dev
       - name: Run Ruff
-        run: ruff check --output-format=github .
+        run: uv run ruff check --output-format=github .
   publish:
-    needs: ["test", "lint"]
+    needs: ["test", "lint", "sanitize"]
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
     name: Upload release to PyPI
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-3.13-publish-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-3.13-
+            ${{ runner.os }}-pip-
       - name: Install hatch
         run: |
           pip install hatch

.gitignore

@@ -1,5 +1,6 @@
 # daggerml
 .dml/
+.opencode/
 # scm writes version info now...
 __about__.py
 
@@ -86,6 +87,8 @@ instance/
 # Sphinx documentation
 docs/_build/
 docs/_autosummary/
+docs/contrib/plans/
+docs/contrib/*-plan.local.md
 
 # PyBuilder
 .pybuilder/
@@ -174,3 +177,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+uv.lock

AGENTS.md

@@ -0,0 +1,6 @@
+# Agent Instructions
+
+- Before editing code, consult relevant docs via `docs/DOC_MAP.md`.
+- In your final summary/PR notes, list the docs you consulted.
+- When a function runs through the script executor (`@api.funkify(uri="script", ...)`), only the function source and explicitly injected `extra_objs`/`extra_lines` are available in the worker; module-level imports/globals are not.
+- Keep script-executed functions self-contained: import dependencies inside the function body or inject them explicitly, or runtime `NameError` failures can appear at the call site.

CMakeLists.txt

@@ -0,0 +1,116 @@
+cmake_minimum_required(VERSION 3.18)
+project(daggerml LANGUAGES C)
+
+option(DML_ENABLE_ASAN "Enable AddressSanitizer" OFF)
+option(DML_ENABLE_UBSAN "Enable UndefinedBehaviorSanitizer" OFF)
+
+set(_dml_sanitize_flags "")
+if(DML_ENABLE_ASAN)
+  list(APPEND _dml_sanitize_flags "-fsanitize=address")
+endif()
+if(DML_ENABLE_UBSAN)
+  list(APPEND _dml_sanitize_flags "-fsanitize=undefined")
+endif()
+if(_dml_sanitize_flags)
+  list(APPEND _dml_sanitize_flags "-fno-omit-frame-pointer")
+endif()
+
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+
+add_library(lmdb STATIC
+  c/third_party/lmdb/libraries/liblmdb/mdb.c
+  c/third_party/lmdb/libraries/liblmdb/midl.c
+)
+target_include_directories(lmdb PUBLIC c/third_party/lmdb/libraries/liblmdb)
+set_property(TARGET lmdb PROPERTY POSITION_INDEPENDENT_CODE ON)
+if(_dml_sanitize_flags)
+  target_compile_options(lmdb PRIVATE ${_dml_sanitize_flags})
+  target_link_options(lmdb PRIVATE ${_dml_sanitize_flags})
+endif()
+
+add_library(msgpackc STATIC
+  c/third_party/msgpack/src/objectc.c
+  c/third_party/msgpack/src/unpack.c
+  c/third_party/msgpack/src/vrefbuffer.c
+  c/third_party/msgpack/src/zone.c
+  c/third_party/msgpack/src/version.c
+)
+target_include_directories(msgpackc PUBLIC c/third_party/msgpack/include)
+set_property(TARGET msgpackc PROPERTY POSITION_INDEPENDENT_CODE ON)
+if(_dml_sanitize_flags)
+  target_compile_options(msgpackc PRIVATE ${_dml_sanitize_flags})
+  target_link_options(msgpackc PRIVATE ${_dml_sanitize_flags})
+endif()
+
+add_library(sha256 STATIC
+  c/third_party/sha256/sha256.c
+)
+target_include_directories(sha256 PUBLIC c/third_party/sha256)
+set_property(TARGET sha256 PROPERTY POSITION_INDEPENDENT_CODE ON)
+if(_dml_sanitize_flags)
+  target_compile_options(sha256 PRIVATE ${_dml_sanitize_flags})
+  target_link_options(sha256 PRIVATE ${_dml_sanitize_flags})
+endif()
+
+add_library(daggerml_core STATIC
+  c/src/dml_db.c
+  c/src/dml_hash.c
+  c/src/dml_msgpack.c
+  c/src/dml_value.c
+)
+target_include_directories(daggerml_core PUBLIC c/include)
+target_include_directories(daggerml_core PUBLIC c/third_party/msgpack/include)
+set_property(TARGET daggerml_core PROPERTY POSITION_INDEPENDENT_CODE ON)
+if(_dml_sanitize_flags)
+  target_compile_options(daggerml_core PRIVATE ${_dml_sanitize_flags})
+  target_link_options(daggerml_core PRIVATE ${_dml_sanitize_flags})
+endif()
+
+set(_db_pyx ${CMAKE_CURRENT_SOURCE_DIR}/src/daggerml/_internal/_db.pyx)
+set(_db_c ${CMAKE_CURRENT_BINARY_DIR}/_db.c)
+
+add_custom_command(
+  OUTPUT ${_db_c}
+  COMMAND ${Python_EXECUTABLE} -m cython -3 --output-file ${_db_c} ${_db_pyx}
+  DEPENDS ${_db_pyx}
+  COMMENT "Cythonizing _db.pyx"
+  VERBATIM
+)
+
+add_library(_db MODULE
+  ${_db_c}
+)
+set(DB_EXPORTS "${CMAKE_CURRENT_BINARY_DIR}/db.exports")
+if(APPLE)
+  file(WRITE "${DB_EXPORTS}" "_PyInit__db\n")
+  target_link_options(_db PRIVATE
+    "-Wl,-exported_symbols_list,${DB_EXPORTS}"
+  )
+else()
+  file(WRITE "${DB_EXPORTS}" "{_PyInit__db;};")
+  target_link_options(_db PRIVATE
+    "-Wl,--version-script=${DB_EXPORTS}"
+  )
+endif()
+target_include_directories(_db PRIVATE c/third_party/lmdb/libraries/liblmdb)
+target_include_directories(_db PRIVATE c/third_party/msgpack/include)
+target_include_directories(_db PRIVATE c/include)
+target_link_libraries(_db PRIVATE daggerml_core lmdb msgpackc sha256 Python::Module)
+if(_dml_sanitize_flags)
+  target_compile_options(_db PRIVATE ${_dml_sanitize_flags})
+  target_link_options(_db PRIVATE ${_dml_sanitize_flags})
+  if(DML_ENABLE_ASAN AND NOT APPLE AND CMAKE_C_COMPILER_ID MATCHES "Clang")
+    target_link_options(_db PRIVATE "-shared-libasan")
+  endif()
+  if(DML_ENABLE_UBSAN AND CMAKE_C_COMPILER_ID MATCHES "Clang")
+    target_compile_options(_db PRIVATE "-fno-sanitize=function")
+  endif()
+endif()
+
+set_target_properties(_db PROPERTIES PREFIX "" OUTPUT_NAME "_db")
+
+install(TARGETS _db
+  LIBRARY DESTINATION daggerml/_internal/
+  RUNTIME DESTINATION daggerml/_internal/
+  ARCHIVE DESTINATION daggerml/_internal/
+)

CONTRIBUTING.md

@@ -18,7 +18,7 @@ requests and appreciate your help in improving this project.
 1. Create a new branch for your feature or bugfix (with the github issue in the name).
 2. Clone the repository and set it up:
    ```bash
-   git clone --recurse-submodules https://github.com/daggerml/python-lib.git
+   git clone https://github.com/daggerml/python-lib.git
    ```
 3. Make your changes in the new branch.
 4. Write or update tests as needed.
@@ -37,30 +37,47 @@ requests and appreciate your help in improving this project.
 
 - Add or update unit tests for any new features or bug fixes.
 - Use [pytest](https://pytest.org/) for running tests.
-- The testing requirements are included in the `test` feature for the library.
-  - You can run tests using [hatch](https://hatch.pypa.io/):
-    ```
-    hatch run pytest .
-    ```
-  - If you're using vscode, you can create a venv with the `test` feature and run tests with the command palette:
-    ```
-    Python: Run Tests
-    ```
-  - Or install the `test` feature with pip and run tests:
-    ```
-    pip install -e </path/to/library>[test]
-    pytest .
-    ```
+- Standard local dev command pattern is:
+  ```bash
+  uv run --dev <python command>
+  ```
+- When a command needs optional dependencies, include all extras:
+  ```bash
+  uv run --dev --all-extras <python command>
+  ```
+- Run tests with:
+  ```bash
+  uv run --dev --all-extras pytest .
+  ```
+- Run lint with:
+  ```bash
+  uv run --dev --all-extras ruff check .
+  ```
 - We mark tests with `@pytest.mark.slow` for those that take longer to run. You can run only the fast tests with:
   ```
-  pytest -m "not slow" .
+  uv run --dev --all-extras pytest -m "not slow" .
   ```
+- Contract-first test layout guidance:
+  - fast, isolated contract tests live under `tests/contracts/`,
+  - integration or infrastructure-heavy coverage lives under `tests/integration/` and should be marked `@pytest.mark.slow`,
+  - canonical contract IDs should be embedded directly in test names or parameterized case IDs for migrated contract suites.
+- CI continues to run the full suite (`uv run pytest .`) to preserve complete coverage while local quick loops use `-m "not slow"`.
 - We mark tests that require `daggerml-cli` to be installed with `@pytest.mark.needs_dml`. You can exclude those tests with:
   ```
-  pytest -m "not needs_dml" .
+  uv run --dev --all-extras pytest -m "not needs_dml" .
   ```
 - Run all tests locally before submitting a pull request:
 - Ensure your code passes all tests and does not decrease code coverage.
 - If your changes introduce new dependencies, please update `pyproject.toml`, but we prefer to keep the dependencies to a minimum.
 
+## Migration Rollout Policy
+
+When migrating storage or execution paths, use phased rollouts with tests at each phase:
+
+1. Implement the new destination path first and test it.
+2. Write to both old and new paths and test.
+3. Read from the new path and test.
+4. Stop writing to the old path and test.
+5. Remove the old path and test.
+
 Thank you for helping make this project better!