From e26a8a356655e24145ee04e81647713c9c9f973e Mon Sep 17 00:00:00 2001 From: jross Date: Wed, 24 Jun 2026 16:07:52 -0600 Subject: [PATCH 1/4] Fix Dagster+ PEX deploy: nmuwd path needs trailing slash The serverless PEX build failed parsing '..' as a dependency: Problem parsing '..' as a requirement: Expected package name The dagster-cloud PEX builder resolves [tool.uv.sources] path deps to a local package only when the path starts with "./", "../", or "/". Bare ".." falls through to the dependency resolver and crashes. Use "../" so nmuwd is bundled into the source PEX as intended. Co-Authored-By: Claude Opus 4.8 --- orchestration/pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/orchestration/pyproject.toml b/orchestration/pyproject.toml index ca41e35..2856778 100644 --- a/orchestration/pyproject.toml +++ b/orchestration/pyproject.toml @@ -17,8 +17,11 @@ dependencies = [ "nmuwd", ] +# Trailing slash is required: the dagster-cloud PEX builder only treats a +# uv.sources path as a local package when it starts with "./", "../", or "/". +# Bare ".." is parsed as a (broken) dependency string instead. [tool.uv.sources] -nmuwd = { path = "..", editable = true } +nmuwd = { path = "../", editable = true } [tool.hatch.build.targets.wheel] packages = ["orchestration"] From 09d8a1569ea54ab0cf4e12a33cf03162709366d7 Mon Sep 17 00:00:00 2001 From: jross Date: Wed, 24 Jun 2026 16:09:23 -0600 Subject: [PATCH 2/4] Remove duplicate Dagster+ deploy workflows dagster-cloud-deploy.yml / dagster-cloud-branch-deployments.yml duplicate the official quickstart workflows (deploy.yml / branch_deployments.yml): same concurrency groups and job names, so the two pairs cancelled each other and double-ran. Keep the official pair (DAGSTER_CLOUD_URL set, pinned actions); drop the redundant pair. Co-Authored-By: Claude Opus 4.8 --- .../dagster-cloud-branch-deployments.yml | 86 ------------------ .github/workflows/dagster-cloud-deploy.yml | 89 ------------------- 2 files changed, 175 deletions(-) delete mode 100644 .github/workflows/dagster-cloud-branch-deployments.yml delete mode 100644 .github/workflows/dagster-cloud-deploy.yml diff --git a/.github/workflows/dagster-cloud-branch-deployments.yml b/.github/workflows/dagster-cloud-branch-deployments.yml deleted file mode 100644 index 005a063..0000000 --- a/.github/workflows/dagster-cloud-branch-deployments.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Dagster+ Serverless Branch Deployments - -on: - pull_request: - types: [opened, synchronize, reopened, closed] - -concurrency: - group: ${{ github.ref }}/branch_deployments - cancel-in-progress: true - -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.10' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-22.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - steps: - - name: Check Dagster+ credentials present - id: guard - run: | - if [ -n "${{ secrets.DAGSTER_CLOUD_URL }}" ] && [ -n "${{ secrets.DAGSTER_CLOUD_API_TOKEN }}" ]; then - echo "enabled=true" >> "$GITHUB_OUTPUT" - else - echo "::notice::Dagster+ secrets (DAGSTER_CLOUD_URL / DAGSTER_CLOUD_API_TOKEN) not set; skipping deploy." - echo "enabled=false" >> "$GITHUB_OUTPUT" - fi - - - name: Prerun Checks - id: prerun - if: steps.guard.outputs.enabled == 'true' - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-22.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster+ serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/dagster-cloud-deploy.yml b/.github/workflows/dagster-cloud-deploy.yml deleted file mode 100644 index 6a9fd6f..0000000 --- a/.github/workflows/dagster-cloud-deploy.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Dagster+ Serverless Deploy - -on: - push: - branches: - - "main" - -concurrency: - group: ${{ github.ref }}/deploy - cancel-in-progress: true - -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - # PEX fast deploy. No system GDAL needed (only shapely, which ships a wheel - # with bundled GEOS), so a Docker build would be pure overhead. - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.10' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-22.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - steps: - - name: Check Dagster+ credentials present - id: guard - run: | - if [ -n "${{ secrets.DAGSTER_CLOUD_URL }}" ] && [ -n "${{ secrets.DAGSTER_CLOUD_API_TOKEN }}" ]; then - echo "enabled=true" >> "$GITHUB_OUTPUT" - else - echo "::notice::Dagster+ secrets (DAGSTER_CLOUD_URL / DAGSTER_CLOUD_API_TOKEN) not set; skipping deploy." - echo "enabled=false" >> "$GITHUB_OUTPUT" - fi - - - name: Prerun Checks - id: prerun - if: steps.guard.outputs.enabled == 'true' - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-22.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster+ serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 3bd5c97850865b06a7e919a15d47be94d7b73297 Mon Sep 17 00:00:00 2001 From: jross Date: Wed, 24 Jun 2026 16:12:20 -0600 Subject: [PATCH 3/4] Add dagster-cloud to orchestration deps The serverless PEX build asserts the deps pex contains dagster_cloud: ValueError: The dagster_cloud package dependency was expected but not found. It was missing from orchestration/pyproject.toml; add it. Co-Authored-By: Claude Opus 4.8 --- orchestration/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/orchestration/pyproject.toml b/orchestration/pyproject.toml index 2856778..50a6288 100644 --- a/orchestration/pyproject.toml +++ b/orchestration/pyproject.toml @@ -9,6 +9,7 @@ description = "DIE Dagster orchestration (internal, not published)" requires-python = ">=3.10" dependencies = [ "dagster>=1.8", + "dagster-cloud", "dagster-gcp>=0.24", "dagster-webserver>=1.8", "google-cloud-storage", From e988b5d940800e9593c7198b0e4805385b7bcd60 Mon Sep 17 00:00:00 2001 From: jross Date: Wed, 24 Jun 2026 16:21:43 -0600 Subject: [PATCH 4/4] Fix Dagster+ code location load: build from repo root Deploy synced but the location failed to import: ModuleNotFoundError: No module named 'orchestration' The source PEX copies build.directory contents into an importable working_directory/root. With directory: orchestration, that root held orchestration's *contents* flat, so there was no `orchestration` package and `orchestration.definitions` (plus its `from backend...` imports) couldn't resolve. Build from the repo root instead: working_directory/root now contains orchestration/ and backend/ as top-level packages. Runtime deps move to a root requirements.txt (dagster stack), since the root pyproject is the nmuwd package and doesn't carry them. Co-Authored-By: Claude Opus 4.8 --- dagster_cloud.yaml | 9 +++++---- requirements.txt | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 requirements.txt diff --git a/dagster_cloud.yaml b/dagster_cloud.yaml index 7896743..14165e0 100644 --- a/dagster_cloud.yaml +++ b/dagster_cloud.yaml @@ -3,7 +3,8 @@ locations: code_source: module_name: orchestration.definitions build: - # PEX fast-deploy build root. orchestration/pyproject.toml declares the - # dagster deps and the `nmuwd` path-dependency (tool.uv.sources); the - # builder recurses into the repo root to collect backend/ + its deps. - directory: orchestration + # Build from the repo root so the source PEX's working_directory holds + # both orchestration/ and backend/ as importable top-level packages + # (module_name orchestration.definitions imports from both). Runtime + # deps come from requirements.txt here at the root. + directory: . diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7dff568 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +# Runtime dependencies for the Dagster+ serverless code location. +# dagster_cloud.yaml builds from the repo root, so the PEX dependency +# resolver reads this file (plus the root pyproject.toml, which supplies the +# backend/nmuwd deps). The dagster stack lives here because it is not part of +# the published nmuwd package. +# +# NOTE: must NOT contain editable/path entries (e.g. `-e .`) — the PEX deps +# builder cannot resolve those. The local packages (backend/, orchestration/) +# are bundled into the source PEX automatically. +dagster>=1.8 +dagster-cloud +dagster-gcp>=0.24 +google-cloud-storage +google-cloud-secret-manager