diff --git a/.github/workflows/dagster-cloud-branch-deployments.yml b/.github/workflows/dagster-cloud-branch-deployments.yml deleted file mode 100644 index 005a063..0000000 --- a/.github/workflows/dagster-cloud-branch-deployments.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Dagster+ Serverless Branch Deployments - -on: - pull_request: - types: [opened, synchronize, reopened, closed] - -concurrency: - group: ${{ github.ref }}/branch_deployments - cancel-in-progress: true - -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.10' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-22.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - steps: - - name: Check Dagster+ credentials present - id: guard - run: | - if [ -n "${{ secrets.DAGSTER_CLOUD_URL }}" ] && [ -n "${{ secrets.DAGSTER_CLOUD_API_TOKEN }}" ]; then - echo "enabled=true" >> "$GITHUB_OUTPUT" - else - echo "::notice::Dagster+ secrets (DAGSTER_CLOUD_URL / DAGSTER_CLOUD_API_TOKEN) not set; skipping deploy." - echo "enabled=false" >> "$GITHUB_OUTPUT" - fi - - - name: Prerun Checks - id: prerun - if: steps.guard.outputs.enabled == 'true' - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-22.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster+ serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/dagster-cloud-deploy.yml b/.github/workflows/dagster-cloud-deploy.yml deleted file mode 100644 index 6a9fd6f..0000000 --- a/.github/workflows/dagster-cloud-deploy.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Dagster+ Serverless Deploy - -on: - push: - branches: - - "main" - -concurrency: - group: ${{ github.ref }}/deploy - cancel-in-progress: true - -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - # PEX fast deploy. No system GDAL needed (only shapely, which ships a wheel - # with bundled GEOS), so a Docker build would be pure overhead. - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.10' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-22.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - steps: - - name: Check Dagster+ credentials present - id: guard - run: | - if [ -n "${{ secrets.DAGSTER_CLOUD_URL }}" ] && [ -n "${{ secrets.DAGSTER_CLOUD_API_TOKEN }}" ]; then - echo "enabled=true" >> "$GITHUB_OUTPUT" - else - echo "::notice::Dagster+ secrets (DAGSTER_CLOUD_URL / DAGSTER_CLOUD_API_TOKEN) not set; skipping deploy." - echo "enabled=false" >> "$GITHUB_OUTPUT" - fi - - - name: Prerun Checks - id: prerun - if: steps.guard.outputs.enabled == 'true' - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-22.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster+ serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/dagster_cloud.yaml b/dagster_cloud.yaml index 7896743..14165e0 100644 --- a/dagster_cloud.yaml +++ b/dagster_cloud.yaml @@ -3,7 +3,8 @@ locations: code_source: module_name: orchestration.definitions build: - # PEX fast-deploy build root. orchestration/pyproject.toml declares the - # dagster deps and the `nmuwd` path-dependency (tool.uv.sources); the - # builder recurses into the repo root to collect backend/ + its deps. - directory: orchestration + # Build from the repo root so the source PEX's working_directory holds + # both orchestration/ and backend/ as importable top-level packages + # (module_name orchestration.definitions imports from both). Runtime + # deps come from requirements.txt here at the root. + directory: . diff --git a/orchestration/pyproject.toml b/orchestration/pyproject.toml index ca41e35..50a6288 100644 --- a/orchestration/pyproject.toml +++ b/orchestration/pyproject.toml @@ -9,6 +9,7 @@ description = "DIE Dagster orchestration (internal, not published)" requires-python = ">=3.10" dependencies = [ "dagster>=1.8", + "dagster-cloud", "dagster-gcp>=0.24", "dagster-webserver>=1.8", "google-cloud-storage", @@ -17,8 +18,11 @@ dependencies = [ "nmuwd", ] +# Trailing slash is required: the dagster-cloud PEX builder only treats a +# uv.sources path as a local package when it starts with "./", "../", or "/". +# Bare ".." is parsed as a (broken) dependency string instead. [tool.uv.sources] -nmuwd = { path = "..", editable = true } +nmuwd = { path = "../", editable = true } [tool.hatch.build.targets.wheel] packages = ["orchestration"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7dff568 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +# Runtime dependencies for the Dagster+ serverless code location. +# dagster_cloud.yaml builds from the repo root, so the PEX dependency +# resolver reads this file (plus the root pyproject.toml, which supplies the +# backend/nmuwd deps). The dagster stack lives here because it is not part of +# the published nmuwd package. +# +# NOTE: must NOT contain editable/path entries (e.g. `-e .`) — the PEX deps +# builder cannot resolve those. The local packages (backend/, orchestration/) +# are bundled into the source PEX automatically. +dagster>=1.8 +dagster-cloud +dagster-gcp>=0.24 +google-cloud-storage +google-cloud-secret-manager