diff --git a/.github/actions/load-ci-tool-versions/action.yml b/.github/actions/load-ci-tool-versions/action.yml new file mode 100644 index 000000000..431a85370 --- /dev/null +++ b/.github/actions/load-ci-tool-versions/action.yml @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Load CI Tool Versions +description: Load shared CI tool versions from .github/ci-tool-versions.env + +outputs: + go_version: + description: Go version + value: ${{ steps.load.outputs.go_version }} + default_python_version: + description: Default Python version + value: ${{ steps.load.outputs.default_python_version }} + rust_version: + description: Rust toolchain version + value: ${{ steps.load.outputs.rust_version }} + node_version: + description: Node.js version + value: ${{ steps.load.outputs.node_version }} + uv_version: + description: uv version + value: ${{ steps.load.outputs.uv_version }} + just_version: + description: just version + value: ${{ steps.load.outputs.just_version }} + wasm_pack_version: + description: wasm-pack version + value: ${{ steps.load.outputs.wasm_pack_version }} + cargo_llvm_cov_version: + description: cargo-llvm-cov version + value: ${{ steps.load.outputs.cargo_llvm_cov_version }} + cargo_nextest_version: + description: cargo-nextest version + value: ${{ steps.load.outputs.cargo_nextest_version }} + cargo_deny_version: + description: cargo-deny version + value: ${{ steps.load.outputs.cargo_deny_version }} + cargo_about_version: + description: cargo-about version + value: ${{ steps.load.outputs.cargo_about_version }} + +runs: + using: composite + steps: + - name: Load versions + id: load + shell: bash + run: | + set -euo pipefail + source "${GITHUB_ACTION_PATH}/../../ci-tool-versions.env" + + required=( + NEMO_FLOW_CI_GO_VERSION + NEMO_FLOW_CI_DEFAULT_PYTHON_VER + NEMO_FLOW_CI_RUST_VERSION + NEMO_FLOW_CI_NODE_VERSION + NEMO_FLOW_CI_UV_VERSION + NEMO_FLOW_CI_JUST_VERSION + NEMO_FLOW_CI_WASM_PACK_VERSION + NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION + NEMO_FLOW_CI_CARGO_NEXTEST_VERSION + NEMO_FLOW_CI_CARGO_DENY_VERSION + NEMO_FLOW_CI_CARGO_ABOUT_VERSION + ) + + for name in "${required[@]}"; do + if [[ -z "${!name:-}" ]]; then + echo "Error: ${name} is missing from .github/ci-tool-versions.env" >&2 + exit 1 + fi + done + + { + printf 'go_version=%s\n' "$NEMO_FLOW_CI_GO_VERSION" + printf 'default_python_version=%s\n' "$NEMO_FLOW_CI_DEFAULT_PYTHON_VER" + printf 'rust_version=%s\n' "$NEMO_FLOW_CI_RUST_VERSION" + printf 'node_version=%s\n' "$NEMO_FLOW_CI_NODE_VERSION" + printf 'uv_version=%s\n' "$NEMO_FLOW_CI_UV_VERSION" + printf 'just_version=%s\n' "$NEMO_FLOW_CI_JUST_VERSION" + printf 'wasm_pack_version=%s\n' "$NEMO_FLOW_CI_WASM_PACK_VERSION" + printf 'cargo_llvm_cov_version=%s\n' "$NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION" + printf 'cargo_nextest_version=%s\n' "$NEMO_FLOW_CI_CARGO_NEXTEST_VERSION" + printf 'cargo_deny_version=%s\n' "$NEMO_FLOW_CI_CARGO_DENY_VERSION" + printf 'cargo_about_version=%s\n' "$NEMO_FLOW_CI_CARGO_ABOUT_VERSION" + } >> "$GITHUB_OUTPUT" diff --git a/.github/ci-tool-versions.env b/.github/ci-tool-versions.env new file mode 100644 index 000000000..da0cba284 --- /dev/null +++ b/.github/ci-tool-versions.env @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +NEMO_FLOW_CI_GO_VERSION=1.26.1 +NEMO_FLOW_CI_DEFAULT_PYTHON_VER=3.11 +NEMO_FLOW_CI_RUST_VERSION=1.93.0 +NEMO_FLOW_CI_NODE_VERSION=24 +NEMO_FLOW_CI_UV_VERSION=0.9.28 +NEMO_FLOW_CI_JUST_VERSION=1.47.1 +NEMO_FLOW_CI_WASM_PACK_VERSION=0.14.0 +NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION=0.8.5 +NEMO_FLOW_CI_CARGO_NEXTEST_VERSION=0.9.133 +NEMO_FLOW_CI_CARGO_DENY_VERSION=0.19.1 +NEMO_FLOW_CI_CARGO_ABOUT_VERSION=0.8.4 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b8b90b265..9fca4fb32 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,19 +17,12 @@ concurrency: group: '${{ github.workflow }} @ ${{ github.event_name }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true -env: - NEMO_FLOW_CI_DEFAULT_PYTHON_VER: "3.11" - NEMO_FLOW_CI_RUST_VERSION: "1.93.0" - NEMO_FLOW_CI_NODE_VERSION: "24" - NEMO_FLOW_CI_UV_VERSION: "0.9.28" - NEMO_FLOW_CI_JUST_VERSION: "1.47.1" - jobs: pr-builder: needs: - prepare - - ci_pipe + - ci_required if: ${{ needs.prepare.outputs.publish_docs != 'true' }} permissions: contents: read @@ -90,6 +83,7 @@ jobs: printf 'full_ci=%s\n' "$full_ci" >> "$GITHUB_OUTPUT" printf 'publish_docs=%s\n' "$publish_docs" >> "$GITHUB_OUTPUT" printf 'publish_packages=%s\n' "$publish_packages" >> "$GITHUB_OUTPUT" + outputs: full_ci: ${{ steps.policy.outputs.full_ci }} is_pr: ${{ startsWith(github.ref_name, 'pull-request/') }} @@ -100,29 +94,190 @@ jobs: pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }} - ci_pipe: - name: CI Pipeline + ci_changes: + name: Changes needs: [prepare] - uses: ./.github/workflows/ci_pipe.yml + uses: ./.github/workflows/ci_changes.yml if: ${{ ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }} permissions: contents: read - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} with: # Info about the PR. Empty for non PR branches. Useful for extracting PR number, title, etc. pr_info: ${{ needs.prepare.outputs.pr_info }} full_ci: ${{ needs.prepare.outputs.full_ci == 'true' }} + ref_name: ${{ github.ref_name }} + default_branch: ${{ github.event.repository.default_branch }} + + ci_check: + name: Check + needs: [prepare, ci_changes] + uses: ./.github/workflows/ci_check.yml + if: ${{ needs.ci_changes.result == 'success' }} + permissions: + contents: read + with: + full_ci: ${{ needs.prepare.outputs.full_ci == 'true' }} + base: ${{ needs.ci_changes.outputs.base }} + + ci_docs: + name: Documentation + needs: [prepare, ci_changes, ci_check] + uses: ./.github/workflows/ci_docs.yml + if: >- + ${{ + !cancelled() + && needs.ci_changes.result == 'success' + && needs.ci_check.result == 'success' + && needs.ci_changes.outputs.run_docs == 'true' + }} + permissions: + contents: read + with: ref_type: ${{ github.ref_type }} ref_name: ${{ github.ref_name }} publish_docs: ${{ needs.prepare.outputs.publish_docs == 'true' }} + ci_rust: + name: Rust + needs: [prepare, ci_changes, ci_check] + uses: ./.github/workflows/ci_rust.yml + if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_rust == 'true' }} + permissions: + contents: read + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + ci_go: + name: Go + needs: [prepare, ci_changes, ci_check] + uses: ./.github/workflows/ci_go.yml + if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_go == 'true' }} + permissions: + contents: read + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + ci_node: + name: Node.js + needs: [prepare, ci_changes, ci_check] + uses: ./.github/workflows/ci_node.yml + if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_node == 'true' }} + permissions: + contents: read + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + ref_type: ${{ github.ref_type }} + ref_name: ${{ github.ref_name }} + + ci_python: + name: Python + needs: [prepare, ci_changes, ci_check] + uses: ./.github/workflows/ci_python.yml + if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_python == 'true' }} + permissions: + contents: read + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + ref_type: ${{ github.ref_type }} + ref_name: ${{ github.ref_name }} + + ci_wasm: + name: WebAssembly + needs: [prepare, ci_changes, ci_check] + uses: ./.github/workflows/ci_wasm.yml + if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_wasm == 'true' }} + permissions: + contents: read + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + ref_type: ${{ github.ref_type }} + ref_name: ${{ github.ref_name }} + + ci_required: + name: CI Pipeline + needs: + - prepare + - ci_changes + - ci_check + - ci_docs + - ci_rust + - ci_go + - ci_node + - ci_python + - ci_wasm + if: ${{ always() && !cancelled() && needs.prepare.result == 'success' && ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }} + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Verify required CI jobs + env: + CHANGES_RESULT: ${{ needs.ci_changes.result }} + CHECK_RESULT: ${{ needs.ci_check.result }} + DOCS_RESULT: ${{ needs.ci_docs.result }} + RUST_RESULT: ${{ needs.ci_rust.result }} + GO_RESULT: ${{ needs.ci_go.result }} + NODE_RESULT: ${{ needs.ci_node.result }} + PYTHON_RESULT: ${{ needs.ci_python.result }} + WEBASSEMBLY_RESULT: ${{ needs.ci_wasm.result }} + publish_docs: ${{ needs.prepare.outputs.publish_docs }} + publish_packages: ${{ needs.prepare.outputs.publish_packages }} + run: | + set -euo pipefail + failed=false + + require_success() { + local name="$1" + local result="$2" + if [[ "$result" != "success" ]]; then + echo "Error: ${name} finished with result '${result}', expected success" >&2 + failed=true + fi + } + + allow_success_or_skipped() { + local name="$1" + local result="$2" + if [[ "$result" != "success" && "$result" != "skipped" ]]; then + echo "Error: ${name} finished with result '${result}', expected success or skipped" >&2 + failed=true + fi + } + + require_success "Changes" "$CHANGES_RESULT" + require_success "Check" "$CHECK_RESULT" + + if [[ "$publish_docs" == "true" ]]; then + require_success "Documentation" "$DOCS_RESULT" + else + allow_success_or_skipped "Documentation" "$DOCS_RESULT" + fi + + if [[ "$publish_packages" == "true" ]]; then + require_success "Rust" "$RUST_RESULT" + require_success "Node.js" "$NODE_RESULT" + require_success "Python" "$PYTHON_RESULT" + require_success "WebAssembly" "$WEBASSEMBLY_RESULT" + else + allow_success_or_skipped "Rust" "$RUST_RESULT" + allow_success_or_skipped "Node.js" "$NODE_RESULT" + allow_success_or_skipped "Python" "$PYTHON_RESULT" + allow_success_or_skipped "WebAssembly" "$WEBASSEMBLY_RESULT" + fi + + allow_success_or_skipped "Go" "$GO_RESULT" + + if [[ "$failed" == "true" ]]; then + exit 1 + fi + deploy-docs: name: Deploy Documentation - needs: [prepare, ci_pipe] - if: ${{ needs.prepare.outputs.publish_docs == 'true' && needs.ci_pipe.result == 'success' }} + needs: [prepare, ci_required] + if: ${{ needs.prepare.outputs.publish_docs == 'true' && needs.ci_required.result == 'success' }} runs-on: ubuntu-latest permissions: pages: write @@ -137,8 +292,8 @@ jobs: publish-rust: name: Publish / crates.io - needs: [prepare, ci_pipe] - if: ${{ needs.prepare.outputs.publish_packages == 'true' && needs.ci_pipe.result == 'success' }} + needs: [prepare, ci_required] + if: ${{ needs.prepare.outputs.publish_packages == 'true' && needs.ci_required.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -148,31 +303,33 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ github.workspace }}/NeMo-Flow + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 with: cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} + toolchain: ${{ steps.ci-config.outputs.rust_version }} - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 with: - version: ${{ env.NEMO_FLOW_CI_UV_VERSION }} + version: ${{ steps.ci-config.outputs.uv_version }} enable-cache: true - cache-dependency-glob: ${{ github.workspace }}/NeMo-Flow/uv.lock + cache-dependency-glob: ${{ github.workspace }}/uv.lock - name: Install managed Python run: | set -e - UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} + UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 with: - tool: just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} + tool: just@${{ steps.ci-config.outputs.just_version }} - name: Set project release version - working-directory: ${{ github.workspace }}/NeMo-Flow + working-directory: ${{ github.workspace }} run: just set-version "${{ github.ref_name }}" - name: Authenticate to crates.io with trusted publishing @@ -180,7 +337,7 @@ jobs: uses: rust-lang/crates-io-auth-action@bbd81622f20ce9e2dd9622e3218b975523e45bbe # v1.0.4 - name: Publish to crates.io with trusted publishing - working-directory: ${{ github.workspace }}/NeMo-Flow + working-directory: ${{ github.workspace }} env: CARGO_REGISTRY_TOKEN: ${{ steps.crates-io-auth.outputs.token }} run: | @@ -191,8 +348,8 @@ jobs: publish-python: name: Publish / PyPI - needs: [prepare, ci_pipe] - if: ${{ needs.prepare.outputs.publish_packages == 'true' && needs.ci_pipe.result == 'success' }} + needs: [prepare, ci_required] + if: ${{ needs.prepare.outputs.publish_packages == 'true' && needs.ci_required.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -212,8 +369,8 @@ jobs: publish-npm: name: Publish / npm - needs: [prepare, ci_pipe] - if: ${{ needs.prepare.outputs.publish_packages == 'true' && needs.ci_pipe.result == 'success' }} + needs: [prepare, ci_required] + if: ${{ needs.prepare.outputs.publish_packages == 'true' && needs.ci_required.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -221,9 +378,16 @@ jobs: id-token: write environment: npm steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 with: - node-version: ${{ env.NEMO_FLOW_CI_NODE_VERSION }} + node-version: ${{ steps.ci-config.outputs.node_version }} registry-url: "https://registry.npmjs.org" - name: Download consolidated Node package artifact @@ -232,7 +396,7 @@ jobs: name: npm-consolidated path: . - - name: Download WASM artifact + - name: Download WebAssembly artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: wasm-bundler @@ -255,7 +419,7 @@ jobs: ls -la combined/package/*.node npm publish ./combined/package --access public --tag "${NEMO_FLOW_NPM_DIST_TAG}" - - name: Publish WASM package to npm + - name: Publish WebAssembly package to npm run: | set -e for pkg in ./wasm-package/*.tgz; do diff --git a/.github/workflows/ci_changes.yml b/.github/workflows/ci_changes.yml new file mode 100644 index 000000000..ebf428394 --- /dev/null +++ b/.github/workflows/ci_changes.yml @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Changes + +on: + workflow_call: + inputs: + pr_info: + description: 'The JSON string containing the PR information' + required: true + type: string + full_ci: + description: 'Whether to run the full CI matrix without path filtering' + required: false + default: false + type: boolean + ref_name: + description: 'The ref name from the triggering workflow' + required: true + type: string + default_branch: + description: 'The repository default branch used as the fallback comparison base' + required: true + type: string + outputs: + base: + description: 'The comparison base used for filtered checks' + value: ${{ jobs.changes.outputs.base }} + run_docs: + description: 'Whether documentation jobs should run' + value: ${{ jobs.changes.outputs.run_docs }} + run_go: + description: 'Whether Go jobs should run' + value: ${{ jobs.changes.outputs.run_go }} + run_node: + description: 'Whether Node.js jobs should run' + value: ${{ jobs.changes.outputs.run_node }} + run_python: + description: 'Whether Python jobs should run' + value: ${{ jobs.changes.outputs.run_python }} + run_rust: + description: 'Whether Rust jobs should run' + value: ${{ jobs.changes.outputs.run_rust }} + run_wasm: + description: 'Whether WebAssembly jobs should run' + value: ${{ jobs.changes.outputs.run_wasm }} + +jobs: + changes: + name: Detect + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + base: ${{ steps.comparison-base.outputs.base }} + run_docs: ${{ (inputs.full_ci || startsWith(inputs.ref_name, 'pull-request/')) && (inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.docs == 'true') }} + run_go: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.go == 'true' }} + run_node: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' }} + run_python: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.python == 'true' }} + run_rust: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.rust == 'true' }} + run_wasm: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.wasm == 'true' }} + + steps: + - name: Checkout + if: ${{ ! inputs.full_ci }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: Resolve comparison base + id: comparison-base + if: ${{ ! inputs.full_ci }} + env: + DEFAULT_BRANCH: ${{ inputs.default_branch }} + EVENT_BEFORE: ${{ github.event.before }} + PR_INFO: ${{ inputs.pr_info }} + REF_NAME: ${{ inputs.ref_name }} + run: | + set -euo pipefail + base="" + if [[ "$REF_NAME" == pull-request/* ]]; then + if [[ -n "$PR_INFO" ]]; then + base="$(jq -r '.base.ref // empty' <<< "$PR_INFO")" + fi + if [[ -z "$base" ]]; then + base="origin/${DEFAULT_BRANCH}" + fi + elif [[ -n "$EVENT_BEFORE" && "$EVENT_BEFORE" != "0000000000000000000000000000000000000000" ]]; then + base="$EVENT_BEFORE" + else + base="origin/${DEFAULT_BRANCH}" + fi + printf 'base=%s\n' "$base" >> "$GITHUB_OUTPUT" + + - name: Detect changed paths + id: filter + if: ${{ ! inputs.full_ci }} + uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.01 + with: + base: ${{ steps.comparison-base.outputs.base }} + filters: .github/ci-path-filters.yml diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml new file mode 100644 index 000000000..f1010ae36 --- /dev/null +++ b/.github/workflows/ci_check.yml @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Check + +on: + workflow_call: + inputs: + full_ci: + description: 'Whether to run checks across all files' + required: false + default: false + type: boolean + base: + description: 'The comparison base used for filtered pre-commit checks' + required: false + default: '' + type: string +defaults: + run: + shell: bash + +env: + GH_TOKEN: "${{ github.token }}" + GIT_COMMIT: "${{ github.sha }}" + NEMO_FLOW_CI_WORKSPACE: "${{ github.workspace }}" + NEMO_FLOW_CI_WORKSPACE_TMP: "${{ github.workspace }}/tmp" + UV_PYTHON_DOWNLOADS: never + +jobs: + check: + name: Run + runs-on: linux-amd64-cpu4 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + with: + version: ${{ steps.ci-config.outputs.uv_version }} + enable-cache: true + cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: ${{ steps.ci-config.outputs.go_version }} + cache: false + + - name: Install managed Python + run: | + set -e + UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: ${{ steps.ci-config.outputs.node_version }} + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + components: clippy, rustfmt + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + save-if: false + + - name: Install cargo-deny + uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: cargo-deny@${{ steps.ci-config.outputs.cargo_deny_version }} + + - name: Install cargo-about + uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: cargo-about@${{ steps.ci-config.outputs.cargo_about_version }} + + - name: pre-commit + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + env: + FULL_CI: ${{ inputs.full_ci }} + PRE_COMMIT_BASE: ${{ inputs.base }} + # The attribution hook syncs docs deps; do not apply repo warning policy to third-party Rust builds. + RUSTFLAGS: "" + run: | + set -e + uv tool install pre-commit==4.5 + if [[ "$FULL_CI" == "true" || -z "$PRE_COMMIT_BASE" ]]; then + pre-commit run --all-files --show-diff-on-failure + else + if [[ "$PRE_COMMIT_BASE" =~ ^[0-9a-f]{40}$ || "$PRE_COMMIT_BASE" == refs/* ]]; then + base_ref="$PRE_COMMIT_BASE" + else + base_ref="origin/$PRE_COMMIT_BASE" + fi + if [[ ! "$PRE_COMMIT_BASE" =~ ^[0-9a-f]{40}$ && "$base_ref" != "$GITHUB_REF" ]]; then + base_ref="$(git merge-base "$base_ref" HEAD)" + fi + pre-commit run --from-ref "$base_ref" --to-ref HEAD --show-diff-on-failure + fi diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml new file mode 100644 index 000000000..d637fe762 --- /dev/null +++ b/.github/workflows/ci_docs.yml @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Documentation + +on: + workflow_call: + inputs: + ref_type: + description: 'The ref type from the triggering workflow' + required: true + type: string + ref_name: + description: 'The ref name from the triggering workflow' + required: true + type: string + publish_docs: + description: 'Whether to upload a GitHub Pages artifact for deployment' + required: false + default: false + type: boolean +defaults: + run: + shell: bash + +env: + GH_TOKEN: "${{ github.token }}" + GIT_COMMIT: "${{ github.sha }}" + NEMO_FLOW_CI_WORKSPACE: "${{ github.workspace }}" + NEMO_FLOW_CI_WORKSPACE_TMP: "${{ github.workspace }}/tmp" + UV_PYTHON_DOWNLOADS: never + +jobs: + docs: + name: Build + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + with: + version: ${{ steps.ci-config.outputs.uv_version }} + enable-cache: true + cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: ${{ steps.ci-config.outputs.node_version }} + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + rustflags: "" + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + save-if: false + + - name: Install managed Python + run: | + set -e + UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: just@${{ steps.ci-config.outputs.just_version }} + + - name: Install Documentation Dependencies + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: uv sync --no-default-groups --group docs --no-install-project + + - name: Install Node.js Documentation Dependencies + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/crates/node + run: npm ci --ignore-scripts + + - name: Materialize Main Branch For Versioned Docs + if: ${{ inputs.ref_type == 'tag' }} + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: git fetch --force origin +refs/heads/main:refs/heads/main + + - name: Check Documentation Links + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + env: + NEMO_FLOW_DOCS_DEPS_READY: "1" + run: just docs-linkcheck + + - name: Build Documentation Site + if: ${{ startsWith(inputs.ref_name, 'pull-request/') || inputs.ref_name == 'main' }} + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + env: + NEMO_FLOW_DOCS_DEPS_READY: "1" + run: just docs + + - name: Build Versioned Documentation Site + if: ${{ inputs.publish_docs }} + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + env: + NEMO_FLOW_DOCS_DEPS_READY: "1" + run: just docs-github-pages + + - name: Upload GitHub Pages Artifact + if: ${{ inputs.publish_docs }} + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + with: + path: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/docs/_build/pages diff --git a/.github/workflows/ci_go.yml b/.github/workflows/ci_go.yml new file mode 100644 index 000000000..457af9746 --- /dev/null +++ b/.github/workflows/ci_go.yml @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Go + +on: + workflow_call: + secrets: + CODECOV_TOKEN: + required: false + +defaults: + run: + shell: bash + +env: + GH_TOKEN: "${{ github.token }}" + GIT_COMMIT: "${{ github.sha }}" + NEMO_FLOW_CI_WORKSPACE: "${{ github.workspace }}" + NEMO_FLOW_CI_WORKSPACE_TMP: "${{ github.workspace }}/tmp" + UV_PYTHON_DOWNLOADS: never + +jobs: + test-go: + name: Test / ${{ matrix.platform }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: linux-amd64-cpu4 + - platform: linux-arm64 + runner: linux-arm64-cpu4 + - platform: macos-arm64 + runner: macos-15 + - platform: windows-amd64 + runner: windows-2022 + - platform: windows-arm64 + runner: windows-11-arm + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: cargo-llvm-cov@${{ steps.ci-config.outputs.cargo_llvm_cov_version }},just@${{ steps.ci-config.outputs.just_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + save-if: false + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: ${{ steps.ci-config.outputs.go_version }} + cache: false + + - name: Run Go tests with coverage + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: | + set -e + just --set ci true --set output_dir "${{ github.workspace }}" test-go + + - name: Upload Go coverage to Codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 + with: + token: ${{ secrets.CODECOV_TOKEN }} + disable_search: true + files: go-coverage.xml + flags: go-${{ matrix.platform }} + name: go-${{ matrix.platform }} + verbose: true diff --git a/.github/workflows/ci_node.yml b/.github/workflows/ci_node.yml new file mode 100644 index 000000000..d838316b4 --- /dev/null +++ b/.github/workflows/ci_node.yml @@ -0,0 +1,274 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Node.js + +on: + workflow_call: + inputs: + ref_type: + description: 'The ref type from the triggering workflow' + required: true + type: string + ref_name: + description: 'The ref name from the triggering workflow' + required: true + type: string + secrets: + CODECOV_TOKEN: + required: false + +defaults: + run: + shell: bash + +env: + GH_TOKEN: "${{ github.token }}" + GIT_COMMIT: "${{ github.sha }}" + NEMO_FLOW_CI_WORKSPACE: "${{ github.workspace }}" + NEMO_FLOW_CI_WORKSPACE_TMP: "${{ github.workspace }}/tmp" + UV_PYTHON_DOWNLOADS: never + +jobs: + test-node: + name: Test / ${{ matrix.platform }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: linux-amd64-cpu4 + - platform: linux-arm64 + runner: linux-arm64-cpu4 + - platform: macos-arm64 + runner: macos-15 + - platform: windows-amd64 + runner: windows-2022 + - platform: windows-arm64 + runner: windows-11-arm + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + save-if: false + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: cargo-llvm-cov@${{ steps.ci-config.outputs.cargo_llvm_cov_version }},just@${{ steps.ci-config.outputs.just_version }} + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: ${{ steps.ci-config.outputs.node_version }} + + - name: Run Node tests with coverage + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: just --set ci true --set output_dir "${{ github.workspace }}" test-node + + - name: Upload Node coverage to Codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 + if: ${{ !startsWith(matrix.platform, 'windows') }} + with: + token: ${{ secrets.CODECOV_TOKEN }} + disable_search: true + files: node-coverage.xml,node-rust.xml + flags: node-${{ matrix.platform }} + name: node-${{ matrix.platform }} + verbose: true + + package-node: + name: Package / ${{ matrix.platform }} + needs: [test-node] + if: ${{ !cancelled() && needs.test-node.result == 'success' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: linux-amd64-cpu4 + - platform: linux-arm64 + runner: linux-arm64-cpu4 + - platform: macos-arm64 + runner: macos-15 + - platform: windows-amd64 + runner: windows-2022 + - platform: windows-arm64 + runner: windows-11-arm + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + save-if: false + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + if: ${{ startsWith(matrix.platform, 'linux-') }} + with: + version: ${{ steps.ci-config.outputs.uv_version }} + enable-cache: true + cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock + + - name: Install managed Python for Zig packaging + if: ${{ startsWith(matrix.platform, 'linux-') }} + run: | + set -e + UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: ${{ steps.ci-config.outputs.node_version }} + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: just@${{ steps.ci-config.outputs.just_version }} + + - name: Create packaging output directory + run: | + set -e + mkdir -p "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/npm" + + - name: Derive Node package version + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: | + set -e + version="$(node -e 'const fs = require("fs"); const pkg = JSON.parse(fs.readFileSync("crates/node/package.json", "utf8")); if (!pkg.version) { throw new Error("crates/node/package.json missing version field"); } console.log(pkg.version);')" + sha="${GITHUB_SHA::8}" + if [ "${{ inputs.ref_type }}" = "tag" ]; then + version="${{ inputs.ref_name }}" + else + version="${version}-${sha}" + fi + printf 'NEMO_FLOW_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" + + - name: Package Node + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: | + set -e + just \ + --set ci true \ + --set output_dir "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}" \ + --set ref_name "${NEMO_FLOW_PACKAGE_VERSION}" \ + package-node + + - name: Upload npm package artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: npm-${{ matrix.platform }} + path: ${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/npm/*.tgz + if-no-files-found: error + + consolidate-node: + name: Consolidate + needs: [package-node] + if: ${{ !cancelled() && needs.package-node.result == 'success' }} + runs-on: linux-amd64-cpu4 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Download npm linux-amd64 artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-linux-amd64 + path: npm-linux-amd64/ + + - name: Download npm linux-arm64 artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-linux-arm64 + path: npm-linux-arm64/ + + - name: Download npm macos-arm64 artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-macos-arm64 + path: npm-macos-arm64/ + + - name: Download npm windows-amd64 artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-windows-amd64 + path: npm-windows-amd64/ + + - name: Download npm windows-arm64 artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-windows-arm64 + path: npm-windows-arm64/ + + - name: Consolidate Node package directory + run: | + set -euo pipefail + mkdir -p combined + first=true + for dir in npm-linux-amd64 npm-linux-arm64 npm-macos-arm64 npm-windows-amd64 npm-windows-arm64; do + shopt -s nullglob + tgzs=("${dir}"/*.tgz) + shopt -u nullglob + if [ "${#tgzs[@]}" -eq 0 ]; then + echo "Error: no npm package artifact found in ${dir}" >&2 + exit 1 + fi + for tgz in "${tgzs[@]}"; do + if [ "${first}" = true ]; then + tar xzf "${tgz}" -C combined + first=false + else + tar xzf "${tgz}" -C combined --wildcards '*.node' + fi + done + done + echo "Platform binaries included:" + ls -la combined/package/*.node + + - name: Archive consolidated Node package + run: | + set -euo pipefail + cd combined + zip -rq "${{ github.workspace }}/consolidated.zip" package + + - name: Upload consolidated Node package artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: npm-consolidated + path: ${{ github.workspace }}/consolidated.zip + if-no-files-found: error diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml deleted file mode 100644 index 9506a51a8..000000000 --- a/.github/workflows/ci_pipe.yml +++ /dev/null @@ -1,1141 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: CI Pipeline -run-name: CI Pipeline - -on: - workflow_call: - inputs: - pr_info: - description: 'The JSON string containing the PR information' - required: true - type: string - full_ci: - description: 'Whether to run the full CI matrix without path filtering' - required: false - default: false - type: boolean - ref_type: - description: 'The ref type from the triggering workflow' - required: true - type: string - ref_name: - description: 'The ref name from the triggering workflow' - required: true - type: string - publish_docs: - description: 'Whether to upload a GitHub Pages artifact for deployment' - required: false - default: false - type: boolean - secrets: - CODECOV_TOKEN: - required: false - SONAR_HOST_URL: - required: false - SONAR_TOKEN: - required: false - -defaults: - run: - shell: bash - -env: - GH_TOKEN: "${{ github.token }}" - GIT_COMMIT: "${{ github.sha }}" - NEMO_FLOW_CI_GO_VERSION: "1.26.1" - NEMO_FLOW_CI_DEFAULT_PYTHON_VER: "3.11" - NEMO_FLOW_CI_RUST_VERSION: "1.93.0" - NEMO_FLOW_CI_NODE_VERSION: "24" - NEMO_FLOW_CI_UV_VERSION: "0.9.28" - NEMO_FLOW_CI_JUST_VERSION: "1.47.1" - NEMO_FLOW_CI_WASM_PACK_VERSION: "0.14.0" - NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION: "0.8.5" - NEMO_FLOW_CI_CARGO_NEXTEST_VERSION: "0.9.133" - NEMO_FLOW_CI_CARGO_DENY_VERSION: "0.19.1" - NEMO_FLOW_CI_CARGO_ABOUT_VERSION: "0.8.4" - NEMO_FLOW_CI_SONAR_SCANNER_VERSION: "7.0.2.4839" - NEMO_FLOW_CI_WORKSPACE: "${{ github.workspace }}/NeMo-Flow" - NEMO_FLOW_CI_WORKSPACE_TMP: "${{ github.workspace }}/tmp" - UV_PYTHON_DOWNLOADS: never - -jobs: - changes: - name: Detect Changes - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - outputs: - base: ${{ steps.comparison-base.outputs.base }} - run_docs: ${{ (inputs.full_ci || startsWith(inputs.ref_name, 'pull-request/')) && (inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.docs == 'true') }} - run_go: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.go == 'true' }} - run_node: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' }} - run_python: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.python == 'true' }} - run_rust: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.rust == 'true' }} - run_wasm: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.wasm == 'true' }} - - steps: - - name: Checkout - if: ${{ ! inputs.full_ci }} - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - fetch-depth: 0 - path: NeMo-Flow - - - name: Resolve comparison base - id: comparison-base - if: ${{ ! inputs.full_ci }} - env: - EVENT_BEFORE: ${{ github.event.before }} - PR_INFO: ${{ inputs.pr_info }} - REF_NAME: ${{ inputs.ref_name }} - run: | - set -euo pipefail - base="" - if [[ "$REF_NAME" == pull-request/* ]]; then - base="$(jq -r '.base.ref // empty' <<< "$PR_INFO")" - if [[ -z "$base" ]]; then - echo "Error: failed to resolve PR base branch from pr_info" >&2 - exit 1 - fi - elif [[ -n "$EVENT_BEFORE" && "$EVENT_BEFORE" != "0000000000000000000000000000000000000000" ]]; then - base="$EVENT_BEFORE" - else - base="${GITHUB_REF}" - fi - printf 'base=%s\n' "$base" >> "$GITHUB_OUTPUT" - - - name: Detect changed paths - id: filter - if: ${{ ! inputs.full_ci }} - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.01 - with: - base: ${{ steps.comparison-base.outputs.base }} - filters: .github/ci-path-filters.yml - working-directory: NeMo-Flow - - check: - name: Check - needs: [changes] - runs-on: linux-amd64-cpu4 - timeout-minutes: 30 - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - fetch-depth: 0 - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 - with: - version: ${{ env.NEMO_FLOW_CI_UV_VERSION }} - enable-cache: true - cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock - - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 - with: - go-version: ${{ env.NEMO_FLOW_CI_GO_VERSION }} - cache: false - - - name: Install managed Python - run: | - set -e - UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 - with: - node-version: ${{ env.NEMO_FLOW_CI_NODE_VERSION }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - components: clippy, rustfmt - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - save-if: false - - - name: Install cargo-deny - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: cargo-deny@${{ env.NEMO_FLOW_CI_CARGO_DENY_VERSION }} - - - name: Install cargo-about - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: cargo-about@${{ env.NEMO_FLOW_CI_CARGO_ABOUT_VERSION }} - - - name: pre-commit - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - env: - FULL_CI: ${{ inputs.full_ci }} - PRE_COMMIT_BASE: ${{ needs.changes.outputs.base }} - # The attribution hook syncs docs deps; do not apply repo warning policy to third-party Rust builds. - RUSTFLAGS: "" - run: | - set -e - uv tool install pre-commit==4.5 - if [[ "$FULL_CI" == "true" || -z "$PRE_COMMIT_BASE" ]]; then - pre-commit run --all-files --show-diff-on-failure - else - if [[ "$PRE_COMMIT_BASE" =~ ^[0-9a-f]{40}$ || "$PRE_COMMIT_BASE" == refs/* ]]; then - base_ref="$PRE_COMMIT_BASE" - else - base_ref="origin/$PRE_COMMIT_BASE" - fi - if [[ ! "$PRE_COMMIT_BASE" =~ ^[0-9a-f]{40}$ && "$base_ref" != "$GITHUB_REF" ]]; then - base_ref="$(git merge-base "$base_ref" HEAD)" - fi - pre-commit run --from-ref "$base_ref" --to-ref HEAD --show-diff-on-failure - fi - - docs: - name: Documentation - needs: [changes, check] - if: >- - ${{ - !cancelled() - && needs.changes.result == 'success' - && needs.check.result == 'success' - && needs.changes.outputs.run_docs == 'true' - }} - runs-on: linux-amd64-cpu4 - timeout-minutes: 60 - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - fetch-depth: 0 - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 - with: - version: ${{ env.NEMO_FLOW_CI_UV_VERSION }} - enable-cache: true - cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock - - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 - with: - node-version: ${{ env.NEMO_FLOW_CI_NODE_VERSION }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - rustflags: "" - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - save-if: false - - - name: Install managed Python - run: | - set -e - UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} - - - name: Install Documentation Dependencies - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: uv sync --no-default-groups --group docs --no-install-project - - - name: Install Node.js Documentation Dependencies - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/crates/node - run: npm ci --ignore-scripts - - - name: Materialize Main Branch For Versioned Docs - if: ${{ inputs.ref_type == 'tag' }} - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: git fetch --force origin +refs/heads/main:refs/heads/main - - - name: Check Documentation Links - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - env: - NEMO_FLOW_DOCS_DEPS_READY: "1" - run: just docs-linkcheck - - - name: Build Documentation Site - if: ${{ startsWith(inputs.ref_name, 'pull-request/') || inputs.ref_name == 'main' }} - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - env: - NEMO_FLOW_DOCS_DEPS_READY: "1" - run: just docs - - - name: Build Versioned Documentation Site - if: ${{ inputs.publish_docs }} - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - env: - NEMO_FLOW_DOCS_DEPS_READY: "1" - run: just docs-github-pages - - - name: Upload GitHub Pages Artifact - if: ${{ inputs.publish_docs }} - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/docs/_build/pages - - test-rust: - name: Test / Rust / ${{ matrix.platform }} - needs: [changes, check] - if: ${{ needs.check.result == 'success' && needs.changes.outputs.run_rust == 'true' }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} - permissions: - contents: read - services: - redis: - image: ${{ matrix.redis_service_image }} - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping || exit 1" - --health-interval 5s - --health-timeout 3s - --health-retries 10 - strategy: - fail-fast: false - matrix: - include: - - platform: linux-amd64 - runner: linux-amd64-cpu4 - run_redis_tests: true - redis_service_image: redis:7.4.7-alpine - - platform: linux-arm64 - runner: linux-arm64-cpu4 - run_redis_tests: false - redis_service_image: '' - - platform: macos-arm64 - runner: macos-15 - run_redis_tests: false - redis_service_image: '' - - platform: windows-amd64 - runner: windows-2022 - run_redis_tests: false - redis_service_image: '' - - platform: windows-arm64 - runner: windows-11-arm - run_redis_tests: false - redis_service_image: '' - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - - - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 - with: - version: ${{ env.NEMO_FLOW_CI_UV_VERSION }} - enable-cache: true - cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock - - - name: Set up Windows ARM Python - if: ${{ matrix.platform == 'windows-arm64' }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 - with: - python-version: ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - architecture: arm64 - - - name: Select setup-python interpreter for uv - if: ${{ matrix.platform == 'windows-arm64' }} - run: | - set -e - python - <<'PY' >> "$GITHUB_ENV" - import sys - print(f"UV_PYTHON={sys.executable}") - PY - - - name: Install managed Python - if: ${{ matrix.platform != 'windows-arm64' }} - run: | - set -e - UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - - - name: Add Python DLL directory to PATH on Windows - if: runner.os == 'Windows' - run: | - set -e - python_exe="$(uv python find)" - python_home="$("$python_exe" -c 'import sys; print(sys.base_prefix)')" - echo "$python_home" >> "$GITHUB_PATH" - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: cargo-llvm-cov@${{ env.NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION }},cargo-nextest@${{ env.NEMO_FLOW_CI_CARGO_NEXTEST_VERSION }},just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} - - - name: Run Rust tests with coverage - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: | - set -e - args=( - --set ci true - --set output_dir "${{ github.workspace }}" - ) - if [ "${{ matrix.run_redis_tests }}" = "true" ]; then - export NEMO_FLOW_RUN_REDIS_TESTS=1 - fi - just "${args[@]}" test-rust - - - name: Upload Rust coverage to Codecov - if: ${{ matrix.platform != 'windows-arm64' }} - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 - with: - token: ${{ secrets.CODECOV_TOKEN }} - disable_search: true - files: rust-workspace.xml - flags: rust-${{ matrix.platform }} - name: rust-${{ matrix.platform }} - verbose: true - - test-go: - name: Test / Go / ${{ matrix.platform }} - needs: [changes, check] - if: ${{ needs.check.result == 'success' && needs.changes.outputs.run_go == 'true' }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - platform: linux-amd64 - runner: linux-amd64-cpu4 - - platform: linux-arm64 - runner: linux-arm64-cpu4 - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: cargo-llvm-cov@${{ env.NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION }},just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - save-if: false - - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 - with: - go-version: ${{ env.NEMO_FLOW_CI_GO_VERSION }} - cache: false - - - name: Run Go tests with coverage - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: | - set -e - just --set ci true --set output_dir "${{ github.workspace }}" test-go - - - name: Upload Go coverage to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 - with: - token: ${{ secrets.CODECOV_TOKEN }} - disable_search: true - files: go-coverage.xml - flags: go-${{ matrix.platform }} - name: go-${{ matrix.platform }} - verbose: true - - test-node: - name: Test / Node / ${{ matrix.platform }} - needs: [changes, check] - if: ${{ needs.check.result == 'success' && needs.changes.outputs.run_node == 'true' }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - platform: linux-amd64 - runner: linux-amd64-cpu4 - - platform: linux-arm64 - runner: linux-arm64-cpu4 - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - save-if: false - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: cargo-llvm-cov@${{ env.NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION }},just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} - - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 - with: - node-version: ${{ env.NEMO_FLOW_CI_NODE_VERSION }} - - - name: Run Node tests with coverage - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: just --set ci true --set output_dir "${{ github.workspace }}" test-node - - - name: Upload Node coverage to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 - if: ${{ !startsWith(matrix.platform, 'windows') }} - with: - token: ${{ secrets.CODECOV_TOKEN }} - disable_search: true - files: node-coverage.xml,node-rust.xml - flags: node-${{ matrix.platform }} - name: node-${{ matrix.platform }} - verbose: true - - test-python: - name: Test / Python / ${{ matrix.platform }} - needs: [changes, check] - if: ${{ needs.check.result == 'success' && needs.changes.outputs.run_python == 'true' }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - platform: linux-amd64 - runner: linux-amd64-cpu4 - - platform: linux-arm64 - runner: linux-arm64-cpu4 - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - save-if: false - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: cargo-llvm-cov@${{ env.NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION }},just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} - - - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 - with: - version: ${{ env.NEMO_FLOW_CI_UV_VERSION }} - enable-cache: true - cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock - - - name: Set up Windows ARM Python - if: ${{ matrix.platform == 'windows-arm64' }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 - with: - python-version: ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - architecture: arm64 - - - name: Select setup-python interpreter for uv - if: ${{ matrix.platform == 'windows-arm64' }} - run: | - set -e - python - <<'PY' >> "$GITHUB_ENV" - import sys - print(f"UV_PYTHON={sys.executable}") - PY - - - name: Install managed Python - if: ${{ matrix.platform != 'windows-arm64' }} - run: | - set -e - UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - - - name: Run Python tests with coverage - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: just --set ci true --set output_dir "${{ github.workspace }}" test-python - - - name: Upload Python coverage to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 - with: - token: ${{ secrets.CODECOV_TOKEN }} - disable_search: true - files: ${{ matrix.platform == 'windows-arm64' && 'python-coverage.xml' || 'python-coverage.xml,python-rust.xml' }} - flags: python-${{ matrix.platform }} - name: python-${{ matrix.platform }} - verbose: true - - - name: Prune uv cache - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: uv cache prune --ci - - test-wasm: - name: Test / WASM / ${{ matrix.platform }} - needs: [changes, check] - if: ${{ needs.check.result == 'success' && needs.changes.outputs.run_wasm == 'true' }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - platform: linux-amd64 - runner: linux-amd64-cpu4 - - platform: linux-arm64 - runner: linux-arm64-cpu4 - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - save-if: false - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: cargo-llvm-cov@${{ env.NEMO_FLOW_CI_CARGO_LLVM_COV_VERSION }},just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} - - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 - with: - node-version: ${{ env.NEMO_FLOW_CI_NODE_VERSION }} - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: wasm-pack@${{ env.NEMO_FLOW_CI_WASM_PACK_VERSION }} - - - name: Run WASM tests with coverage - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: just --set ci true --set output_dir "${{ github.workspace }}" test-wasm - - - name: Upload WASM coverage to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 - with: - token: ${{ secrets.CODECOV_TOKEN }} - disable_search: true - files: wasm-js.xml - flags: wasm-${{ matrix.platform }} - name: wasm-${{ matrix.platform }} - verbose: true - - sonarqube: - if: false - name: Analyze / SonarQube - needs: [check] - runs-on: linux-amd64-cpu4 - timeout-minutes: 30 - permissions: - contents: read - - steps: - - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - fetch-depth: 0 - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - components: clippy - - - name: SonarQube Scan - uses: SonarSource/sonarqube-scan-action@299e4b793aaa83bf2aba7c9c14bedbb485688ec4 # v7 - with: - projectBaseDir: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - scannerVersion: ${{ env.NEMO_FLOW_CI_SONAR_SCANNER_VERSION }} - env: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} - - package-node: - name: Package / Node / ${{ matrix.platform }} - needs: [changes, test-node] - if: >- - ${{ - !cancelled() - && needs.changes.result == 'success' - && needs.test-node.result == 'success' - && needs.changes.outputs.run_node == 'true' - }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - platform: linux-amd64 - runner: linux-amd64-cpu4 - - platform: linux-arm64 - runner: linux-arm64-cpu4 - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - save-if: false - - - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 - if: ${{ startsWith(matrix.platform, 'linux-') }} - with: - version: ${{ env.NEMO_FLOW_CI_UV_VERSION }} - enable-cache: true - cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock - - - name: Install managed Python for Zig packaging - if: ${{ startsWith(matrix.platform, 'linux-') }} - run: | - set -e - UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 - with: - node-version: ${{ env.NEMO_FLOW_CI_NODE_VERSION }} - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} - - - name: Create packaging output directory - run: | - set -e - mkdir -p "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/npm" - - - name: Derive Node package version - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: | - set -e - version="$(node -e 'const fs = require("fs"); const pkg = JSON.parse(fs.readFileSync("crates/node/package.json", "utf8")); if (!pkg.version) { throw new Error("crates/node/package.json missing version field"); } console.log(pkg.version);')" - sha="${GITHUB_SHA::8}" - if [ "${{ inputs.ref_type }}" = "tag" ]; then - version="${{ inputs.ref_name }}" - else - version="${version}-${sha}" - fi - printf 'NEMO_FLOW_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" - - - name: Package Node - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: | - set -e - just \ - --set ci true \ - --set output_dir "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}" \ - --set ref_name "${NEMO_FLOW_PACKAGE_VERSION}" \ - package-node - - - name: Upload npm package artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: npm-${{ matrix.platform }} - path: ${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/npm/*.tgz - if-no-files-found: error - - consolidate-node: - name: Consolidate / Node - needs: [package-node] - if: ${{ !cancelled() && needs.package-node.result == 'success' }} - runs-on: linux-amd64-cpu4 - timeout-minutes: 30 - permissions: - contents: read - steps: - - name: Download npm linux-amd64 artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: npm-linux-amd64 - path: npm-linux-amd64/ - - - name: Download npm linux-arm64 artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: npm-linux-arm64 - path: npm-linux-arm64/ - - - name: Download npm macos-arm64 artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: npm-macos-arm64 - path: npm-macos-arm64/ - - - name: Download npm windows-amd64 artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: npm-windows-amd64 - path: npm-windows-amd64/ - - - name: Download npm windows-arm64 artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: npm-windows-arm64 - path: npm-windows-arm64/ - - - name: Consolidate Node package directory - run: | - set -euo pipefail - mkdir -p combined - first=true - for dir in npm-linux-amd64 npm-linux-arm64 npm-macos-arm64 npm-windows-amd64 npm-windows-arm64; do - shopt -s nullglob - tgzs=("${dir}"/*.tgz) - shopt -u nullglob - if [ "${#tgzs[@]}" -eq 0 ]; then - echo "Error: no npm package artifact found in ${dir}" >&2 - exit 1 - fi - for tgz in "${tgzs[@]}"; do - if [ "${first}" = true ]; then - tar xzf "${tgz}" -C combined - first=false - else - tar xzf "${tgz}" -C combined --wildcards '*.node' - fi - done - done - echo "Platform binaries included:" - ls -la combined/package/*.node - - - name: Archive consolidated Node package - run: | - set -euo pipefail - cd combined - zip -rq "${{ github.workspace }}/consolidated.zip" package - - - name: Upload consolidated Node package artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: npm-consolidated - path: ${{ github.workspace }}/consolidated.zip - if-no-files-found: error - - package-python: - name: Package / Python / ${{ matrix.platform }} - needs: [changes, test-python] - if: >- - ${{ - !cancelled() - && needs.changes.result == 'success' - && needs.test-python.result == 'success' - && needs.changes.outputs.run_python == 'true' - }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - platform: linux-amd64 - runner: linux-amd64-cpu4 - - platform: linux-arm64 - runner: linux-arm64-cpu4 - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - - - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 - with: - version: ${{ env.NEMO_FLOW_CI_UV_VERSION }} - enable-cache: true - cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock - - - name: Set up Windows ARM Python - if: ${{ matrix.platform == 'windows-arm64' }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 - with: - python-version: ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - architecture: arm64 - - - name: Select setup-python interpreter for uv - if: ${{ matrix.platform == 'windows-arm64' }} - run: | - set -e - python - <<'PY' >> "$GITHUB_ENV" - import sys - print(f"UV_PYTHON={sys.executable}") - PY - - - name: Install managed Python - if: ${{ matrix.platform != 'windows-arm64' }} - run: | - set -e - UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ env.NEMO_FLOW_CI_DEFAULT_PYTHON_VER }} - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: just@${{ env.NEMO_FLOW_CI_JUST_VERSION }} - - - name: Create packaging output directory - run: | - set -e - mkdir -p "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/wheels" - - - name: Derive Python package version - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: | - set -e - version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" - if [ -z "$version" ]; then - echo "Error: failed to read workspace version from Cargo.toml" >&2 - exit 1 - fi - sha="${GITHUB_SHA::8}" - if [ "${{ inputs.ref_type }}" = "tag" ]; then - version="${{ inputs.ref_name }}" - else - version="${version}+${sha}" - fi - printf 'NEMO_FLOW_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" - - - name: Package Python wheel - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: | - set -e - just \ - --set output_dir "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}" \ - --set ref_name "${NEMO_FLOW_PACKAGE_VERSION}" \ - package-python - - - name: Upload wheel artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wheel-${{ matrix.platform }} - path: ${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/wheels/*.whl - if-no-files-found: error - - - name: Prune uv cache - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: uv cache prune --ci - - package-wasm: - name: Package / WASM / linux-amd64 - needs: [changes, test-wasm] - if: >- - ${{ - !cancelled() - && needs.changes.result == 'success' - && needs.test-wasm.result == 'success' - && needs.changes.outputs.run_wasm == 'true' - }} - runs-on: linux-amd64-cpu4 - timeout-minutes: 30 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - - - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - toolchain: ${{ env.NEMO_FLOW_CI_RUST_VERSION }} - - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ env.NEMO_FLOW_CI_RUST_VERSION }} - workspaces: NeMo-Flow -> target - cache-all-crates: true - cache-bin: false - save-if: false - - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 - with: - node-version: ${{ env.NEMO_FLOW_CI_NODE_VERSION }} - - - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 - with: - tool: just@${{ env.NEMO_FLOW_CI_JUST_VERSION }},wasm-pack@${{ env.NEMO_FLOW_CI_WASM_PACK_VERSION }} - - - name: Create packaging output directory - run: | - set -e - mkdir -p "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/wasm" - - - name: Derive WASM package version - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: | - set -e - version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" - if [ -z "$version" ]; then - echo "Error: failed to read workspace version from Cargo.toml" >&2 - exit 1 - fi - sha="${GITHUB_SHA::8}" - if [ "${{ inputs.ref_type }}" = "tag" ]; then - version="${{ inputs.ref_name }}" - else - version="${version}-${sha}" - fi - printf 'NEMO_FLOW_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" - - - name: Package WASM bundler - working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: | - set -e - just \ - --set output_dir "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}" \ - --set ref_name "${NEMO_FLOW_PACKAGE_VERSION}" \ - package-wasm - - - name: Upload WASM package artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wasm-bundler - path: ${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/wasm/*.tgz - if-no-files-found: error diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml new file mode 100644 index 000000000..f0af23e36 --- /dev/null +++ b/.github/workflows/ci_python.yml @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Python + +on: + workflow_call: + inputs: + ref_type: + description: 'The ref type from the triggering workflow' + required: true + type: string + ref_name: + description: 'The ref name from the triggering workflow' + required: true + type: string + secrets: + CODECOV_TOKEN: + required: false + +defaults: + run: + shell: bash + +env: + GH_TOKEN: "${{ github.token }}" + GIT_COMMIT: "${{ github.sha }}" + NEMO_FLOW_CI_WORKSPACE: "${{ github.workspace }}" + NEMO_FLOW_CI_WORKSPACE_TMP: "${{ github.workspace }}/tmp" + UV_PYTHON_DOWNLOADS: never + +jobs: + test-python: + name: Test / ${{ matrix.platform }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: linux-amd64-cpu4 + - platform: linux-arm64 + runner: linux-arm64-cpu4 + - platform: macos-arm64 + runner: macos-15 + - platform: windows-amd64 + runner: windows-2022 + - platform: windows-arm64 + runner: windows-11-arm + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + save-if: false + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: cargo-llvm-cov@${{ steps.ci-config.outputs.cargo_llvm_cov_version }},just@${{ steps.ci-config.outputs.just_version }} + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + with: + version: ${{ steps.ci-config.outputs.uv_version }} + enable-cache: true + cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock + + - name: Set up Windows ARM Python + if: ${{ matrix.platform == 'windows-arm64' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ steps.ci-config.outputs.default_python_version }} + architecture: arm64 + + - name: Select setup-python interpreter for uv + if: ${{ matrix.platform == 'windows-arm64' }} + run: | + set -e + python - <<'PY' >> "$GITHUB_ENV" + import sys + print(f"UV_PYTHON={sys.executable}") + PY + + - name: Install managed Python + if: ${{ matrix.platform != 'windows-arm64' }} + run: | + set -e + UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + + - name: Run Python tests with coverage + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: just --set ci true --set output_dir "${{ github.workspace }}" test-python + + - name: Upload Python coverage to Codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 + with: + token: ${{ secrets.CODECOV_TOKEN }} + disable_search: true + files: ${{ matrix.platform == 'windows-arm64' && 'python-coverage.xml' || 'python-coverage.xml,python-rust.xml' }} + flags: python-${{ matrix.platform }} + name: python-${{ matrix.platform }} + verbose: true + + - name: Prune uv cache + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: uv cache prune --ci + + package-python: + name: Package / ${{ matrix.platform }} + needs: [test-python] + if: ${{ !cancelled() && needs.test-python.result == 'success' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: linux-amd64-cpu4 + - platform: linux-arm64 + runner: linux-arm64-cpu4 + - platform: macos-arm64 + runner: macos-15 + - platform: windows-amd64 + runner: windows-2022 + - platform: windows-arm64 + runner: windows-11-arm + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + with: + version: ${{ steps.ci-config.outputs.uv_version }} + enable-cache: true + cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock + + - name: Set up Windows ARM Python + if: ${{ matrix.platform == 'windows-arm64' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ steps.ci-config.outputs.default_python_version }} + architecture: arm64 + + - name: Select setup-python interpreter for uv + if: ${{ matrix.platform == 'windows-arm64' }} + run: | + set -e + python - <<'PY' >> "$GITHUB_ENV" + import sys + print(f"UV_PYTHON={sys.executable}") + PY + + - name: Install managed Python + if: ${{ matrix.platform != 'windows-arm64' }} + run: | + set -e + UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: just@${{ steps.ci-config.outputs.just_version }} + + - name: Create packaging output directory + run: | + set -e + mkdir -p "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/wheels" + + - name: Derive Python package version + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: | + set -e + version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" + if [ -z "$version" ]; then + echo "Error: failed to read workspace version from Cargo.toml" >&2 + exit 1 + fi + sha="${GITHUB_SHA::8}" + if [ "${{ inputs.ref_type }}" = "tag" ]; then + version="${{ inputs.ref_name }}" + else + version="${version}+${sha}" + fi + printf 'NEMO_FLOW_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" + + - name: Package Python wheel + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: | + set -e + just \ + --set output_dir "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}" \ + --set ref_name "${NEMO_FLOW_PACKAGE_VERSION}" \ + package-python + + - name: Upload wheel artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheel-${{ matrix.platform }} + path: ${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/wheels/*.whl + if-no-files-found: error + + - name: Prune uv cache + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: uv cache prune --ci diff --git a/.github/workflows/ci_rust.yml b/.github/workflows/ci_rust.yml new file mode 100644 index 000000000..1e8ad6903 --- /dev/null +++ b/.github/workflows/ci_rust.yml @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Rust + +on: + workflow_call: + secrets: + CODECOV_TOKEN: + required: false + +defaults: + run: + shell: bash + +env: + GH_TOKEN: "${{ github.token }}" + GIT_COMMIT: "${{ github.sha }}" + NEMO_FLOW_CI_WORKSPACE: "${{ github.workspace }}" + NEMO_FLOW_CI_WORKSPACE_TMP: "${{ github.workspace }}/tmp" + UV_PYTHON_DOWNLOADS: never + +jobs: + test-rust: + name: Test / ${{ matrix.platform }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} + permissions: + contents: read + services: + redis: + image: ${{ matrix.redis_service_image }} + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping || exit 1" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: linux-amd64-cpu4 + run_redis_tests: true + redis_service_image: redis:7.4.7-alpine + - platform: linux-arm64 + runner: linux-arm64-cpu4 + run_redis_tests: false + redis_service_image: '' + - platform: macos-arm64 + runner: macos-15 + run_redis_tests: false + redis_service_image: '' + - platform: windows-amd64 + runner: windows-2022 + run_redis_tests: false + redis_service_image: '' + - platform: windows-arm64 + runner: windows-11-arm + run_redis_tests: false + redis_service_image: '' + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + with: + version: ${{ steps.ci-config.outputs.uv_version }} + enable-cache: true + cache-dependency-glob: ${{ env.NEMO_FLOW_CI_WORKSPACE }}/uv.lock + + - name: Set up Windows ARM Python + if: ${{ matrix.platform == 'windows-arm64' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ steps.ci-config.outputs.default_python_version }} + architecture: arm64 + + - name: Select setup-python interpreter for uv + if: ${{ matrix.platform == 'windows-arm64' }} + run: | + set -e + python - <<'PY' >> "$GITHUB_ENV" + import sys + print(f"UV_PYTHON={sys.executable}") + PY + + - name: Install managed Python + if: ${{ matrix.platform != 'windows-arm64' }} + run: | + set -e + UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + + - name: Add Python DLL directory to PATH on Windows + if: runner.os == 'Windows' + run: | + set -e + python_exe="$(uv python find)" + python_home="$("$python_exe" -c 'import sys; print(sys.base_prefix)')" + echo "$python_home" >> "$GITHUB_PATH" + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: cargo-llvm-cov@${{ steps.ci-config.outputs.cargo_llvm_cov_version }},cargo-nextest@${{ steps.ci-config.outputs.cargo_nextest_version }},just@${{ steps.ci-config.outputs.just_version }} + + - name: Run Rust tests with coverage + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: | + set -e + args=( + --set ci true + --set output_dir "${{ github.workspace }}" + ) + if [ "${{ matrix.run_redis_tests }}" = "true" ]; then + export NEMO_FLOW_RUN_REDIS_TESTS=1 + fi + just "${args[@]}" test-rust + + - name: Upload Rust coverage to Codecov + if: ${{ matrix.platform != 'windows-arm64' }} + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 + with: + token: ${{ secrets.CODECOV_TOKEN }} + disable_search: true + files: rust-workspace.xml + flags: rust-${{ matrix.platform }} + name: rust-${{ matrix.platform }} + verbose: true diff --git a/.github/workflows/ci_wasm.yml b/.github/workflows/ci_wasm.yml new file mode 100644 index 000000000..e0b16a813 --- /dev/null +++ b/.github/workflows/ci_wasm.yml @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: WebAssembly + +on: + workflow_call: + inputs: + ref_type: + description: 'The ref type from the triggering workflow' + required: true + type: string + ref_name: + description: 'The ref name from the triggering workflow' + required: true + type: string + secrets: + CODECOV_TOKEN: + required: false + +defaults: + run: + shell: bash + +env: + GH_TOKEN: "${{ github.token }}" + GIT_COMMIT: "${{ github.sha }}" + NEMO_FLOW_CI_WORKSPACE: "${{ github.workspace }}" + NEMO_FLOW_CI_WORKSPACE_TMP: "${{ github.workspace }}/tmp" + UV_PYTHON_DOWNLOADS: never + +jobs: + test-wasm: + name: Test / ${{ matrix.platform }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: linux-amd64-cpu4 + - platform: linux-arm64 + runner: linux-arm64-cpu4 + - platform: macos-arm64 + runner: macos-15 + - platform: windows-amd64 + runner: windows-2022 + - platform: windows-arm64 + runner: windows-11-arm + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + save-if: false + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: cargo-llvm-cov@${{ steps.ci-config.outputs.cargo_llvm_cov_version }},just@${{ steps.ci-config.outputs.just_version }} + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: ${{ steps.ci-config.outputs.node_version }} + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: wasm-pack@${{ steps.ci-config.outputs.wasm_pack_version }} + + - name: Run WebAssembly tests with coverage + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: just --set ci true --set output_dir "${{ github.workspace }}" test-wasm + + - name: Upload WebAssembly coverage to Codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 + with: + token: ${{ secrets.CODECOV_TOKEN }} + disable_search: true + files: wasm-js.xml + flags: wasm-${{ matrix.platform }} + name: wasm-${{ matrix.platform }} + verbose: true + + package-wasm: + name: Package / linux-amd64 + needs: [test-wasm] + if: ${{ !cancelled() && needs.test-wasm.result == 'success' }} + runs-on: linux-amd64-cpu4 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: nemo-flow-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + save-if: false + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: ${{ steps.ci-config.outputs.node_version }} + + - uses: taiki-e/install-action@5939f3337e40968c39aa70f5ecb1417a92fb25a0 # v2.75.15 + with: + tool: just@${{ steps.ci-config.outputs.just_version }},wasm-pack@${{ steps.ci-config.outputs.wasm_pack_version }} + + - name: Create packaging output directory + run: | + set -e + mkdir -p "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/wasm" + + - name: Derive WebAssembly package version + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: | + set -e + version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" + if [ -z "$version" ]; then + echo "Error: failed to read workspace version from Cargo.toml" >&2 + exit 1 + fi + sha="${GITHUB_SHA::8}" + if [ "${{ inputs.ref_type }}" = "tag" ]; then + version="${{ inputs.ref_name }}" + else + version="${version}-${sha}" + fi + printf 'NEMO_FLOW_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" + + - name: Package WebAssembly bundler + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: | + set -e + just \ + --set output_dir "${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}" \ + --set ref_name "${NEMO_FLOW_PACKAGE_VERSION}" \ + package-wasm + + - name: Upload WebAssembly package artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wasm-bundler + path: ${{ env.NEMO_FLOW_CI_WORKSPACE_TMP }}/wasm/*.tgz + if-no-files-found: error