From 8192740c48d7145bc87c3775989d6b1e938363a6 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Mon, 13 Jul 2026 17:47:55 +0000 Subject: [PATCH 1/4] ci: skip unsupported platform services wheel on python 3.14 Signed-off-by: Matt Kornfield --- .github/workflows/ci.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 85121f2e90..eda051c9a4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1013,6 +1013,13 @@ jobs: matrix: package: [nemo-platform, nemo-platform-plugin] python-version: ["3.11", "3.12", "3.13", "3.14"] + # nemo-platform[services] currently pulls nvidia-nat-langchain -> + # langchain-litellm -> litellm. The CVE-fixed litellm range is not + # installable on Python 3.14 yet: pre-1.92 releases declare <3.14, and + # 1.92.0 falls back to a PyO3 source build that only supports <=3.13. + exclude: + - package: nemo-platform + python-version: "3.14" steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -1055,6 +1062,11 @@ jobs: matrix: package: [nemo-platform, nemo-platform-plugin] python-version: ["3.11", "3.12", "3.13", "3.14"] + # Keep the full-services wheel smoke aligned with the build matrix + # above until the litellm/NAT dependency chain supports Python 3.14. + exclude: + - package: nemo-platform + python-version: "3.14" steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 From 9c27d91f06cc5f0dc343d316c97946093ff10a45 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Mon, 13 Jul 2026 17:49:02 +0000 Subject: [PATCH 2/4] ci: enable pyo3 compatibility for python 3.14 wheel smoke Signed-off-by: Matt Kornfield --- .github/workflows/ci.yaml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eda051c9a4..26d3bda3f2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1013,13 +1013,6 @@ jobs: matrix: package: [nemo-platform, nemo-platform-plugin] python-version: ["3.11", "3.12", "3.13", "3.14"] - # nemo-platform[services] currently pulls nvidia-nat-langchain -> - # langchain-litellm -> litellm. The CVE-fixed litellm range is not - # installable on Python 3.14 yet: pre-1.92 releases declare <3.14, and - # 1.92.0 falls back to a PyO3 source build that only supports <=3.13. - exclude: - - package: nemo-platform - python-version: "3.14" steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -1062,11 +1055,6 @@ jobs: matrix: package: [nemo-platform, nemo-platform-plugin] python-version: ["3.11", "3.12", "3.13", "3.14"] - # Keep the full-services wheel smoke aligned with the build matrix - # above until the litellm/NAT dependency chain supports Python 3.14. - exclude: - - package: nemo-platform - python-version: "3.14" steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -1108,6 +1096,13 @@ jobs: uv pip install --python "${VENV}/bin/python" \ --constraint "${GITHUB_WORKSPACE}/.github/wheel-constraints/nemo-platform-services.txt" \ "${WHEEL}[services]" + if [[ "${PYTHON_VERSION}" == "3.14" ]]; then + # litellm 1.92.0 builds its Rust bridge from source on Python 3.14. + # Its pinned PyO3 supports the stable ABI, but rejects 3.14 unless + # this forward-compatibility opt-in is present. + export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 + fi + uv tool install --force --python "${PYTHON_VERSION}" "${WHEEL}[services]" cd "${RUNNER_TEMP}/wheelcheck" unset PYTHONPATH VIRTUAL_ENV export PATH="${VENV}/bin:${PATH}" From ab9fb0c84516bc3466b73f38cb79a0f062ce8673 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Mon, 13 Jul 2026 19:20:53 +0000 Subject: [PATCH 3/4] docs: document python 3.14 install compatibility Signed-off-by: Matt Kornfield --- README.md | 8 ++++++++ docs/cli/index.mdx | 8 ++++++++ docs/get-started/setup.mdx | 8 ++++++++ docs/requirements.mdx | 2 +- docs/support-matrix.mdx | 2 +- packages/nemo_platform/README.md | 8 ++++++++ .../src/nemo_platform_ext/cli/commands/services/cli.py | 5 ++++- .../src/nemo_platform_ext/cli/commands/setup.py | 2 ++ .../src/nemo_platform_plugin/README.md | 8 ++++++++ .../src/nemo_platform/cli/commands/services/cli.py | 5 ++++- .../nemo-platform/src/nemo_platform/cli/commands/setup.py | 2 ++ 11 files changed, 54 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 098ef03e60..cdc1fc0cb2 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,14 @@ pip install "nemo-platform[all]" nemo setup ``` +On Python 3.14, prefix the install command with +`PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` while the transitive `litellm` Rust +extension catches up to Python 3.14: + +```bash +PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install "nemo-platform[all]" +``` + Source checkout for development: ```bash diff --git a/docs/cli/index.mdx b/docs/cli/index.mdx index 789ad763f5..0ff58c8f1b 100644 --- a/docs/cli/index.mdx +++ b/docs/cli/index.mdx @@ -42,6 +42,14 @@ Or with uv: uv pip install "nemo-platform[all]" ``` +On Python 3.14, prefix the install command with +`PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` while the transitive `litellm` Rust +extension catches up to Python 3.14: + +```bash +PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install "nemo-platform[all]" +``` + When installed in a virtual environment, the `nemo` command is only available when the environment is activated. diff --git a/docs/get-started/setup.mdx b/docs/get-started/setup.mdx index 8e85300644..995d6bdf03 100644 --- a/docs/get-started/setup.mdx +++ b/docs/get-started/setup.mdx @@ -29,6 +29,14 @@ source .venv/bin/activate pip install "nemo-platform[all]" ``` +On Python 3.14, prefix the install command with +`PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` while the transitive `litellm` Rust +extension catches up to Python 3.14: + +```bash +PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install "nemo-platform[all]" +``` + For SDK and CLI usage only, install the base package: ```bash diff --git a/docs/requirements.mdx b/docs/requirements.mdx index 2b5998e5b6..ed8326d5b2 100644 --- a/docs/requirements.mdx +++ b/docs/requirements.mdx @@ -10,7 +10,7 @@ The OSS 0.2.0 documentation is scoped to local setup with the Python package and | Component | Requirement | Notes | |-----------|-------------|-------| -| Python | 3.11, 3.12, 3.13, or 3.14 (`>=3.11,<3.15`) | Use an isolated virtual environment. Python 3.15 and later are not part of the OSS local-install support range. | +| Python | 3.11, 3.12, 3.13, or 3.14 (`>=3.11,<3.15`) | Use an isolated virtual environment. On Python 3.14, prefix `pip install "nemo-platform[all]"` with `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` while the transitive `litellm` Rust extension catches up to Python 3.14. Python 3.15 and later are not part of the OSS local-install support range. | | Package installer | `uv` recommended; `pip` supported | The one-line installer sets up `uv` automatically when needed. | | Operating system | Recent Linux or macOS release | See the [Support Matrix](/documentation/reference/support-matrix) for the supported OS list. | | Memory | 8 GB RAM minimum | 16 GB or more is recommended for larger local workflows. | diff --git a/docs/support-matrix.mdx b/docs/support-matrix.mdx index 8dd50d1cb3..6c9f9bd9fb 100644 --- a/docs/support-matrix.mdx +++ b/docs/support-matrix.mdx @@ -11,7 +11,7 @@ Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of | Area | Supported | Notes | |------|-----------|-------| | Deployment mode | `nemo setup` local install | Setup starts local services, registers a model provider, and configures the CLI and SDK. | -| Python | 3.11, 3.12, 3.13, 3.14 (`>=3.11,<3.15`) | Use an isolated virtual environment. Python 3.11 is the lowest supported Python version and Python 3.14 is the highest supported Python version for the OSS local install path. | +| Python | 3.11, 3.12, 3.13, 3.14 (`>=3.11,<3.15`) | Use an isolated virtual environment. On Python 3.14, prefix `pip install "nemo-platform[all]"` with `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` while the transitive `litellm` Rust extension catches up to Python 3.14. Python 3.11 is the lowest supported Python version and Python 3.14 is the highest supported Python version for the OSS local install path. | | Linux | Ubuntu 22.04 LTS, Ubuntu 24.04 LTS, RHEL 9, Rocky Linux 9, Debian 12 | Supported for CLI, SDK, local services, and NVIDIA GPU workloads. | | macOS | [macOS Tahoe 26](https://support.apple.com/en-us/122868) and macOS Sequoia 15 | Supported for CLI, SDK, local services, and cloud/provider workflows. Local NVIDIA GPU workloads are not supported on macOS. | | Architecture | x86_64 Linux; Apple Silicon and Intel macOS | NVIDIA GPU workloads require x86_64 Linux. | diff --git a/packages/nemo_platform/README.md b/packages/nemo_platform/README.md index 402d6f65fd..91a7f3aa38 100644 --- a/packages/nemo_platform/README.md +++ b/packages/nemo_platform/README.md @@ -31,6 +31,14 @@ pip install nemo-platform pip install "nemo-platform[all]" ``` +On Python 3.14, prefix the `nemo-platform[all]` install with +`PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` while the transitive `litellm` Rust +extension catches up to Python 3.14: + +```bash +PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install "nemo-platform[all]" +``` + Then bring up the platform: ```bash diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py index f14f462b0c..d5e4a7cf66 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py @@ -71,7 +71,10 @@ def _require_services_extra() -> None: "Running local platform services needs extra components that aren't installed yet.\n" "\n" "Install them with:\n" - " pip install 'nemo-platform[all]'\n", + " pip install 'nemo-platform[all]'\n" + "\n" + "On Python 3.14, use:\n" + " PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install 'nemo-platform[all]'\n", err=True, ) raise typer.Exit(1) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py index 5133eafe4d..1c52e750b1 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py @@ -762,6 +762,8 @@ def _maybe_start_services( console.print(f"{CROSS} Local services require extra dependencies that aren't installed.") console.print(" Install them with:") console.print(" [cyan]pip install 'nemo-platform\\[all]'[/cyan]") + console.print(" On Python 3.14, use:") + console.print(" [cyan]PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install 'nemo-platform\\[all]'[/cyan]") raise typer.Exit(1) if already_running: diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/README.md b/packages/nemo_platform_plugin/src/nemo_platform_plugin/README.md index 730f19cb0c..251a75506f 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/README.md +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/README.md @@ -31,6 +31,14 @@ To run a local NeMo Platform that loads and serves your plugin while you develop pip install "nemo-platform[all]" ``` +On Python 3.14, prefix the `nemo-platform[all]` install with +`PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` while the transitive `litellm` Rust +extension catches up to Python 3.14: + +```bash +PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install "nemo-platform[all]" +``` + ## A minimal plugin A complete NeMo Platform plugin that contributes one HTTP route — a `pyproject.toml` and a service module: diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py index e9ee1b915e..4df852bb8e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py @@ -71,7 +71,10 @@ def _require_services_extra() -> None: "Running local platform services needs extra components that aren't installed yet.\n" "\n" "Install them with:\n" - " pip install 'nemo-platform[all]'\n", + " pip install 'nemo-platform[all]'\n" + "\n" + "On Python 3.14, use:\n" + " PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install 'nemo-platform[all]'\n", err=True, ) raise typer.Exit(1) diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py index 03943305c9..29081aa867 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py @@ -762,6 +762,8 @@ def _maybe_start_services( console.print(f"{CROSS} Local services require extra dependencies that aren't installed.") console.print(" Install them with:") console.print(" [cyan]pip install 'nemo-platform\\[all]'[/cyan]") + console.print(" On Python 3.14, use:") + console.print(" [cyan]PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install 'nemo-platform\\[all]'[/cyan]") raise typer.Exit(1) if already_running: From e286d73f3346ab21feca9248b0fdf66e16455ca5 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Mon, 13 Jul 2026 19:44:16 +0000 Subject: [PATCH 4/4] chore: solve merge conflicts Signed-off-by: Matt Kornfield --- .github/workflows/ci.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26d3bda3f2..ac42b6d37c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1086,6 +1086,12 @@ jobs: echo "::error::Expected exactly one wheel, found ${#wheels[@]}" exit 1 fi + if [[ "${PYTHON_VERSION}" == "3.14" ]]; then + # litellm 1.92.0 builds its Rust bridge from source on Python 3.14. + # Its pinned PyO3 supports the stable ABI, but rejects 3.14 unless + # this forward-compatibility opt-in is present. + export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 + fi WHEEL="${wheels[0]}" # Supply-chain: partial-pin the wheel's direct external deps to a committed constraints # file (deep transitives resolve normally, so they stay py3.14-compatible and can't @@ -1096,13 +1102,6 @@ jobs: uv pip install --python "${VENV}/bin/python" \ --constraint "${GITHUB_WORKSPACE}/.github/wheel-constraints/nemo-platform-services.txt" \ "${WHEEL}[services]" - if [[ "${PYTHON_VERSION}" == "3.14" ]]; then - # litellm 1.92.0 builds its Rust bridge from source on Python 3.14. - # Its pinned PyO3 supports the stable ABI, but rejects 3.14 unless - # this forward-compatibility opt-in is present. - export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 - fi - uv tool install --force --python "${PYTHON_VERSION}" "${WHEEL}[services]" cd "${RUNNER_TEMP}/wheelcheck" unset PYTHONPATH VIRTUAL_ENV export PATH="${VENV}/bin:${PATH}"