From 7267306df68f2b2ebc5e0ab8ea289abc1ce6f7eb Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 08:51:09 +0000 Subject: [PATCH 01/32] Bump pip in devcontainer --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ca9447684..982e4c1b3 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -20,7 +20,7 @@ ENV PYTHONUSERBASE="/home/dev/.local" COPY --chown=dev:dev . . -RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.1.1 \ +RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.2 \ && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts] \ && pre-commit install --install-hooks From ff46eda6cdef17a8c53a42d4d26a669f1cce6193 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 10:15:00 +0000 Subject: [PATCH 02/32] Pin upload action --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index a197ff103..b851caa62 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -39,7 +39,7 @@ jobs: - name: Build a binary wheel and a source tarball run: python3 -m build - name: Store the distribution packages - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: python-package-distributions path: dist/ From 6ab4ec1847050961e27b3abdd941deeefd2422de Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 13:52:43 +0000 Subject: [PATCH 03/32] Add tooling for building with Nuitka --- .devcontainer/Dockerfile | 4 +++- pyproject.toml | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 982e4c1b3..949f1e11b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,7 +3,9 @@ FROM mcr.microsoft.com/devcontainers/python:3.13-bullseye@sha256:e6d1214434cb015 # Install dependencies # pv is required for asciicasts RUN apt-get update && apt-get install --no-install-recommends -y \ + ccache=4.2-1 \ pv=1.6.6-1+b1 \ + patchelf=0.12-1 \ subversion=1.14.1-3+deb11u2 && \ rm -rf /var/lib/apt/lists/* @@ -21,7 +23,7 @@ ENV PYTHONUSERBASE="/home/dev/.local" COPY --chown=dev:dev . . RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.2 \ - && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts] \ + && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts,build] \ && pre-commit install --install-hooks # Set bash as the default shell diff --git a/pyproject.toml b/pyproject.toml index 5bc3adae2..97115dabe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,10 @@ docs = [ ] test = ['pytest==8.4.2', 'pytest-cov==7.0.0', 'behave==1.3.3'] casts = ['asciinema==2.4.0'] - +build = [ + 'nuitka==2.7.16', + "tomli; python_version < '3.11'", # Tomllib is default in 3.11, required for letting codespell read the pyproject.toml] +] [project.scripts] dfetch = "dfetch.__main__:main" From ee3ffc22230f8ecd551aa6842a621df613351d97 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 14:25:33 +0000 Subject: [PATCH 04/32] Create & release standalone binaries Fixes #701 --- .github/workflows/build.yml | 82 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 17 ++++++++ script/build.py | 61 +++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 script/build.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..2bf3d94a2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,82 @@ +name: Build + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +jobs: + + build: + strategy: + matrix: + platform: [ubuntu-latest] #, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} + permissions: + contents: read + security-events: write + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + with: + egress-policy: audit + + - uses: actions/checkout@ff7abcd0c3c05ccf6adc123a8cd1fd4fb30fb493 # v5.0.0 + + - name: Setup Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.14' + + - name: Install patchelf (patchelf) + if: matrix.platform == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install -y patchelf=0.12-1 ccache=4.2-1 + + # - name: Install Subversion (SVN) + # if: matrix.platform == 'macos-latest' + # run: | + # brew install svn + # svn --version # Verify installation + + # - name: Install Subversion (SVN) + # if: matrix.platform == 'windows-latest' + # run: | + # choco install svn -y + # $env:PATH = "C:\Program Files (x86)\Subversion\bin;$env:PATH" + # echo "C:\Program Files (x86)\Subversion\bin" >> $env:GITHUB_PATH + # svn --version # Verify installation + + - name: Create binary + run: | + pip install .[build] + python script/build.py + + - name: Store the distribution packages + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: binary-distribution + path: build/dfetch + + - run: build/dfetch environment + - run: build/dfetch validate + - run: build/dfetch check + - run: build/dfetch update + - run: build/dfetch update + - run: build/dfetch report -t sbom + + - name: Run example + working-directory: ./example + env: + CI: 'false' + run: | + build/dfetch update + build/dfetch update + build/dfetch report diff --git a/pyproject.toml b/pyproject.toml index 97115dabe..a68509e6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,3 +170,20 @@ standard = ["dfetch", "features"] reportMissingImports = false reportMissingModuleSource = false pythonVersion = "3.9" + +[tool.nuitka] +mode = "standalone" # Switch this between standalone and onefile as needed +jobs = "4" +assume-yes-for-downloads = true + +# Include data files (local path = target path in standalone) +include-data-dir = [{ source = "dfetch/resources", destination = "resources" }] + +output-dir = "build" +output-filename-win = "dfetch.exe" +output-filename-linux = "dfetch" +output-filename-macos = "dfetch" + +# windows-icon-from-ico = "static/favicon.ico" +# windows-company-name = "dfetch-org" +show-progress = true diff --git a/script/build.py b/script/build.py new file mode 100644 index 000000000..6b9c39999 --- /dev/null +++ b/script/build.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""This script builds the dfetch executable using Nuitka.""" +import os +import subprocess +import sys +import tomllib as toml +from typing import Union + + +def parse_option( + option_name: str, option_value: Union[bool, str, list, dict] +) -> list[str]: + """ + Convert a config value to CLI args for Nuitka. + Handles booleans, strings, lists, and nested dicts. + """ + args = [] + cli_key = f"--{option_name.replace('_','-')}" + + if isinstance(option_value, bool): + if option_value: + args.append(cli_key) + elif isinstance(option_value, str): + args.append(f"{cli_key}={option_value}") + elif isinstance(option_value, list): + for v in option_value: + if isinstance(v, dict): + parts = [f"{v[k]}" for k in v] + args.append(f"{cli_key}={'='.join(parts)}") + else: + args.append(f"{cli_key}={v}") + else: + args.append(f"{cli_key}={option_value}") + + return args + + +# Load pyproject.toml +with open("pyproject.toml", "r", encoding="UTF-8") as pyproject_file: + pyproject = toml.loads(pyproject_file.read()) +nuitka_opts = pyproject.get("tool", {}).get("nuitka", {}) + + +if sys.platform.startswith("win"): + nuitka_opts["output-filename"] = nuitka_opts["output-filename-win"] +elif sys.platform.startswith("linux"): + nuitka_opts["output-filename"] = nuitka_opts["output-filename-linux"] +elif sys.platform.startswith("darwin"): + nuitka_opts["output-filename"] = nuitka_opts["output-filename-macos"] + +for key in ["output-filename-win", "output-filename-linux", "output-filename-macos"]: + nuitka_opts.pop(key, None) + +command = [sys.executable, "-m", "nuitka"] +for key, value in nuitka_opts.items(): + command.extend(parse_option(key, value)) + +command.append(os.path.join("dfetch", "__main__.py")) + +print(command) +subprocess.check_call(command) From 8fbb692419045b8d08da3911dec3f0260ae481d6 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 14:28:17 +0000 Subject: [PATCH 05/32] Unpin deps in build.yaml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2bf3d94a2..173760b1d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: if: matrix.platform == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install -y patchelf=0.12-1 ccache=4.2-1 + sudo apt-get install -y patchelf ccache # - name: Install Subversion (SVN) # if: matrix.platform == 'macos-latest' From 9bdd871660b3680de2e84763e7cd6a9669d89c2a Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 14:32:26 +0000 Subject: [PATCH 06/32] Use 3.13 for build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 173760b1d..910d1fcc4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Python uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - python-version: '3.14' + python-version: '3.13' - name: Install patchelf (patchelf) if: matrix.platform == 'ubuntu-latest' From 2afe2f6870f829a473be203aca0060362e0af9f1 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 14:44:39 +0000 Subject: [PATCH 07/32] Hide debug output --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a68509e6b..abc86b561 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,4 +186,5 @@ output-filename-macos = "dfetch" # windows-icon-from-ico = "static/favicon.ico" # windows-company-name = "dfetch-org" -show-progress = true +# Enable below for debugging +# show-progress = true From d3243d2d4795be531925ed0509fab27566b3ddd2 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 14:44:53 +0000 Subject: [PATCH 08/32] Split test to clean environment --- .github/workflows/build.yml | 43 ++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 910d1fcc4..03acd97c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,18 +65,31 @@ jobs: name: binary-distribution path: build/dfetch - - run: build/dfetch environment - - run: build/dfetch validate - - run: build/dfetch check - - run: build/dfetch update - - run: build/dfetch update - - run: build/dfetch report -t sbom - - - name: Run example - working-directory: ./example - env: - CI: 'false' - run: | - build/dfetch update - build/dfetch update - build/dfetch report + test-binary: + name: Run some commands with binary + needs: + - build + runs-on: ubuntu-latest + + steps: + - name: Download the binary artifact + uses: actions/download-artifact@v5 + with: + name: binary-distribution + path: build/dfetch + + - run: build/dfetch environment + - run: build/dfetch validate + - run: build/dfetch check + - run: build/dfetch update + - run: build/dfetch update + - run: build/dfetch report -t sbom + + - name: Run example + working-directory: ./example + env: + CI: 'false' + run: | + build/dfetch update + build/dfetch update + build/dfetch report From 70325be14a73bb8e0908d5300ed15b023034fc98 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 16:06:12 +0000 Subject: [PATCH 09/32] Fix build --- .github/workflows/build.yml | 22 +++++++++++----------- pyproject.toml | 18 ++++++++++-------- script/build.py | 7 ++++--- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03acd97c0..bd11f6c93 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,7 +63,7 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: binary-distribution - path: build/dfetch + path: build/dfetch-* test-binary: name: Run some commands with binary @@ -76,20 +76,20 @@ jobs: uses: actions/download-artifact@v5 with: name: binary-distribution - path: build/dfetch + path: build/dfetch-* - - run: build/dfetch environment - - run: build/dfetch validate - - run: build/dfetch check - - run: build/dfetch update - - run: build/dfetch update - - run: build/dfetch report -t sbom + - run: build/dfetch-* environment + - run: build/dfetch-* validate + - run: build/dfetch-* check + - run: build/dfetch-* update + - run: build/dfetch-* update + - run: build/dfetch-* report -t sbom - name: Run example working-directory: ./example env: CI: 'false' run: | - build/dfetch update - build/dfetch update - build/dfetch report + build/dfetch-* update + build/dfetch-* update + build/dfetch-* report diff --git a/pyproject.toml b/pyproject.toml index abc86b561..083d2b46d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -172,19 +172,21 @@ reportMissingModuleSource = false pythonVersion = "3.9" [tool.nuitka] -mode = "standalone" # Switch this between standalone and onefile as needed +mode = "onefile" # Switch this between standalone and onefile as needed jobs = "4" +# Enable below for debugging +# show-progress = true assume-yes-for-downloads = true -# Include data files (local path = target path in standalone) -include-data-dir = [{ source = "dfetch/resources", destination = "resources" }] +include-package-data="dfetch,infer_license" +include-module="infer_license.licenses" + +# python-flag = ["-OO"] output-dir = "build" -output-filename-win = "dfetch.exe" -output-filename-linux = "dfetch" -output-filename-macos = "dfetch" +output-filename-win = "dfetch-{VERSION}.exe" +output-filename-linux = "dfetch-{VERSION}-x86_64" +output-filename-macos = "dfetch-{VERSION}-osx" # windows-icon-from-ico = "static/favicon.ico" # windows-company-name = "dfetch-org" -# Enable below for debugging -# show-progress = true diff --git a/script/build.py b/script/build.py index 6b9c39999..502c8a2e0 100644 --- a/script/build.py +++ b/script/build.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 """This script builds the dfetch executable using Nuitka.""" -import os import subprocess import sys import tomllib as toml from typing import Union +from dfetch import __version__ + def parse_option( option_name: str, option_value: Union[bool, str, list, dict] @@ -21,7 +22,7 @@ def parse_option( if option_value: args.append(cli_key) elif isinstance(option_value, str): - args.append(f"{cli_key}={option_value}") + args.append(f"{cli_key}={option_value}".replace("{VERSION}", __version__)) elif isinstance(option_value, list): for v in option_value: if isinstance(v, dict): @@ -55,7 +56,7 @@ def parse_option( for key, value in nuitka_opts.items(): command.extend(parse_option(key, value)) -command.append(os.path.join("dfetch", "__main__.py")) +command.append("dfetch") print(command) subprocess.check_call(command) From 5c1c7b68c45c956646c489a17402fc02897df0a8 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 16:20:45 +0000 Subject: [PATCH 10/32] Fix build --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd11f6c93..252d4dcca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: path: build/dfetch-* test-binary: - name: Run some commands with binary + name: Run binary needs: - build runs-on: ubuntu-latest @@ -76,8 +76,9 @@ jobs: uses: actions/download-artifact@v5 with: name: binary-distribution - path: build/dfetch-* + path: . + - run: tree . - run: build/dfetch-* environment - run: build/dfetch-* validate - run: build/dfetch-* check From 344383902df99f0a7f31dbf73fd180424c34a7fb Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 16:22:32 +0000 Subject: [PATCH 11/32] Patchelf is already installed & ccache unneeded for ephermeral agent --- .github/workflows/build.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 252d4dcca..ed6b67b48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,12 +34,6 @@ jobs: with: python-version: '3.13' - - name: Install patchelf (patchelf) - if: matrix.platform == 'ubuntu-latest' - run: | - sudo apt-get update - sudo apt-get install -y patchelf ccache - # - name: Install Subversion (SVN) # if: matrix.platform == 'macos-latest' # run: | From 048ae655f7cdfd7e8ec1e5c7234629aa67f5c549 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 16:43:41 +0000 Subject: [PATCH 12/32] Optimize binary --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 083d2b46d..b9ba65f1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,7 +173,7 @@ pythonVersion = "3.9" [tool.nuitka] mode = "onefile" # Switch this between standalone and onefile as needed -jobs = "4" +jobs = "2" # Enable below for debugging # show-progress = true assume-yes-for-downloads = true @@ -181,7 +181,7 @@ assume-yes-for-downloads = true include-package-data="dfetch,infer_license" include-module="infer_license.licenses" -# python-flag = ["-OO"] +python-flag = ["-OO"] output-dir = "build" output-filename-win = "dfetch-{VERSION}.exe" From 1c41986e756cae5512c8dc7241d546cb6e6994fb Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 16:43:51 +0000 Subject: [PATCH 13/32] Fix path in binary test --- .github/workflows/build.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed6b67b48..8ccf0722c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,18 +73,19 @@ jobs: path: . - run: tree . - - run: build/dfetch-* environment - - run: build/dfetch-* validate - - run: build/dfetch-* check - - run: build/dfetch-* update - - run: build/dfetch-* update - - run: build/dfetch-* report -t sbom + - run: dfetch-* init + - run: dfetch-* environment + - run: dfetch-* validate + - run: dfetch-* check + - run: dfetch-* update + - run: dfetch-* update + - run: dfetch-* report -t sbom - name: Run example working-directory: ./example env: CI: 'false' run: | - build/dfetch-* update - build/dfetch-* update - build/dfetch-* report + dfetch-* update + dfetch-* update + dfetch-* report From 528ccdf8dfb0c48d3166e9cc980521c5e52729ff Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 17:37:36 +0000 Subject: [PATCH 14/32] Fix build with symlink --- .github/workflows/build.yml | 23 ++++++++++++----------- pyproject.toml | 4 ++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ccf0722c..57442f320 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,20 +72,21 @@ jobs: name: binary-distribution path: . - - run: tree . - - run: dfetch-* init - - run: dfetch-* environment - - run: dfetch-* validate - - run: dfetch-* check - - run: dfetch-* update - - run: dfetch-* update - - run: dfetch-* report -t sbom + - run: ln -sf $(ls dfetch-*) dfetch + - run: ls -la . + - run: dfetch init + - run: dfetch environment + - run: dfetch validate + - run: dfetch check + - run: dfetch update + - run: dfetch update + - run: dfetch report -t sbom - name: Run example working-directory: ./example env: CI: 'false' run: | - dfetch-* update - dfetch-* update - dfetch-* report + dfetch update + dfetch update + dfetch report diff --git a/pyproject.toml b/pyproject.toml index b9ba65f1c..3d53c9515 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,7 +173,7 @@ pythonVersion = "3.9" [tool.nuitka] mode = "onefile" # Switch this between standalone and onefile as needed -jobs = "2" +jobs = "4" # Enable below for debugging # show-progress = true assume-yes-for-downloads = true @@ -181,7 +181,7 @@ assume-yes-for-downloads = true include-package-data="dfetch,infer_license" include-module="infer_license.licenses" -python-flag = ["-OO"] +# python-flag = ["-OO"] # Cannot optimize (yet) commands rely on __doc__ being present output-dir = "build" output-filename-win = "dfetch-{VERSION}.exe" From e6318dca045f4a95dae3e18cf78491c762e78cc9 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 17:53:08 +0000 Subject: [PATCH 15/32] Make sure dfetch can be executed --- .github/workflows/build.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57442f320..38e05c578 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,20 +73,21 @@ jobs: path: . - run: ln -sf $(ls dfetch-*) dfetch + - run: chmod +x dfetch - run: ls -la . - - run: dfetch init - - run: dfetch environment - - run: dfetch validate - - run: dfetch check - - run: dfetch update - - run: dfetch update - - run: dfetch report -t sbom + - run: ./dfetch init + - run: ./dfetch environment + - run: ./dfetch validate + - run: ./dfetch check + - run: ./dfetch update + - run: ./dfetch update + - run: ./dfetch report -t sbom - name: Run example working-directory: ./example env: CI: 'false' run: | - dfetch update - dfetch update - dfetch report + ./dfetch update + ./dfetch update + ./dfetch report From 0af09be63e56ad1cc793d7303c8e767d5e6ef507 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 18:16:01 +0000 Subject: [PATCH 16/32] Don't return non-zero exit code if tool not found during environment --- CHANGELOG.rst | 1 + dfetch/project/git.py | 8 +++++--- dfetch/project/svn.py | 6 +++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 635ecdde6..52c0a1141 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,7 @@ Release 0.11.0 (unreleased) * Add evidence to sbom report (#788) * Let action work outside of dfetch repo (#816) * Handle SVN tags with special characters (#811) +* Don't return non-zero exit code if tool not found during environment (#701) Release 0.10.0 (released 2025-03-12) ==================================== diff --git a/dfetch/project/git.py b/dfetch/project/git.py index 49ed8284f..22308beac 100644 --- a/dfetch/project/git.py +++ b/dfetch/project/git.py @@ -62,9 +62,11 @@ def revision_is_enough() -> bool: @staticmethod def list_tool_info() -> None: """Print out version information.""" - tool, version = get_git_version() - - VCS._log_tool(tool, version) + try: + tool, version = get_git_version() + VCS._log_tool(tool, version) + except RuntimeError: + VCS._log_tool("git", "") def _fetch_impl(self, version: Version) -> Version: """Get the revision of the remote and place it at the local path.""" diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index d81cfb5d9..0616d5486 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -173,7 +173,11 @@ def _list_of_tags(self) -> List[str]: @staticmethod def list_tool_info() -> None: """Print out version information.""" - result = run_on_cmdline(logger, "svn --version") + try: + result = run_on_cmdline(logger, "svn --version") + except RuntimeError: + VCS._log_tool("svn", "") + return first_line = result.stdout.decode().split("\n")[0] tool, version = first_line.replace(",", "").split("version", maxsplit=1) From 7609f0db14320671dfc4b3c014cc200533a83efc Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 18:16:38 +0000 Subject: [PATCH 17/32] Cleanup build script --- script/build.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/script/build.py b/script/build.py index 502c8a2e0..2010abdca 100644 --- a/script/build.py +++ b/script/build.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """This script builds the dfetch executable using Nuitka.""" -import subprocess +import subprocess # nosec import sys import tomllib as toml from typing import Union @@ -12,8 +12,13 @@ def parse_option( option_name: str, option_value: Union[bool, str, list, dict] ) -> list[str]: """ - Convert a config value to CLI args for Nuitka. - Handles booleans, strings, lists, and nested dicts. + Convert a config value to Nuitka CLI arguments. + + Handles booleans (--flag), strings (--flag=value), lists (multiple --flag=value), + and nested dicts (--flag=key1=val1=key2=val2). + + Returns: + list[str]: Nuitka CLI arguments in the format ['--flag', '--key=value'] """ args = [] cli_key = f"--{option_name.replace('_','-')}" @@ -37,8 +42,8 @@ def parse_option( # Load pyproject.toml -with open("pyproject.toml", "r", encoding="UTF-8") as pyproject_file: - pyproject = toml.loads(pyproject_file.read()) +with open("pyproject.toml", "rb") as pyproject_file: + pyproject = toml.load(pyproject_file) nuitka_opts = pyproject.get("tool", {}).get("nuitka", {}) @@ -49,8 +54,13 @@ def parse_option( elif sys.platform.startswith("darwin"): nuitka_opts["output-filename"] = nuitka_opts["output-filename-macos"] -for key in ["output-filename-win", "output-filename-linux", "output-filename-macos"]: - nuitka_opts.pop(key, None) + +nuitka_opts = { + k: v + for k, v in nuitka_opts.items() + if k + not in {"output-filename-win", "output-filename-linux", "output-filename-macos"} +} command = [sys.executable, "-m", "nuitka"] for key, value in nuitka_opts.items(): @@ -59,4 +69,4 @@ def parse_option( command.append("dfetch") print(command) -subprocess.check_call(command) +subprocess.check_call(command) # nosec From db6a5293a761ff7e3620f84bba06c7e540a366d8 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 18:20:52 +0000 Subject: [PATCH 18/32] Add ccache that caches between builds --- .github/workflows/build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 38e05c578..3562a69b3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,6 +48,12 @@ jobs: # echo "C:\Program Files (x86)\Subversion\bin" >> $env:GITHUB_PATH # svn --version # Verify installation + - name: ccache + uses: hendrikmuhs/ccache-action@30ae3502c7f2d3200209bf2f90eccef2357896cf # v1.2 + with: + key: ${{ github.job }}-${{ matrix.platform }} + create-symlink: true + - name: Create binary run: | pip install .[build] @@ -67,7 +73,7 @@ jobs: steps: - name: Download the binary artifact - uses: actions/download-artifact@v5 + uses: actions/download-artifact@4a24838f3d5601fd639834081e118c2995d51e1c # v5 with: name: binary-distribution path: . From 8f7024ddc72a7ad7f48bc872db2e21aee4bcbe7b Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 18:48:00 +0000 Subject: [PATCH 19/32] Also find projects with comments --- dfetch/manifest/manifest.py | 4 +++- tests/test_manifest.py | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/dfetch/manifest/manifest.py b/dfetch/manifest/manifest.py index db9050301..eebe88aab 100644 --- a/dfetch/manifest/manifest.py +++ b/dfetch/manifest/manifest.py @@ -344,7 +344,9 @@ def find_name_in_manifest(self, name: str) -> ManifestEntryLocation: raise FileNotFoundError("No manifest text available") for line_nr, line in enumerate(self.__text.splitlines(), start=1): - match = re.search(rf"^\s+-\s*name:\s*(?P{re.escape(name)})\s*$", line) + match = re.search( + rf"^\s+-\s*name:\s*(?P{re.escape(name)})\s*#?.*$", line + ) if match: return ManifestEntryLocation( diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 7f2a80930..4726fd7f8 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -12,6 +12,7 @@ from dfetch.manifest.manifest import ( Manifest, ManifestDict, + ManifestEntryLocation, RequestedProjectNotFoundError, find_manifest, get_childmanifests, @@ -177,3 +178,43 @@ def test_single_suggestion_not_found() -> None: exception = RequestedProjectNotFoundError(["irst", "1234"], ["first", "other"]) assert ["first"] == exception._guess_project(["irst", "1234"]) + + +@pytest.mark.parametrize( + "name, manifest, project_name, result", + [ + ( + "match", + " - name: foo", + "foo", + ManifestEntryLocation(line_number=1, start=10, end=12), + ), + ( + "no match", + " - name: foo", + "baz", + RuntimeError, + ), + ( + "with comment", + " - name: foo # some comment", + "foo", + ManifestEntryLocation(line_number=1, start=10, end=12), + ), + ( + "no spaces", + " -name:foo #some comment", + "foo", + ManifestEntryLocation(line_number=1, start=8, end=10), + ), + ], +) +def test_get_manifest_location(name, manifest, project_name, result) -> None: + + manifest = Manifest(DICTIONARY_MANIFEST, text=manifest) + + if result == RuntimeError: + with pytest.raises(RuntimeError): + manifest.find_name_in_manifest(project_name) + else: + assert manifest.find_name_in_manifest(project_name) == result From 32d7c4055cd2d795c4a695f760f5330c4ac41ecc Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 19:02:09 +0000 Subject: [PATCH 20/32] There is no example folder --- .github/workflows/build.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3562a69b3..ccd80c0b6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -88,12 +88,3 @@ jobs: - run: ./dfetch update - run: ./dfetch update - run: ./dfetch report -t sbom - - - name: Run example - working-directory: ./example - env: - CI: 'false' - run: | - ./dfetch update - ./dfetch update - ./dfetch report From 77cac4dfc61d09bd95ed88f81858514b74e48131 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 19:16:59 +0000 Subject: [PATCH 21/32] Try configuring ccache --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ccd80c0b6..7f905e146 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,6 +56,9 @@ jobs: - name: Create binary run: | + echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV + echo "NUITKA_CCACHE_BINARY=/usr/bin/ccache" >> $GITHUB_ENV + echo "NUITKA_CACHE_DIR_CCACHE=$HOME/.ccache" >> $GITHUB_ENV pip install .[build] python script/build.py From a6645b2a71e2fb74c30770cdb4e7fd7bdf8862c2 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 19:21:55 +0000 Subject: [PATCH 22/32] Pin more actions --- .github/workflows/python-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index b851caa62..5c91c3eef 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -59,12 +59,12 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v5 + uses: actions/download-artifact@4a24838f3d5601fd639834081e118c2995d51e1c # v5 with: name: python-package-distributions path: dist/ - name: Publish distribution 📦 to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ab69e431e9c9f48a3310be0a56527c679f56e04d # v1 with: repository-url: https://test.pypi.org/legacy/ @@ -86,9 +86,9 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v5 + uses: actions/download-artifact@4a24838f3d5601fd639834081e118c2995d51e1c # v5 with: name: python-package-distributions path: dist/ - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ab69e431e9c9f48a3310be0a56527c679f56e04d # v1 From a49bde78fb129ec498cf256cfe3b97aac2c1f92a Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 19:23:56 +0000 Subject: [PATCH 23/32] Don't fail the build if the exact same test version already exists --- .github/workflows/python-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 5c91c3eef..fffc8b312 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -67,6 +67,7 @@ jobs: uses: pypa/gh-action-pypi-publish@ab69e431e9c9f48a3310be0a56527c679f56e04d # v1 with: repository-url: https://test.pypi.org/legacy/ + skip-existing: true - name: Test install from TestPyPI run: | From 22ac492c3d642a9474315395f1887d2bf0f8efd4 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 19:33:01 +0000 Subject: [PATCH 24/32] Add verbose ccache --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f905e146..2cc927aab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,6 +53,7 @@ jobs: with: key: ${{ github.job }}-${{ matrix.platform }} create-symlink: true + verbose: 1 - name: Create binary run: | From cfaf3896187b885fde1bf59d9e06c4ce9d92e43f Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 19:53:47 +0000 Subject: [PATCH 25/32] Improve ccache settings --- .github/workflows/build.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2cc927aab..28d473d03 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,14 +52,16 @@ jobs: uses: hendrikmuhs/ccache-action@30ae3502c7f2d3200209bf2f90eccef2357896cf # v1.2 with: key: ${{ github.job }}-${{ matrix.platform }} - create-symlink: true verbose: 1 + create-symlink: true - name: Create binary + env: + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_NOHASHDIR: true + NUITKA_CACHE_DIR_CCACHE: $HOME/.ccache + NUITKA_CCACHE_BINARY: /usr/bin/ccache run: | - echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV - echo "NUITKA_CCACHE_BINARY=/usr/bin/ccache" >> $GITHUB_ENV - echo "NUITKA_CACHE_DIR_CCACHE=$HOME/.ccache" >> $GITHUB_ENV pip install .[build] python script/build.py From 31aa02ae76e6a8b71a61ffd73549a6a6daf366da Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 11 Oct 2025 21:25:34 +0000 Subject: [PATCH 26/32] Build binaries on windows & macos --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 28d473d03..050829f39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - platform: [ubuntu-latest] #, macos-latest, windows-latest] + platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} permissions: contents: read @@ -49,6 +49,7 @@ jobs: # svn --version # Verify installation - name: ccache + if: matrix.platform == 'ubuntu-latest' uses: hendrikmuhs/ccache-action@30ae3502c7f2d3200209bf2f90eccef2357896cf # v1.2 with: key: ${{ github.job }}-${{ matrix.platform }} @@ -84,7 +85,7 @@ jobs: name: binary-distribution path: . - - run: ln -sf $(ls dfetch-*) dfetch + - run: ln -sf $(ls dfetch-*-x86_64) dfetch - run: chmod +x dfetch - run: ls -la . - run: ./dfetch init From ec42d662db5159251e2c114c4e851882a221eb23 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 12 Oct 2025 09:58:56 +0000 Subject: [PATCH 27/32] Make artifacts unique --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 050829f39..8cbca5466 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,7 +69,7 @@ jobs: - name: Store the distribution packages uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: binary-distribution + name: binary-distribution-${{ matrix.platform }} path: build/dfetch-* test-binary: @@ -82,7 +82,7 @@ jobs: - name: Download the binary artifact uses: actions/download-artifact@4a24838f3d5601fd639834081e118c2995d51e1c # v5 with: - name: binary-distribution + name: binary-distribution-ubuntu-latest path: . - run: ln -sf $(ls dfetch-*-x86_64) dfetch From fff51a09e80ba834c9dd106ada64fd66701a5430 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 12 Oct 2025 10:16:34 +0000 Subject: [PATCH 28/32] Add more logging --- dfetch/project/git.py | 5 ++++- dfetch/project/svn.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dfetch/project/git.py b/dfetch/project/git.py index 22308beac..b5563600c 100644 --- a/dfetch/project/git.py +++ b/dfetch/project/git.py @@ -65,7 +65,10 @@ def list_tool_info() -> None: try: tool, version = get_git_version() VCS._log_tool(tool, version) - except RuntimeError: + except RuntimeError as exc: + logger.debug( + f"Something went wrong trying to get the version of git: {exc}" + ) VCS._log_tool("git", "") def _fetch_impl(self, version: Version) -> Version: diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index 0616d5486..7b874676a 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -175,7 +175,10 @@ def list_tool_info() -> None: """Print out version information.""" try: result = run_on_cmdline(logger, "svn --version") - except RuntimeError: + except RuntimeError as exc: + logger.debug( + f"Something went wrong trying to get the version of svn: {exc}" + ) VCS._log_tool("svn", "") return From a5f40be9d04e8d843bae6913f46b606ef2ade41b Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 12 Oct 2025 10:17:47 +0000 Subject: [PATCH 29/32] Don't limit jobs an dhopefully speed up jobs --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3d53c9515..e388453f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,7 +173,7 @@ pythonVersion = "3.9" [tool.nuitka] mode = "onefile" # Switch this between standalone and onefile as needed -jobs = "4" +# jobs = "4" # Can be used to reduce memory usage, in case of compilation issues # Enable below for debugging # show-progress = true assume-yes-for-downloads = true From c1ba4aa89dd3e721960810b395b2ee98d52ccadb Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 12 Oct 2025 10:24:59 +0000 Subject: [PATCH 30/32] Test binaries on all OS'es --- .github/workflows/build.yml | 38 +++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8cbca5466..b950f7138 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,21 +73,47 @@ jobs: path: build/dfetch-* test-binary: - name: Run binary + name: test binary needs: - build - runs-on: ubuntu-latest + strategy: + matrix: + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} steps: - name: Download the binary artifact uses: actions/download-artifact@4a24838f3d5601fd639834081e118c2995d51e1c # v5 with: - name: binary-distribution-ubuntu-latest + name: binary-distribution-${{ matrix.platform }} path: . - - run: ln -sf $(ls dfetch-*-x86_64) dfetch - - run: chmod +x dfetch - - run: ls -la . + - name: Prepare binary + if: matrix.platform == 'ubuntu-latest' + run: | + binary=$(ls dfetch-*-x86_64) + ln -sf "$binary" dfetch + chmod +x dfetch + ls -la . + shell: bash + + - name: Prepare binary + if: matrix.platform == 'macos-latest' + run: | + binary=$(ls dfetch-*-osx) + ln -sf "$binary" dfetch + chmod +x dfetch + ls -la . + shell: bash + + - name: Prepare binary on Windows + if: matrix.platform == 'windows-latest' + run: | + $binary = Get-ChildItem dfetch-*-x86_64.exe | Select-Object -First 1 + Copy-Item $binary -Destination dfetch.exe -Force + Get-ChildItem + shell: pwsh + - run: ./dfetch init - run: ./dfetch environment - run: ./dfetch validate From 87b700760b3511513f36ac34e87924104f636ee9 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 12 Oct 2025 10:29:09 +0000 Subject: [PATCH 31/32] ccache action should work on all platforms --- .github/workflows/build.yml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b950f7138..07e4a5203 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,27 +34,12 @@ jobs: with: python-version: '3.13' - # - name: Install Subversion (SVN) - # if: matrix.platform == 'macos-latest' - # run: | - # brew install svn - # svn --version # Verify installation - - # - name: Install Subversion (SVN) - # if: matrix.platform == 'windows-latest' - # run: | - # choco install svn -y - # $env:PATH = "C:\Program Files (x86)\Subversion\bin;$env:PATH" - # echo "C:\Program Files (x86)\Subversion\bin" >> $env:GITHUB_PATH - # svn --version # Verify installation - - name: ccache - if: matrix.platform == 'ubuntu-latest' uses: hendrikmuhs/ccache-action@30ae3502c7f2d3200209bf2f90eccef2357896cf # v1.2 with: key: ${{ github.job }}-${{ matrix.platform }} verbose: 1 - create-symlink: true + create-symlink: ${{ matrix.platform != 'windows-latest' }} - name: Create binary env: @@ -109,7 +94,7 @@ jobs: - name: Prepare binary on Windows if: matrix.platform == 'windows-latest' run: | - $binary = Get-ChildItem dfetch-*-x86_64.exe | Select-Object -First 1 + $binary = Get-ChildItem dfetch-*.exe | Select-Object -First 1 Copy-Item $binary -Destination dfetch.exe -Force Get-ChildItem shell: pwsh From 0c4c802fd1873bf6f1f6ada39ef5a1a78966e8b6 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 12 Oct 2025 11:45:53 +0000 Subject: [PATCH 32/32] Update changelog --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 52c0a1141..c35d573af 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,7 @@ Release 0.11.0 (unreleased) * Let action work outside of dfetch repo (#816) * Handle SVN tags with special characters (#811) * Don't return non-zero exit code if tool not found during environment (#701) +* Create standalone binaries for Linux, Mac & Windows (#705) Release 0.10.0 (released 2025-03-12) ====================================