Use Pre-Commit in CI (#159)

This PR unifies the linting and formatting with `pre-commit`. Before, we used a mixture of `pre-commit` and custom-installed tools. However, this could lead to version inconsistencies between different users and the CI. 

**I suggest to only look at 563138b. The other commit just reformatted all the files.**

## Changed
- Switch CI to use pre-commit for linting
- Update contributing guide
- Adjust Make targets
- Formatted all files

## Fixed
- Fix missing dependency in pre-commit-config

Author: Xeratec

.github/workflows/ci-lint.yml

@@ -13,55 +13,32 @@ name: CI • Lint & Licenses
       - "v*.*.*"
   pull_request:
   workflow_dispatch:
-    inputs:
-      docker_image_deeploy:
-        description: "Deeploy Image to use"
-        required: false
-        default: "ghcr.io/pulp-platform/deeploy:devel"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
 
 jobs:
-  select-env:
-    uses: ./.github/workflows/_select-env.yml
-    with:
-      docker_image_deeploy: ${{ inputs.docker_image_deeploy }}
-
   linting:
-    needs: select-env
-    runs-on: ${{ needs.select-env.outputs.runner }}
-    container:
-      image: ${{ needs.select-env.outputs.image }}
+    name: Run pre-commit
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
         with:
-          submodules: recursive
-      - name: Build Deeploy
-        shell: bash
-        run: |
-          pip install .
-          pip install -r requirements-dev.txt
-      - name: Format Python
-        shell: bash
-        run: |
-          yapf -rpd -e "*/TEST_*/" -e "*/third_party/" -e "install/" -e "toolchain/" .
-      - name: Format Python Imports
-        shell: bash
-        run: |
-          isort --quiet --sg "**/TEST_*/*" --sg "**/third_party/*" --sg "install/*" --sg "toolchain/*" ./ -c
-          autoflake --quiet -c -r --remove-all-unused-imports --ignore-init-module-imports --exclude "**/third_party/*,**/install/*,**/toolchain/*" .
-      - name: Format C
-        shell: bash
-        run: |
-          python scripts/run_clang_format.py -e "*/TEST_*/*" -e "*/third_party/*" -e "*/install/*" -e "*/toolchain/*" -r --clang-format-executable=${LLVM_INSTALL_DIR}/bin/clang-format . scripts
-      - name: Format YAML
-        shell: bash
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
         run: |
-          yamllint .
-      - name: Check Licenses
-        shell: bash
+          python -m pip install --upgrade pip
+          pip install pre-commit
+
+      - name: Run pre-commit (all files)
         run: |
-          python scripts/reuse_skip_wrapper.py $(find . \( -name '*.py' -o -name '*.c' -o -name '*.h' -o -name '*.html' -o -name '*.rst' -o -name '*.yml' -o -name '*.yaml' \) -not -path "*toolchain*" -not -path "*third_party*" -not -path "*prebuilt*" -not -path "*.git/*" -not -path "*install/*" -type f)
+          # Show diffs on failure and run across the entire repository to ensure consistency
+          pre-commit run --show-diff-on-failure --all-files
+
+      - name: Show git status (debug)
+        if: ${{ always() }}
+        run: git status --porcelain || true

.github/workflows/package-publish.yml

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+---
 name: Package • Publish to PyPi
 
 on:

.pre-commit-config.yaml

@@ -13,6 +13,7 @@ exclude: |
     | .*TEST_.*
     | .*TestFiles.*
     | .*runtime.*
+    | .*prebuilt/.*
   )
 
 repos:
@@ -28,6 +29,7 @@ repos:
       - id: reuse
         name: Check SPDX License Headers
         entry: scripts/reuse_skip_wrapper.py
+        additional_dependencies: ["reuse==6.2.0"]
         language: python
         stages: [pre-commit, pre-merge-commit, pre-push, manual]
         types: [text]

.vscode/launch.json

@@ -67,7 +67,7 @@
         },
         "labelTransform": {
           "text": "${fileDirname}",
-          "find" : "${workspaceFolder}/DeeployTest/Tests/",
+          "find": "${workspaceFolder}/DeeployTest/Tests/",
           "replace": ""
         },
         "valueTransform": {

CHANGELOG.md

@@ -5,18 +5,21 @@ This file contains the changelog for the Deeploy project. The changelog is divid
 
 
 ### List of Pull Requests
+- Use Pre-Commit in CI [#159](https://github.com/pulp-platform/Deeploy/pull/159)
 - Deeploy-GAP9 Platform [#143](https://github.com/pulp-platform/Deeploy/pull/143)
 - Update CLI interface Across Project, Fix Tutorial, and Remove Legacy Test [#157](https://github.com/pulp-platform/Deeploy/pull/157)
 
 ### Added
 - Added GAP9 Platform Support: Deployer, Bindings, Templates, Tiler, DMA (L3Dma/MchanDma), target library, CI workflows
 
 ### Changed
+- Switch CI to use pre-commit for linting
 - Update `pulp-nnx` and `pulp-nn-mixed` submodules to their latest versions
 - PULP-NN moved to TargetLibraries third-party folder
 - Aligned CLI commands across the project
 
 ### Fixed
+- Fix missing dependency in pre-commit-config
 - Fix test paths in Deeploy 101 tutorial
 
 ### Removed

CONTRIBUTING.md

@@ -36,42 +36,36 @@ Additionally, add the title and link to the pull request in the list of pull req
 - Remove the link to the precompiled LLVM 12 in the `deeployRunner` for Snitch and in the CI.
 [...]
 ```
-
 ## Style guide
 
-Deeploy mainly consists of code implemented in C, Makefile, and Python. To facilitate efficient collaboration among users and contributors, it is important to maintain a consistent coding style. To achieve this, it is strongly recommend to use autoformatting tools with the provided configuration files. Additionally, the Continuous Integration (CI) system checks the adherence to the style guide for each pushed commit. Currently configuration for C using `clang-format` and for Python using `yapf` and `isort` are provided.
-
-You can format all relevant files by running:
-```bash
-make format
-```
-
-Alternatively, to only lint the files without modifying them, you can run:
-```bash
-make lint
-```
-
-### Pre-commit
+Deeploy mainly consists of code implemented in C, Makefile, and Python. To facilitate efficient collaboration among users and contributors, it is important to maintain a consistent coding style. We use [pre-commit](https://pre-commit.com) with autoformatting tools to maintain this consistency. Configuration is provided for C using `clang-format` and for Python using `yapf` and `isort`.
 
-Additionally, we provide the [pre-commit](https://pre-commit.com) configuration file which you can use to install github hooks that execute the formatting commands on your changes.
+### Setting up pre-commit
 
-You will need to manually install pre-commit since it's not added as a dependency to the `pyproject.toml`:
+Install pre-commit (not included in `pyproject.toml`):
 ```bash
 pip install pre-commit
 ```
 
-The configuration sets the default stage for all the hooks to `pre-push` so to install the git hooks run:
+Install the git hooks configured to run at the `pre-push` stage:
 ```bash
 pre-commit install --hook-type pre-push
 ```
-The hooks will run before each push, making sure the pushed code can pass linting checks and not fail the CI on linting.
 
-If you change your mind and don't want the git hooks:
+The hooks will automatically format your code before each push, ensuring it passes linting checks and CI validation.
+
+To uninstall the git hooks:
 ```bash
 pre-commit uninstall
 ```
 
-_Note:_ This configures only the python formatting git hooks. The c formatting is not supported at the moment.
+You can also manually run formatting without pushing:
+```bash
+pre-commit run --all-files
+
+# Or by running the Makefile target:
+make format
+```
 
 ## Licensing
 

Container/Dockerfile.deeploy

@@ -81,7 +81,7 @@ ENV SOFTHIER_INSTALL_DIR=/app/install/softhier
 ENV MINIMALLOC_INSTALL_DIR=/app/install/minimalloc
 ENV MEMPOOL_HOME=/app/install/mempool
 ENV BENDER_INSTALL_DIR=/app/install/bender
-ENV PATH=/app/install/qemu/bin:/app/install/banshee:/app/install/bender:$PATH 
+ENV PATH=/app/install/qemu/bin:/app/install/banshee:/app/install/bender:$PATH
 
 WORKDIR /app
 

Deeploy/Targets/PULPOpen/Templates/FloatLayernormTemplate.py

@@ -24,19 +24,19 @@
 int8_t ${nodeName}_log2Core = log2(NUM_CORES);
 
 int32_t ${nodeName}_seq_length = ${size} / ${lastDimLength};
-int32_t ${nodeName}_chunk = (${nodeName}_seq_length >> ${nodeName}_log2Core) + 
+int32_t ${nodeName}_chunk = (${nodeName}_seq_length >> ${nodeName}_log2Core) +
                           ((${nodeName}_seq_length & (NUM_CORES-1)) != 0);
 int32_t ${nodeName}_start = MIN(${nodeName}_chunk * ${nodeName}_core_id, ${nodeName}_seq_length);
 int32_t ${nodeName}_end = MIN(${nodeName}_start + ${nodeName}_chunk, ${nodeName}_seq_length);
- 
+
 int32_t ${nodeName}_elem_start = ${nodeName}_start * ${lastDimLength};
 int32_t ${nodeName}_elem_end = ${nodeName}_end * ${lastDimLength};
 int32_t ${nodeName}_elem_count = ${nodeName}_elem_end - ${nodeName}_elem_start;
- 
+
 const float${grad_in_type.referencedType.typeWidth}_t* ${nodeName}_grad_in_ptr = ${grad_in} + ${nodeName}_elem_start;
 const float${data_in_type.referencedType.typeWidth}_t* ${nodeName}_data_in_ptr = ${data_in} + ${nodeName}_elem_start;
 float${grad_out_type.referencedType.typeWidth}_t* ${nodeName}_grad_out_ptr = ${grad_out} + ${nodeName}_elem_start;
- 
+
 if (${nodeName}_elem_count > 0) {
   LayernormGrad_fp${grad_in_type.referencedType.typeWidth}_fp${grad_out_type.referencedType.typeWidth}(
       ${nodeName}_grad_in_ptr,     // Upstream gradient (dy)

Deeploy/Targets/PULPOpen/Templates/SGDTemplate.py

@@ -31,7 +31,7 @@
     temp_mul[3] = learning_rate * ref_${grad}[i+3];
     temp_mul[4] = learning_rate * ref_${grad}[i+4];
     temp_mul[5] = learning_rate * ref_${grad}[i+5];
-    
+
     // Unrolled subtraction operations
     ref_${weight_updated}[i] = ref_${weight}[i] - temp_mul[0];
     ref_${weight_updated}[i+1] = ref_${weight}[i+1] - temp_mul[1];

DeeployTest/testUtils/core/execution.py

@@ -17,7 +17,7 @@
 def generate_network(config: DeeployTestConfig, skip: bool = False) -> None:
     """
     Generate network code from ONNX model.
-        
+
     Raises:
         RuntimeError: If network generation fails
     """
@@ -151,7 +151,7 @@ def build_binary(config: DeeployTestConfig) -> None:
 def run_simulation(config: DeeployTestConfig, skip: bool = False) -> TestResult:
     """
     Run simulation and parse output.
-        
+
     Raises:
         RuntimeError: If simulation cannot be executed
     """