From 6e95aa0d69371ef85cd16a2049af84c40e9387dd Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 10:27:22 +0100
Subject: [PATCH 01/25] build(github): move all tests to free runners

---
 .github/workflows/tests.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c823d0cd061..e0206bb2474 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -85,9 +85,11 @@ jobs:
       package: ${{ matrix.rs-package }}
       # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-22.04"' }}
       # FIXME: Clippy fails on github hosted runners, most likely due to RAM usage. Using self-hosted runners for now.
-      lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      # lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      lint-runner: '["ubuntu-24.04"]'
       # Run drive tests on self-hosted 4x
-      test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      # test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      test-runner: '["ubuntu-24.04"]'
       check-each-feature: ${{ contains(fromJSON('["dash-sdk","rs-dapi-client","dapi-grpc","dpp","drive-abci"]'), matrix.rs-package) }}
 
   rs-crates-security:

From ca00d6e8480e15af084d9ffb52450b12d16dd968 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 11:06:37 +0100
Subject: [PATCH 02/25] feat(github): action to install librocksdb

---
 .github/actions/librocksdb/action.yaml | 54 ++++++++++++++++++++++++++
 .github/workflows/tests-rs-package.yml | 14 +++++++
 .github/workflows/tests.yml            | 12 +++++-
 3 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 .github/actions/librocksdb/action.yaml

diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
new file mode 100644
index 00000000000..a20d51c81dc
--- /dev/null
+++ b/.github/actions/librocksdb/action.yaml
@@ -0,0 +1,54 @@
+---
+name: "librocksdb"
+description: "Build and install librocksdb"
+inputs:
+  runs-on:
+    description: Target platform
+    required: false
+    default: linux-24.04
+  cache:
+    description: Enable cache
+    required: false
+    default: "true"
+  version:
+    description: RocksDB version
+    required: false
+    default: "8.10.2"
+runs:
+  using: composite
+  steps:
+    - name: Load cached librocksdb
+      id: librocksdb-cache
+      uses: actions/cache@v3
+      with:
+        path: /opt/rocksdb
+        key: librocksdb-${{ runner.os }}-${{ inputs.version }}
+        restore-keys: |
+          librocksdb-${{ runner.os }}-${{ inputs.version }}
+          librocksdb-${{ runner.os }}-
+
+    - if: ${{ steps.cache-npm.outputs.cache-hit != 'true' }}
+      shell: bash
+      name: Build librocksdb
+      run: |
+        set -ex
+        WORKDIR=/tmp/rocksdb-build
+        mkdir -p ${WORKDIR}
+        mkdir -p /opt/rocksdb/usr/local/lib/
+        pushd $WORKDIR
+
+        # building rocksdb
+        mkdir -p ${WORKDIR}/rocksdb
+        cd ${WORKDIR}/rocksdb
+        [[ -d .git ]] || git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 .
+        make -j4 static_lib
+        make DESTDIR=/opt/rocksdb install-static
+        set +x
+
+        echo Done.
+        echo Configuration:
+        echo
+        echo "ROCKSDB_STATIC='/opt/rocksdb/usr/local/lib/librocksdb.a'"
+        echo "ROCKSDB_LIB_DIR='/opt/rocksdb/usr/local/lib'"
+
+        popd
diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml
index e666491ebcf..e72d72c8893 100644
--- a/.github/workflows/tests-rs-package.yml
+++ b/.github/workflows/tests-rs-package.yml
@@ -184,6 +184,11 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/rust
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+        with:
+          version: 8.10.2
+
       - name: Run tests
         run: cargo test --package=${{ inputs.package }} --all-features --locked
         env:
@@ -191,6 +196,8 @@ jobs:
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
 
   check_each_feature:
     name: Check each feature
@@ -211,6 +218,11 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/rust
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+        with:
+          version: 8.10.2
+
       - name: Get crate ${{ runner.arch }} info
         id: crate_info
         uses: ./.github/actions/crate_info
@@ -223,6 +235,8 @@ jobs:
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
         run: |
           echo Verify all features disabled
           set -ex
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index e0206bb2474..f3c3b9e9228 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -22,13 +22,13 @@ jobs:
   changes:
     name: Determine changed packages
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       js-packages: ${{ steps.filter-js.outputs.changes }}
       rs-packages: ${{ steps.filter-rs.outputs.changes }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -42,6 +42,13 @@ jobs:
         with:
           filters: .github/package-filters/rs-packages.yml
 
+  rust-deps:
+    name: Prebuild and cache some Rust dependencies
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Precompile librocksdb
+        uses: ./.github/actions/librocksdb
+
   build-js:
     name: Build JS packages
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
@@ -75,6 +82,7 @@ jobs:
     name: Rust packages
     needs:
       - changes
+      - rust-deps
     secrets: inherit
     strategy:
       fail-fast: false

From 25abfd6c436d2356fb4befb70158120cd5482192 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 11:17:03 +0100
Subject: [PATCH 03/25] fix: add checkout

---
 .github/workflows/tests.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index f3c3b9e9228..b3b98b72ea8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -46,6 +46,10 @@ jobs:
     name: Prebuild and cache some Rust dependencies
     runs-on: ubuntu-24.04
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - name: Precompile librocksdb
         uses: ./.github/actions/librocksdb
 

From dafc31124e0267b05ba656b6d986d69130dde492 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 11:42:14 +0100
Subject: [PATCH 04/25] fix: don't rebuild cache every time

---
 .github/actions/librocksdb/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
index a20d51c81dc..b22ee39cfcf 100644
--- a/.github/actions/librocksdb/action.yaml
+++ b/.github/actions/librocksdb/action.yaml
@@ -27,7 +27,7 @@ runs:
           librocksdb-${{ runner.os }}-${{ inputs.version }}
           librocksdb-${{ runner.os }}-
 
-    - if: ${{ steps.cache-npm.outputs.cache-hit != 'true' }}
+    - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' }}
       shell: bash
       name: Build librocksdb
       run: |

From 9b093df453f2ba5a64c439b4cfaec006205a99d7 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:39:38 +0100
Subject: [PATCH 05/25] build(docker): Prebuild dependencies image

---
 .github/workflows/tests-build-image.yml |  3 ++-
 .github/workflows/tests.yml             | 17 +++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml
index f8b7c830f42..a0c7d7379f3 100644
--- a/.github/workflows/tests-build-image.yml
+++ b/.github/workflows/tests-build-image.yml
@@ -17,7 +17,8 @@ on:
 jobs:
   build-image:
     name: Build ${{ inputs.name }} image
-    runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
+    # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index b3b98b72ea8..a81cfe5b9d8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -53,6 +53,21 @@ jobs:
       - name: Precompile librocksdb
         uses: ./.github/actions/librocksdb
 
+  docker-deps:
+    name: Prebuild and cache some Docker dependencies
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Prebuild Docker dependencies
+        uses: ./.github/workflows/tests-build-image.yml
+        with:
+          name: Dependencies
+          image_name: deps
+          target: deps
+
   build-js:
     name: Build JS packages
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
@@ -61,6 +76,8 @@ jobs:
 
   build-images:
     name: Build Docker images
+    needs:
+      - docker-deps
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
     secrets: inherit
     strategy:

From 39eb45c190a0e03f74adcf29a8034ce91b3c133a Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:47:07 +0100
Subject: [PATCH 06/25] chore: fix docker caching job

---
 .github/workflows/tests.yml | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a81cfe5b9d8..dbd7aa1a4a1 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -61,12 +61,30 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - name: Prebuild Docker dependencies
-        uses: ./.github/workflows/tests-build-image.yml
+      - name: Configure AWS credentials and bucket region
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to ECR
+        run: aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
+        shell: bash
+
+      - name: Build and push by SHA
+        uses: ./.github/actions/docker
         with:
-          name: Dependencies
           image_name: deps
+          image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
           target: deps
+          platform: linux/arm64
+          push_tags: true
+          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+          region: ${{ secrets.AWS_REGION }}
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
   build-js:
     name: Build JS packages

From 249f8b5c55566e512f90b4835e559da227c5c96a Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:59:43 +0100
Subject: [PATCH 07/25] fix(docker): rm fails

---
 Dockerfile | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 5039a062b69..e20d1a53a0b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -145,6 +145,19 @@ ENV CARGO_INCREMENTAL=${CARGO_INCREMENTAL}
 ARG AWS_ACCESS_KEY_ID
 ARG AWS_SECRET_ACCESS_KEY
 
+#
+# BUILD ROCKSDB STATIC LIBRARY
+#
+FROM deps-${RUSTC_WRAPPER:-base} AS rocksdb
+
+RUN mkdir -p /tmp/rocksdb
+WORKDIR /tmp/rocksdb
+RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \
+    make static_lib && \
+    make DESTDIR=/opt/rocksdb install-static && \
+    cd / && \
+    rm -rf /tmp/rocksdb
+
 #
 # DEPS: FULL DEPENDENCIES LIST
 #
@@ -154,6 +167,14 @@ FROM deps-${RUSTC_WRAPPER:-base} AS deps
 ARG SCCACHE_S3_KEY_PREFIX
 ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl
 
+# Install prebuilt rocksdb library
+
+COPY --from=rocksdb /opt/rocksdb /opt/rocksdb
+# Set env variables so that Rust's rocksdb-sys will not build rocksdb from scratch
+
+ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a
+ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib
+
 WORKDIR /platform
 
 # Install wasm-bindgen-cli in the same profile as other components, to sacrifice some performance & disk space to gain
@@ -171,12 +192,17 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM
     # Meanwhile if you want to update wasm-bindgen you also need to update version in:
     #  - packages/wasm-dpp/Cargo.toml
     #  - packages/wasm-dpp/scripts/build-wasm.sh
-    cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked
+    cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked  
+
 
 #
 # Rust build planner to speed up builds
 #
 FROM deps AS build-planner
+
+ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a
+ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib
+
 WORKDIR /platform
 COPY . .
 RUN source $HOME/.cargo/env && \
@@ -196,6 +222,9 @@ SHELL ["/bin/bash", "-o", "pipefail","-e", "-x", "-c"]
 ARG SCCACHE_S3_KEY_PREFIX
 ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl
 
+ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a
+ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib
+
 WORKDIR /platform
 
 COPY --from=build-planner /platform/recipe.json recipe.json
@@ -251,6 +280,9 @@ FROM deps AS build-js
 ARG SCCACHE_S3_KEY_PREFIX
 ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/wasm/wasm32
 
+ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a
+ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib
+
 WORKDIR /platform
 
 COPY --from=build-planner /platform/recipe.json recipe.json

From 488a7aeecdad690ec16d482d1f1e8f0a23903ed1 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 13:00:44 +0100
Subject: [PATCH 08/25] feat(github): use prebuilt rocksdb everywhere

---
 .github/workflows/tests-rs-package.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml
index e72d72c8893..7a2229a215d 100644
--- a/.github/workflows/tests-rs-package.yml
+++ b/.github/workflows/tests-rs-package.yml
@@ -42,6 +42,11 @@ jobs:
         with:
           components: clippy
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+        with:
+          version: 8.10.2
+
       - uses: clechasseur/rs-clippy-check@v3
         with:
           args: --package ${{ inputs.package }} --all-features --locked -- --no-deps
@@ -50,6 +55,8 @@ jobs:
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
 
   formatting:
     name: Formatting
@@ -65,7 +72,19 @@ jobs:
           components: rustfmt
           cache: false
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+        with:
+          version: 8.10.2
+
       - name: Check formatting
+        env:
+          RUSTC_WRAPPER: sccache
+          SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
+          SCCACHE_REGION: ${{ secrets.AWS_REGION }}
+          SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
         run: cargo fmt --check --package=${{ inputs.package }}
 
   unused_deps:
@@ -89,6 +108,11 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/rust
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+        with:
+          version: 8.10.2
+
       - name: Get crate ${{ inputs.package }} info
         id: crate_info
         uses: ./.github/actions/crate_info
@@ -102,6 +126,8 @@ jobs:
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
         with:
           args: ${{ steps.crate_info.outputs.cargo_manifest_dir }}
 

From 8ccb0671ba6e23149655f96964bb825073e9f85b Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 13:10:12 +0100
Subject: [PATCH 09/25] chore: remove librocksdb from formatting step

---
 .github/workflows/tests-rs-package.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml
index 7a2229a215d..017adb14a07 100644
--- a/.github/workflows/tests-rs-package.yml
+++ b/.github/workflows/tests-rs-package.yml
@@ -72,10 +72,7 @@ jobs:
           components: rustfmt
           cache: false
 
-      - name: Install librocksdb
-        uses: ./.github/actions/librocksdb
-        with:
-          version: 8.10.2
+      #  This step doesn't need librocksdb, so we don't install it
 
       - name: Check formatting
         env:
@@ -83,8 +80,6 @@ jobs:
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
-          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
-          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
         run: cargo fmt --check --package=${{ inputs.package }}
 
   unused_deps:

From e608408da0dd776cb550d72d9b68818b52f8def6 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 13:36:27 +0100
Subject: [PATCH 10/25] build(docker): move tests to amd64

---
 .github/workflows/tests-build-image.yml | 2 +-
 .github/workflows/tests.yml             | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml
index a0c7d7379f3..d21ddd9c132 100644
--- a/.github/workflows/tests-build-image.yml
+++ b/.github/workflows/tests-build-image.yml
@@ -42,7 +42,7 @@ jobs:
           image_name: ${{ inputs.image_name }}
           image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
           target: ${{ inputs.target }}
-          platform: linux/arm64
+          platform: linux/amd64
           push_tags: true
           dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index dbd7aa1a4a1..46bcd52e29c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -56,6 +56,7 @@ jobs:
   docker-deps:
     name: Prebuild and cache some Docker dependencies
     runs-on: ubuntu-24.04
+    # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -78,7 +79,7 @@ jobs:
           image_name: deps
           image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
           target: deps
-          platform: linux/arm64
+          platform: linux/amd64
           push_tags: true
           dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}

From eafa0404cacf55d2f2066d2723b2a2f2942e5bc7 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 15:36:02 +0100
Subject: [PATCH 11/25] build(docker): fix rocksdb build

---
 Dockerfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e20d1a53a0b..cb63cf3b5d1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -153,12 +153,13 @@ FROM deps-${RUSTC_WRAPPER:-base} AS rocksdb
 RUN mkdir -p /tmp/rocksdb
 WORKDIR /tmp/rocksdb
 RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \
-    make static_lib && \
-    make DESTDIR=/opt/rocksdb install-static && \
+    make -j 4 static_lib && \
+    mkdir -p /opt/rocksdb/usr/local/lib && \
+    cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \
+    cp -r include /opt/rocksdb/usr/local/lib/ && \
     cd / && \
     rm -rf /tmp/rocksdb
 
-#
 # DEPS: FULL DEPENDENCIES LIST
 #
 # This is separate from `deps` to use sccache for caching

From 9a57f6f00b12a58432657e5da292ae8505a64173 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 15:41:02 +0100
Subject: [PATCH 12/25] build: adjust -j flag

---
 .github/actions/librocksdb/action.yaml | 2 +-
 Dockerfile                             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
index b22ee39cfcf..ef64fd3ee2d 100644
--- a/.github/actions/librocksdb/action.yaml
+++ b/.github/actions/librocksdb/action.yaml
@@ -41,7 +41,7 @@ runs:
         mkdir -p ${WORKDIR}/rocksdb
         cd ${WORKDIR}/rocksdb
         [[ -d .git ]] || git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 .
-        make -j4 static_lib
+        make -j$(nproc) static_lib
         make DESTDIR=/opt/rocksdb install-static
         set +x
 
diff --git a/Dockerfile b/Dockerfile
index cb63cf3b5d1..59622a7086b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -153,7 +153,7 @@ FROM deps-${RUSTC_WRAPPER:-base} AS rocksdb
 RUN mkdir -p /tmp/rocksdb
 WORKDIR /tmp/rocksdb
 RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \
-    make -j 4 static_lib && \
+    make -j$(nproc) static_lib && \
     mkdir -p /opt/rocksdb/usr/local/lib && \
     cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \
     cp -r include /opt/rocksdb/usr/local/lib/ && \

From 355e344c973528a1bb2415661b08be84522f8506 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 16:31:49 +0100
Subject: [PATCH 13/25] chore(github): remove docker-deps from tests.yml, we
 will trust layer cache

---
 .github/workflows/tests.yml | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 46bcd52e29c..b3b98b72ea8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -53,40 +53,6 @@ jobs:
       - name: Precompile librocksdb
         uses: ./.github/actions/librocksdb
 
-  docker-deps:
-    name: Prebuild and cache some Docker dependencies
-    runs-on: ubuntu-24.04
-    # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-      - name: Configure AWS credentials and bucket region
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ secrets.AWS_REGION }}
-
-      - name: Login to ECR
-        run: aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
-        shell: bash
-
-      - name: Build and push by SHA
-        uses: ./.github/actions/docker
-        with:
-          image_name: deps
-          image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
-          target: deps
-          platform: linux/amd64
-          push_tags: true
-          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-          region: ${{ secrets.AWS_REGION }}
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
   build-js:
     name: Build JS packages
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
@@ -95,8 +61,6 @@ jobs:
 
   build-images:
     name: Build Docker images
-    needs:
-      - docker-deps
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
     secrets: inherit
     strategy:

From 02396930ae9507123444d009401a2a8573e1726c Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 16:40:42 +0100
Subject: [PATCH 14/25] build: add job to prebuild deps on push to master

---
 .github/workflows/prebuild-deps.yml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 .github/workflows/prebuild-deps.yml

diff --git a/.github/workflows/prebuild-deps.yml b/.github/workflows/prebuild-deps.yml
new file mode 100644
index 00000000000..dd6a2fc0595
--- /dev/null
+++ b/.github/workflows/prebuild-deps.yml
@@ -0,0 +1,21 @@
+---
+## This workflow is used to prebuild and cache some dependencies for the build process.
+name: Prebuild dependencies
+on:
+  push:
+    branches:
+      - master
+      - build/optimize-memory # todo: remove this line before merging
+  schedule:
+    - cron: "30 4 * * 0"
+jobs:
+  deps-rocksdb:
+    name: Prebuild and cache librocksdb
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Precompile librocksdb
+        uses: ./.github/actions/librocksdb

From 0a73edcc60125bd91908fdec2a78ec4ef805bc11 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 6 Nov 2024 16:46:34 +0100
Subject: [PATCH 15/25] chore: self-review

---
 .github/actions/librocksdb/action.yaml | 18 +++++-------------
 .github/workflows/tests-rs-package.yml |  8 --------
 2 files changed, 5 insertions(+), 21 deletions(-)

diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
index ef64fd3ee2d..fb7e8b9dea0 100644
--- a/.github/actions/librocksdb/action.yaml
+++ b/.github/actions/librocksdb/action.yaml
@@ -6,12 +6,9 @@ inputs:
     description: Target platform
     required: false
     default: linux-24.04
-  cache:
-    description: Enable cache
-    required: false
-    default: "true"
+
   version:
-    description: RocksDB version
+    description: RocksDB version, eg. "8.10.2"
     required: false
     default: "8.10.2"
 runs:
@@ -23,9 +20,6 @@ runs:
       with:
         path: /opt/rocksdb
         key: librocksdb-${{ runner.os }}-${{ inputs.version }}
-        restore-keys: |
-          librocksdb-${{ runner.os }}-${{ inputs.version }}
-          librocksdb-${{ runner.os }}-
 
     - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' }}
       shell: bash
@@ -33,14 +27,12 @@ runs:
       run: |
         set -ex
         WORKDIR=/tmp/rocksdb-build
-        mkdir -p ${WORKDIR}
+        mkdir -p ${WORKDIR}/rocksdb
         mkdir -p /opt/rocksdb/usr/local/lib/
-        pushd $WORKDIR
+        pushd ${WORKDIR}/rocksdb
 
         # building rocksdb
-        mkdir -p ${WORKDIR}/rocksdb
-        cd ${WORKDIR}/rocksdb
-        [[ -d .git ]] || git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 .
+        [[ -d .git ]] || git clone https://github.com/facebook/rocksdb.git -b v${{ inputs.version }} --depth 1 .
         make -j$(nproc) static_lib
         make DESTDIR=/opt/rocksdb install-static
         set +x
diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml
index 017adb14a07..04a4ae187bf 100644
--- a/.github/workflows/tests-rs-package.yml
+++ b/.github/workflows/tests-rs-package.yml
@@ -44,8 +44,6 @@ jobs:
 
       - name: Install librocksdb
         uses: ./.github/actions/librocksdb
-        with:
-          version: 8.10.2
 
       - uses: clechasseur/rs-clippy-check@v3
         with:
@@ -105,8 +103,6 @@ jobs:
 
       - name: Install librocksdb
         uses: ./.github/actions/librocksdb
-        with:
-          version: 8.10.2
 
       - name: Get crate ${{ inputs.package }} info
         id: crate_info
@@ -207,8 +203,6 @@ jobs:
 
       - name: Install librocksdb
         uses: ./.github/actions/librocksdb
-        with:
-          version: 8.10.2
 
       - name: Run tests
         run: cargo test --package=${{ inputs.package }} --all-features --locked
@@ -241,8 +235,6 @@ jobs:
 
       - name: Install librocksdb
         uses: ./.github/actions/librocksdb
-        with:
-          version: 8.10.2
 
       - name: Get crate ${{ runner.arch }} info
         id: crate_info

From 8318995026a91e6434c87d56a054de74e4dade0f Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:23:35 +0100
Subject: [PATCH 16/25] chore: librocksdb cached in s3 instead of gh cache

---
 .github/actions/librocksdb/action.yaml | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
index fb7e8b9dea0..760d6f2603b 100644
--- a/.github/actions/librocksdb/action.yaml
+++ b/.github/actions/librocksdb/action.yaml
@@ -6,20 +6,26 @@ inputs:
     description: Target platform
     required: false
     default: linux-24.04
-
   version:
     description: RocksDB version, eg. "8.10.2"
     required: false
     default: "8.10.2"
+  bucket:
+    description: S3 bucket to use for caching
+    required: false
+    default: multi-runner-cache-x1xibo9c
+
 runs:
   using: composite
   steps:
-    - name: Load cached librocksdb
+    # Cache librocksdb using s3 bucket
+    - name: Restore local network data
       id: librocksdb-cache
-      uses: actions/cache@v3
+      uses: strophy/actions-cache@opendal-update
       with:
+        bucket: ${{ inputs.bucket }}
         path: /opt/rocksdb
-        key: librocksdb-${{ runner.os }}-${{ inputs.version }}
+        key: local-network-volumes/${{ steps.dashmate-fingerprint.outputs.sha }}
 
     - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' }}
       shell: bash

From 97bcd47b1f7ec2bb12dd9d5081da097c15b9227e Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:26:47 +0100
Subject: [PATCH 17/25] chore: rename rust-deps step

---
 .github/workflows/tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index b3b98b72ea8..24fb99e3ce2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -42,7 +42,7 @@ jobs:
         with:
           filters: .github/package-filters/rs-packages.yml
 
-  rust-deps:
+  build-rust-deps:
     name: Prebuild and cache some Rust dependencies
     runs-on: ubuntu-24.04
     steps:
@@ -86,7 +86,7 @@ jobs:
     name: Rust packages
     needs:
       - changes
-      - rust-deps
+      - build-rust-deps
     secrets: inherit
     strategy:
       fail-fast: false

From 40e8ad58b64c16d25088c3a68b7ec1cc12000593 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:33:38 +0100
Subject: [PATCH 18/25] chore: fix cache key

---
 .github/actions/librocksdb/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
index 760d6f2603b..705a3e6d9db 100644
--- a/.github/actions/librocksdb/action.yaml
+++ b/.github/actions/librocksdb/action.yaml
@@ -25,7 +25,7 @@ runs:
       with:
         bucket: ${{ inputs.bucket }}
         path: /opt/rocksdb
-        key: local-network-volumes/${{ steps.dashmate-fingerprint.outputs.sha }}
+        key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }}
 
     - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' }}
       shell: bash

From 9b8e0c20bd7d935c404cc5b8b1fd75ac83e4d388 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:34:08 +0100
Subject: [PATCH 19/25] chore: remove weekly rebuild

---
 .github/workflows/prebuild-deps.yml | 21 ---------------------
 1 file changed, 21 deletions(-)
 delete mode 100644 .github/workflows/prebuild-deps.yml

diff --git a/.github/workflows/prebuild-deps.yml b/.github/workflows/prebuild-deps.yml
deleted file mode 100644
index dd6a2fc0595..00000000000
--- a/.github/workflows/prebuild-deps.yml
+++ /dev/null
@@ -1,21 +0,0 @@
----
-## This workflow is used to prebuild and cache some dependencies for the build process.
-name: Prebuild dependencies
-on:
-  push:
-    branches:
-      - master
-      - build/optimize-memory # todo: remove this line before merging
-  schedule:
-    - cron: "30 4 * * 0"
-jobs:
-  deps-rocksdb:
-    name: Prebuild and cache librocksdb
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-      - name: Precompile librocksdb
-        uses: ./.github/actions/librocksdb

From fcd3d6a1ecc6d810d3c8a80d6ff54c189b9f628e Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:48:22 +0100
Subject: [PATCH 20/25] chore: set aws region and secrets

---
 .github/workflows/tests.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 6ab326e1d3b..9341728e4ff 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -39,6 +39,11 @@ jobs:
   build-rust-deps:
     name: Prebuild and cache some Rust dependencies
     runs-on: ubuntu-24.04
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4

From 169a655d2de48ea604d0ee21e6271ded24636a44 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 10:07:00 +0100
Subject: [PATCH 21/25] chore: move docker changes to another pr

---
 .github/workflows/tests-build-image.yml |  5 ++--
 Dockerfile                              | 35 +------------------------
 2 files changed, 3 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml
index d21ddd9c132..f8b7c830f42 100644
--- a/.github/workflows/tests-build-image.yml
+++ b/.github/workflows/tests-build-image.yml
@@ -17,8 +17,7 @@ on:
 jobs:
   build-image:
     name: Build ${{ inputs.name }} image
-    # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
-    runs-on: ubuntu-24.04
+    runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
@@ -42,7 +41,7 @@ jobs:
           image_name: ${{ inputs.image_name }}
           image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
           target: ${{ inputs.target }}
-          platform: linux/amd64
+          platform: linux/arm64
           push_tags: true
           dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/Dockerfile b/Dockerfile
index 59622a7086b..5039a062b69 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -146,20 +146,6 @@ ARG AWS_ACCESS_KEY_ID
 ARG AWS_SECRET_ACCESS_KEY
 
 #
-# BUILD ROCKSDB STATIC LIBRARY
-#
-FROM deps-${RUSTC_WRAPPER:-base} AS rocksdb
-
-RUN mkdir -p /tmp/rocksdb
-WORKDIR /tmp/rocksdb
-RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \
-    make -j$(nproc) static_lib && \
-    mkdir -p /opt/rocksdb/usr/local/lib && \
-    cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \
-    cp -r include /opt/rocksdb/usr/local/lib/ && \
-    cd / && \
-    rm -rf /tmp/rocksdb
-
 # DEPS: FULL DEPENDENCIES LIST
 #
 # This is separate from `deps` to use sccache for caching
@@ -168,14 +154,6 @@ FROM deps-${RUSTC_WRAPPER:-base} AS deps
 ARG SCCACHE_S3_KEY_PREFIX
 ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl
 
-# Install prebuilt rocksdb library
-
-COPY --from=rocksdb /opt/rocksdb /opt/rocksdb
-# Set env variables so that Rust's rocksdb-sys will not build rocksdb from scratch
-
-ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a
-ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib
-
 WORKDIR /platform
 
 # Install wasm-bindgen-cli in the same profile as other components, to sacrifice some performance & disk space to gain
@@ -193,17 +171,12 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM
     # Meanwhile if you want to update wasm-bindgen you also need to update version in:
     #  - packages/wasm-dpp/Cargo.toml
     #  - packages/wasm-dpp/scripts/build-wasm.sh
-    cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked  
-
+    cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked
 
 #
 # Rust build planner to speed up builds
 #
 FROM deps AS build-planner
-
-ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a
-ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib
-
 WORKDIR /platform
 COPY . .
 RUN source $HOME/.cargo/env && \
@@ -223,9 +196,6 @@ SHELL ["/bin/bash", "-o", "pipefail","-e", "-x", "-c"]
 ARG SCCACHE_S3_KEY_PREFIX
 ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl
 
-ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a
-ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib
-
 WORKDIR /platform
 
 COPY --from=build-planner /platform/recipe.json recipe.json
@@ -281,9 +251,6 @@ FROM deps AS build-js
 ARG SCCACHE_S3_KEY_PREFIX
 ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/wasm/wasm32
 
-ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a
-ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib
-
 WORKDIR /platform
 
 COPY --from=build-planner /platform/recipe.json recipe.json

From c939f44b307cbe8f1a23e2055f5e62b8af51d92f Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 10:08:54 +0100
Subject: [PATCH 22/25] chore: fix

---
 .github/actions/librocksdb/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
index 705a3e6d9db..7b02d596756 100644
--- a/.github/actions/librocksdb/action.yaml
+++ b/.github/actions/librocksdb/action.yaml
@@ -19,7 +19,7 @@ runs:
   using: composite
   steps:
     # Cache librocksdb using s3 bucket
-    - name: Restore local network data
+    - name: Restore cached librocksdb from S3
       id: librocksdb-cache
       uses: strophy/actions-cache@opendal-update
       with:

From dd18c7d71e75e9533845e5c9137bc3277b63949e Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 10:36:05 +0100
Subject: [PATCH 23/25] chore: rebuild cached deps

---
 .github/actions/librocksdb/action.yaml | 15 +++++++++------
 .github/workflows/cached.yml           | 23 +++++++++++++++++++++++
 2 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/cached.yml

diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml
index 7b02d596756..895185f43dd 100644
--- a/.github/actions/librocksdb/action.yaml
+++ b/.github/actions/librocksdb/action.yaml
@@ -1,11 +1,10 @@
 ---
+# This action builds and caches librocksdb. If we find that this solution consumes too much time, we can consider
+# prebuilding librocksdb outside of the pipeline (eg. in the grovedb release process), publish as an artifact, and
+# download it in the pipeline.
 name: "librocksdb"
 description: "Build and install librocksdb"
 inputs:
-  runs-on:
-    description: Target platform
-    required: false
-    default: linux-24.04
   version:
     description: RocksDB version, eg. "8.10.2"
     required: false
@@ -14,6 +13,10 @@ inputs:
     description: S3 bucket to use for caching
     required: false
     default: multi-runner-cache-x1xibo9c
+  force:
+    description: Force rebuild
+    required: false
+    default: "false"
 
 runs:
   using: composite
@@ -27,7 +30,7 @@ runs:
         path: /opt/rocksdb
         key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }}
 
-    - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' }}
+    - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' || inputs.force == 'true' }}
       shell: bash
       name: Build librocksdb
       run: |
@@ -38,7 +41,7 @@ runs:
         pushd ${WORKDIR}/rocksdb
 
         # building rocksdb
-        [[ -d .git ]] || git clone https://github.com/facebook/rocksdb.git -b v${{ inputs.version }} --depth 1 .
+        git clone https://github.com/facebook/rocksdb.git -b v${{ inputs.version }} --depth 1 .
         make -j$(nproc) static_lib
         make DESTDIR=/opt/rocksdb install-static
         set +x
diff --git a/.github/workflows/cached.yml b/.github/workflows/cached.yml
new file mode 100644
index 00000000000..12eed8dd3e8
--- /dev/null
+++ b/.github/workflows/cached.yml
@@ -0,0 +1,23 @@
+---
+name: Rebuild cached dependencies
+
+on:
+  workflow_dispatch:
+jobs:
+  build-rust-deps:
+    name: Prebuild and cache some Rust dependencies
+    runs-on: ubuntu-24.04
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Precompile librocksdb
+        uses: ./.github/actions/librocksdb
+        with:
+          force: true

From 39e9c410ca637fe1bba3cf19cef35f36a4694b35 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 10:36:34 +0100
Subject: [PATCH 24/25] chore: switch to ubuntu 24.04

---
 .github/workflows/docs.yml                 |  2 +-
 .github/workflows/manage-runs.yml          |  2 +-
 .github/workflows/pr.yml                   |  2 +-
 .github/workflows/release-docker-image.yml |  2 +-
 .github/workflows/release.yml              |  6 +++---
 .github/workflows/tests-codeql.yml         |  2 +-
 .github/workflows/tests-js-package.yml     |  4 ++--
 .github/workflows/tests-rs-package.yml     |  8 ++++----
 .github/workflows/tests.yml                | 13 +++++++++----
 9 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 12f512b7208..d8e24ef706f 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -7,7 +7,7 @@ on:
 jobs:
   build:
     name: Deploy docs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout main
         uses: actions/checkout@v4
diff --git a/.github/workflows/manage-runs.yml b/.github/workflows/manage-runs.yml
index 2c07af5b4f3..29bc43aa535 100644
--- a/.github/workflows/manage-runs.yml
+++ b/.github/workflows/manage-runs.yml
@@ -7,7 +7,7 @@ on:
 jobs:
   cancel-merged-or-closed-pr-runs:
     name: Cancel runs for merged or closed PRs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: octokit/request-action@v2.x
         id: get_active_workflows
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d475a3eef8f..e75151c8413 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -10,7 +10,7 @@ on:
 jobs:
   pr-title:
     name: PR title
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Validate conventional PR title
         uses: amannn/action-semantic-pull-request@v5
diff --git a/.github/workflows/release-docker-image.yml b/.github/workflows/release-docker-image.yml
index 5c56ade272f..54e4cb465a3 100644
--- a/.github/workflows/release-docker-image.yml
+++ b/.github/workflows/release-docker-image.yml
@@ -89,7 +89,7 @@ jobs:
   publish-manifest:
     name: Publish image tags
     needs: build-image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Download digests
         uses: actions/download-artifact@v3
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index dba5d592f56..4c43639d3ab 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -208,11 +208,11 @@ jobs:
       matrix:
         include:
           - package_type: tarballs
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: win
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: deb
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: macos
             os: macos-14
     steps:
diff --git a/.github/workflows/tests-codeql.yml b/.github/workflows/tests-codeql.yml
index d00a66c8dfd..78a3f53aeb7 100644
--- a/.github/workflows/tests-codeql.yml
+++ b/.github/workflows/tests-codeql.yml
@@ -4,7 +4,7 @@ on:
 jobs:
   codeql:
     name: Run Code QL
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       actions: read
       contents: read
diff --git a/.github/workflows/tests-js-package.yml b/.github/workflows/tests-js-package.yml
index ef508ec0738..bdffc8cd41f 100644
--- a/.github/workflows/tests-js-package.yml
+++ b/.github/workflows/tests-js-package.yml
@@ -17,7 +17,7 @@ on:
 jobs:
   lint:
     name: Linting
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       id-token: write
       contents: read
@@ -51,7 +51,7 @@ jobs:
 
   test:
     name: Tests
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       id-token: write
       contents: read
diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml
index 04a4ae187bf..e36d66dfa86 100644
--- a/.github/workflows/tests-rs-package.yml
+++ b/.github/workflows/tests-rs-package.yml
@@ -12,7 +12,7 @@ on:
       lint-runner:
         description: Runner for linting. Must be JSON valid string.
         type: string
-        default: '"ubuntu-22.04"'
+        default: '"ubuntu-24.04"'
       check-each-feature:
         description: If true, try to build each individual feature for this crate
         type: boolean
@@ -58,7 +58,7 @@ jobs:
 
   formatting:
     name: Formatting
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 5
     steps:
       - name: Check out repo
@@ -82,7 +82,7 @@ jobs:
 
   unused_deps:
     name: Unused dependencies
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       id-token: write
       contents: read
@@ -124,7 +124,7 @@ jobs:
 
   detect_structure_changes:
     name: Detect immutable structure changes
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     # FIXME: as we use `gh pr view` below, this check can only
     # run on pull requests. We should find a way to run it
     # when manual triggers are used.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 9341728e4ff..5f6cec2c08d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -2,6 +2,11 @@ name: Tests
 
 on:
   workflow_dispatch:
+    inputs:
+      rebuild-deps:
+        description: "Rebuild cached Rust dependencies"
+        required: false
+        default: "false"
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
     branches:
@@ -94,7 +99,7 @@ jobs:
     uses: ./.github/workflows/tests-rs-package.yml
     with:
       package: ${{ matrix.rs-package }}
-      # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-22.04"' }}
+      # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-24.04"' }}
       # FIXME: Clippy fails on github hosted runners, most likely due to RAM usage. Using self-hosted runners for now.
       # lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
       lint-runner: '["ubuntu-24.04"]'
@@ -106,7 +111,7 @@ jobs:
   rs-crates-security:
     name: Rust crates security audit
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
@@ -135,7 +140,7 @@ jobs:
   js-deps-versions:
     name: JS dependency versions check
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
@@ -154,7 +159,7 @@ jobs:
   js-npm-security:
     name: JS NPM security audit
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repo
         uses: actions/checkout@v4

From 691afb21ea7dbbeb3c515dd0fce6d10a6c056ce7 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Fri, 8 Nov 2024 10:38:01 +0100
Subject: [PATCH 25/25] chore: actions/checkout v4

---
 .github/workflows/tests-rs-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml
index e36d66dfa86..a05e68ee6ce 100644
--- a/.github/workflows/tests-rs-package.yml
+++ b/.github/workflows/tests-rs-package.yml
@@ -221,7 +221,7 @@ jobs:
     if: ${{ inputs.check-each-feature }}
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure AWS credentials and bucket region
         uses: aws-actions/configure-aws-credentials@v4