From 9ed4690af02e9bf62c6cbe52c39e6afc7e5e332f Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 12:01:36 -0400 Subject: [PATCH 1/9] Second attempt creating specialized caches. --- .circleci/config.yml | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c597797d3..2751c2c19 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,14 +11,25 @@ jobs: apt install -y build-essential zlib1g-dev uuid-dev libdigest-sha-perl libelf-dev bc bzip2 bison flex git gnupg iasl m4 nasm patch python wget gnat cpio ccache pkg-config cmake libusb-1.0-0-dev autoconf texinfo ncurses-dev - checkout - - restore_cache: - key: heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} - - run: name: git reset command: | git reset --hard "$CIRCLE_SHA1" \ + - run: + name: Creating all modules and patches digest + command: | + find ./patches/ ./modules/ | sort -h |xargs sha256sum > /tmp/all_modules_patches.sha256sums \ + + - restore_cache: + keys: + #Restore existing cache for modules checksums validated to be exactly the same as in github current commit + - heads_modules_and_patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit + - heads_cross-musl-{{ checksum "./modules/musl-cross" }} + #If precedent fails. Restore statically built cache on first successful build for this CI instance with CACHE_VERSION configured on CI + - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} + # linuxboot steps need something to pass in the kernel header path # skipping for now # - run: @@ -153,12 +164,26 @@ jobs: path: build/qemu-coreboot - save_cache: - key: heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} + #Generate generic cache on first run if hash doesn't exist + key: heads-{{ .Environment.CIRCLE_USERNAME }}{{ .Environment.CACHE_VERSION }} + paths: + - packages + - crossgcc + - build + - save_cache: + #Generate cache for the same musl-cross module definition if hash is not previously existing + key: heads_cross-musl-{{ checksum "modules/musl-cross" }} + paths: + - packages + - crossgcc + - build/musl-cross-* + - - save_cache: + #Generate cache for the exact same modules definitions if hash is not previously existing + key: heads_modules_and_patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} paths: - packages - crossgcc - build - workflows: version: 2 build_and_test: From 223d03ab216b940abd936e9edca22fe01dc33473 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 12:31:59 -0400 Subject: [PATCH 2/9] changing underscore for dashes in naming... --- .circleci/config.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2751c2c19..28427e0a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,9 +24,9 @@ jobs: - restore_cache: keys: #Restore existing cache for modules checksums validated to be exactly the same as in github current commit - - heads_modules_and_patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit - - heads_cross-musl-{{ checksum "./modules/musl-cross" }} + - heads-cross-musl-{{ checksum "./modules/musl-cross" }} #If precedent fails. Restore statically built cache on first successful build for this CI instance with CACHE_VERSION configured on CI - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} @@ -170,16 +170,18 @@ jobs: - packages - crossgcc - build + - save_cache: #Generate cache for the same musl-cross module definition if hash is not previously existing - key: heads_cross-musl-{{ checksum "modules/musl-cross" }} + key: heads-cross-musl-{{ checksum "modules/musl-cross" }} paths: - packages - crossgcc - build/musl-cross-* + - - save_cache: #Generate cache for the exact same modules definitions if hash is not previously existing - key: heads_modules_and_patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + key: heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} paths: - packages - crossgcc From abc593f542c0a3e907b00ff56c224fbcc3e39066 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 12:36:49 -0400 Subject: [PATCH 3/9] removing dashes at restore_cache keymap statements --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 28427e0a3..779d99a66 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,11 +24,11 @@ jobs: - restore_cache: keys: #Restore existing cache for modules checksums validated to be exactly the same as in github current commit - - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit - - heads-cross-musl-{{ checksum "./modules/musl-cross" }} + heads-cross-musl-{{ checksum "./modules/musl-cross" }} #If precedent fails. Restore statically built cache on first successful build for this CI instance with CACHE_VERSION configured on CI - - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} + heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} # linuxboot steps need something to pass in the kernel header path # skipping for now From d3fb00005ed40c9a1d0be950d05232e8163dea29 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 12:49:01 -0400 Subject: [PATCH 4/9] Simplifying and fixing a statement error. May partly revert this if this works --- .circleci/config.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 779d99a66..b0d8475e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,11 +24,9 @@ jobs: - restore_cache: keys: #Restore existing cache for modules checksums validated to be exactly the same as in github current commit - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} - #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit - heads-cross-musl-{{ checksum "./modules/musl-cross" }} + - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} #If precedent fails. Restore statically built cache on first successful build for this CI instance with CACHE_VERSION configured on CI - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} + - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} # linuxboot steps need something to pass in the kernel header path # skipping for now @@ -165,21 +163,13 @@ jobs: - save_cache: #Generate generic cache on first run if hash doesn't exist - key: heads-{{ .Environment.CIRCLE_USERNAME }}{{ .Environment.CACHE_VERSION }} + key: heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} paths: - packages - crossgcc - build - save_cache: - #Generate cache for the same musl-cross module definition if hash is not previously existing - key: heads-cross-musl-{{ checksum "modules/musl-cross" }} - paths: - - packages - - crossgcc - - build/musl-cross-* - - - - save_cache: #Generate cache for the exact same modules definitions if hash is not previously existing key: heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} paths: From a6c15129abb11d44ea88bb8ab6dc91e6167fcc67 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 12:51:22 -0400 Subject: [PATCH 5/9] Fixing only what was the real error, reverting to cache/restore desired logic. --- .circleci/config.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b0d8475e5..b369c1e6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,9 +24,11 @@ jobs: - restore_cache: keys: #Restore existing cache for modules checksums validated to be exactly the same as in github current commit - - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit + heads-cross-musl-{{ checksum "./modules/musl-cross" }} #If precedent fails. Restore statically built cache on first successful build for this CI instance with CACHE_VERSION configured on CI - - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} + heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} # linuxboot steps need something to pass in the kernel header path # skipping for now @@ -169,6 +171,14 @@ jobs: - crossgcc - build + - save_cache: + #Generate cache for the same musl-cross module definition if hash is not previously existing + key: heads-cross-musl-{{ checksum "modules/musl-cross" }} + paths: + - packages + - crossgcc + - build/musl-cross-* + - save_cache: #Generate cache for the exact same modules definitions if hash is not previously existing key: heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} From dde4b421d42dfc2887c7919f0d871369ac9aa3c8 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 13:03:10 -0400 Subject: [PATCH 6/9] find files only, no dir to respect pipefail --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b369c1e6a..ce7aae5af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,7 +19,7 @@ jobs: - run: name: Creating all modules and patches digest command: | - find ./patches/ ./modules/ | sort -h |xargs sha256sum > /tmp/all_modules_patches.sha256sums \ + find ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > /tmp/all_modules_patches.sha256sums \ - restore_cache: keys: From 676b3a560de747aead4e4b4e8aad8f4922a5b3e6 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 13:06:23 -0400 Subject: [PATCH 7/9] forgot dashes in front of the savecache statements... --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce7aae5af..dc32ea6ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,11 +24,11 @@ jobs: - restore_cache: keys: #Restore existing cache for modules checksums validated to be exactly the same as in github current commit - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit - heads-cross-musl-{{ checksum "./modules/musl-cross" }} + - heads-cross-musl-{{ checksum "./modules/musl-cross" }} #If precedent fails. Restore statically built cache on first successful build for this CI instance with CACHE_VERSION configured on CI - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} + - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} # linuxboot steps need something to pass in the kernel header path # skipping for now From f4957e7547a21fcbcda21540da33e840944924de Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 13:19:57 -0400 Subject: [PATCH 8/9] Replacing the generic cache bound to CircleCI user to have two caches levels. One for musl-cross and its patchsets, one for all modules and their patchset being the same. So if modules changed, we use the cache for musl-cross as a fallback to economize one hour of precious build time out of two, while most of Heads changes are on the scripts and can be built on top of packages+patches cache --- .circleci/config.yml | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc32ea6ea..ff9d1488a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,16 +19,20 @@ jobs: - run: name: Creating all modules and patches digest command: | - find ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > /tmp/all_modules_patches.sha256sums \ + find ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > /tmp/all_modules_and_patches.sha256sums \ + + - run: + name: Creating musl-cross-make and musl-cross-make patches digest + command: | + find ./patches/musl-cross-* modules/musl-cross* -type f | sort | xargs sha256sum > /tmp/musl-cross_module_and_patches.sha256sums \ + - restore_cache: keys: #Restore existing cache for modules checksums validated to be exactly the same as in github current commit - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit - - heads-cross-musl-{{ checksum "./modules/musl-cross" }} - #If precedent fails. Restore statically built cache on first successful build for this CI instance with CACHE_VERSION configured on CI - - heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} + - heads-cross-musl-{{ checksum "/tmp/musl-cross_module_and_patches.sha256sums" }} # linuxboot steps need something to pass in the kernel header path # skipping for now @@ -163,25 +167,16 @@ jobs: - store-artifacts: path: build/qemu-coreboot - - save_cache: - #Generate generic cache on first run if hash doesn't exist - key: heads-{{ .Environment.CIRCLE_USERNAME }}-{{ .Environment.CACHE_VERSION }} - paths: - - packages - - crossgcc - - build - - save_cache: #Generate cache for the same musl-cross module definition if hash is not previously existing - key: heads-cross-musl-{{ checksum "modules/musl-cross" }} + key: heads-cross-musl-{{ checksum "/tmp/musl-cross_module_and_patches.sha256sums" }} paths: - - packages - crossgcc - build/musl-cross-* - save_cache: #Generate cache for the exact same modules definitions if hash is not previously existing - key: heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + key: heads-modules-and-patches-{{ checksum "/tmp/all_modules_and_patches.sha256sums" }} paths: - packages - crossgcc From e427cc79da40820cd9ae9320b78900f14a23a152 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 11 Aug 2020 13:25:29 -0400 Subject: [PATCH 9/9] Correction of naming of sha256sum digests --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff9d1488a..0a0d96531 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,13 +24,13 @@ jobs: - run: name: Creating musl-cross-make and musl-cross-make patches digest command: | - find ./patches/musl-cross-* modules/musl-cross* -type f | sort | xargs sha256sum > /tmp/musl-cross_module_and_patches.sha256sums \ + find ./patches/musl-cross-* modules/musl-cross* -type f | sort -h | xargs sha256sum > /tmp/musl-cross_module_and_patches.sha256sums \ - restore_cache: keys: #Restore existing cache for modules checksums validated to be exactly the same as in github current commit - - heads-modules-and-patches-{{ checksum "/tmp/all_modules_patches.sha256sums" }} + - heads-modules-and-patches-{{ checksum "/tmp/all_modules_and_patches.sha256sums" }} #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit - heads-cross-musl-{{ checksum "/tmp/musl-cross_module_and_patches.sha256sums" }}