ci/packagespec (#9653)

* add packagespec build system

- The majority of changes in this commit are files generated
  by packagespec (everything in the packages-oss.lock directory).

* add .yamllint

* update to packagespec@fd54aea4

* ci: bust packagespec cache

- Change to packagespec results in package IDs that can use
  git tag refs, not just commit refs.

* update to packagepsec@5fc121d0

- This busts all caches, because of a change to the way we
  no longer traverse from tag refs to commit refs, due to
  the potential confusion this can cause.
- See hashicorp/packagespec@fd54aea
  for the original change to packagespec necessitating this.

* update to packagespec@5e6c87b6

- This completes the change to allowing git tag refs
  to be used for source IDs, begun in f130b940a8fbe3e9398225b08ea1d63420bef7b6

* update to packagespec@4d3c9e8b

- This busts cache, needed to apply previous change.

* remove RELEASE_BUILD_WORKFLOW_NAME

* update packagespec, add watch-ci target

* fix package names (do not refer to EDITION)

* remove EDITION input from packages-oss.yml

* bump package cache, update packagespec

* update packagespec, add 'aliases' target

* update packagespec; less output noise

* ci: give release bundle file a better name

- When performing a release build, this will include the build ID
  as part of the name, making it easier to distinguish from other
  builds.

* ci: create aliases tarball artifact

* ci: cache package metadata files

* ci: add metadata artifact

* ci: bust circleci package cache

* Revert "ci: bust circleci package cache"

This reverts commit 1320d182613466f0999d63f5742db12ac17f8e92.

* ci: remove aliases artifact

* ci: use buildID not workflowName to id artifacts

* packages: add BUNDLE_NAME metadata

* do not cache package metadata with binaries

* ci: bump package cache

* ci: debugging

* ci: fix package cache; update packagespec

* ci: update packagespec to 10e7beb2

* ci: write package metadata and aliases

* ci: switch to .zip artifacts

* switch package bundle back to tar.gz (from zip)

- Because of the way zip works, the zip archive was over 2GB rather than under 750MB as with tar.gz.

* bump packagespec, adds list-staged-builds

* update packagespec

* add publish stub + general tidy up

* bump packagespec

* bump packagespec; add make publish-config

* Makefile: tidy up packagespec targets

* pass PRODUCT_REPO_ROOT to packagespec

* bump go to 1.14.6

* packages-oss.yml: use more explicit base image

* bump packagespec to b899b7c1

* bump packagespec to f040ce8f

* packages-oss.yml: pin base image to digest

- This digest is pointed to by debian:buster-20200720
- Using a specific digest ensures that builds use the same
  base image in all contexts

* add release-repo; bump packagespec

* remove BUILD_TAGS and -tags flag

* bump packagespec to e444f742

* bump to go1.14.7

* ci: bump to go1.14.7

Author: samsalisbury

.circleci/config.yml

@@ -0,0 +1,229 @@
+{{- $data := (datasource "package-list") -}}
+{{- /*
+  BUILD_ID is set by the staging workflow to produce an identifiable build.
+*/ -}}
+{{- $buildID := (env.Getenv "BUILD_ID" "standalone") -}}
+{{- $workflowName := (env.Getenv "RELEASE_BUILD_WORKFLOW_NAME" "build-standalone") -}}
+{{- $packages := $data.packages -}}
+{{- $layers := $data.layers -}}
+{{- $revision := (env.Getenv "PRODUCT_REVISION") -}}
+{{- define "cache-key"}}{{template "cache-version"}}-{{.}}{{end -}}
+{{- define "cache-version"}}cache002{{end -}}
+{{- /*
+  Any change to cache-version invalidates all build layer and package caches.
+*/ -}}
+# Current cache version: {{template "cache-version"}}
+
+executors:
+  releaser:
+    docker:
+      - image: circleci/buildpack-deps
+    environment:
+      PRODUCT_REVISION: "{{if $revision}}{{$revision}}{{end}}"
+      AUTO_INSTALL_TOOLS: 'YES'
+    shell: /usr/bin/env bash -euo pipefail -c
+
+workflows:
+  {{$workflowName}}:
+    jobs:
+      - cache-builder-images:
+          filters:
+            branches:
+              only:
+                - /build-.*/
+                - /ci.*/
+      {{- range $packages}}
+      - {{.meta.BUILD_JOB_NAME}}: { requires: [ cache-builder-images ] }
+      {{- end }}
+      - bundle-releases:
+          requires:
+            {{- range $packages}}
+            - {{.meta.BUILD_JOB_NAME}}{{end}}
+jobs:
+  cache-builder-images:
+    executor: releaser
+    steps:
+      - setup_remote_docker
+      - checkout
+      - write-build-layer-cache-keys
+
+      # Load best available cached image.
+
+      {{- $targetLayerType := "build-static-assets" }}
+      {{- $targetLayer := .}}
+      {{- range $layers}}
+      {{- if eq .type $targetLayerType }}
+      {{- $targetLayer = .}}
+      - restore_cache:
+          keys:
+            {{- range .meta.circleci.CACHE_KEY_PREFIX_LIST}}
+            - {{template "cache-key" .}}
+            {{- end}}
+      - run:
+          name: Finish early if loaded exact match from cache.
+          command: |
+            if [ -f {{.archivefile}} ]; then
+              echo "Exact match found in cache, skipping build."
+              circleci-agent step halt
+            else
+              echo "No exact match found, proceeding with build."
+            fi
+      - run: LAYER_SPEC_ID={{.name}} make -C packages*.lock load-builder-cache
+      {{- end}}{{end}}
+
+      # No exact match was found, so build each layer up to target type.
+
+      {{- $finished := false }}
+      {{- range $layers}}
+      {{- if not $finished }}
+      {{- $finished = eq .type $targetLayerType}}
+      - run: make -f packages*.lock/layer.mk {{.name}}-image
+      {{- end}}
+      {{- end}}
+
+      # Save the target layer archive.
+      
+      - run: make -f packages*.lock/layer.mk {{$targetLayer.name}}-save
+      
+      # Store the target layer archive as all the relevant cache names.
+
+      {{- $lastArchive := $targetLayer.archivefile}}
+      {{- range $i, $l := $targetLayer.meta.builtin.LAYER_LIST}}
+      {{- $currentArchive := $l.archive}}
+      {{- if ne $currentArchive $lastArchive }}
+      - run: mv {{$lastArchive}} {{$currentArchive}}
+      {{- end}}
+      - save_cache:
+          key: {{template "cache-key" (index $targetLayer.meta.circleci.CACHE_KEY_PREFIX_LIST $i)}}
+          paths:
+            - {{$currentArchive}}
+      {{- $lastArchive = $currentArchive }}
+      {{- end}}
+
+{{- range $packages}}
+  {{.meta.BUILD_JOB_NAME}}:
+    executor: releaser
+    environment:
+      - PACKAGE_SPEC_ID: {{.packagespecid}}
+    steps:
+      - setup_remote_docker
+      - checkout
+
+      # Restore the package cache first, we might not need to rebuild.
+      - write-package-cache-key
+      - restore_cache:
+          key: '{{template "cache-key" .meta.circleci.PACKAGE_CACHE_KEY}}'
+      - run:
+          name: Check the cache status.
+          command: |
+            if ! { PKG=$(find .buildcache/packages/store -maxdepth 1 -mindepth 1 -name '*.zip' 2> /dev/null) && [ -n "$PKG" ]; }; then
+              echo "No package found, continuing with build."
+              exit 0
+            fi
+            echo "Package already cached, skipping build."
+            circleci-agent step halt
+
+      # We need to rebuild, so load the builder cache.
+      - write-build-layer-cache-keys
+      - restore_cache:
+          keys:
+          {{- range .meta.circleci.BUILDER_CACHE_KEY_PREFIX_LIST}}
+          - {{template "cache-key" .}}
+          {{- end}}
+      - run: make -C packages*.lock load-builder-cache
+      - run: make -C packages*.lock package
+      - run: ls -lahR .buildcache/packages
+      # Save package cache.
+      - save_cache:
+          key: '{{template "cache-key" .meta.circleci.PACKAGE_CACHE_KEY}}'
+          paths:
+            - .buildcache/packages/store
+      # Save builder image cache if necessary.
+      # The range should only iterate over a single layer.
+      {{- $pkg := . -}}
+      {{- range $idx, $layerInfo := .meta.builtin.BUILD_LAYERS }}
+      {{- if eq $layerInfo.type "warm-go-build-vendor-cache" }}
+      {{- with $layerInfo }}
+      {{- $circleCICacheKey := (index $pkg.meta.circleci.BUILDER_CACHE_KEY_PREFIX_LIST $idx) }}
+      - run:
+          name: Check builder cache status
+          command: |
+            if [ -f {{.archive}} ]; then
+              echo "Builder image already cached, skipping cache step."
+              circleci-agent step halt
+            fi
+      - run: make -f packages*.lock/layer.mk {{.name}}-save
+      - save_cache:
+          key: '{{template "cache-key" $circleCICacheKey}}'
+          paths:
+            - {{.archive}}
+      {{- end}}
+      {{- end}}
+      {{- end}}
+{{end}}
+
+  bundle-releases:
+    executor: releaser
+    steps:
+      - checkout
+      - write-all-package-cache-keys
+      {{- range $packages}}
+      - load-{{.meta.BUILD_JOB_NAME}}
+      - run:
+          environment:
+            PACKAGE_SPEC_ID: {{.packagespecid}}
+          name: Write package metadata for {{.meta.BUILD_JOB_NAME}}
+          command: |
+            make package-meta
+      {{- end}}
+      - run:
+          name: Write package aliases
+          command:
+            make aliases
+      - run:
+          name: List Build Cache
+          command: ls -lahR .buildcache
+
+      # Surface the package store directory as an artifact.
+      # This makes each zipped package separately downloadable.
+      - store_artifacts:
+          path: .buildcache/packages
+          destination: packages-{{$buildID}}
+
+      # Surface a tarball of the whole package store as an artifact.
+      - run: tar -czf packages-{{$buildID}}.tar.gz .buildcache/packages
+      - store_artifacts:
+          path: packages-{{$buildID}}.tar.gz
+          destination: packages-{{$buildID}}.tar.gz
+
+      # Surface a tarball of just the metadata files.
+      - run: tar -czf meta-{{$buildID}}.tar.gz .buildcache/packages/store/*.json
+      - store_artifacts:
+          path: meta-{{$buildID}}.tar.gz
+          destination: meta-{{$buildID}}.tar.gz
+
+commands:
+  {{- range $packages }}
+  load-{{.meta.BUILD_JOB_NAME}}:
+    steps:
+      - restore_cache:
+          key: '{{template "cache-key" .meta.circleci.PACKAGE_CACHE_KEY}}'
+  {{end}}
+
+  write-build-layer-cache-keys:
+    steps:
+      - run:
+          name: Write builder layer cache keys
+          command: make -C packages*.lock write-builder-cache-keys
+
+  write-package-cache-key:
+    steps:
+      - run:
+          name: Write package cache key
+          command: make -C packages*.lock write-package-cache-key
+
+  write-all-package-cache-keys:
+    steps:
+      - run:
+          name: Write package cache key
+          command: make -C packages*.lock write-all-package-cache-keys

.circleci/config/@build-release.yml

@@ -1,52 +1,6 @@
 ---
 version: 2.1
 
-references:
-  images:
-    node: &NODE_IMAGE node:10-buster # Pin Node.js to major version (ex: 10)
+orbs:
+  slack: circleci/slack@3.2.0
 
-  cache:
-    go-sum: &GO_SUM_CACHE_KEY go-sum-v1-{{ checksum "go.sum" }}
-    yarn-lock: &YARN_LOCK_CACHE_KEY yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
-
-# more commands defined in commands/
-commands:
-  restore_yarn_cache:
-    steps:
-      - restore_cache:
-          key: *YARN_LOCK_CACHE_KEY
-  save_yarn_cache:
-    steps:
-      - save_cache:
-          key: *YARN_LOCK_CACHE_KEY
-          paths:
-            - ui/node_modules
-
-executors:
-  go-machine:
-    machine: true
-    shell: /usr/bin/env bash -euo pipefail -c
-    environment:
-      GO111MODULE: "off"
-      CIRCLECI_CLI_VERSION: 0.1.5546  # Pin CircleCI CLI to patch version (ex: 1.2.3)
-      GO_VERSION: 1.14.7  # Pin Go to patch version (ex: 1.2.3)
-      GOTESTSUM_VERSION: 0.5.2  # Pin gotestsum to patch version (ex: 1.2.3)
-      GO_TAGS: ""
-    working_directory: /go/src/github.com/hashicorp/vault
-  node:
-    docker:
-      - image: *NODE_IMAGE
-        environment:
-            JOBS: 2
-    shell: /usr/bin/env bash -euo pipefail -c
-    working_directory: /go/src/github.com/hashicorp/vault
-  python:
-    docker:
-     - image: python:3-alpine
-    shell: /usr/bin/env bash -euo pipefail -c
-    working_directory: /go/src/github.com/hashicorp/vault
-  alpine:
-    docker:
-      - image: alpine:3.10.2
-    shell: /bin/sh
-    working_directory: /go/src/github.com/hashicorp/vault

.circleci/config/@build-release.yml.tpl

@@ -0,0 +1,10 @@
+restore_yarn_cache:
+  steps:
+    - restore_cache:
+        key: &YARN_LOCK_CACHE_KEY yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
+save_yarn_cache:
+  steps:
+    - save_cache:
+        key: *YARN_LOCK_CACHE_KEY
+        paths:
+          - ui/node_modules

.circleci/config/@config.yml

@@ -0,0 +1,25 @@
+go-machine:
+  machine: true
+  shell: /usr/bin/env bash -euo pipefail -c
+  environment:
+    GO111MODULE: "off"
+    CIRCLECI_CLI_VERSION: 0.1.5546  # Pin CircleCI CLI to patch version (ex: 1.2.3)
+    GO_VERSION: 1.14.7  # Pin Go to patch version (ex: 1.2.3)
+    GOTESTSUM_VERSION: 0.5.2  # Pin gotestsum to patch version (ex: 1.2.3)
+    GO_TAGS: ""
+  working_directory: /go/src/github.com/hashicorp/vault
+node:
+  docker:
+    - image: node:10-buster
+  shell: /usr/bin/env bash -euo pipefail -c
+  working_directory: /go/src/github.com/hashicorp/vault
+python:
+  docker:
+    - image: python:3-alpine
+  shell: /usr/bin/env bash -euo pipefail -c
+  working_directory: /go/src/github.com/hashicorp/vault
+alpine:
+  docker:
+    - image: alpine:3.10.2
+  shell: /bin/sh
+  working_directory: /go/src/github.com/hashicorp/vault

.circleci/config/commands/@yarn-cache.yml

@@ -0,0 +1,10 @@
+---
+
+extends: relaxed 
+
+rules:
+  comments: disable
+  comments-indentation: disable
+  line-length: disable
+  commas:
+  	max-spaces-after: -1