From 5902273325b4a00a75a2b14d648b1f5219fc508c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 13 Jul 2022 18:13:39 +0100 Subject: [PATCH 1/3] Add rekor test harness to presubmit tests This will test critical user journeys against the last three versions of rekor released. Signed-off-by: Priya Wadhwa --- .github/workflows/main.yml | 40 +++++++++++++++++++++ tests/e2e_test.go | 2 +- tests/rekor-harness.sh | 72 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100755 tests/rekor-harness.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 161d9a9c8..d34a47bbb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,3 +132,43 @@ jobs: with: name: Docker Compose logs path: /tmp/*docker-compose.log + + get-versions: + runs-on: ubuntu-20.04 + outputs: + versions: ${{ steps.versions.outputs.versions }} + steps: + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2 + - name: get versions to test harness against + id: versions + shell: bash + run: | + git fetch origin + VERSIONS=$(git tag --sort=-version:refname | head -n 3 | jq --raw-input . | jq --slurp) + echo "Testing versions: $VERSIONS" + echo "::set-output name=versions::$(echo $VERSIONS)" + + harness: + runs-on: ubuntu-20.04 + needs: [get-versions, build] + continue-on-error: true + strategy: + matrix: + server-version: ${{fromJson(needs.get-versions.outputs.versions)}} + env: + SERVER_VERSION: ${{ matrix.server-version }} + steps: + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2 + - name: Extract version of Go to use + run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV + - uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v3.1.0 + with: + go-version: ${{ env.GOVERSION }} + - name: Run test harness + run: ./tests/rekor-harness.sh + - name: Upload logs if they exist + uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3 + if: failure() + with: + name: E2E Docker Compose logs + path: /tmp/docker-compose.log diff --git a/tests/e2e_test.go b/tests/e2e_test.go index 5810c292c..62a826b3b 100644 --- a/tests/e2e_test.go +++ b/tests/e2e_test.go @@ -203,7 +203,7 @@ type getOut struct { IntegratedTime int64 } -func TestGet(t *testing.T) { +func TestGetCLI(t *testing.T) { // Create something and add it to the log artifactPath := filepath.Join(t.TempDir(), "artifact") sigPath := filepath.Join(t.TempDir(), "signature.asc") diff --git a/tests/rekor-harness.sh b/tests/rekor-harness.sh new file mode 100755 index 000000000..27bdea9f8 --- /dev/null +++ b/tests/rekor-harness.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Copyright 2021 The Sigstore Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +if [ -z "$SERVER_VERSION" ]; then + echo "Please indicate which version of rekor to test against by setting SERVER_VERSION" + exit 1 +fi + +HARNESS_TESTS="TestUploadVerify TestLogInfo TestGetCLI TestSSH TestJAR TestAPK TestIntoto TestX509 TestEntryUpload" + +testdir=$(dirname "$0") + +echo "building CLI and server" +go build -o rekor-cli ./cmd/rekor-cli + +echo "starting services with version $SERVER_VERSION" +git fetch origin +current_branch=$(git rev-parse --abbrev-ref HEAD) +git checkout $SERVER_VERSION +docker-compose up -d --build +git checkout $current_branch + +count=0 + +echo -n "waiting up to 60 sec for system to start" +until [ $(docker-compose ps | grep -c "(healthy)") == 3 ]; +do + if [ $count -eq 6 ]; then + echo "! timeout reached" + exit 1 + else + echo -n "." + sleep 10 + let 'count+=1' + fi +done + +echo +echo "running tests $HARNESS_TESTS" +REKORTMPDIR="$(mktemp -d -t rekor_test.XXXXXX)" +touch $REKORTMPDIR.rekor.yaml +trap "rm -rf $REKORTMPDIR" EXIT + + +for test in $HARNESS_TESTS +do + echo $test + if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/; then + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 + fi + if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then + # if we're here, we found a panic + echo "Failing due to panics detected in logs" + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 + fi +done From 026d608ae3605683a4f51cf55610e2e933d8b546 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 18 Jul 2022 13:23:04 -0700 Subject: [PATCH 2/3] Set up matrix for testing Signed-off-by: Priya Wadhwa --- .github/workflows/main.yml | 23 +------ .gitignore | 1 + tests/rekor-harness.sh | 121 +++++++++++++++++++++++-------------- 3 files changed, 78 insertions(+), 67 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d34a47bbb..312a808ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -133,30 +133,9 @@ jobs: name: Docker Compose logs path: /tmp/*docker-compose.log - get-versions: - runs-on: ubuntu-20.04 - outputs: - versions: ${{ steps.versions.outputs.versions }} - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2 - - name: get versions to test harness against - id: versions - shell: bash - run: | - git fetch origin - VERSIONS=$(git tag --sort=-version:refname | head -n 3 | jq --raw-input . | jq --slurp) - echo "Testing versions: $VERSIONS" - echo "::set-output name=versions::$(echo $VERSIONS)" - harness: runs-on: ubuntu-20.04 - needs: [get-versions, build] - continue-on-error: true - strategy: - matrix: - server-version: ${{fromJson(needs.get-versions.outputs.versions)}} - env: - SERVER_VERSION: ${{ matrix.server-version }} + needs: build steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2 - name: Extract version of Go to use diff --git a/.gitignore b/.gitignore index f5ff5d8b8..b04fee058 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ rekorServerImagerefs rekorCliImagerefs trillianServerImagerefs trillianSignerImagerefs +logs diff --git a/tests/rekor-harness.sh b/tests/rekor-harness.sh index 27bdea9f8..9adb0afd2 100755 --- a/tests/rekor-harness.sh +++ b/tests/rekor-harness.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2021 The Sigstore Authors. +# Copyright 2022 The Sigstore Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,58 +15,89 @@ # limitations under the License. set -e -if [ -z "$SERVER_VERSION" ]; then - echo "Please indicate which version of rekor to test against by setting SERVER_VERSION" - exit 1 -fi +function start_server () { + server_version=$1 + current_branch=$(git rev-parse --abbrev-ref HEAD) + git checkout $server_version + if [ $(docker-compose ps | grep -c "(healthy)") == 0 ]; then + echo "starting services with version $server_version" + docker-compose up -d --build + else + echo "turning down rekor and restarting at version $server_version" + docker stop $(docker ps --filter name=rekor-server --format {{.ID}}) + docker-compose up -d --build rekor-server + fi + git checkout $current_branch -HARNESS_TESTS="TestUploadVerify TestLogInfo TestGetCLI TestSSH TestJAR TestAPK TestIntoto TestX509 TestEntryUpload" + count=0 + echo -n "waiting up to 60 sec for system to start" + until [ $(docker-compose ps | grep -c "(healthy)") == 3 ]; + do + if [ $count -eq 6 ]; then + echo "! timeout reached" + exit 1 + else + echo -n "." + sleep 10 + let 'count+=1' + fi + done + echo +} + +function build_cli () { + echo "Building CLI at version $cli_version" + cli_version=$1 + current_branch=$(git rev-parse --abbrev-ref HEAD) + git checkout $cli_version + go build -o rekor-cli ./cmd/rekor-cli + git checkout $current_branch +} -testdir=$(dirname "$0") +function run_tests () { + REKORTMPDIR="$(mktemp -d -t rekor_test.XXXXXX)" + touch $REKORTMPDIR.rekor.yaml + trap "rm -rf $REKORTMPDIR" EXIT -echo "building CLI and server" -go build -o rekor-cli ./cmd/rekor-cli -echo "starting services with version $SERVER_VERSION" + go clean -testcache + for test in $HARNESS_TESTS + do + if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/ > logs ; then + cat logs + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 + fi + if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then + # if we're here, we found a panic + echo "Failing due to panics detected in logs" + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 + fi + done +} + +# Get last 3 server versions git fetch origin -current_branch=$(git rev-parse --abbrev-ref HEAD) -git checkout $SERVER_VERSION -docker-compose up -d --build -git checkout $current_branch +VERSIONS=$(git tag --sort=-version:refname | head -n 3 | tac) +echo $VERSIONS -count=0 +HARNESS_TESTS="TestUploadVerify TestLogInfo TestGetCLI TestSSH TestJAR TestAPK TestIntoto TestX509 TestEntryUpload" -echo -n "waiting up to 60 sec for system to start" -until [ $(docker-compose ps | grep -c "(healthy)") == 3 ]; +for server_version in $VERSIONS do - if [ $count -eq 6 ]; then - echo "! timeout reached" - exit 1 - else - echo -n "." - sleep 10 - let 'count+=1' - fi -done + start_server $server_version + for cli_version in $VERSIONS + do + echo "=======================================================" + echo "Running tests with server version $server_version and CLI version $cli_version" -echo -echo "running tests $HARNESS_TESTS" -REKORTMPDIR="$(mktemp -d -t rekor_test.XXXXXX)" -touch $REKORTMPDIR.rekor.yaml -trap "rm -rf $REKORTMPDIR" EXIT + build_cli $cli_version + run_tests - -for test in $HARNESS_TESTS -do - echo $test - if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/; then - docker-compose logs --no-color > /tmp/docker-compose.log - exit 1 - fi - if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then - # if we're here, we found a panic - echo "Failing due to panics detected in logs" - docker-compose logs --no-color > /tmp/docker-compose.log - exit 1 - fi + echo "Tests passed successfully." + echo "=======================================================" + done done + +echo "Harness testing successful :)" From 603cc15dc0194d3f27bdf2df0cb90a4ba92b4663 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 25 Jul 2022 15:23:09 -0700 Subject: [PATCH 3/3] Code review comments Signed-off-by: Priya Wadhwa --- .gitignore | 1 - tests/rekor-harness.sh | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index b04fee058..f5ff5d8b8 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,3 @@ rekorServerImagerefs rekorCliImagerefs trillianServerImagerefs trillianSignerImagerefs -logs diff --git a/tests/rekor-harness.sh b/tests/rekor-harness.sh index 9adb0afd2..13c4c5f80 100755 --- a/tests/rekor-harness.sh +++ b/tests/rekor-harness.sh @@ -50,7 +50,7 @@ function build_cli () { cli_version=$1 current_branch=$(git rev-parse --abbrev-ref HEAD) git checkout $cli_version - go build -o rekor-cli ./cmd/rekor-cli + make rekor-cli git checkout $current_branch } @@ -63,8 +63,8 @@ function run_tests () { go clean -testcache for test in $HARNESS_TESTS do - if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/ > logs ; then - cat logs + if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/ > $REKORTMPDIR/logs ; then + cat $REKORTMPDIR/logs docker-compose logs --no-color > /tmp/docker-compose.log exit 1 fi