Set up matrix for testing

Author: priyawadhwa

.github/workflows/main.yml

@@ -133,30 +133,9 @@ jobs:
           name: Docker Compose logs
           path: /tmp/*docker-compose.log
 
-  get-versions:
-    runs-on: ubuntu-20.04
-    outputs:
-      versions: ${{ steps.versions.outputs.versions }}
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2
-    - name: get versions to test harness against
-      id: versions
-      shell: bash
-      run: |
-        git fetch origin
-        VERSIONS=$(git tag --sort=-version:refname | head -n 3 | jq --raw-input . | jq --slurp)
-        echo "Testing versions: $VERSIONS" 
-        echo "::set-output name=versions::$(echo $VERSIONS)"
-
   harness:
     runs-on: ubuntu-20.04
-    needs: [get-versions, build]
-    continue-on-error: true
-    strategy:
-        matrix: 
-          server-version: ${{fromJson(needs.get-versions.outputs.versions)}}
-    env:
-      SERVER_VERSION: ${{ matrix.server-version }}
+    needs: build
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2
     - name: Extract version of Go to use

.gitignore

@@ -17,3 +17,4 @@ rekorServerImagerefs
 rekorCliImagerefs
 trillianServerImagerefs
 trillianSignerImagerefs
+logs

tests/rekor-harness.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2021 The Sigstore Authors.
+# Copyright 2022 The Sigstore Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,58 +15,92 @@
 # limitations under the License.
 set -e
 
-if [ -z "$SERVER_VERSION" ]; then
-    echo "Please indicate which version of rekor to test against by setting SERVER_VERSION"
-    exit 1
-fi
+function start_server () {
+    server_version=$1
+    current_branch=$(git rev-parse --abbrev-ref HEAD)
+    git checkout $server_version
+    if [ $(docker-compose ps | grep -c "(healthy)") == 0 ]; then
+        echo "starting services with version $server_version"
+        docker-compose up -d --build
+    else
+        echo "turning down rekor and restarting at version $server_version"
+        docker stop $(docker ps --filter name=rekor-server --format {{.ID}})
+        docker-compose up -d --build rekor-server
+    fi
+    git checkout $current_branch
 
-HARNESS_TESTS="TestUploadVerify TestLogInfo TestGetCLI TestSSH TestJAR TestAPK TestIntoto TestX509 TestEntryUpload"
+    count=0
+    echo -n "waiting up to 60 sec for system to start"
+    until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
+    do
+        if [ $count -eq 6 ]; then
+            echo "! timeout reached"
+            exit 1
+        else
+            echo -n "."
+            sleep 10
+            let 'count+=1'
+        fi
+    done
+    echo
+}
 
-testdir=$(dirname "$0")
+function build_cli () {
+    echo "Building CLI at version $cli_version"
+    cli_version=$1
+    current_branch=$(git rev-parse --abbrev-ref HEAD)
+    git checkout $cli_version
+    go build -o rekor-cli ./cmd/rekor-cli
+    git checkout $current_branch
+}
 
-echo "building CLI and server"
-go build -o rekor-cli ./cmd/rekor-cli
+function run_tests () {
+    server_version=$1
+    cli_version=$2
 
-echo "starting services with version $SERVER_VERSION"
-git fetch origin
-current_branch=$(git rev-parse --abbrev-ref HEAD)
-git checkout $SERVER_VERSION
-docker-compose up -d --build
-git checkout $current_branch
+    REKORTMPDIR="$(mktemp -d -t rekor_test.XXXXXX)"
+    touch $REKORTMPDIR.rekor.yaml
+    trap "rm -rf $REKORTMPDIR" EXIT
 
-count=0
 
-echo -n "waiting up to 60 sec for system to start"
-until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
-do
-    if [ $count -eq 6 ]; then
-       echo "! timeout reached"
-       exit 1
-    else
-       echo -n "."
-       sleep 10
-       let 'count+=1'
-    fi
-done
+    go clean -testcache
+    for test in $HARNESS_TESTS
+    do
+        if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/ > logs ; then 
+            cat logs
+            docker-compose logs --no-color > /tmp/docker-compose.log
+            exit 1
+        fi
+        if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then
+            # if we're here, we found a panic
+            echo "Failing due to panics detected in logs"
+            docker-compose logs --no-color > /tmp/docker-compose.log
+            exit 1
+        fi
+    done
+}
 
-echo
-echo "running tests $HARNESS_TESTS"
-REKORTMPDIR="$(mktemp -d -t rekor_test.XXXXXX)"
-touch $REKORTMPDIR.rekor.yaml
-trap "rm -rf $REKORTMPDIR" EXIT
+# Get last 3 server versions
+git fetch origin
+VERSIONS=$(git tag --sort=-version:refname | head -n 3 | tac)
+echo $VERSIONS
 
+HARNESS_TESTS="TestUploadVerify TestLogInfo TestGetCLI TestSSH TestJAR TestAPK TestIntoto TestX509 TestEntryUpload"
 
-for test in $HARNESS_TESTS
+for server_version in $VERSIONS 
 do
-    echo $test
-    if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/; then 
-        docker-compose logs --no-color > /tmp/docker-compose.log
-        exit 1
-    fi
-    if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then
-        # if we're here, we found a panic
-        echo "Failing due to panics detected in logs"
-        docker-compose logs --no-color > /tmp/docker-compose.log
-        exit 1
-    fi
+    start_server $server_version
+    for cli_version in $VERSIONS 
+    do
+        echo "======================================================="
+        echo "Running tests with server version $server_version and CLI version $cli_version"
+
+        build_cli $cli_version
+        run_tests $server_version $cli_version
+
+        echo "Tests passed successfully."
+        echo "======================================================="
+    done
 done
+
+echo "Harness testing successful :)"