Add rekor harness tests

.github/workflows/main.yml

@@ -138,6 +138,8 @@ jobs:
     needs: build
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2
+    - name: Create git branch
+      run: git switch -c harness-test-branch
     - name: Extract version of Go to use
       run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV
     - uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v3.1.0

tests/harness_test.go

@@ -0,0 +1,211 @@
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/in-toto/in-toto-golang/in_toto"
+	slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2"
+	"github.com/secure-systems-lab/go-securesystemslib/dsse"
+	"github.com/sigstore/rekor/pkg/generated/models"
+	"github.com/sigstore/rekor/pkg/types"
+)
+
+// Make sure we can add an entry
+func TestHarnessAddEntry(t *testing.T) {
+	// Create a random artifact and sign it.
+	artifactPath := filepath.Join(t.TempDir(), "artifact")
+	sigPath := filepath.Join(t.TempDir(), "signature.asc")
+
+	createdX509SignedArtifact(t, artifactPath, sigPath)
+	dataBytes, _ := ioutil.ReadFile(artifactPath)
+	h := sha256.Sum256(dataBytes)
+	dataSHA := hex.EncodeToString(h[:])
+
+	// Write the public key to a file
+	pubPath := filepath.Join(t.TempDir(), "pubKey.asc")
+	if err := ioutil.WriteFile(pubPath, []byte(rsaCert), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify should fail initially
+	runCliErr(t, "verify", "--type=hashedrekord", "--pki-format=x509", "--artifact-hash", dataSHA, "--signature", sigPath, "--public-key", pubPath)
+
+	// It should upload successfully.
+	out := runCli(t, "upload", "--type=hashedrekord", "--pki-format=x509", "--artifact-hash", dataSHA, "--signature", sigPath, "--public-key", pubPath)
+	outputContains(t, out, "Created entry at")
+
+	// Now we should be able to verify it.
+	out = runCli(t, "verify", "--type=hashedrekord", "--pki-format=x509", "--artifact-hash", dataSHA, "--signature", sigPath, "--public-key", pubPath)
+	outputContains(t, out, "Inclusion Proof:")
+}
+
+// Make sure we can add an intoto entry
+func TestHarnessAddIntoto(t *testing.T) {
+	td := t.TempDir()
+	attestationPath := filepath.Join(td, "attestation.json")
+	pubKeyPath := filepath.Join(td, "pub.pem")
+
+	// Get some random data so it's unique each run
+	d := randomData(t, 10)
+	id := base64.StdEncoding.EncodeToString(d)
+
+	it := in_toto.ProvenanceStatement{
+		StatementHeader: in_toto.StatementHeader{
+			Type:          in_toto.StatementInTotoV01,
+			PredicateType: slsa.PredicateSLSAProvenance,
+			Subject: []in_toto.Subject{
+				{
+					Name: "foobar",
+					Digest: slsa.DigestSet{
+						"foo": "bar",
+					},
+				},
+			},
+		},
+		Predicate: slsa.ProvenancePredicate{
+			Builder: slsa.ProvenanceBuilder{
+				ID: "foo" + id,
+			},
+		},
+	}
+
+	b, err := json.Marshal(it)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pb, _ := pem.Decode([]byte(ecdsaPriv))
+	priv, err := x509.ParsePKCS8PrivateKey(pb.Bytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	signer, err := dsse.NewEnvelopeSigner(&IntotoSigner{
+		priv: priv.(*ecdsa.PrivateKey),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	env, err := signer.SignPayload("application/vnd.in-toto+json", b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	eb, err := json.Marshal(env)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	write(t, string(eb), attestationPath)
+	write(t, ecdsaPub, pubKeyPath)
+
+	// If we do it twice, it should already exist
+	out := runCli(t, "upload", "--artifact", attestationPath, "--type", "intoto", "--public-key", pubKeyPath)
+	outputContains(t, out, "Created entry at")
+	uuid := getUUIDFromUploadOutput(t, out)
+
+	out = runCli(t, "get", "--uuid", uuid, "--format=json")
+	g := getOut{}
+	if err := json.Unmarshal([]byte(out), &g); err != nil {
+		t.Fatal(err)
+	}
+	// The attestation should be stored at /var/run/attestations/sha256:digest
+
+	got := in_toto.ProvenanceStatement{}
+	if err := json.Unmarshal([]byte(g.Attestation), &got); err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(it, got); diff != "" {
+		t.Errorf("diff: %s", diff)
+	}
+
+	attHash := sha256.Sum256(b)
+
+	intotoModel := &models.IntotoV001Schema{}
+	if err := types.DecodeEntry(g.Body.(map[string]interface{})["IntotoObj"], intotoModel); err != nil {
+		t.Errorf("could not convert body into intoto type: %v", err)
+	}
+	if intotoModel.Content == nil || intotoModel.Content.PayloadHash == nil {
+		t.Errorf("could not find hash over attestation %v", intotoModel)
+	}
+	recordedPayloadHash, err := hex.DecodeString(*intotoModel.Content.PayloadHash.Value)
+	if err != nil {
+		t.Errorf("error converting attestation hash to []byte: %v", err)
+	}
+
+	if !bytes.Equal(attHash[:], recordedPayloadHash) {
+		t.Fatal(fmt.Errorf("attestation hash %v doesnt match the payload we sent %v", hex.EncodeToString(attHash[:]),
+			*intotoModel.Content.PayloadHash.Value))
+	}
+
+	out = runCli(t, "upload", "--artifact", attestationPath, "--type", "intoto", "--public-key", pubKeyPath)
+	outputContains(t, out, "Entry already exists")
+}
+
+// Make sure we can get and verify all entries
+// For attestations, make sure we can see the attestation
+func TestHarnessGetAllEntriesLogIndex(t *testing.T) {
+	treeSize := activeTreeSize(t)
+	if treeSize == 0 {
+		t.Fatal("There are 0 entries in the log, there should be at least 2")
+	}
+	for i := 0; i < treeSize; i++ {
+		out := runCli(t, "get", "--log-index", fmt.Sprintf("%d", i), "--format", "json")
+		if !strings.Contains(out, "IntotoObj") {
+			continue
+		}
+		var intotoObj struct {
+			Attestation string
+		}
+		if err := json.Unmarshal([]byte(out), &intotoObj); err != nil {
+			t.Fatal(err)
+		}
+		if intotoObj.Attestation == "" {
+			t.Log(out)
+			t.Fatalf("intotoObj attestation is empty for log index %d", i)
+		}
+		t.Log("Got IntotoObj type with attestation")
+	}
+}
+
+func activeTreeSize(t *testing.T) int {
+	out := runCliStdout(t, "loginfo", "--format", "json", "--store_tree_state", "false")
+	t.Log(string(out))
+	var s struct {
+		ActiveTreeSize int
+	}
+	if err := json.Unmarshal([]byte(out), &s); err != nil {
+		t.Fatal(err)
+	}
+	return s.ActiveTreeSize
+}

tests/rekor-harness.sh

@@ -15,33 +15,47 @@
 # limitations under the License.
 set -e
 
+TREE_ID=""
+
 function start_server () {
     server_version=$1
     current_branch=$(git rev-parse --abbrev-ref HEAD)
     git checkout $server_version
     if [ $(docker-compose ps | grep -c "(healthy)") == 0 ]; then
         echo "starting services with version $server_version"
         docker-compose up -d --build
+        sleep 30
+        make rekor-cli
+        export TREE_ID=$(./rekor-cli loginfo --format json --rekor_server http://localhost:3000 --store_tree_state=false | jq -r .TreeID)
     else
         echo "turning down rekor and restarting at version $server_version"
         docker stop $(docker ps --filter name=rekor-server --format {{.ID}})
+
+        # Replace log in docker-compose.yml with the Tree ID we want
+        search="# Uncomment this for production logging"
+        replace="\"--trillian_log_server.tlog_id=$TREE_ID\","
+        sed -i "s/$search/$replace/" docker-compose.yml
+
         docker-compose up -d --build rekor-server
     fi
-    git checkout $current_branch
 
     count=0
     echo -n "waiting up to 60 sec for system to start"
     until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
     do
         if [ $count -eq 6 ]; then
             echo "! timeout reached"
+            cat docker-compose.yml
+            docker-compose logs --no-color > /tmp/docker-compose.log
             exit 1
         else
             echo -n "."
             sleep 10
             let 'count+=1'
         fi
     done
+    git checkout $server_version .
+    git checkout $current_branch
     echo
 }
 
@@ -51,6 +65,7 @@ function build_cli () {
     current_branch=$(git rev-parse --abbrev-ref HEAD)
     git checkout $cli_version
     make rekor-cli
+    git checkout $cli_version .
     git checkout $current_branch
 }
 
@@ -59,30 +74,25 @@ function run_tests () {
     touch $REKORTMPDIR.rekor.yaml
     trap "rm -rf $REKORTMPDIR" EXIT
 
-
     go clean -testcache
-    for test in $HARNESS_TESTS
-    do
-        if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/ > $REKORTMPDIR/logs ; then 
-            cat $REKORTMPDIR/logs
-            docker-compose logs --no-color > /tmp/docker-compose.log
-            exit 1
-        fi
-        if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then
-            # if we're here, we found a panic
-            echo "Failing due to panics detected in logs"
-            docker-compose logs --no-color > /tmp/docker-compose.log
-            exit 1
-        fi
-    done
+    if ! REKORTMPDIR=$REKORTMPDIR go test -run TestHarness -v -tags=e2e ./tests/ ; then 
+        docker-compose logs --no-color > /tmp/docker-compose.log
+        exit 1
+    fi
+    if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then
+        # if we're here, we found a panic
+        echo "Failing due to panics detected in logs"
+        docker-compose logs --no-color > /tmp/docker-compose.log
+        exit 1
+    fi
 }
 
 # Get last 3 server versions
 git fetch origin
-VERSIONS=$(git tag --sort=-version:refname | head -n 3 | tac)
+NUM_VERSIONS_TO_TEST=3
+VERSIONS=$(git tag --sort=-version:refname | head -n $NUM_VERSIONS_TO_TEST | tac)
 echo $VERSIONS
 
-HARNESS_TESTS="TestUploadVerify TestLogInfo TestGetCLI TestSSH TestJAR TestAPK TestIntoto TestX509 TestEntryUpload"
 
 for server_version in $VERSIONS 
 do
@@ -100,4 +110,13 @@ do
     done
 done
 
+# Since we add two entries to the log for every test, once all tests are run we should have 2*($NUM_VERSIONS_TO_TEST^2) entries
+make rekor-cli
+actual=$(./rekor-cli loginfo --rekor_server http://localhost:3000 --format json --store_tree_state=false | jq -r .ActiveTreeSize)
+expected=$((2*$NUM_VERSIONS_TO_TEST*$NUM_VERSIONS_TO_TEST))
+if [[ ! "$expected" == "$actual" ]]; then
+    echo "ERROR: Log had $actual entries instead of expected $expected"
+    exit 1
+fi
+
 echo "Harness testing successful :)"

tests/util.go

@@ -74,6 +74,23 @@ func runCli(t *testing.T, arg ...string) string {
 	return run(t, "", cli, arg...)
 }
 
+func runCliStdout(t *testing.T, arg ...string) string {
+	t.Helper()
+	arg = append(arg, rekorServerFlag())
+	c := exec.Command(cli, arg...)
+
+	if os.Getenv("REKORTMPDIR") != "" {
+		// ensure that we use a clean state.json file for each run
+		c.Env = append(c.Env, "HOME="+os.Getenv("REKORTMPDIR"))
+	}
+	b, err := c.Output()
+	if err != nil {
+		t.Log(string(b))
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
 func runCliErr(t *testing.T, arg ...string) string {
 	t.Helper()
 	arg = append(arg, rekorServerFlag())