Convert STH to checkpoint format (sigstore#322)

* Convert STH to checkpoint format

This switches away from sending the (now deprecated) Trillian LogRootV1
format over to the checkpoint format documented at
https://github.com/google/trillian-examples/tree/master/formats/log

Fixes: sigstore#313

Signed-off-by: Bob Callaway <bob.callaway@gmail.com>

Author: bobcallaway

cmd/rekor-cli/app/log_info.go

@@ -22,7 +22,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"strings"
 	"time"
 
 	"github.com/google/trillian/merkle/logverifier"
@@ -34,7 +33,7 @@ import (
 	"github.com/sigstore/rekor/cmd/rekor-cli/app/state"
 	"github.com/sigstore/rekor/pkg/generated/client/tlog"
 	"github.com/sigstore/rekor/pkg/log"
-	"github.com/sigstore/rekor/pkg/verify"
+	"github.com/sigstore/rekor/pkg/util"
 )
 
 type logInfoCmdOutput struct {
@@ -72,14 +71,11 @@ var logInfoCmd = &cobra.Command{
 
 		logInfo := result.GetPayload()
 
-		logRoot := *logInfo.SignedTreeHead.LogRoot
-		if logRoot == nil {
-			return nil, errors.New("logroot should not be nil")
-		}
-		signature := *logInfo.SignedTreeHead.Signature
-		if signature == nil {
-			return nil, errors.New("signature should not be nil")
+		sth := util.RekorSTH{}
+		if err := sth.UnmarshalText([]byte(*logInfo.SignedTreeHead)); err != nil {
+			return nil, err
 		}
+
 		publicKey := viper.GetString("rekor_server_public_key")
 		if publicKey == "" {
 			// fetch key from server
@@ -100,33 +96,25 @@ var logInfoCmd = &cobra.Command{
 			return nil, err
 		}
 
-		lr, err := verify.SignedLogRoot(pub, logRoot, signature)
-		if err != nil {
-			return nil, err
+		if !sth.Verify(pub) {
+			return nil, errors.New("signature on tree head did not verify")
 		}
+
 		cmdOutput := &logInfoCmdOutput{
 			TreeSize:       *logInfo.TreeSize,
 			RootHash:       *logInfo.RootHash,
-			TimestampNanos: lr.TimestampNanos,
-		}
-
-		if lr.TreeSize != uint64(*logInfo.TreeSize) {
-			return nil, errors.New("tree size in signed tree head does not match value returned in API call")
-		}
-
-		if !strings.EqualFold(hex.EncodeToString(lr.RootHash), *logInfo.RootHash) {
-			return nil, errors.New("root hash in signed tree head does not match value returned in API call")
+			TimestampNanos: sth.GetTimestamp(),
 		}
 
 		oldState := state.Load(serverURL)
 		if oldState != nil {
-			persistedSize := oldState.TreeSize
-			if persistedSize < lr.TreeSize {
-				log.CliLogger.Infof("Found previous log state, proving consistency between %d and %d", oldState.TreeSize, lr.TreeSize)
+			persistedSize := oldState.Size
+			if persistedSize < sth.Size {
+				log.CliLogger.Infof("Found previous log state, proving consistency between %d and %d", oldState.Size, sth.Size)
 				params := tlog.NewGetLogProofParams()
 				firstSize := int64(persistedSize)
 				params.FirstSize = &firstSize
-				params.LastSize = int64(lr.TreeSize)
+				params.LastSize = int64(sth.Size)
 				proof, err := rekorClient.Tlog.GetLogProof(params)
 				if err != nil {
 					return nil, err
@@ -137,25 +125,25 @@ var logInfoCmd = &cobra.Command{
 					hashes = append(hashes, b)
 				}
 				v := logverifier.New(rfc6962.DefaultHasher)
-				if err := v.VerifyConsistencyProof(firstSize, int64(lr.TreeSize), oldState.RootHash,
-					lr.RootHash, hashes); err != nil {
+				if err := v.VerifyConsistencyProof(firstSize, int64(sth.Size), oldState.Hash,
+					sth.Hash, hashes); err != nil {
 					return nil, err
 				}
 				log.CliLogger.Infof("Consistency proof valid!")
-			} else if persistedSize == lr.TreeSize {
-				if !bytes.Equal(oldState.RootHash, lr.RootHash) {
+			} else if persistedSize == sth.Size {
+				if !bytes.Equal(oldState.Hash, sth.Hash) {
 					return nil, errors.New("root hash returned from server does not match previously persisted state")
 				}
 				log.CliLogger.Infof("Persisted log state matches the current state of the log")
-			} else if persistedSize > lr.TreeSize {
-				return nil, fmt.Errorf("current size of tree reported from server %d is less than previously persisted state %d", lr.TreeSize, persistedSize)
+			} else if persistedSize > sth.Size {
+				return nil, fmt.Errorf("current size of tree reported from server %d is less than previously persisted state %d", sth.Size, persistedSize)
 			}
 		} else {
 			log.CliLogger.Infof("No previous log state stored, unable to prove consistency")
 		}
 
 		if viper.GetBool("store_tree_state") {
-			if err := state.Dump(serverURL, lr); err != nil {
+			if err := state.Dump(serverURL, &sth); err != nil {
 				log.CliLogger.Infof("Unable to store previous state: %v", err)
 			}
 		}

cmd/rekor-cli/app/root.go

@@ -131,7 +131,10 @@ func GetRekorClient(rekorServerURL string) (*client.Rekor, error) {
 	if viper.GetString("api-key") != "" {
 		rt.DefaultAuthentication = httptransport.APIKeyAuth("apiKey", "query", viper.GetString("api-key"))
 	}
-	return client.New(rt, strfmt.Default), nil
+
+	registry := strfmt.Default
+	registry.Add("signedCheckpoint", &util.SignedCheckpoint{}, util.SignedCheckpointValidator)
+	return client.New(rt, registry), nil
 }
 
 type urlFlag struct {

cmd/rekor-cli/app/state/state.go

@@ -21,13 +21,13 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/google/trillian/types"
 	"github.com/mitchellh/go-homedir"
+	"github.com/sigstore/rekor/pkg/util"
 )
 
-type persistedState map[string]*types.LogRootV1
+type persistedState map[string]*util.RekorSTH
 
-func Dump(url string, lr *types.LogRootV1) error {
+func Dump(url string, sth *util.RekorSTH) error {
 	rekorDir, err := getRekorDir()
 	if err != nil {
 		return err
@@ -38,7 +38,7 @@ func Dump(url string, lr *types.LogRootV1) error {
 	if state == nil {
 		state = make(persistedState)
 	}
-	state[url] = lr
+	state[url] = sth
 
 	b, err := json.Marshal(&state)
 	if err != nil {
@@ -67,7 +67,7 @@ func loadStateFile() persistedState {
 	return result
 }
 
-func Load(url string) *types.LogRootV1 {
+func Load(url string) *util.RekorSTH {
 	if state := loadStateFile(); state != nil {
 		return state[url]
 	}

cmd/rekor-server/app/root.go

@@ -59,6 +59,7 @@ func init() {
 	rootCmd.PersistentFlags().String("trillian_log_server.address", "127.0.0.1", "Trillian log server address")
 	rootCmd.PersistentFlags().Uint16("trillian_log_server.port", 8090, "Trillian log server port")
 	rootCmd.PersistentFlags().Uint("trillian_log_server.tlog_id", 0, "Trillian tree id")
+	rootCmd.PersistentFlags().String("rekor_server.hostname", "rekor.sigstore.dev", "public hostname of instance")
 	rootCmd.PersistentFlags().String("rekor_server.address", "127.0.0.1", "Address to bind to")
 	rootCmd.PersistentFlags().String("rekor_server.signer", "memory", "Rekor signer to use. Current valid options include: [gcpkms, memory]")
 	rootCmd.PersistentFlags().String("rekor_server.timestamp_chain", "", "PEM encoded cert chain to use for timestamping")

cmd/rekor-server/app/watch.go

@@ -29,17 +29,15 @@ import (
 	_ "gocloud.dev/blob/fileblob" // fileblob
 	_ "gocloud.dev/blob/gcsblob"
 
-	"github.com/google/trillian/types"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"gocloud.dev/blob"
 
 	"github.com/sigstore/rekor/cmd/rekor-cli/app"
 	"github.com/sigstore/rekor/pkg/generated/client"
-	"github.com/sigstore/rekor/pkg/generated/models"
 	"github.com/sigstore/rekor/pkg/log"
-	"github.com/sigstore/rekor/pkg/verify"
+	"github.com/sigstore/rekor/pkg/util"
 )
 
 const rekorSthBucketEnv = "REKOR_STH_BUCKET"
@@ -109,10 +107,10 @@ var watchCmd = &cobra.Command{
 				log.Logger.Warnf("error verifiying tree: %s", err)
 				continue
 			}
-			log.Logger.Infof("Found and verified state at %d %d", lr.VerifiedLogRoot.TreeSize, lr.VerifiedLogRoot.TimestampNanos)
-			if last != nil && last.VerifiedLogRoot.TreeSize == lr.VerifiedLogRoot.TreeSize {
+			log.Logger.Infof("Found and verified state at %d", lr.VerifiedLogRoot.Size)
+			if last != nil && last.VerifiedLogRoot.Size == lr.VerifiedLogRoot.Size {
 				log.Logger.Infof("Last tree size is the same as the current one: %d %d",
-					last.VerifiedLogRoot.TreeSize, lr.VerifiedLogRoot.TreeSize)
+					last.VerifiedLogRoot.Size, lr.VerifiedLogRoot.Size)
 				// If it's the same, it shouldn't have changed but we'll still upload anyway
 				// in case that failed.
 			}
@@ -136,23 +134,17 @@ func doCheck(c *client.Rekor, pub crypto.PublicKey) (*SignedAndUnsignedLogRoot,
 	if err != nil {
 		return nil, errors.Wrap(err, "getting log info")
 	}
-	logRoot := *li.Payload.SignedTreeHead.LogRoot
-	if logRoot == nil {
-		return nil, errors.New("logroot should not be nil")
-	}
-	signature := *li.Payload.SignedTreeHead.Signature
-	if signature == nil {
-		return nil, errors.New("signature should not be nil")
+	sth := util.RekorSTH{}
+	if err := sth.UnmarshalText([]byte(*li.Payload.SignedTreeHead)); err != nil {
+		return nil, errors.Wrap(err, "unmarshalling tree head")
 	}
 
-	verifiedLogRoot, err := verify.SignedLogRoot(pub, logRoot, signature)
-	if err != nil {
-		return nil, errors.Wrap(err, "signing log root")
+	if !sth.Verify(pub) {
+		return nil, errors.Wrap(err, "signed tree head failed verification")
 	}
 
 	return &SignedAndUnsignedLogRoot{
-		SignedLogRoot:   li.GetPayload().SignedTreeHead,
-		VerifiedLogRoot: verifiedLogRoot,
+		VerifiedLogRoot: &sth,
 	}, nil
 }
 
@@ -162,7 +154,7 @@ func uploadToBlobStorage(ctx context.Context, bucket *blob.Bucket, lr *SignedAnd
 		return err
 	}
 
-	objName := fmt.Sprintf("sth-%d.json", lr.VerifiedLogRoot.TreeSize)
+	objName := fmt.Sprintf("sth-%d.json", lr.VerifiedLogRoot.Size)
 	w, err := bucket.NewWriter(ctx, objName, nil)
 	if err != nil {
 		return err
@@ -176,6 +168,5 @@ func uploadToBlobStorage(ctx context.Context, bucket *blob.Bucket, lr *SignedAnd
 
 // For JSON marshalling
 type SignedAndUnsignedLogRoot struct {
-	SignedLogRoot   *models.LogInfoSignedTreeHead
-	VerifiedLogRoot *types.LogRootV1
+	VerifiedLogRoot *util.RekorSTH
 }

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/go-openapi/swag v0.19.15
 	github.com/go-openapi/validate v0.20.2
 	github.com/go-playground/validator v9.31.0+incompatible
-	github.com/google/go-cmp v0.5.5
+	github.com/google/go-cmp v0.5.6
 	github.com/google/rpmpack v0.0.0-20210107155803-d6befbf05148
 	github.com/google/trillian v1.3.14-0.20210413093047-5e12fb368c8f
 	github.com/in-toto/in-toto-golang v0.1.1-0.20210528150343-f7dc21abaccf
@@ -40,6 +40,7 @@ require (
 	go.uber.org/zap v1.17.0
 	gocloud.dev v0.23.0
 	golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf
+	golang.org/x/mod v0.4.2
 	golang.org/x/net v0.0.0-20210505214959-0714010a04ed
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect

go.sum

@@ -568,8 +568,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-containerregistry v0.4.1 h1:Lrcj2AOoZ7WKawsoKAh2O0dH0tBqMW2lTEmozmK4Z3k=
 github.com/google/go-containerregistry v0.4.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0=
 github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=

openapi.yaml

@@ -444,25 +444,9 @@ definitions:
         description: The current number of nodes in the merkle tree
         minimum: 1
       signedTreeHead:
-        type: object
+        type: string
+        format: signedCheckpoint
         description: The current signed tree head
-        properties: 
-          keyHint:
-            type: string
-            description: Key hint
-            format: byte
-          logRoot:
-            type: string
-            description: Log root
-            format: byte
-          signature:
-            type: string
-            description: Signature for log root
-            format: byte
-        required:
-          - keyHint
-          - logRoot
-          - signature
     required:
       - rootHash
       - treeSize

pkg/api/error.go

@@ -45,6 +45,7 @@ const (
 	lastSizeGreaterThanKnown          = "The tree size requested(%d) was greater than what is currently observable(%d)"
 	signingError                      = "Error signing"
 	failedToGenerateTimestampResponse = "Error generating timestamp response"
+	sthGenerateError                  = "Error generating signed tree head"
 )
 
 func errorMsg(message string, code int) *models.Error {

pkg/api/tlog.go

@@ -16,17 +16,22 @@
 package api
 
 import (
+	"encoding/base64"
+	"encoding/binary"
 	"encoding/hex"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/go-openapi/runtime/middleware"
-	"github.com/go-openapi/strfmt"
 	"github.com/google/trillian/types"
+	"github.com/spf13/viper"
+	"golang.org/x/mod/sumdb/note"
 	"google.golang.org/grpc/codes"
 
 	"github.com/sigstore/rekor/pkg/generated/models"
 	"github.com/sigstore/rekor/pkg/generated/restapi/operations/tlog"
+	"github.com/sigstore/rekor/pkg/util"
 )
 
 // GetLogInfoHandler returns the current size of the tree and the STH
@@ -46,25 +51,42 @@ func GetLogInfoHandler(params tlog.GetLogInfoParams) middleware.Responder {
 
 	hashString := hex.EncodeToString(root.RootHash)
 	treeSize := int64(root.TreeSize)
-	logRoot := strfmt.Base64(result.SignedLogRoot.GetLogRoot())
+
+	sth := util.RekorSTH{
+		SignedCheckpoint: util.SignedCheckpoint{
+			Checkpoint: util.Checkpoint{
+				Ecosystem: "Rekor",
+				Size:      root.TreeSize,
+				Hash:      root.RootHash,
+			},
+		},
+	}
+	sth.SetTimestamp(uint64(time.Now().UnixNano()))
+	// TODO: once api.signer implements crypto.Signer, switch to using Sign() API on Checkpoint
 
 	// sign the log root ourselves to get the log root signature
-	sig, _, err := api.signer.Sign(params.HTTPRequest.Context(), result.SignedLogRoot.GetLogRoot())
+	cpString, _ := sth.Checkpoint.MarshalText()
+	sig, _, err := api.signer.Sign(params.HTTPRequest.Context(), []byte(cpString))
 	if err != nil {
-		return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("signing error: %w", err), trillianCommunicationError)
+		return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("signing error: %w", err), signingError)
 	}
 
-	signature := strfmt.Base64(sig)
+	sth.Signatures = append(sth.Signatures, note.Signature{
+		Name:   viper.GetString("rekor_server.hostname"),
+		Hash:   binary.BigEndian.Uint32([]byte(api.pubkeyHash)[0:4]),
+		Base64: base64.StdEncoding.EncodeToString(sig),
+	})
 
-	sth := models.LogInfoSignedTreeHead{
-		LogRoot:   &logRoot,
-		Signature: &signature,
+	scBytes, err := sth.MarshalText()
+	if err != nil {
+		return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("marshalling error: %w", err), sthGenerateError)
 	}
+	scString := string(scBytes)
 
 	logInfo := models.LogInfo{
 		RootHash:       &hashString,
 		TreeSize:       &treeSize,
-		SignedTreeHead: &sth,
+		SignedTreeHead: &scString,
 	}
 	return tlog.NewGetLogInfoOK().WithPayload(&logInfo)
 }