name stored attestations by digest instead of UUID (sigstore#769)

bobcallaway · web-flow · commit e375eb461cae · 2022-05-24T12:30:48.000-04:00
* store attestations by digest instead of by UUID

Signed-off-by: Bob Callaway &lt;bcallaway@google.com&gt;

* fix typo

Signed-off-by: Bob Callaway &lt;bcallaway@google.com&gt;

* remove github.com/pkg/errors

Signed-off-by: Bob Callaway &lt;bcallaway@google.com&gt;
diff --git a/pkg/api/entries.go b/pkg/api/entries.go
@@ -26,6 +26,7 @@ import (
 	"strconv"
 
 	"github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer"
+	"github.com/go-openapi/runtime"
 	"github.com/go-openapi/runtime/middleware"
 	"github.com/go-openapi/strfmt"
 	"github.com/go-openapi/swag"
@@ -97,12 +98,35 @@ func logEntryFromLeaf(ctx context.Context, signer signature.Signer, tc TrillianC
 
 	uuid := hex.EncodeToString(leaf.MerkleLeafHash)
 	if viper.GetBool("enable_attestation_storage") {
-		att, err := storageClient.FetchAttestation(ctx, uuid)
+		pe, err := models.UnmarshalProposedEntry(bytes.NewReader(leaf.LeafValue), runtime.JSONConsumer())
 		if err != nil {
-			log.Logger.Errorf("error fetching attestation: %s %s", uuid, err)
-		} else {
-			logEntryAnon.Attestation = &models.LogEntryAnonAttestation{
-				Data: att,
+			return nil, err
+		}
+		eimpl, err := types.NewEntry(pe)
+		if err != nil {
+			return nil, err
+		}
+		attKey := eimpl.AttestationKey()
+		if attKey != "" {
+			att, err := storageClient.FetchAttestation(ctx, attKey)
+			if err != nil {
+				log.Logger.Errorf("error fetching attestation by key, trying by UUID: %s %s", attKey, err)
+				// the original attestation implementation stored this by uuid instead of by digest
+				activeTree := fmt.Sprintf("%x", tc.logID)
+				entryIDstruct, err := sharding.CreateEntryIDFromParts(activeTree, uuid)
+				if err != nil {
+					err := fmt.Errorf("error creating EntryID from active treeID %v and uuid %v: %w", activeTree, uuid, err)
+					return nil, err
+				}
+				att, err = storageClient.FetchAttestation(ctx, entryIDstruct.UUID)
+				if err != nil {
+					log.Logger.Errorf("error fetching attestation by uuid: %s %s", entryIDstruct.UUID, err)
+				}
+			}
+			if err == nil {
+				logEntryAnon.Attestation = &models.LogEntryAnonAttestation{
+					Data: att,
+				}
 			}
 		}
 	}
@@ -224,14 +248,16 @@ func createLogEntry(params entries.CreateLogEntryParams) (models.LogEntry, middl
 	if viper.GetBool("enable_attestation_storage") {
 
 		go func() {
-			attestation := entry.Attestation()
-			if attestation == nil {
-				log.RequestIDLogger(params.HTTPRequest).Infof("no attestation for %s", entryID)
+			attKey, attVal := entry.AttestationKeyValue()
+			if attVal == nil {
+				log.RequestIDLogger(params.HTTPRequest).Infof("no attestation for %s", uuid)
 				return
 			}
-			// TODO stop using uuid and use attestation hash
-			if err := storeAttestation(context.Background(), entryIDstruct.UUID, attestation); err != nil {
+			if err := storeAttestation(context.Background(), attKey, attVal); err != nil {
+				// entryIDstruct.UUID
 				log.RequestIDLogger(params.HTTPRequest).Errorf("error storing attestation: %s", err)
+			} else {
+				log.RequestIDLogger(params.HTTPRequest).Infof("stored attestation for uuid %s with filename %s", entryIDstruct.UUID, attKey)
 			}
 		}()
 	}
diff --git a/pkg/types/alpine/v0.0.1/entry.go b/pkg/types/alpine/v0.0.1/entry.go
@@ -287,8 +287,12 @@ func (v V001Entry) validate() error {
 	return nil
 }
 
-func (v V001Entry) Attestation() []byte {
-	return nil
+func (v V001Entry) AttestationKey() string {
+	return ""
+}
+
+func (v V001Entry) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/pkg/types/entries.go b/pkg/types/entries.go
@@ -35,7 +35,8 @@ type EntryImpl interface {
 	IndexKeys() ([]string, error)                     // the keys that should be added to the external index for this entry
 	Canonicalize(ctx context.Context) ([]byte, error) // marshal the canonical entry to be put into the tlog
 	Unmarshal(e models.ProposedEntry) error           // unmarshal the abstract entry into the specific struct for this versioned type
-	Attestation() []byte
+	AttestationKey() string                           // returns the key used to look up the attestation from storage (should be sha256:digest)
+	AttestationKeyValue() (string, []byte)            // returns the key to be used when storing the attestation as well as the attestation itself
 	CreateFromArtifactProperties(context.Context, ArtifactProperties) (models.ProposedEntry, error)
 }
 
diff --git a/pkg/types/hashedrekord/v0.0.1/entry.go b/pkg/types/hashedrekord/v0.0.1/entry.go
@@ -183,8 +183,12 @@ func (v *V001Entry) validate() (pki.Signature, pki.PublicKey, error) {
 	return sigObj, keyObj, nil
 }
 
-func (v V001Entry) Attestation() []byte {
-	return nil
+func (v V001Entry) AttestationKey() string {
+	return ""
+}
+
+func (v V001Entry) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/pkg/types/helm/v0.0.1/entry.go b/pkg/types/helm/v0.0.1/entry.go
@@ -280,8 +280,12 @@ func (v V001Entry) validate() error {
 	return nil
 }
 
-func (v V001Entry) Attestation() []byte {
-	return nil
+func (v V001Entry) AttestationKey() string {
+	return ""
+}
+
+func (v V001Entry) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/pkg/types/intoto/v0.0.1/entry.go b/pkg/types/intoto/v0.0.1/entry.go
@@ -182,16 +182,11 @@ func (v *V001Entry) Canonicalize(ctx context.Context) ([]byte, error) {
 			},
 		},
 	}
-	attestation := v.Attestation()
-	if attestation != nil {
-		decodedAttestation, err := base64.StdEncoding.DecodeString(string(attestation))
-		if err != nil {
-			return nil, fmt.Errorf("decoding attestation: %w", err)
-		}
-		attH := sha256.Sum256(decodedAttestation)
+	attKey, attValue := v.AttestationKeyValue()
+	if attValue != nil {
 		canonicalEntry.Content.PayloadHash = &models.IntotoV001SchemaContentPayloadHash{
 			Algorithm: swag.String(models.IntotoV001SchemaContentHashAlgorithmSha256),
-			Value:     swag.String(hex.EncodeToString(attH[:])),
+			Value:     swag.String(strings.Replace(attKey, fmt.Sprintf("%s:", models.IntotoV001SchemaContentHashAlgorithmSha256), "", 1)),
 		}
 	}
 
@@ -237,13 +232,25 @@ func (v *V001Entry) validate() error {
 	return nil
 }
 
-func (v *V001Entry) Attestation() []byte {
+// AttestationKey returns the digest of the attestation that was uploaded, to be used to lookup the attestation from storage
+func (v *V001Entry) AttestationKey() string {
+	if v.IntotoObj.Content != nil && v.IntotoObj.Content.PayloadHash != nil {
+		return fmt.Sprintf("%s:%s", *v.IntotoObj.Content.PayloadHash.Algorithm, *v.IntotoObj.Content.PayloadHash.Value)
+	}
+	return ""
+}
+
+// AttestationKeyValue returns both the key and value to be persisted into attestation storage
+func (v *V001Entry) AttestationKeyValue() (string, []byte) {
 	storageSize := base64.StdEncoding.DecodedLen(len(v.env.Payload))
 	if storageSize > viper.GetInt("max_attestation_size") {
 		log.Logger.Infof("Skipping attestation storage, size %d is greater than max %d", storageSize, viper.GetInt("max_attestation_size"))
-		return nil
+		return "", nil
 	}
-	return []byte(v.env.Payload)
+	attBytes, _ := base64.StdEncoding.DecodeString(v.env.Payload)
+	attHash := sha256.Sum256(attBytes)
+	attKey := fmt.Sprintf("%s:%s", models.IntotoV001SchemaContentHashAlgorithmSha256, hex.EncodeToString(attHash[:]))
+	return attKey, attBytes
 }
 
 type verifier struct {
diff --git a/pkg/types/jar/v0.0.1/entry.go b/pkg/types/jar/v0.0.1/entry.go
@@ -271,8 +271,12 @@ func extractPKCS7SignatureFromJAR(inz *zip.Reader) ([]byte, error) {
 	return nil, errors.New("unable to locate signature in JAR file")
 }
 
-func (v *V001Entry) Attestation() []byte {
-	return nil
+func (v V001Entry) AttestationKey() string {
+	return ""
+}
+
+func (v V001Entry) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (v *V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/pkg/types/rekord/v0.0.1/entry.go b/pkg/types/rekord/v0.0.1/entry.go
@@ -332,8 +332,12 @@ func (v V001Entry) validate() error {
 	return nil
 }
 
-func (v V001Entry) Attestation() []byte {
-	return nil
+func (v V001Entry) AttestationKey() string {
+	return ""
+}
+
+func (v V001Entry) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/pkg/types/rfc3161/v0.0.1/entry.go b/pkg/types/rfc3161/v0.0.1/entry.go
@@ -173,8 +173,12 @@ func (v V001Entry) validate() error {
 	return nil
 }
 
-func (v V001Entry) Attestation() []byte {
-	return nil
+func (v V001Entry) AttestationKey() string {
+	return ""
+}
+
+func (v V001Entry) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (v V001Entry) CreateFromArtifactProperties(_ context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/pkg/types/rpm/v0.0.1/entry.go b/pkg/types/rpm/v0.0.1/entry.go
@@ -307,8 +307,12 @@ func (v V001Entry) validate() error {
 	return nil
 }
 
-func (v V001Entry) Attestation() []byte {
-	return nil
+func (v V001Entry) AttestationKey() string {
+	return ""
+}
+
+func (v V001Entry) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/pkg/types/test_util.go b/pkg/types/test_util.go
@@ -48,8 +48,12 @@ func (u BaseUnmarshalTester) Validate() error {
 	return nil
 }
 
-func (u BaseUnmarshalTester) Attestation() []byte {
-	return nil
+func (u BaseUnmarshalTester) AttestationKey() string {
+	return ""
+}
+
+func (u BaseUnmarshalTester) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (u BaseUnmarshalTester) CreateFromArtifactProperties(_ context.Context, _ ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/pkg/types/tuf/v0.0.1/entry.go b/pkg/types/tuf/v0.0.1/entry.go
@@ -294,8 +294,12 @@ func (v V001Entry) Validate() error {
 	return nil
 }
 
-func (v *V001Entry) Attestation() []byte {
-	return nil
+func (v V001Entry) AttestationKey() string {
+	return ""
+}
+
+func (v V001Entry) AttestationKeyValue() (string, []byte) {
+	return "", nil
 }
 
 func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {
diff --git a/tests/e2e_test.go b/tests/e2e_test.go
@@ -196,7 +196,7 @@ func TestLogInfo(t *testing.T) {
 }
 
 type getOut struct {
-	Attestation     []byte
+	Attestation     string
 	AttestationType string
 	Body            interface{}
 	LogIndex        int
@@ -447,17 +447,17 @@ func TestIntoto(t *testing.T) {
 	if err := json.Unmarshal([]byte(out), &g); err != nil {
 		t.Fatal(err)
 	}
-	// The attestation should be stored at /var/run/attestations/$uuid
+	// The attestation should be stored at /var/run/attestations/sha256:digest
 
 	got := in_toto.ProvenanceStatement{}
-	if err := json.Unmarshal(g.Attestation, &got); err != nil {
+	if err := json.Unmarshal([]byte(g.Attestation), &got); err != nil {
 		t.Fatal(err)
 	}
 	if diff := cmp.Diff(it, got); diff != "" {
 		t.Errorf("diff: %s", diff)
 	}
 
-	attHash := sha256.Sum256(g.Attestation)
+	attHash := sha256.Sum256(b)
 
 	intotoModel := &models.IntotoV001Schema{}
 	if err := types.DecodeEntry(g.Body.(map[string]interface{})["IntotoObj"], intotoModel); err != nil {

Original file line number	Diff line number	Diff line change
`@@ -287,8 +287,12 @@ func (v V001Entry) validate() error {`
`287`	`287`	`return nil`
`288`	`288`	`}`
`289`	`289`
`290`		`-func (v V001Entry) Attestation() []byte {`
`291`		`- return nil`
	`290`	`+func (v V001Entry) AttestationKey() string {`
	`291`	`+ return ""`
	`292`	`+}`
	`293`	`+`
	`294`	`+func (v V001Entry) AttestationKeyValue() (string, []byte) {`
	`295`	`+ return "", nil`
`292`	`296`	`}`
`293`	`297`
`294`	`298`	`func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,8 @@ type EntryImpl interface {`
`35`	`35`	`IndexKeys() ([]string, error) // the keys that should be added to the external index for this entry`
`36`	`36`	`Canonicalize(ctx context.Context) ([]byte, error) // marshal the canonical entry to be put into the tlog`
`37`	`37`	`Unmarshal(e models.ProposedEntry) error // unmarshal the abstract entry into the specific struct for this versioned type`
`38`		`- Attestation() []byte`
	`38`	`+ AttestationKey() string // returns the key used to look up the attestation from storage (should be sha256:digest)`
	`39`	`+ AttestationKeyValue() (string, []byte) // returns the key to be used when storing the attestation as well as the attestation itself`
`39`	`40`	`CreateFromArtifactProperties(context.Context, ArtifactProperties) (models.ProposedEntry, error)`
`40`	`41`	`}`
`41`	`42`
Original file line number	Diff line number	Diff line change
`@@ -183,8 +183,12 @@ func (v *V001Entry) validate() (pki.Signature, pki.PublicKey, error) {`
`183`	`183`	`return sigObj, keyObj, nil`
`184`	`184`	`}`
`185`	`185`
`186`		`-func (v V001Entry) Attestation() []byte {`
`187`		`- return nil`
	`186`	`+func (v V001Entry) AttestationKey() string {`
	`187`	`+ return ""`
	`188`	`+}`
	`189`	`+`
	`190`	`+func (v V001Entry) AttestationKeyValue() (string, []byte) {`
	`191`	`+ return "", nil`
`188`	`192`	`}`
`189`	`193`
`190`	`194`	`func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {`
Original file line number	Diff line number	Diff line change
`@@ -280,8 +280,12 @@ func (v V001Entry) validate() error {`
`280`	`280`	`return nil`
`281`	`281`	`}`
`282`	`282`
`283`		`-func (v V001Entry) Attestation() []byte {`
`284`		`- return nil`
	`283`	`+func (v V001Entry) AttestationKey() string {`
	`284`	`+ return ""`
	`285`	`+}`
	`286`	`+`
	`287`	`+func (v V001Entry) AttestationKeyValue() (string, []byte) {`
	`288`	`+ return "", nil`
`285`	`289`	`}`
`286`	`290`
`287`	`291`	`func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {`
Original file line number	Diff line number	Diff line change
`@@ -271,8 +271,12 @@ func extractPKCS7SignatureFromJAR(inz *zip.Reader) ([]byte, error) {`
`271`	`271`	`return nil, errors.New("unable to locate signature in JAR file")`
`272`	`272`	`}`
`273`	`273`
`274`		`-func (v *V001Entry) Attestation() []byte {`
`275`		`- return nil`
	`274`	`+func (v V001Entry) AttestationKey() string {`
	`275`	`+ return ""`
	`276`	`+}`
	`277`	`+`
	`278`	`+func (v V001Entry) AttestationKeyValue() (string, []byte) {`
	`279`	`+ return "", nil`
`276`	`280`	`}`
`277`	`281`
`278`	`282`	`func (v *V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {`
Original file line number	Diff line number	Diff line change
`@@ -332,8 +332,12 @@ func (v V001Entry) validate() error {`
`332`	`332`	`return nil`
`333`	`333`	`}`
`334`	`334`
`335`		`-func (v V001Entry) Attestation() []byte {`
`336`		`- return nil`
	`335`	`+func (v V001Entry) AttestationKey() string {`
	`336`	`+ return ""`
	`337`	`+}`
	`338`	`+`
	`339`	`+func (v V001Entry) AttestationKeyValue() (string, []byte) {`
	`340`	`+ return "", nil`
`337`	`341`	`}`
`338`	`342`
`339`	`343`	`func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {`
Original file line number	Diff line number	Diff line change
`@@ -173,8 +173,12 @@ func (v V001Entry) validate() error {`
`173`	`173`	`return nil`
`174`	`174`	`}`
`175`	`175`
`176`		`-func (v V001Entry) Attestation() []byte {`
`177`		`- return nil`
	`176`	`+func (v V001Entry) AttestationKey() string {`
	`177`	`+ return ""`
	`178`	`+}`
	`179`	`+`
	`180`	`+func (v V001Entry) AttestationKeyValue() (string, []byte) {`
	`181`	`+ return "", nil`
`178`	`182`	`}`
`179`	`183`
`180`	`184`	`func (v V001Entry) CreateFromArtifactProperties(_ context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {`
Original file line number	Diff line number	Diff line change
`@@ -307,8 +307,12 @@ func (v V001Entry) validate() error {`
`307`	`307`	`return nil`
`308`	`308`	`}`
`309`	`309`
`310`		`-func (v V001Entry) Attestation() []byte {`
`311`		`- return nil`
	`310`	`+func (v V001Entry) AttestationKey() string {`
	`311`	`+ return ""`
	`312`	`+}`
	`313`	`+`
	`314`	`+func (v V001Entry) AttestationKeyValue() (string, []byte) {`
	`315`	`+ return "", nil`
`312`	`316`	`}`
`313`	`317`
`314`	`318`	`func (v V001Entry) CreateFromArtifactProperties(ctx context.Context, props types.ArtifactProperties) (models.ProposedEntry, error) {`