Skip to content

Commit f3db95b

Browse files
Hayden-IOHayden-IO
authored
Cache checkpoint for inactive shards (#2332)
* Cache checkpoint for inactive shards On service startup, Rekor will sign checkpoints for the inactive shards, since inactive tree lengths do not change. The calls to CreateAndSignCheckpoint that were not updated are because the checkpoint is being signed only for the active shard, e.g. on entry upload and when returning the active shard checkpoint. Signed-off-by: Hayden Blauzvern <hblauzvern@google.com> * reorder PublicKey Signed-off-by: Hayden Blauzvern <hblauzvern@google.com> * update error message Signed-off-by: Hayden Blauzvern <hblauzvern@google.com> --------- Signed-off-by: Hayden Blauzvern <hblauzvern@google.com>
1 parent 1cb78ca commit f3db95b

File tree

5 files changed

+75
-22
lines changed

5 files changed

+75
-22
lines changed

‎pkg/api/api.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@ import (
2525
"strings"
2626

2727
"github.com/google/trillian"
28+
"github.com/google/trillian/types"
2829
"github.com/redis/go-redis/v9"
2930
"github.com/spf13/viper"
3031
"golang.org/x/exp/slices"
3132
"google.golang.org/grpc"
33+
"google.golang.org/grpc/codes"
3234
"google.golang.org/grpc/credentials"
3335
"google.golang.org/grpc/credentials/insecure"
3436

@@ -39,6 +41,7 @@ import (
3941
"github.com/sigstore/rekor/pkg/signer"
4042
"github.com/sigstore/rekor/pkg/storage"
4143
"github.com/sigstore/rekor/pkg/trillianclient"
44+
"github.com/sigstore/rekor/pkg/util"
4245
"github.com/sigstore/rekor/pkg/witness"
4346

4447
_ "github.com/sigstore/rekor/pkg/pubsub/gcp" // Load GCP pubsub implementation
@@ -95,6 +98,11 @@ type API struct {
9598
// Publishes notifications when new entries are added to the log. May be
9699
// nil if no publisher is configured.
97100
newEntryPublisher pubsub.Publisher
101+
// Stores map of inactive tree IDs to checkpoints
102+
// Inactive shards will always return the same checkpoint,
103+
// so we can fetch the checkpoint on service startup to
104+
// minimize signature generations
105+
cachedCheckpoints map[int64]string
98106
}
99107

100108
func NewAPI(treeID uint) (*API, error) {
@@ -132,6 +140,26 @@ func NewAPI(treeID uint) (*API, error) {
132140
return nil, fmt.Errorf("unable get sharding details from sharding config: %w", err)
133141
}
134142

143+
cachedCheckpoints := make(map[int64]string)
144+
for _, r := range ranges.GetInactive() {
145+
tc := trillianclient.NewTrillianClient(ctx, logClient, r.TreeID)
146+
resp := tc.GetLatest(0)
147+
if resp.Status != codes.OK {
148+
return nil, fmt.Errorf("error fetching latest tree head for inactive shard %d: resp code is %d, err is %w", r.TreeID, resp.Status, resp.Err)
149+
}
150+
result := resp.GetLatestResult
151+
root := &types.LogRootV1{}
152+
if err := root.UnmarshalBinary(result.SignedLogRoot.LogRoot); err != nil {
153+
return nil, fmt.Errorf("error unmarshalling root: %w", err)
154+
}
155+
156+
cp, err := util.CreateAndSignCheckpoint(ctx, viper.GetString("rekor_server.hostname"), r.TreeID, uint64(r.TreeLength), root.RootHash, r.Signer)
157+
if err != nil {
158+
return nil, fmt.Errorf("error signing checkpoint for inactive shard %d: %w", r.TreeID, err)
159+
}
160+
cachedCheckpoints[r.TreeID] = string(cp)
161+
}
162+
135163
var newEntryPublisher pubsub.Publisher
136164
if p := viper.GetString("rekor_server.new_entry_publisher"); p != "" {
137165
if !viper.GetBool("rekor_server.publish_events_protobuf") && !viper.GetBool("rekor_server.publish_events_json") {
@@ -151,6 +179,7 @@ func NewAPI(treeID uint) (*API, error) {
151179
logRanges: ranges,
152180
// Utility functionality not required for operation of the core service
153181
newEntryPublisher: newEntryPublisher,
182+
cachedCheckpoints: cachedCheckpoints,
154183
}, nil
155184
}
156185

‎pkg/api/entries.go‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ func signEntry(ctx context.Context, signer signature.Signer, entry models.LogEnt
7474
}
7575

7676
// logEntryFromLeaf creates a signed LogEntry struct from trillian structs
77-
func logEntryFromLeaf(ctx context.Context, _ trillianclient.TrillianClient, leaf *trillian.LogLeaf,
78-
signedLogRoot *trillian.SignedLogRoot, proof *trillian.Proof, tid int64, ranges sharding.LogRanges) (models.LogEntry, error) {
77+
func logEntryFromLeaf(ctx context.Context, leaf *trillian.LogLeaf, signedLogRoot *trillian.SignedLogRoot,
78+
proof *trillian.Proof, tid int64, ranges sharding.LogRanges, cachedCheckpoints map[int64]string) (models.LogEntry, error) {
7979

8080
log.ContextLogger(ctx).Debugf("log entry from leaf %d", leaf.GetLeafIndex())
8181
root := &ttypes.LogRootV1{}
@@ -105,17 +105,25 @@ func logEntryFromLeaf(ctx context.Context, _ trillianclient.TrillianClient, leaf
105105
return nil, fmt.Errorf("signing entry error: %w", err)
106106
}
107107

108-
scBytes, err := util.CreateAndSignCheckpoint(ctx, viper.GetString("rekor_server.hostname"), tid, root.TreeSize, root.RootHash, logRange.Signer)
109-
if err != nil {
110-
return nil, err
108+
// If tree ID is inactive, use cached checkpoint
109+
var sc string
110+
val, ok := cachedCheckpoints[tid]
111+
if ok {
112+
sc = val
113+
} else {
114+
scBytes, err := util.CreateAndSignCheckpoint(ctx, viper.GetString("rekor_server.hostname"), tid, root.TreeSize, root.RootHash, logRange.Signer)
115+
if err != nil {
116+
return nil, err
117+
}
118+
sc = string(scBytes)
111119
}
112120

113121
inclusionProof := models.InclusionProof{
114122
TreeSize: swag.Int64(int64(root.TreeSize)),
115123
RootHash: swag.String(hex.EncodeToString(root.RootHash)),
116124
LogIndex: swag.Int64(proof.GetLeafIndex()),
117125
Hashes: hashes,
118-
Checkpoint: stringPointer(string(scBytes)),
126+
Checkpoint: stringPointer(sc),
119127
}
120128

121129
uuid := hex.EncodeToString(leaf.MerkleLeafHash)
@@ -515,8 +523,7 @@ func SearchLogQueryHandler(params entries.SearchLogQueryParams) middleware.Respo
515523
if leafResp == nil {
516524
continue
517525
}
518-
tcs := trillianclient.NewTrillianClient(httpReqCtx, api.logClient, shard)
519-
logEntry, err := logEntryFromLeaf(httpReqCtx, tcs, leafResp.Leaf, leafResp.SignedLogRoot, leafResp.Proof, shard, api.logRanges)
526+
logEntry, err := logEntryFromLeaf(httpReqCtx, leafResp.Leaf, leafResp.SignedLogRoot, leafResp.Proof, shard, api.logRanges, api.cachedCheckpoints)
520527
if err != nil {
521528
return handleRekorAPIError(params, http.StatusInternalServerError, err, err.Error())
522529
}
@@ -563,7 +570,7 @@ func retrieveLogEntryByIndex(ctx context.Context, logIndex int) (models.LogEntry
563570
return models.LogEntry{}, ErrNotFound
564571
}
565572

566-
return logEntryFromLeaf(ctx, tc, leaf, result.SignedLogRoot, result.Proof, tid, api.logRanges)
573+
return logEntryFromLeaf(ctx, leaf, result.SignedLogRoot, result.Proof, tid, api.logRanges, api.cachedCheckpoints)
567574
}
568575

569576
// Retrieve a Log Entry
@@ -628,7 +635,7 @@ func retrieveUUIDFromTree(ctx context.Context, uuid string, tid int64) (models.L
628635
return models.LogEntry{}, err
629636
}
630637

631-
logEntry, err := logEntryFromLeaf(ctx, tc, result.Leaf, result.SignedLogRoot, result.Proof, tid, api.logRanges)
638+
logEntry, err := logEntryFromLeaf(ctx, result.Leaf, result.SignedLogRoot, result.Proof, tid, api.logRanges, api.cachedCheckpoints)
632639
if err != nil {
633640
return models.LogEntry{}, fmt.Errorf("could not create log entry from leaf: %w", err)
634641
}

‎pkg/api/tlog.go‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ import (
3333
"github.com/sigstore/rekor/pkg/log"
3434
"github.com/sigstore/rekor/pkg/trillianclient"
3535
"github.com/sigstore/rekor/pkg/util"
36-
"github.com/sigstore/sigstore/pkg/signature"
3736
)
3837

3938
// GetLogInfoHandler returns the current size of the tree and the STH
@@ -44,7 +43,7 @@ func GetLogInfoHandler(params tlog.GetLogInfoParams) middleware.Responder {
4443
var inactiveShards []*models.InactiveShardLogInfo
4544
for _, shard := range api.logRanges.GetInactive() {
4645
// Get details for this inactive shard
47-
is, err := inactiveShardLogInfo(params.HTTPRequest.Context(), shard.TreeID, shard.Signer)
46+
is, err := inactiveShardLogInfo(params.HTTPRequest.Context(), shard.TreeID, api.cachedCheckpoints)
4847
if err != nil {
4948
return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("inactive shard error: %w", err), unexpectedInactiveShardError)
5049
}
@@ -168,7 +167,7 @@ func GetLogProofHandler(params tlog.GetLogProofParams) middleware.Responder {
168167
return tlog.NewGetLogProofOK().WithPayload(&consistencyProof)
169168
}
170169

171-
func inactiveShardLogInfo(ctx context.Context, tid int64, signer signature.Signer) (*models.InactiveShardLogInfo, error) {
170+
func inactiveShardLogInfo(ctx context.Context, tid int64, cachedCheckpoints map[int64]string) (*models.InactiveShardLogInfo, error) {
172171
tc := trillianclient.NewTrillianClient(ctx, api.logClient, tid)
173172
resp := tc.GetLatest(0)
174173
if resp.Status != codes.OK {
@@ -184,16 +183,11 @@ func inactiveShardLogInfo(ctx context.Context, tid int64, signer signature.Signe
184183
hashString := hex.EncodeToString(root.RootHash)
185184
treeSize := int64(root.TreeSize)
186185

187-
scBytes, err := util.CreateAndSignCheckpoint(ctx, viper.GetString("rekor_server.hostname"), tid, root.TreeSize, root.RootHash, signer)
188-
if err != nil {
189-
return nil, err
190-
}
191-
192186
m := models.InactiveShardLogInfo{
193187
RootHash: &hashString,
194188
TreeSize: &treeSize,
195189
TreeID: stringPointer(fmt.Sprintf("%d", tid)),
196-
SignedTreeHead: stringPointer(string(scBytes)),
190+
SignedTreeHead: stringPointer(cachedCheckpoints[tid]),
197191
}
198192
return &m, nil
199193
}

‎pkg/sharding/ranges.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -256,13 +256,14 @@ func (l *LogRanges) PublicKey(treeID string) (string, error) {
256256
return "", err
257257
}
258258

259+
if tid == int(l.GetActive().TreeID) {
260+
return l.active.PemPubKey, nil
261+
}
262+
259263
for _, i := range l.inactive {
260264
if int(i.TreeID) == tid {
261265
return i.PemPubKey, nil
262266
}
263267
}
264-
if tid == int(l.GetActive().TreeID) {
265-
return l.active.PemPubKey, nil
266-
}
267268
return "", fmt.Errorf("%d is not a valid tree ID and doesn't have an associated public key", tid)
268269
}

‎tests/sharding-e2e-test.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,18 @@ function stringsMatch () {
6363
fi
6464
}
6565

66+
function stringsNotMatch () {
67+
one=$1
68+
two=$2
69+
70+
if [[ "$one" != "$two" ]]; then
71+
echo "Strings do not match"
72+
else
73+
echo "Strings $one match but shouldn't"
74+
exit 1
75+
fi
76+
}
77+
6678
function waitForRekorServer () {
6779
count=0
6880

@@ -278,4 +290,14 @@ echo
278290
echo "Testing rekor-cli verification via Entry ID..."
279291
DEBUG=1 $REKOR_CLI verify --uuid $ENTRY_ID_1 --rekor_server http://localhost:3000
280292

293+
# Verify that the checkpoint/SignedTreeHead for inactive shards is cached between calls
294+
ACTIVE_SHARD_CHECKPOINT=$(curl "http://localhost:3000/api/v1/log" | jq .signedTreeHead | base64 -w 0)
295+
INACTIVE_SHARD_CHECKPOINT=$(curl "http://localhost:3000/api/v1/log" | jq .inactiveShards[0].signedTreeHead | base64 -w 0)
296+
ACTIVE_SHARD_CHECKPOINT_NOT_CACHED=$(curl "http://localhost:3000/api/v1/log" | jq .signedTreeHead | base64 -w 0)
297+
INACTIVE_SHARD_CHECKPOINT_CACHED=$(curl "http://localhost:3000/api/v1/log" | jq .inactiveShards[0].signedTreeHead | base64 -w 0)
298+
# inactive shard checkpoint is cached
299+
stringsMatch $INACTIVE_SHARD_CHECKPOINT $INACTIVE_SHARD_CHECKPOINT_CACHED
300+
# active shard checkpoint is not cached
301+
stringsNotMatch $ACTIVE_SHARD_CHECKPOINT $ACTIVE_SHARD_CHECKPOINT_NOT_CACHED
302+
281303
echo "Test passed successfully :)"

0 commit comments

Comments
 (0)