From 041f1e581f984de455e9f0ed478f6f08527e3b4d Mon Sep 17 00:00:00 2001 From: Maggie Lou Date: Tue, 10 Feb 2026 14:57:45 -0600 Subject: [PATCH] [SS] Fix local snapshot check --- .../containers/firecracker/BUILD | 1 + .../containers/firecracker/firecracker.go | 18 +-- .../firecracker/firecracker_test.go | 116 ++++++++++++++++++ .../remote_execution/snaploader/snaploader.go | 38 +++--- .../snaploader/snaploader_test.go | 60 +++++---- .../upload_local_snapshot.go | 5 +- 6 files changed, 187 insertions(+), 51 deletions(-) diff --git a/enterprise/server/remote_execution/containers/firecracker/BUILD b/enterprise/server/remote_execution/containers/firecracker/BUILD index d068cd50b62..2940f75c580 100644 --- a/enterprise/server/remote_execution/containers/firecracker/BUILD +++ b/enterprise/server/remote_execution/containers/firecracker/BUILD @@ -150,6 +150,7 @@ go_test( "//enterprise/server/remote_execution/container", "//enterprise/server/remote_execution/copy_on_write", "//enterprise/server/remote_execution/filecache", + "//enterprise/server/remote_execution/snaploader", "//enterprise/server/remote_execution/snaputil", "//enterprise/server/remote_execution/workspace", "//enterprise/server/testutil/testcontainer", diff --git a/enterprise/server/remote_execution/containers/firecracker/firecracker.go b/enterprise/server/remote_execution/containers/firecracker/firecracker.go index 117a6086e4c..dae0d65d275 100644 --- a/enterprise/server/remote_execution/containers/firecracker/firecracker.go +++ b/enterprise/server/remote_execution/containers/firecracker/firecracker.go @@ -825,8 +825,9 @@ func NewContainer(ctx context.Context, env environment.Env, task *repb.Execution return nil, err } snap, err := loader.GetSnapshot(ctx, c.snapshotKeySet, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: c.supportsRemoteSnapshots, - ReadPolicy: readPolicy, + SupportsRemoteChunks: c.supportsRemoteSnapshots, + SupportsRemoteManifest: c.supportsRemoteSnapshots, + ReadPolicy: readPolicy, }) c.createFromSnapshot = err == nil label := "" @@ -1282,7 +1283,8 @@ func (c *FirecrackerContainer) LoadSnapshot(ctx context.Context) error { maxFallbackAge = d } snap, err := c.loader.GetSnapshot(ctx, c.snapshotKeySet, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: c.supportsRemoteSnapshots, + SupportsRemoteChunks: c.supportsRemoteSnapshots, + SupportsRemoteManifest: c.supportsRemoteSnapshots, ReadPolicy: readPolicy, MaxStaleFallbackSnapshotAge: maxFallbackAge, }) @@ -3347,8 +3349,9 @@ func (c *FirecrackerContainer) hasRemoteSnapshotForKey(ctx context.Context, load return false } _, err = loader.GetSnapshot(ctx, &fcpb.SnapshotKeySet{BranchKey: key}, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: c.supportsRemoteSnapshots, - ReadPolicy: readPolicy, + SupportsRemoteChunks: c.supportsRemoteSnapshots, + SupportsRemoteManifest: c.supportsRemoteSnapshots, + ReadPolicy: readPolicy, }) return err == nil } @@ -3358,8 +3361,9 @@ func (c *FirecrackerContainer) hasLocalSnapshotForKey(ctx context.Context, loade return false } _, err = loader.GetSnapshot(ctx, &fcpb.SnapshotKeySet{BranchKey: key}, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: false, - ReadPolicy: readPolicy, + SupportsRemoteChunks: c.supportsRemoteSnapshots, + SupportsRemoteManifest: false, + ReadPolicy: readPolicy, }) return err == nil } diff --git a/enterprise/server/remote_execution/containers/firecracker/firecracker_test.go b/enterprise/server/remote_execution/containers/firecracker/firecracker_test.go index 58033911395..139dfb47cdb 100644 --- a/enterprise/server/remote_execution/containers/firecracker/firecracker_test.go +++ b/enterprise/server/remote_execution/containers/firecracker/firecracker_test.go @@ -29,6 +29,7 @@ import ( "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/containers/firecracker" "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/copy_on_write" "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/filecache" + "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/snaploader" "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/snaputil" "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/workspace" "github.com/buildbuddy-io/buildbuddy/enterprise/server/testutil/testcontainer" @@ -803,6 +804,121 @@ func TestFirecracker_LocalSnapshotSharing_DontResave(t *testing.T) { } } +func TestFirecracker_LocalSnapshotSharing_DontResave_RemoteChunkFallback(t *testing.T) { + ctx := context.Background() + env := getTestEnv(ctx, t, envOpts{}) + env.SetAuthenticator(testauth.NewTestAuthenticator(t, testauth.TestUsers("US1", "GR1"))) + + filecacheRoot := testfs.MakeTempDir(t) + fc, err := filecache.NewFileCache(filecacheRoot, fileCacheSize, false) + require.NoError(t, err) + fc.WaitForDirectoryScanToComplete() + env.SetFileCache(fc) + + rootDir := testfs.MakeTempDir(t) + cfg := getExecutorConfig(t) + + var containersToCleanup []*firecracker.FirecrackerContainer + t.Cleanup(func() { + for _, vm := range containersToCleanup { + err := vm.Remove(ctx) + assert.NoError(t, err) + } + }) + + task := &repb.ExecutionTask{ + Command: &repb.Command{ + Platform: &repb.Platform{Properties: []*repb.Platform_Property{ + {Name: "recycle-runner", Value: "true"}, + // Only a snapshot it none exists + {Name: platform.SnapshotSavePolicyPropertyName, Value: platform.OnlySaveNonDefaultSnapshotIfNoneAvailable}, + }}, + // Use ci_runner so it supports remote snapshots + Arguments: []string{"./buildbuddy_ci_runner"}, + EnvironmentVariables: []*repb.Command_EnvironmentVariable{ + {Name: "GIT_REPO_DEFAULT_BRANCH", Value: "main"}, + {Name: "GIT_BRANCH", Value: "pr"}, + }, + }, + } + + workDir := testfs.MakeDirAll(t, rootDir, "work") + opts := firecracker.ContainerOpts{ + ContainerImage: busyboxImage, + ActionWorkingDirectory: workDir, + VMConfiguration: &fcpb.VMConfiguration{ + NumCpus: 1, + MemSizeMb: minMemSizeMB, + NetworkMode: fcpb.NetworkMode_NETWORK_MODE_OFF, + ScratchDiskSizeMb: 100, + }, + ExecutorConfig: cfg, + } + baseVM, err := firecracker.NewContainer(ctx, env, task, opts) + require.NoError(t, err) + containersToCleanup = append(containersToCleanup, baseVM) + err = container.PullImageIfNecessary(ctx, env, baseVM, oci.Credentials{}, opts.ContainerImage) + require.NoError(t, err) + err = baseVM.Create(ctx, opts.ActionWorkingDirectory) + require.NoError(t, err) + + // Create an initial snapshot. Data written to this snapshot should persist + // when other VMs reuse the snapshot + cmd := appendToLog("Base") + res := baseVM.Exec(ctx, cmd, nil /*=stdio*/) + require.NoError(t, res.Error) + require.Equal(t, "Base\n", string(res.Stdout)) + assert.True(t, res.VMMetadata.GetSavedLocalSnapshot()) + err = baseVM.Pause(ctx) + require.NoError(t, err) + + // Delete some chunks from the local filecache. The chunks should still + // exist in the remote cache, so the local manifest should still be valid. + loader, err := snaploader.New(env) + require.NoError(t, err) + snapshotKeys := baseVM.SnapshotKeySet() + snap, err := loader.GetSnapshot(ctx, snapshotKeys, &snaploader.GetSnapshotOptions{ + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, + }) + require.NoError(t, err) + + deletedCount := 0 + for _, cf := range snap.GetChunkedFiles() { + for i, c := range cf.GetChunks() { + // Delete every other chunk + if i%2 == 0 { + deleted := fc.DeleteFile(ctx, &repb.FileNode{Digest: c.GetDigest()}) + if deleted { + deletedCount++ + } + } + } + } + require.Greater(t, deletedCount, 0, "should have deleted at least one chunk") + + // Create a new VM that loads from the snapshot. Even though some chunks + // are missing locally, they should be fetched from the remote cache. + workDir2 := testfs.MakeDirAll(t, rootDir, "fork") + opts.ActionWorkingDirectory = workDir2 + forkVM, err := firecracker.NewContainer(ctx, env, task, opts) + require.NoError(t, err) + containersToCleanup = append(containersToCleanup, forkVM) + + err = forkVM.Unpause(ctx) + require.NoError(t, err) + + cmd = appendToLog("Fork") + res = forkVM.Exec(ctx, cmd, nil /*=stdio*/) + require.NoError(t, res.Error) + require.Equal(t, "Base\nFork\n", string(res.Stdout)) + + // The fork should not save a new local snapshot because the local manifest + // exists and all chunks are available (locally or remotely). + assert.False(t, res.VMMetadata.GetSavedLocalSnapshot()) +} + func TestFirecracker_RemoteSnapshotSharing_SavePolicy(t *testing.T) { tests := []struct { name string diff --git a/enterprise/server/remote_execution/snaploader/snaploader.go b/enterprise/server/remote_execution/snaploader/snaploader.go index 785f2f29e6f..e0dfad675a4 100644 --- a/enterprise/server/remote_execution/snaploader/snaploader.go +++ b/enterprise/server/remote_execution/snaploader/snaploader.go @@ -268,7 +268,7 @@ func KeysetDebugString(ctx context.Context, env environment.Env, s *fcpb.Snapsho } func SnapshotDebugString(ctx context.Context, env environment.Env, s *Snapshot) string { - return snapshotDebugString(ctx, env, s.GetKey(), s.remoteEnabled, s.GetVMMetadata().GetSnapshotId()) + return snapshotDebugString(ctx, env, s.GetKey(), s.supportsRemoteChunks, s.GetVMMetadata().GetSnapshotId()) } func fileDigest(filePath string) (*repb.Digest, error) { @@ -282,10 +282,10 @@ func fileDigest(filePath string) (*repb.Digest, error) { // Snapshot holds a snapshot manifest along with the corresponding cache key. type Snapshot struct { - key *fcpb.SnapshotKey - manifest *fcpb.SnapshotManifest - remoteEnabled bool - manifestFetchSource snaputil.ChunkSource + key *fcpb.SnapshotKey + manifest *fcpb.SnapshotManifest + supportsRemoteChunks bool + manifestFetchSource snaputil.ChunkSource } func (s *Snapshot) GetKey() *fcpb.SnapshotKey { @@ -317,7 +317,8 @@ func (s *Snapshot) GetManifestFetchSource() snaputil.ChunkSource { } type GetSnapshotOptions struct { - RemoteReadEnabled bool + SupportsRemoteManifest bool + SupportsRemoteChunks bool ReadPolicy string MaxStaleFallbackSnapshotAge time.Duration } @@ -429,10 +430,10 @@ func (l *FileCacheLoader) GetSnapshot(ctx context.Context, keys *fcpb.SnapshotKe continue } return &Snapshot{ - key: key, - manifest: manifest, - remoteEnabled: opts.RemoteReadEnabled, - manifestFetchSource: manifestFetchSource, + key: key, + manifest: manifest, + supportsRemoteChunks: opts.SupportsRemoteChunks, + manifestFetchSource: manifestFetchSource, }, nil } return nil, lastErr @@ -448,7 +449,7 @@ func (l *FileCacheLoader) getSnapshot(ctx context.Context, key *fcpb.SnapshotKey // Note that if platform.SnapshotReadPolicy=newest, the master snapshot is // never cached locally. if *snaputil.EnableLocalSnapshotSharing { - supportsRemoteFallback := opts.RemoteReadEnabled && *snaputil.EnableRemoteSnapshotSharing + supportsRemoteFallback := opts.SupportsRemoteChunks && *snaputil.EnableRemoteSnapshotSharing manifest, err := l.getLocalManifest(ctx, key, supportsRemoteFallback) if err == nil { if validateLocalSnapshot(ctx, manifest, opts, isFallback) { @@ -457,14 +458,14 @@ func (l *FileCacheLoader) getSnapshot(ctx context.Context, key *fcpb.SnapshotKey } } else { log.CtxInfof(ctx, "Failed to get local manifest for key %s: %s", key, err) - if !opts.RemoteReadEnabled || !*snaputil.EnableRemoteSnapshotSharing { + if !opts.SupportsRemoteManifest || !*snaputil.EnableRemoteSnapshotSharing { return nil, snaputil.ChunkSourceUnmapped, err } } // If local snapshot is not valid or couldn't be found, fallback to // fetching a remote snapshot. - } else if !opts.RemoteReadEnabled { - return nil, snaputil.ChunkSourceUnmapped, status.InternalErrorf("invalid state: EnableLocalSnapshotSharing=false and remoteEnabled=false") + } else if !opts.SupportsRemoteManifest { + return nil, snaputil.ChunkSourceUnmapped, status.InternalErrorf("invalid state: EnableLocalSnapshotSharing=false and SupportsRemoteManifest=false") } if opts.ReadPolicy == platform.ReadLocalSnapshotOnly { @@ -684,7 +685,7 @@ func (l *FileCacheLoader) UnpackSnapshot(ctx context.Context, snapshot *Snapshot for _, fileNode := range snapshot.manifest.Files { outputPath := filepath.Join(outputDirectory, fileNode.GetName()) - if _, err := snaputil.GetArtifact(ctx, l.env.GetFileCache(), l.env.GetByteStreamClient(), snapshot.remoteEnabled, fileNode.GetDigest(), snapshot.key.InstanceName, outputPath); err != nil { + if _, err := snaputil.GetArtifact(ctx, l.env.GetFileCache(), l.env.GetByteStreamClient(), snapshot.supportsRemoteChunks, fileNode.GetDigest(), snapshot.key.InstanceName, outputPath); err != nil { return nil, err } } @@ -694,7 +695,7 @@ func (l *FileCacheLoader) UnpackSnapshot(ctx context.Context, snapshot *Snapshot } // Construct COWs from chunks. for _, cf := range snapshot.manifest.ChunkedFiles { - cow, err := l.unpackCOW(ctx, cf, snapshot.key.InstanceName, outputDirectory, snapshot.remoteEnabled) + cow, err := l.unpackCOW(ctx, cf, snapshot.key.InstanceName, outputDirectory, snapshot.supportsRemoteChunks) if err != nil { return nil, status.WrapError(err, "unpack COW") } @@ -1224,8 +1225,9 @@ func UnpackContainerImage(ctx context.Context, l *FileCacheLoader, instanceName, }} snap, err := l.GetSnapshot(ctx, key, &GetSnapshotOptions{ - RemoteReadEnabled: remoteEnabled, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteManifest: remoteEnabled, + SupportsRemoteChunks: remoteEnabled, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) if err != nil && !(status.IsNotFoundError(err) || status.IsUnavailableError(err)) { return nil, err diff --git a/enterprise/server/remote_execution/snaploader/snaploader_test.go b/enterprise/server/remote_execution/snaploader/snaploader_test.go index eb173da1d5a..efa82e66452 100644 --- a/enterprise/server/remote_execution/snaploader/snaploader_test.go +++ b/enterprise/server/remote_execution/snaploader/snaploader_test.go @@ -159,8 +159,9 @@ func TestUnpackFallbackKey(t *testing.T) { // Sanity check that the branch key is not found in cache, to make sure // we're actually falling back. _, err = loader.GetSnapshot(ctx, &fcpb.SnapshotKeySet{BranchKey: forkKeys.GetBranchKey()}, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.Error(t, err) @@ -341,8 +342,9 @@ func TestSnapshotVersioning(t *testing.T) { // We should not be able to find a snapshot for the new key snapshot2, _ := loader.GetSnapshot(ctx, snapshotKey2, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: remoteEnabled, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: remoteEnabled, + SupportsRemoteManifest: remoteEnabled, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.Nil(t, snapshot2) @@ -415,8 +417,9 @@ func TestRemoteSnapshotFetching(t *testing.T) { // Delete some artifacts from the local cache, so we can test fetching artifacts // from both the local and remote cache snapMetadata, err := loader.GetSnapshot(ctx, keys, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.NoError(t, err) for i, f := range snapMetadata.GetFiles() { @@ -480,8 +483,9 @@ func TestRemoteSnapshotFetching_RemoteEviction(t *testing.T) { // Delete some artifacts from the remote cache (arbitrarily the first one for simplicity) snapMetadata, err := loader.GetSnapshot(ctx, keys, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.NoError(t, err) for _, f := range snapMetadata.GetFiles() { @@ -502,8 +506,9 @@ func TestRemoteSnapshotFetching_RemoteEviction(t *testing.T) { // Even though the assets still exist in the local cache, the remote cache // should serve as the source of truth on whether a snapshot is valid _, err = loader.GetSnapshot(ctx, keys, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.Error(t, err) } @@ -543,8 +548,9 @@ func TestGetSnapshot_CacheIsolation(t *testing.T) { // Fetching snapshot with same group id different instance name should fail keysNewInstanceName := keysWithInstanceName(t, ctx, loader, "remote-C") _, err = loader.GetSnapshot(ctx, keysNewInstanceName, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: enableRemote, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: enableRemote, + SupportsRemoteManifest: enableRemote, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.Error(t, err) @@ -552,8 +558,9 @@ func TestGetSnapshot_CacheIsolation(t *testing.T) { ctxNewGroup, err := auth.WithAuthenticatedUser(context.Background(), "US2") require.NoError(t, err) _, err = loader.GetSnapshot(ctxNewGroup, originalKeys, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: enableRemote, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: enableRemote, + SupportsRemoteManifest: enableRemote, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.Error(t, err) } @@ -595,8 +602,9 @@ func TestGetSnapshot_MixOfLocalAndRemoteChunks(t *testing.T) { // snapshot locally only. workDirB := testfs.MakeDirAll(t, workDir, "VM-B") snap, err := loader.GetSnapshot(ctx, keyset, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.NoError(t, err) unpacked, err := loader.UnpackSnapshot(ctx, snap, workDirB) @@ -624,8 +632,9 @@ func TestGetSnapshot_MixOfLocalAndRemoteChunks(t *testing.T) { // GetSnapshot should use the local snapshot manifest, but fallback to the // remote cache for any missing chunks. snap, err = loader.GetSnapshot(ctx, keyset, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.NoError(t, err) unpackedC, err := loader.UnpackSnapshot(ctx, snap, workDirC) @@ -675,8 +684,9 @@ func TestMergeQueueBranch(t *testing.T) { // Make sure we can fetch a snapshot for the merge queue branch snapMetadata, err := loader.GetSnapshot(ctx, mergeBranchKeys, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.NoError(t, err) require.NotNil(t, snapMetadata) @@ -700,8 +710,9 @@ func TestMergeQueueBranch(t *testing.T) { prBranchKeys, err := loader.SnapshotKeySet(ctx, prTask, "config-hash", "") require.NoError(t, err) snapMetadata, err = loader.GetSnapshot(ctx, prBranchKeys, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.NoError(t, err) require.NotNil(t, snapMetadata) @@ -754,8 +765,9 @@ func writeRandomRange(t *testing.T, store *copy_on_write.COWStore) { // originally cached contents. func mustUnpack(t *testing.T, ctx context.Context, loader snaploader.Loader, snapshotKeySet *fcpb.SnapshotKeySet, outDir string, originalSnapshot *snaploader.CacheSnapshotOptions) *snaploader.UnpackedSnapshot { snap, err := loader.GetSnapshot(ctx, snapshotKeySet, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) require.NoError(t, err) unpacked, err := loader.UnpackSnapshot(ctx, snap, outDir) diff --git a/enterprise/tools/upload_local_snapshot/upload_local_snapshot.go b/enterprise/tools/upload_local_snapshot/upload_local_snapshot.go index 2e30296b255..60fd99844c2 100644 --- a/enterprise/tools/upload_local_snapshot/upload_local_snapshot.go +++ b/enterprise/tools/upload_local_snapshot/upload_local_snapshot.go @@ -146,8 +146,9 @@ func main() { // Sanity check that snapshot exists in remote cache flagutil.SetValueForFlagName("executor.enable_remote_snapshot_sharing", true, nil, false) _, err = loader.GetSnapshot(ctx, &fcpb.SnapshotKeySet{BranchKey: key}, &snaploader.GetSnapshotOptions{ - RemoteReadEnabled: true, - ReadPolicy: platform.AlwaysReadNewestSnapshot, + SupportsRemoteChunks: true, + SupportsRemoteManifest: true, + ReadPolicy: platform.AlwaysReadNewestSnapshot, }) if err != nil { log.Fatalf("Snapshot upload failed. Snapshot is not in the remote cache after it should've been uploaded.")