From 87a653f619ae143b52e08bf92ca26e4f9e9bc8b8 Mon Sep 17 00:00:00 2001 From: Nishchay Date: Tue, 10 Mar 2026 16:05:24 -0700 Subject: [PATCH] Use containerd GC for image-fetcher cleanup Mark unpacked pulled layers as GC-eligible during VHD image preloading and trigger a single synchronous containerd GC after the preload batch. This avoids scanning all images per pull while preserving fetch-only image blobs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- image-fetcher/main.go | 84 +++++++++++++++-------- vhdbuilder/packer/install-dependencies.sh | 1 + 2 files changed, 56 insertions(+), 29 deletions(-) diff --git a/image-fetcher/main.go b/image-fetcher/main.go index 4f4185d0998..786c4aea107 100644 --- a/image-fetcher/main.go +++ b/image-fetcher/main.go @@ -5,8 +5,11 @@ import ( "fmt" "os" "runtime" + "time" containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/core/leases" "github.com/containerd/containerd/v2/pkg/namespaces" "github.com/containerd/platforms" ) @@ -23,6 +26,7 @@ const ( func main() { if len(os.Args) < 2 { fmt.Fprintf(os.Stderr, "Usage: %s [image-ref...]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s --gc\n", os.Args[0]) fmt.Fprintf(os.Stderr, "Example: %s mcr.microsoft.com/oss/kubernetes/pause:3.9\n", os.Args[0]) os.Exit(1) } @@ -45,6 +49,15 @@ func main() { ctx := namespaces.WithNamespace(context.Background(), ns) + if len(os.Args) == 2 && os.Args[1] == "--gc" { + if err := triggerGarbageCollection(ctx, client); err != nil { + fmt.Fprintf(os.Stderr, "Failed to trigger containerd GC: %v\n", err) + os.Exit(1) + } + fmt.Println("Triggered containerd GC") + return + } + failed := 0 for _, ref := range os.Args[1:] { if err := fetchImage(ctx, client, ref); err != nil { @@ -68,7 +81,7 @@ func main() { // already-fetched content from the store and handles snapshotter resolution // internally (namespace label → platform default). func fetchImage(ctx context.Context, client *containerd.Client, ref string) error { - fetchOnly := os.Getenv("IMAGE_FETCH_ONLY") == "true" + //fetchOnly := os.Getenv("IMAGE_FETCH_ONLY") == "true" fmt.Printf("Fetching %s ...\n", ref) @@ -79,42 +92,55 @@ func fetchImage(ctx context.Context, client *containerd.Client, ref string) erro } platformMatcher := platforms.OnlyStrict(p) - imageMeta, err := client.Fetch(ctx, ref, + // imageMeta, err := client.Fetch(ctx, ref, + // containerd.WithPlatformMatcher(platformMatcher), + // ) + // if err != nil { + // return fmt.Errorf("fetch failed: %w", err) + // } + + // if fetchOnly { + // fmt.Printf("OK %s -> %s (fetched)\n", imageMeta.Name, imageMeta.Target.Digest) + // return nil + // } + + // image := containerd.NewImage(client, imageMeta) + + // size, err := image.Size(ctx) + // if err != nil { + // fmt.Fprintf(os.Stderr, "WARN %s: could not determine image size, skipping unpack: %v\n", ref, err) + // fmt.Printf("OK %s -> %s (fetched)\n", imageMeta.Name, imageMeta.Target.Digest) + // return nil + // } + + // if size < pullSizeThreshold { + // We use pull here instead of use unpack because some runtimes (e.g. containerd-shim-runsc-v1), + // require pull to trigger unpacking into the correct snapshotter based on the image's platform. + pullOpts := []containerd.RemoteOpt{ containerd.WithPlatformMatcher(platformMatcher), - ) - if err != nil { - return fmt.Errorf("fetch failed: %w", err) + containerd.WithPullUnpack, + containerd.WithChildLabelMap(images.ChildGCLabelsFilterLayers), } - if fetchOnly { - fmt.Printf("OK %s -> %s (fetched)\n", imageMeta.Name, imageMeta.Target.Digest) - return nil + imageMeta, err := client.Pull(ctx, ref, pullOpts...) + if err != nil { + return fmt.Errorf("pull failed: %w", err) } + fmt.Printf("OK %s (pulled)\n", imageMeta.Name) + // } else { + // fmt.Printf("OK %s -> %s (fetched, %s)\n", imageMeta.Name, imageMeta.Target.Digest, formatSize(size)) + // } - image := containerd.NewImage(client, imageMeta) + return nil +} - size, err := image.Size(ctx) +func triggerGarbageCollection(ctx context.Context, client *containerd.Client) error { + ls := client.LeasesService() + l, err := ls.Create(ctx, leases.WithRandomID(), leases.WithExpiration(time.Hour)) if err != nil { - fmt.Fprintf(os.Stderr, "WARN %s: could not determine image size, skipping unpack: %v\n", ref, err) - fmt.Printf("OK %s -> %s (fetched)\n", imageMeta.Name, imageMeta.Target.Digest) - return nil - } - - if size < pullSizeThreshold { - // We use pull here instead of use unpack because some runtimes (e.g. containerd-shim-runsc-v1), - // require pull to trigger unpacking into the correct snapshotter based on the image's platform. - if _, err := client.Pull(ctx, ref, - containerd.WithPlatformMatcher(platformMatcher), - containerd.WithPullUnpack, - ); err != nil { - return fmt.Errorf("pull failed: %w", err) - } - fmt.Printf("OK %s -> %s (pulled, %s)\n", imageMeta.Name, imageMeta.Target.Digest, formatSize(size)) - } else { - fmt.Printf("OK %s -> %s (fetched, %s)\n", imageMeta.Name, imageMeta.Target.Digest, formatSize(size)) + return err } - - return nil + return ls.Delete(ctx, l, leases.SynchronousDelete) } func formatSize(bytes int64) string { diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index e1319774059..e69ef32a145 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -699,6 +699,7 @@ while IFS= read -r imageToBePulled; do done <<< "$ContainerImages" echo "Waiting for container image pulls to finish. PID: ${image_pids[@]}" wait ${image_pids[@]} +/opt/azure/containers/image-fetcher --gc capture_benchmark "${SCRIPT_NAME}_caching_container_images" retagAKSNodeCAWatcher() {