From c03bc8a528b90c28021194759e17fcb0e853cccb Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 22 May 2024 15:37:03 +0000 Subject: [PATCH 1/3] Add extraction progress output to GetFSFromImage/GetFSFromLayers --- pkg/util/fs_util.go | 71 ++++++++++++++++++++++++++++++++++++++-- pkg/util/fs_util_test.go | 8 +++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index c8d5a613aa..e03ac29e14 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -96,8 +96,9 @@ type FileContext struct { type ExtractFunction func(string, *tar.Header, string, io.Reader) error type FSConfig struct { - includeWhiteout bool - extractFunc ExtractFunction + includeWhiteout bool + printExtractionProgress bool + extractFunc ExtractFunction } type FSOpt func(*FSConfig) @@ -126,6 +127,12 @@ func IncludeWhiteout() FSOpt { } } +func PrintExtractionProgress() FSOpt { + return func(opts *FSConfig) { + opts.printExtractionProgress = true + } +} + func ExtractFunc(extractFunc ExtractFunction) FSOpt { return func(opts *FSConfig) { opts.extractFunc = extractFunc @@ -144,7 +151,7 @@ func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]strin return nil, err } - return GetFSFromLayers(root, layers, ExtractFunc(extract)) + return GetFSFromLayers(root, layers, ExtractFunc(extract), PrintExtractionProgress()) } func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, error) { @@ -163,7 +170,23 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e return nil, errors.New("must supply an extract function") } + var totalSize int64 + var layerSizes []int64 + for _, l := range layers { + layerSize, err := l.Size() + if err != nil { + return nil, err + } + layerSizes = append(layerSizes, layerSize) + totalSize += layerSize + } + + if cfg.printExtractionProgress { + logrus.Infof("Extracting image layers to %s (%d bytes)", root, totalSize) + } + extractedFiles := []string{} + var extractedBytes int64 for i, l := range layers { if mediaType, err := l.MediaType(); err == nil { logrus.Tracef("Extracting layer %d of media type %s", i, mediaType) @@ -171,12 +194,26 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e logrus.Tracef("Extracting layer %d", i) } + if cfg.printExtractionProgress { + logrus.Infof("Extracting layer %d %d/%d (%d%%)", i, extractedBytes, totalSize, int(math.Round(float64(extractedBytes)/float64(totalSize)*100))) + } + r, err := l.Uncompressed() if err != nil { return nil, err } defer r.Close() + if cfg.printExtractionProgress { + r = &printAfterReader{ + ReadCloser: r, + after: time.Second, + print: func(int64) { + logrus.Infof("Extracting layer %d...", i) + }, + } + } + tr := tar.NewReader(r) for { hdr, err := tr.Next() @@ -225,10 +262,38 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e extractedFiles = append(extractedFiles, filepath.Join(root, cleanedName)) } + + extractedBytes += layerSizes[i] } + + if cfg.printExtractionProgress { + logrus.Infof("Extraction complete %d/%d (%d%%)", extractedBytes, totalSize, int(math.Round(float64(extractedBytes)/float64(totalSize)*100))) + } + return extractedFiles, nil } +type printAfterReader struct { + io.ReadCloser + n int64 + t time.Time + after time.Duration + print func(int64) +} + +func (r *printAfterReader) Read(p []byte) (n int, err error) { + n, err = r.ReadCloser.Read(p) + r.n += int64(n) + if r.t.IsZero() { + r.t = time.Now() + } + if time.Since(r.t) >= r.after { + r.print(r.n) + r.t = time.Now() + } + return +} + // DeleteFilesystem deletes the extracted image file system func DeleteFilesystem() error { logrus.Info("Deleting filesystem...") diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 9bd44c835c..c21c9bfa0f 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -1067,6 +1067,7 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_enabled(t *testing.T) f(expectedFiles, tw) mockLayer := mockv1.NewMockLayer(ctrl) + mockLayer.EXPECT().Size().Return(int64(buf.Len()), nil) mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil) rc := io.NopCloser(buf) @@ -1082,6 +1083,7 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_enabled(t *testing.T) f(secondLayerFiles, tw) mockLayer2 := mockv1.NewMockLayer(ctrl) + mockLayer2.EXPECT().Size().Return(int64(buf.Len()), nil) mockLayer2.EXPECT().MediaType().Return(types.OCILayer, nil) rc = io.NopCloser(buf) @@ -1175,6 +1177,7 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_disabled(t *testing.T) f(expectedFiles, tw) mockLayer := mockv1.NewMockLayer(ctrl) + mockLayer.EXPECT().Size().Return(int64(buf.Len()), nil) mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil) layerFiles := []string{ filepath.Join(root, "foobar"), @@ -1197,6 +1200,7 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_disabled(t *testing.T) f(secondLayerFiles, tw) mockLayer2 := mockv1.NewMockLayer(ctrl) + mockLayer2.EXPECT().Size().Return(int64(buf.Len()), nil) mockLayer2.EXPECT().MediaType().Return(types.OCILayer, nil) rc = io.NopCloser(buf) @@ -1280,6 +1284,7 @@ func Test_GetFSFromLayers_ignorelist(t *testing.T) { f(expectedFiles, tw) mockLayer := mockv1.NewMockLayer(ctrl) + mockLayer.EXPECT().Size().Return(int64(buf.Len()), nil) mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil) layerFiles := []string{ filepath.Join(root, ".wh.testdir"), @@ -1345,6 +1350,8 @@ func Test_GetFSFromLayers_ignorelist(t *testing.T) { f(layerFiles, tw) + mockLayer.EXPECT().Size().Return(int64(buf.Len()), nil) + rc = io.NopCloser(buf) mockLayer.EXPECT().Uncompressed().Return(rc, nil) @@ -1410,6 +1417,7 @@ func Test_GetFSFromLayers(t *testing.T) { } mockLayer := mockv1.NewMockLayer(ctrl) + mockLayer.EXPECT().Size().Return(int64(buf.Len()), nil) mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil) rc := io.NopCloser(buf) From 612954f3d894a20047c1ea278bb60f86be661ee3 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 23 May 2024 10:32:18 +0000 Subject: [PATCH 2/3] Amend review comments and simplify output --- pkg/util/fs_util.go | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index e03ac29e14..6d5fc33c50 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -171,18 +171,22 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e } var totalSize int64 - var layerSizes []int64 - for _, l := range layers { + layerSizes := make([]int64, 0, len(layers)) + for i, l := range layers { layerSize, err := l.Size() if err != nil { - return nil, err + return nil, errors.Wrap(err, fmt.Sprintf("error checking layer size %d", i)) } layerSizes = append(layerSizes, layerSize) totalSize += layerSize } + printExtractionProgress := cfg.printExtractionProgress + if totalSize == 0 { + printExtractionProgress = false + } - if cfg.printExtractionProgress { - logrus.Infof("Extracting image layers to %s (%d bytes)", root, totalSize) + if printExtractionProgress { + logrus.Infof("Extracting image layers to %s", root) } extractedFiles := []string{} @@ -194,8 +198,9 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e logrus.Tracef("Extracting layer %d", i) } - if cfg.printExtractionProgress { - logrus.Infof("Extracting layer %d %d/%d (%d%%)", i, extractedBytes, totalSize, int(math.Round(float64(extractedBytes)/float64(totalSize)*100))) + progressPerc := float64(extractedBytes) / float64(totalSize) * 100 + if printExtractionProgress { + logrus.Infof("Extracting layer %d (%.1f%%)", i, progressPerc) } r, err := l.Uncompressed() @@ -204,12 +209,12 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e } defer r.Close() - if cfg.printExtractionProgress { + if printExtractionProgress { r = &printAfterReader{ ReadCloser: r, after: time.Second, - print: func(int64) { - logrus.Infof("Extracting layer %d...", i) + print: func(n int) { + logrus.Infof("Extracting layer %d (%.1f%%) %s", i, progressPerc, strings.Repeat(".", n)) }, } } @@ -266,8 +271,8 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e extractedBytes += layerSizes[i] } - if cfg.printExtractionProgress { - logrus.Infof("Extraction complete %d/%d (%d%%)", extractedBytes, totalSize, int(math.Round(float64(extractedBytes)/float64(totalSize)*100))) + if printExtractionProgress { + logrus.Infof("Extraction complete") } return extractedFiles, nil @@ -275,20 +280,20 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e type printAfterReader struct { io.ReadCloser - n int64 t time.Time after time.Duration - print func(int64) + count int + print func(int) } func (r *printAfterReader) Read(p []byte) (n int, err error) { n, err = r.ReadCloser.Read(p) - r.n += int64(n) if r.t.IsZero() { r.t = time.Now() } if time.Since(r.t) >= r.after { - r.print(r.n) + r.count++ + r.print(r.count) r.t = time.Now() } return From b68a9f68b6bb9f3234ffd05d108b53fb085563b8 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 23 May 2024 15:09:49 +0000 Subject: [PATCH 3/3] Print current and total layers --- pkg/util/fs_util.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 6d5fc33c50..e968d62e8e 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -193,14 +193,14 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e var extractedBytes int64 for i, l := range layers { if mediaType, err := l.MediaType(); err == nil { - logrus.Tracef("Extracting layer %d of media type %s", i, mediaType) + logrus.Tracef("Extracting layer %d/%d of media type %s", i+1, len(layers), mediaType) } else { - logrus.Tracef("Extracting layer %d", i) + logrus.Tracef("Extracting layer %d/%d", i+1, len(layers)) } progressPerc := float64(extractedBytes) / float64(totalSize) * 100 if printExtractionProgress { - logrus.Infof("Extracting layer %d (%.1f%%)", i, progressPerc) + logrus.Infof("Extracting layer %d/%d (%.1f%%)", i+1, len(layers), progressPerc) } r, err := l.Uncompressed() @@ -214,7 +214,7 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e ReadCloser: r, after: time.Second, print: func(n int) { - logrus.Infof("Extracting layer %d (%.1f%%) %s", i, progressPerc, strings.Repeat(".", n)) + logrus.Infof("Extracting layer %d/%d (%.1f%%) %s", i+1, len(layers), progressPerc, strings.Repeat(".", n)) }, } }