From c13a0bc16ce6abcd3abd951666c344cda8258ead Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Sun, 23 Jun 2024 13:28:32 +0100 Subject: [PATCH 1/3] pkg/util: CreateFile: add workaround for ETXTBSY on copy --- pkg/util/fs_util.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index e968d62e8e..989d91aa15 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -668,14 +668,30 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode, uid uint32, gid } } + var renamed string dest, err := os.Create(path) if err != nil { - return errors.Wrap(err, "creating file") + if !errors.Is(err, syscall.ETXTBSY) { + return errors.Wrap(err, "creating file") + } + // If the file is busy, just write to a temp file and + // move to dest. + dest, err = os.CreateTemp(os.TempDir(), "") + if err != nil { + return errors.Wrap(err, "create temp file") + } + defer os.Remove(dest.Name()) + renamed = dest.Name() } defer dest.Close() if _, err := io.Copy(dest, reader); err != nil { return errors.Wrap(err, "copying file") } + if renamed != "" { + if err := os.Rename(renamed, path); err != nil { + return errors.Wrap(err, "rename temp file") + } + } return setFilePermissions(path, perm, int(uid), int(gid)) } From c36d9e29c8b540658ba4f56e534c0fc9f1d9e691 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Sun, 23 Jun 2024 13:29:00 +0100 Subject: [PATCH 2/3] pkg/util: Add AllowedPaths to IgnoreListEntry --- pkg/util/fs_util.go | 23 +++++++++++++++++++++++ pkg/util/fs_util_test.go | 29 +++++++++++++++++------------ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 989d91aa15..a2ce270bd2 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -56,24 +56,29 @@ const ( type IgnoreListEntry struct { Path string PrefixMatchOnly bool + // AllowedPaths specifies **exact matches** to ignore, even if they are under Path. + AllowedPaths map[string]struct{} } var defaultIgnoreList = []IgnoreListEntry{ { Path: filepath.Clean(config.KanikoDir), PrefixMatchOnly: false, + AllowedPaths: make(map[string]struct{}), }, { // similarly, we ignore /etc/mtab, since there is no way to know if the file was mounted or came // from the base image Path: "/etc/mtab", PrefixMatchOnly: false, + AllowedPaths: make(map[string]struct{}), }, { // we ingore /tmp/apt-key-gpghome, since the apt keys are added temporarily in this directory. // from the base image Path: "/tmp/apt-key-gpghome", PrefixMatchOnly: true, + AllowedPaths: make(map[string]struct{}), }, } @@ -111,6 +116,7 @@ func AddToIgnoreList(entry IgnoreListEntry) { ignorelist = append(ignorelist, IgnoreListEntry{ Path: filepath.Clean(entry.Path), PrefixMatchOnly: entry.PrefixMatchOnly, + AllowedPaths: make(map[string]struct{}), }) } @@ -118,9 +124,22 @@ func AddToDefaultIgnoreList(entry IgnoreListEntry) { defaultIgnoreList = append(defaultIgnoreList, IgnoreListEntry{ Path: filepath.Clean(entry.Path), PrefixMatchOnly: entry.PrefixMatchOnly, + AllowedPaths: make(map[string]struct{}), }) } +func AddAllowedPathToDefaultIgnoreList(allowPath string) error { + for _, ile := range defaultIgnoreList { + logrus.Debugf("check ignore list entry %+v", ile) + if !strings.HasPrefix(allowPath, ile.Path) { + continue + } + ile.AllowedPaths[allowPath] = struct{}{} + return nil + } + return fmt.Errorf("path %q is not covered by any current entry in ignore list", allowPath) +} + func IncludeWhiteout() FSOpt { return func(opts *FSConfig) { opts.includeWhiteout = true @@ -500,6 +519,9 @@ func IsInIgnoreList(path string) bool { func CheckCleanedPathAgainstProvidedIgnoreList(path string, wl []IgnoreListEntry) bool { for _, wl := range ignorelist { + if _, ok := wl.AllowedPaths[path]; ok { + return false + } if hasCleanedFilepathPrefix(path, wl.Path, wl.PrefixMatchOnly) { return true } @@ -701,6 +723,7 @@ func AddVolumePathToIgnoreList(path string) { AddToIgnoreList(IgnoreListEntry{ Path: path, PrefixMatchOnly: true, + AllowedPaths: map[string]struct{}{}, }) volumes = append(volumes, path) } diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index c21c9bfa0f..97a2a6fcf4 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -58,13 +58,13 @@ func Test_DetectFilesystemSkiplist(t *testing.T) { err := DetectFilesystemIgnoreList(path) expectedSkiplist := []IgnoreListEntry{ - {"/kaniko", false}, - {"/proc", false}, - {"/dev", false}, - {"/dev/pts", false}, - {"/sys", false}, - {"/etc/mtab", false}, - {"/tmp/apt-key-gpghome", true}, + {"/kaniko", false, map[string]struct{}{}}, + {"/proc", false, map[string]struct{}{}}, + {"/dev", false, map[string]struct{}{}}, + {"/dev/pts", false, map[string]struct{}{}}, + {"/sys", false, map[string]struct{}{}}, + {"/etc/mtab", false, map[string]struct{}{}}, + {"/tmp/apt-key-gpghome", true, map[string]struct{}{}}, } actualSkiplist := ignorelist sort.Slice(actualSkiplist, func(i, j int) bool { @@ -274,7 +274,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "file ignored", args: args{ path: "/foo", - ignorelist: []IgnoreListEntry{{"/foo", false}}, + ignorelist: []IgnoreListEntry{{"/foo", false, map[string]struct{}{}}}, }, want: true, }, @@ -282,7 +282,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "directory ignored", args: args{ path: "/foo/bar", - ignorelist: []IgnoreListEntry{{"/foo", false}}, + ignorelist: []IgnoreListEntry{{"/foo", false, map[string]struct{}{}}}, }, want: true, }, @@ -290,7 +290,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "grandparent ignored", args: args{ path: "/foo/bar/baz", - ignorelist: []IgnoreListEntry{{"/foo", false}}, + ignorelist: []IgnoreListEntry{{"/foo", false, map[string]struct{}{}}}, }, want: true, }, @@ -298,7 +298,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "sibling ignored", args: args{ path: "/foo/bar/baz", - ignorelist: []IgnoreListEntry{{"/foo/bat", false}}, + ignorelist: []IgnoreListEntry{{"/foo/bat", false, map[string]struct{}{}}}, }, want: false, }, @@ -306,7 +306,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "prefix match only ", args: args{ path: "/tmp/apt-key-gpghome.xft/gpg.key", - ignorelist: []IgnoreListEntry{{"/tmp/apt-key-gpghome.*", true}}, + ignorelist: []IgnoreListEntry{{"/tmp/apt-key-gpghome.*", true, map[string]struct{}{}}}, }, want: true, }, @@ -1487,22 +1487,27 @@ func TestInitIgnoreList(t *testing.T) { { Path: "/kaniko", PrefixMatchOnly: false, + AllowedPaths: map[string]struct{}{}, }, { Path: "/test/kaniko", PrefixMatchOnly: false, + AllowedPaths: map[string]struct{}{}, }, { Path: "/test/proc", PrefixMatchOnly: false, + AllowedPaths: map[string]struct{}{}, }, { Path: "/etc/mtab", PrefixMatchOnly: false, + AllowedPaths: map[string]struct{}{}, }, { Path: "/tmp/apt-key-gpghome", PrefixMatchOnly: true, + AllowedPaths: map[string]struct{}{}, }, } From 7208a49f5b155d381ae5d67ee15b36df5794c5a0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 24 Jun 2024 10:11:20 +0100 Subject: [PATCH 3/3] make AllowedPaths a []string instead of a map[string]struct{} --- pkg/util/fs_util.go | 27 +++++++++++++++------------ pkg/util/fs_util_test.go | 34 +++++++++++++++++----------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index a2ce270bd2..a27d531bb4 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -56,29 +56,30 @@ const ( type IgnoreListEntry struct { Path string PrefixMatchOnly bool - // AllowedPaths specifies **exact matches** to ignore, even if they are under Path. - AllowedPaths map[string]struct{} + // AllowedPaths specifies **exact matches** to allow, even if they are under + // Path. + AllowedPaths []string } var defaultIgnoreList = []IgnoreListEntry{ { Path: filepath.Clean(config.KanikoDir), PrefixMatchOnly: false, - AllowedPaths: make(map[string]struct{}), + AllowedPaths: nil, }, { // similarly, we ignore /etc/mtab, since there is no way to know if the file was mounted or came // from the base image Path: "/etc/mtab", PrefixMatchOnly: false, - AllowedPaths: make(map[string]struct{}), + AllowedPaths: nil, }, { // we ingore /tmp/apt-key-gpghome, since the apt keys are added temporarily in this directory. // from the base image Path: "/tmp/apt-key-gpghome", PrefixMatchOnly: true, - AllowedPaths: make(map[string]struct{}), + AllowedPaths: nil, }, } @@ -116,7 +117,7 @@ func AddToIgnoreList(entry IgnoreListEntry) { ignorelist = append(ignorelist, IgnoreListEntry{ Path: filepath.Clean(entry.Path), PrefixMatchOnly: entry.PrefixMatchOnly, - AllowedPaths: make(map[string]struct{}), + AllowedPaths: nil, }) } @@ -124,17 +125,16 @@ func AddToDefaultIgnoreList(entry IgnoreListEntry) { defaultIgnoreList = append(defaultIgnoreList, IgnoreListEntry{ Path: filepath.Clean(entry.Path), PrefixMatchOnly: entry.PrefixMatchOnly, - AllowedPaths: make(map[string]struct{}), + AllowedPaths: nil, }) } func AddAllowedPathToDefaultIgnoreList(allowPath string) error { for _, ile := range defaultIgnoreList { - logrus.Debugf("check ignore list entry %+v", ile) if !strings.HasPrefix(allowPath, ile.Path) { continue } - ile.AllowedPaths[allowPath] = struct{}{} + ile.AllowedPaths = append(ile.AllowedPaths, allowPath) return nil } return fmt.Errorf("path %q is not covered by any current entry in ignore list", allowPath) @@ -519,8 +519,10 @@ func IsInIgnoreList(path string) bool { func CheckCleanedPathAgainstProvidedIgnoreList(path string, wl []IgnoreListEntry) bool { for _, wl := range ignorelist { - if _, ok := wl.AllowedPaths[path]; ok { - return false + for _, ap := range wl.AllowedPaths { + if ap == path { + return false + } } if hasCleanedFilepathPrefix(path, wl.Path, wl.PrefixMatchOnly) { return true @@ -578,6 +580,7 @@ func DetectFilesystemIgnoreList(path string) error { AddToIgnoreList(IgnoreListEntry{ Path: lineArr[4], PrefixMatchOnly: false, + AllowedPaths: nil, }) } if err == io.EOF { @@ -723,7 +726,7 @@ func AddVolumePathToIgnoreList(path string) { AddToIgnoreList(IgnoreListEntry{ Path: path, PrefixMatchOnly: true, - AllowedPaths: map[string]struct{}{}, + AllowedPaths: nil, }) volumes = append(volumes, path) } diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 97a2a6fcf4..3e34cd49c9 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -58,13 +58,13 @@ func Test_DetectFilesystemSkiplist(t *testing.T) { err := DetectFilesystemIgnoreList(path) expectedSkiplist := []IgnoreListEntry{ - {"/kaniko", false, map[string]struct{}{}}, - {"/proc", false, map[string]struct{}{}}, - {"/dev", false, map[string]struct{}{}}, - {"/dev/pts", false, map[string]struct{}{}}, - {"/sys", false, map[string]struct{}{}}, - {"/etc/mtab", false, map[string]struct{}{}}, - {"/tmp/apt-key-gpghome", true, map[string]struct{}{}}, + {"/kaniko", false, nil}, + {"/proc", false, nil}, + {"/dev", false, nil}, + {"/dev/pts", false, nil}, + {"/sys", false, nil}, + {"/etc/mtab", false, nil}, + {"/tmp/apt-key-gpghome", true, nil}, } actualSkiplist := ignorelist sort.Slice(actualSkiplist, func(i, j int) bool { @@ -274,7 +274,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "file ignored", args: args{ path: "/foo", - ignorelist: []IgnoreListEntry{{"/foo", false, map[string]struct{}{}}}, + ignorelist: []IgnoreListEntry{{"/foo", false, nil}}, }, want: true, }, @@ -282,7 +282,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "directory ignored", args: args{ path: "/foo/bar", - ignorelist: []IgnoreListEntry{{"/foo", false, map[string]struct{}{}}}, + ignorelist: []IgnoreListEntry{{"/foo", false, nil}}, }, want: true, }, @@ -290,7 +290,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "grandparent ignored", args: args{ path: "/foo/bar/baz", - ignorelist: []IgnoreListEntry{{"/foo", false, map[string]struct{}{}}}, + ignorelist: []IgnoreListEntry{{"/foo", false, nil}}, }, want: true, }, @@ -298,7 +298,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "sibling ignored", args: args{ path: "/foo/bar/baz", - ignorelist: []IgnoreListEntry{{"/foo/bat", false, map[string]struct{}{}}}, + ignorelist: []IgnoreListEntry{{"/foo/bat", false, nil}}, }, want: false, }, @@ -306,7 +306,7 @@ func Test_CheckIgnoreList(t *testing.T) { name: "prefix match only ", args: args{ path: "/tmp/apt-key-gpghome.xft/gpg.key", - ignorelist: []IgnoreListEntry{{"/tmp/apt-key-gpghome.*", true, map[string]struct{}{}}}, + ignorelist: []IgnoreListEntry{{"/tmp/apt-key-gpghome.*", true, nil}}, }, want: true, }, @@ -1487,27 +1487,27 @@ func TestInitIgnoreList(t *testing.T) { { Path: "/kaniko", PrefixMatchOnly: false, - AllowedPaths: map[string]struct{}{}, + AllowedPaths: nil, }, { Path: "/test/kaniko", PrefixMatchOnly: false, - AllowedPaths: map[string]struct{}{}, + AllowedPaths: nil, }, { Path: "/test/proc", PrefixMatchOnly: false, - AllowedPaths: map[string]struct{}{}, + AllowedPaths: nil, }, { Path: "/etc/mtab", PrefixMatchOnly: false, - AllowedPaths: map[string]struct{}{}, + AllowedPaths: nil, }, { Path: "/tmp/apt-key-gpghome", PrefixMatchOnly: true, - AllowedPaths: map[string]struct{}{}, + AllowedPaths: nil, }, }