From c698e2d01202debcf4ffdc169cf9ef687ddbaccd Mon Sep 17 00:00:00 2001 From: chlins Date: Thu, 18 Dec 2025 10:50:13 +0800 Subject: [PATCH] feat(attach,upload): add destination-dir flag for custom file paths Signed-off-by: chlins --- cmd/attach.go | 1 + cmd/upload.go | 1 + pkg/backend/attach.go | 22 +++++---- pkg/backend/attach_test.go | 2 +- pkg/backend/build.go | 8 ++-- pkg/backend/build/builder.go | 8 ++-- pkg/backend/build/builder_test.go | 8 ++-- pkg/backend/build/local.go | 8 +++- pkg/backend/build/local_test.go | 4 +- pkg/backend/build/remote.go | 8 +++- pkg/backend/processor/base.go | 11 ++++- pkg/backend/processor/code.go | 3 +- pkg/backend/processor/code_test.go | 4 +- pkg/backend/processor/doc.go | 3 +- pkg/backend/processor/doc_test.go | 4 +- pkg/backend/processor/model.go | 3 +- pkg/backend/processor/model_config.go | 3 +- pkg/backend/processor/model_config_test.go | 4 +- pkg/backend/processor/model_test.go | 4 +- pkg/backend/upload.go | 2 +- pkg/config/attach.go | 50 +++++++++++++-------- pkg/config/upload.go | 31 +++++++++---- test/mocks/backend/build/builder.go | 31 ++++++------- test/mocks/backend/build/output_strategy.go | 31 ++++++------- 24 files changed, 155 insertions(+), 99 deletions(-) diff --git a/cmd/attach.go b/cmd/attach.go index 18b27f0d..064b579f 100644 --- a/cmd/attach.go +++ b/cmd/attach.go @@ -51,6 +51,7 @@ func init() { flags := attachCmd.Flags() flags.StringVarP(&attachConfig.Source, "source", "s", "", "source model artifact name") flags.StringVarP(&attachConfig.Target, "target", "t", "", "target model artifact name") + flags.StringVarP(&attachConfig.DestinationDir, "destination-dir", "d", "", "destination directory for the attached file should be specified as a relative path; by default, it will match the original directory of the attachment") flags.BoolVarP(&attachConfig.OutputRemote, "output-remote", "", false, "turning on this flag will output model artifact to remote registry directly") flags.BoolVarP(&attachConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS") flags.BoolVarP(&attachConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification") diff --git a/cmd/upload.go b/cmd/upload.go index f8ee95f1..10b50eae 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -53,6 +53,7 @@ func init() { flags.BoolVarP(&uploadConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS") flags.BoolVarP(&uploadConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification") flags.BoolVar(&uploadConfig.Raw, "raw", true, "turning on this flag will upload model artifact layer in raw format") + flags.StringVar(&uploadConfig.DestinationDir, "destination-dir", "", "destination directory for the uploaded file should be specified as a relative path; by default, it will match the original directory of the uploaded file") if err := viper.BindPFlags(flags); err != nil { panic(fmt.Errorf("bind cache list flags to viper: %w", err)) diff --git a/pkg/backend/attach.go b/pkg/backend/attach.go index ed716efa..9cc3281a 100644 --- a/pkg/backend/attach.go +++ b/pkg/backend/attach.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "os" + pathfilepath "path/filepath" "reflect" "slices" "sort" @@ -74,7 +75,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac logrus.Infof("attach: loaded source model config [%+v]", srcModelConfig) - proc := b.getProcessor(filepath, cfg.Raw) + proc := b.getProcessor(cfg.DestinationDir, filepath, cfg.Raw) if proc == nil { return fmt.Errorf("failed to get processor for file %s", filepath) } @@ -88,15 +89,20 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac pb.Start() defer pb.Stop() + destPath := filepath + if cfg.DestinationDir != "" { + destPath = pathfilepath.Join(cfg.DestinationDir, pathfilepath.Base(filepath)) + } + layers := srcManifest.Layers // If attach a normal file, we need to process it and create a new layer. if !cfg.Config { var foundLayer *ocispec.Descriptor for _, layer := range srcManifest.Layers { if anno := layer.Annotations; anno != nil { - if anno[modelspec.AnnotationFilepath] == filepath || anno[legacymodelspec.AnnotationFilepath] == filepath { + if anno[modelspec.AnnotationFilepath] == destPath || anno[legacymodelspec.AnnotationFilepath] == destPath { if !cfg.Force { - return fmt.Errorf("file %s already exists, please use --force to overwrite if you want to attach it forcibly", filepath) + return fmt.Errorf("file %s already exists, please use --force to overwrite if you want to attach it forcibly", destPath) } foundLayer = &layer @@ -299,13 +305,13 @@ func (b *backend) getModelConfig(ctx context.Context, reference string, desc oci return &model, nil } -func (b *backend) getProcessor(filepath string, rawMediaType bool) processor.Processor { +func (b *backend) getProcessor(destDir, filepath string, rawMediaType bool) processor.Processor { if modelfile.IsFileType(filepath, modelfile.ConfigFilePatterns) { mediaType := legacymodelspec.MediaTypeModelWeightConfig if rawMediaType { mediaType = legacymodelspec.MediaTypeModelWeightConfigRaw } - return processor.NewModelConfigProcessor(b.store, mediaType, []string{filepath}) + return processor.NewModelConfigProcessor(b.store, mediaType, []string{filepath}, destDir) } if modelfile.IsFileType(filepath, modelfile.ModelFilePatterns) { @@ -313,7 +319,7 @@ func (b *backend) getProcessor(filepath string, rawMediaType bool) processor.Pro if rawMediaType { mediaType = legacymodelspec.MediaTypeModelWeightRaw } - return processor.NewModelProcessor(b.store, mediaType, []string{filepath}) + return processor.NewModelProcessor(b.store, mediaType, []string{filepath}, destDir) } if modelfile.IsFileType(filepath, modelfile.CodeFilePatterns) { @@ -321,7 +327,7 @@ func (b *backend) getProcessor(filepath string, rawMediaType bool) processor.Pro if rawMediaType { mediaType = legacymodelspec.MediaTypeModelCodeRaw } - return processor.NewCodeProcessor(b.store, mediaType, []string{filepath}) + return processor.NewCodeProcessor(b.store, mediaType, []string{filepath}, destDir) } if modelfile.IsFileType(filepath, modelfile.DocFilePatterns) { @@ -329,7 +335,7 @@ func (b *backend) getProcessor(filepath string, rawMediaType bool) processor.Pro if rawMediaType { mediaType = legacymodelspec.MediaTypeModelDocRaw } - return processor.NewDocProcessor(b.store, mediaType, []string{filepath}) + return processor.NewDocProcessor(b.store, mediaType, []string{filepath}, destDir) } return nil diff --git a/pkg/backend/attach_test.go b/pkg/backend/attach_test.go index 293e4368..cbf38abc 100644 --- a/pkg/backend/attach_test.go +++ b/pkg/backend/attach_test.go @@ -73,7 +73,7 @@ func TestGetProcessor(t *testing.T) { for _, tt := range tests { t.Run(tt.filepath, func(t *testing.T) { - proc := b.getProcessor(tt.filepath, false) + proc := b.getProcessor("", tt.filepath, false) if tt.wantType == "" { assert.Nil(t, proc) } else { diff --git a/pkg/backend/build.go b/pkg/backend/build.go index 0aa1d402..3446e857 100644 --- a/pkg/backend/build.go +++ b/pkg/backend/build.go @@ -170,7 +170,7 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile, cfg *config.Build if cfg.Raw { mediaType = modelspec.MediaTypeModelWeightConfigRaw } - processors = append(processors, processor.NewModelConfigProcessor(b.store, mediaType, configs)) + processors = append(processors, processor.NewModelConfigProcessor(b.store, mediaType, configs, "")) } if models := modelfile.GetModels(); len(models) > 0 { @@ -178,7 +178,7 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile, cfg *config.Build if cfg.Raw { mediaType = modelspec.MediaTypeModelWeightRaw } - processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models)) + processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models, "")) } if codes := modelfile.GetCodes(); len(codes) > 0 { @@ -186,7 +186,7 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile, cfg *config.Build if cfg.Raw { mediaType = modelspec.MediaTypeModelCodeRaw } - processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes)) + processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes, "")) } if docs := modelfile.GetDocs(); len(docs) > 0 { @@ -194,7 +194,7 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile, cfg *config.Build if cfg.Raw { mediaType = modelspec.MediaTypeModelDocRaw } - processors = append(processors, processor.NewDocProcessor(b.store, mediaType, docs)) + processors = append(processors, processor.NewDocProcessor(b.store, mediaType, docs, "")) } return processors diff --git a/pkg/backend/build/builder.go b/pkg/backend/build/builder.go index d971589a..bd2c089a 100644 --- a/pkg/backend/build/builder.go +++ b/pkg/backend/build/builder.go @@ -58,7 +58,7 @@ const ( // Builder is an interface for building artifacts. type Builder interface { // BuildLayer builds the layer blob from the given file path. - BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error) + BuildLayer(ctx context.Context, mediaType, workDir, path, destPath string, hooks hooks.Hooks) (ocispec.Descriptor, error) // BuildConfig builds the config blob of the artifact. BuildConfig(ctx context.Context, config modelspec.Model, hooks hooks.Hooks) (ocispec.Descriptor, error) @@ -69,7 +69,7 @@ type Builder interface { type OutputStrategy interface { // OutputLayer outputs the layer blob to the storage (local or remote). - OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) + OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) // OutputConfig outputs the config blob to the storage (local or remote). OutputConfig(ctx context.Context, mediaType, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) @@ -122,7 +122,7 @@ type abstractBuilder struct { interceptor interceptor.Interceptor } -func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error) { +func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path, destPath string, hooks hooks.Hooks) (ocispec.Descriptor, error) { info, err := os.Stat(path) if err != nil { return ocispec.Descriptor{}, fmt.Errorf("failed to get file info: %w", err) @@ -179,7 +179,7 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p }() } - desc, err := ab.strategy.OutputLayer(ctx, mediaType, relPath, digest, size, reader, hooks) + desc, err := ab.strategy.OutputLayer(ctx, mediaType, relPath, destPath, digest, size, reader, hooks) if err != nil { return desc, err } diff --git a/pkg/backend/build/builder_test.go b/pkg/backend/build/builder_test.go index 332cbabe..8c5b776c 100644 --- a/pkg/backend/build/builder_test.go +++ b/pkg/backend/build/builder_test.go @@ -123,10 +123,10 @@ func (s *BuilderTestSuite) TestBuildLayer() { Size: 100, } - s.mockOutputStrategy.On("OutputLayer", mock.Anything, "test/media-type.tar", "test-file.txt", mock.AnythingOfType("string"), mock.AnythingOfType("int64"), mock.AnythingOfType("*io.PipeReader"), mock.Anything). + s.mockOutputStrategy.On("OutputLayer", mock.Anything, "test/media-type.tar", "test-file.txt", "", mock.AnythingOfType("string"), mock.AnythingOfType("int64"), mock.AnythingOfType("*io.PipeReader"), mock.Anything). Return(expectedDesc, nil) - desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, hooks.NewHooks()) + desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, "", hooks.NewHooks()) s.NoError(err) s.Equal(expectedDesc.MediaType, desc.MediaType) s.Equal(expectedDesc.Digest, desc.Digest) @@ -134,12 +134,12 @@ func (s *BuilderTestSuite) TestBuildLayer() { }) s.Run("file not found", func() { - _, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, filepath.Join(s.tempDir, "non-existent.txt"), hooks.NewHooks()) + _, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, filepath.Join(s.tempDir, "non-existent.txt"), "", hooks.NewHooks()) s.Error(err) }) s.Run("directory not supported", func() { - _, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempDir, hooks.NewHooks()) + _, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempDir, "", hooks.NewHooks()) s.Error(err) s.True(strings.Contains(err.Error(), "is a directory and not supported yet")) }) diff --git a/pkg/backend/build/local.go b/pkg/backend/build/local.go index d56215c2..095d124e 100644 --- a/pkg/backend/build/local.go +++ b/pkg/backend/build/local.go @@ -46,7 +46,7 @@ type localOutput struct { } // OutputLayer outputs the layer blob to the local storage. -func (lo *localOutput) OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) { +func (lo *localOutput) OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) { reader = hooks.OnStart(relPath, size, reader) digest, size, err := lo.store.PushBlob(ctx, lo.repo, reader, ocispec.Descriptor{}) if err != nil { @@ -54,12 +54,16 @@ func (lo *localOutput) OutputLayer(ctx context.Context, mediaType, relPath, dige return ocispec.Descriptor{}, fmt.Errorf("failed to push blob to storage: %w", err) } + if destPath == "" { + destPath = relPath + } + desc := ocispec.Descriptor{ MediaType: mediaType, Digest: godigest.Digest(digest), Size: size, Annotations: map[string]string{ - modelspec.AnnotationFilepath: relPath, + modelspec.AnnotationFilepath: destPath, }, } diff --git a/pkg/backend/build/local_test.go b/pkg/backend/build/local_test.go index 89b46d35..1c275f21 100644 --- a/pkg/backend/build/local_test.go +++ b/pkg/backend/build/local_test.go @@ -72,7 +72,7 @@ func (s *LocalOutputTestSuite) TestOutputLayer() { s.mockStorage.On("PushBlob", s.ctx, "test-repo", mock.Anything, ocispec.Descriptor{}). Return(expectedDigest, expectedSize, nil).Once() - desc, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "test-file.txt", expectedDigest, expectedSize, reader, hooks.NewHooks()) + desc, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "test-file.txt", "", expectedDigest, expectedSize, reader, hooks.NewHooks()) s.NoError(err) s.Equal("test/mediatype", desc.MediaType) @@ -88,7 +88,7 @@ func (s *LocalOutputTestSuite) TestOutputLayer() { s.mockStorage.On("PushBlob", s.ctx, "test-repo", mock.Anything, ocispec.Descriptor{}). Return("", int64(0), errors.New("storage error")).Once() - _, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "/work", "test-file.txt", int64(0), reader, hooks.NewHooks()) + _, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "/work", "test-file.txt", "", int64(0), reader, hooks.NewHooks()) s.Error(err) s.Contains(err.Error(), "failed to push blob to storage") diff --git a/pkg/backend/build/remote.go b/pkg/backend/build/remote.go index 32411da4..7011eb69 100644 --- a/pkg/backend/build/remote.go +++ b/pkg/backend/build/remote.go @@ -51,13 +51,17 @@ type remoteOutput struct { } // OutputLayer outputs the layer blob to the remote storage. -func (ro *remoteOutput) OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) { +func (ro *remoteOutput) OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) { + if destPath == "" { + destPath = relPath + } + desc := ocispec.Descriptor{ MediaType: mediaType, Digest: godigest.Digest(digest), Size: size, Annotations: map[string]string{ - modelspec.AnnotationFilepath: relPath, + modelspec.AnnotationFilepath: destPath, }, } diff --git a/pkg/backend/processor/base.go b/pkg/backend/processor/base.go index 1eee1aca..e1e3fcff 100644 --- a/pkg/backend/processor/base.go +++ b/pkg/backend/processor/base.go @@ -48,6 +48,10 @@ type base struct { mediaType string // patterns is the list of patterns to match. patterns []string + // destDir is the destination dir for the processed content, + // which is used to store in the layer filepath annotation, + // it can be empty and by default is relative path to the workDir. + destDir string } // Process implements the Processor interface, which can be reused by other processors. @@ -140,7 +144,12 @@ func (b *base) Process(ctx context.Context, builder build.Builder, workDir strin if err := retry.Do(func() error { logrus.Debugf("processor: processing %s file %s", b.name, path) - desc, err := builder.BuildLayer(ctx, b.mediaType, workDir, path, hooks.NewHooks( + var destPath string + if b.destDir != "" { + destPath = filepath.Join(b.destDir, filepath.Base(path)) + } + + desc, err := builder.BuildLayer(ctx, b.mediaType, workDir, path, destPath, hooks.NewHooks( hooks.WithOnStart(func(name string, size int64, reader io.Reader) io.Reader { return tracker.Add(internalpb.NormalizePrompt("Building layer"), name, size, reader) }), diff --git a/pkg/backend/processor/code.go b/pkg/backend/processor/code.go index 61a72fe3..5e35e421 100644 --- a/pkg/backend/processor/code.go +++ b/pkg/backend/processor/code.go @@ -30,13 +30,14 @@ const ( ) // NewCodeProcessor creates a new code processor. -func NewCodeProcessor(store storage.Storage, mediaType string, patterns []string) Processor { +func NewCodeProcessor(store storage.Storage, mediaType string, patterns []string, destDir string) Processor { return &codeProcessor{ base: &base{ name: codeProcessorName, store: store, mediaType: mediaType, patterns: patterns, + destDir: destDir, }, } } diff --git a/pkg/backend/processor/code_test.go b/pkg/backend/processor/code_test.go index c0f03685..42c0b573 100644 --- a/pkg/backend/processor/code_test.go +++ b/pkg/backend/processor/code_test.go @@ -44,7 +44,7 @@ type codeProcessorSuite struct { func (s *codeProcessorSuite) SetupTest() { s.mockStore = &storage.Storage{} s.mockBuilder = &buildmock.Builder{} - s.processor = NewCodeProcessor(s.mockStore, modelspec.MediaTypeModelCode, []string{"*.py"}) + s.processor = NewCodeProcessor(s.mockStore, modelspec.MediaTypeModelCode, []string{"*.py"}, "") // generate test files for prorcess. s.workDir = s.Suite.T().TempDir() if err := os.WriteFile(filepath.Join(s.workDir, "test.py"), []byte(""), 0644); err != nil { @@ -58,7 +58,7 @@ func (s *codeProcessorSuite) TestName() { func (s *codeProcessorSuite) TestProcess() { ctx := context.Background() - s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{ + s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{ Digest: godigest.Digest("sha256:1234567890abcdef"), Size: int64(1024), Annotations: map[string]string{ diff --git a/pkg/backend/processor/doc.go b/pkg/backend/processor/doc.go index 5a94597b..b48887a5 100644 --- a/pkg/backend/processor/doc.go +++ b/pkg/backend/processor/doc.go @@ -30,13 +30,14 @@ const ( ) // NewDocProcessor creates a new doc processor. -func NewDocProcessor(store storage.Storage, mediaType string, patterns []string) Processor { +func NewDocProcessor(store storage.Storage, mediaType string, patterns []string, destDir string) Processor { return &docProcessor{ base: &base{ name: docProcessorName, store: store, mediaType: mediaType, patterns: patterns, + destDir: destDir, }, } } diff --git a/pkg/backend/processor/doc_test.go b/pkg/backend/processor/doc_test.go index 190624af..bce04969 100644 --- a/pkg/backend/processor/doc_test.go +++ b/pkg/backend/processor/doc_test.go @@ -44,7 +44,7 @@ type docProcessorSuite struct { func (s *docProcessorSuite) SetupTest() { s.mockStore = &storage.Storage{} s.mockBuilder = &buildmock.Builder{} - s.processor = NewDocProcessor(s.mockStore, modelspec.MediaTypeModelDoc, []string{"LICENSE"}) + s.processor = NewDocProcessor(s.mockStore, modelspec.MediaTypeModelDoc, []string{"LICENSE"}, "") // generate test files for prorcess. s.workDir = s.Suite.T().TempDir() if err := os.WriteFile(filepath.Join(s.workDir, "LICENSE"), []byte(""), 0644); err != nil { @@ -58,7 +58,7 @@ func (s *docProcessorSuite) TestName() { func (s *docProcessorSuite) TestProcess() { ctx := context.Background() - s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{ + s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{ Digest: godigest.Digest("sha256:1234567890abcdef"), Size: int64(1024), Annotations: map[string]string{ diff --git a/pkg/backend/processor/model.go b/pkg/backend/processor/model.go index 993d0675..6049b976 100644 --- a/pkg/backend/processor/model.go +++ b/pkg/backend/processor/model.go @@ -30,13 +30,14 @@ const ( ) // NewModelProcessor creates a new model processor. -func NewModelProcessor(store storage.Storage, mediaType string, patterns []string) Processor { +func NewModelProcessor(store storage.Storage, mediaType string, patterns []string, destDir string) Processor { return &modelProcessor{ base: &base{ name: modelProcessorName, store: store, mediaType: mediaType, patterns: patterns, + destDir: destDir, }, } } diff --git a/pkg/backend/processor/model_config.go b/pkg/backend/processor/model_config.go index b03fa0f3..96c7d939 100644 --- a/pkg/backend/processor/model_config.go +++ b/pkg/backend/processor/model_config.go @@ -30,13 +30,14 @@ const ( ) // NewModelConfigProcessor creates a new model config processor. -func NewModelConfigProcessor(store storage.Storage, mediaType string, patterns []string) Processor { +func NewModelConfigProcessor(store storage.Storage, mediaType string, patterns []string, destDir string) Processor { return &modelConfigProcessor{ base: &base{ name: modelConfigProcessorName, store: store, mediaType: mediaType, patterns: patterns, + destDir: destDir, }, } } diff --git a/pkg/backend/processor/model_config_test.go b/pkg/backend/processor/model_config_test.go index 8ddbee30..25baeb0f 100644 --- a/pkg/backend/processor/model_config_test.go +++ b/pkg/backend/processor/model_config_test.go @@ -44,7 +44,7 @@ type modelConfigProcessorSuite struct { func (s *modelConfigProcessorSuite) SetupTest() { s.mockStore = &storage.Storage{} s.mockBuilder = &buildmock.Builder{} - s.processor = NewModelConfigProcessor(s.mockStore, modelspec.MediaTypeModelWeightConfig, []string{"config"}) + s.processor = NewModelConfigProcessor(s.mockStore, modelspec.MediaTypeModelWeightConfig, []string{"config"}, "") // generate test files for prorcess. s.workDir = s.Suite.T().TempDir() if err := os.WriteFile(filepath.Join(s.workDir, "config"), []byte(""), 0644); err != nil { @@ -58,7 +58,7 @@ func (s *modelConfigProcessorSuite) TestName() { func (s *modelConfigProcessorSuite) TestProcess() { ctx := context.Background() - s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{ + s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{ Digest: godigest.Digest("sha256:1234567890abcdef"), Size: int64(1024), Annotations: map[string]string{ diff --git a/pkg/backend/processor/model_test.go b/pkg/backend/processor/model_test.go index 7c05d15d..d414fb57 100644 --- a/pkg/backend/processor/model_test.go +++ b/pkg/backend/processor/model_test.go @@ -44,7 +44,7 @@ type modelProcessorSuite struct { func (s *modelProcessorSuite) SetupTest() { s.mockStore = &storage.Storage{} s.mockBuilder = &buildmock.Builder{} - s.processor = NewModelProcessor(s.mockStore, modelspec.MediaTypeModelWeight, []string{"model"}) + s.processor = NewModelProcessor(s.mockStore, modelspec.MediaTypeModelWeight, []string{"model"}, "") // generate test files for prorcess. s.workDir = s.Suite.T().TempDir() if err := os.WriteFile(filepath.Join(s.workDir, "model"), []byte(""), 0644); err != nil { @@ -58,7 +58,7 @@ func (s *modelProcessorSuite) TestName() { func (s *modelProcessorSuite) TestProcess() { ctx := context.Background() - s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{ + s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{ Digest: godigest.Digest("sha256:1234567890abcdef"), Size: int64(1024), Annotations: map[string]string{ diff --git a/pkg/backend/upload.go b/pkg/backend/upload.go index e4d3ca62..64d43ea9 100644 --- a/pkg/backend/upload.go +++ b/pkg/backend/upload.go @@ -31,7 +31,7 @@ import ( // Upload uploads the file to a model artifact repository in advance, but will not push config and manifest. func (b *backend) Upload(ctx context.Context, filepath string, cfg *config.Upload) error { logrus.Infof("upload: starting upload operation for file %s [repository: %s]", filepath, cfg.Repo) - proc := b.getProcessor(filepath, cfg.Raw) + proc := b.getProcessor(cfg.DestinationDir, filepath, cfg.Raw) if proc == nil { return fmt.Errorf("failed to get processor for file %s", filepath) } diff --git a/pkg/config/attach.go b/pkg/config/attach.go index 0ac02ebb..53943189 100644 --- a/pkg/config/attach.go +++ b/pkg/config/attach.go @@ -16,31 +16,36 @@ package config -import "fmt" +import ( + "fmt" + "path/filepath" +) type Attach struct { - Source string - Target string - OutputRemote bool - PlainHTTP bool - Insecure bool - Nydusify bool - Force bool - Raw bool - Config bool + Source string + Target string + DestinationDir string + OutputRemote bool + PlainHTTP bool + Insecure bool + Nydusify bool + Force bool + Raw bool + Config bool } func NewAttach() *Attach { return &Attach{ - Source: "", - Target: "", - OutputRemote: false, - PlainHTTP: false, - Insecure: false, - Nydusify: false, - Force: false, - Raw: false, - Config: false, + Source: "", + Target: "", + DestinationDir: "", + OutputRemote: false, + PlainHTTP: false, + Insecure: false, + Nydusify: false, + Force: false, + Raw: false, + Config: false, } } @@ -49,6 +54,13 @@ func (a *Attach) Validate() error { return fmt.Errorf("source and target must be specified") } + // Check if destination directory is relative path. + if a.DestinationDir != "" { + if filepath.IsAbs(a.DestinationDir) { + return fmt.Errorf("destination directory must be relative path") + } + } + if a.Nydusify { if !a.OutputRemote { return fmt.Errorf("nydusify only works with output remote") diff --git a/pkg/config/upload.go b/pkg/config/upload.go index df3187b9..f8e3f738 100644 --- a/pkg/config/upload.go +++ b/pkg/config/upload.go @@ -16,21 +16,27 @@ package config -import "errors" +import ( + "errors" + "fmt" + "path/filepath" +) type Upload struct { - Repo string - PlainHTTP bool - Insecure bool - Raw bool + Repo string + PlainHTTP bool + Insecure bool + Raw bool + DestinationDir string } func NewUpload() *Upload { return &Upload{ - Repo: "", - PlainHTTP: false, - Insecure: false, - Raw: false, + Repo: "", + PlainHTTP: false, + Insecure: false, + Raw: false, + DestinationDir: "", } } @@ -39,5 +45,12 @@ func (u *Upload) Validate() error { return errors.New("repo is required") } + // Check if destination directory is relative path. + if u.DestinationDir != "" { + if filepath.IsAbs(u.DestinationDir) { + return fmt.Errorf("destination directory must be relative path") + } + } + return nil } diff --git a/test/mocks/backend/build/builder.go b/test/mocks/backend/build/builder.go index 805fee0d..34bae0c6 100644 --- a/test/mocks/backend/build/builder.go +++ b/test/mocks/backend/build/builder.go @@ -100,9 +100,9 @@ func (_c *Builder_BuildConfig_Call) RunAndReturn(run func(context.Context, v1.Mo return _c } -// BuildLayer provides a mock function with given fields: ctx, mediaType, workDir, path, _a4 -func (_m *Builder) BuildLayer(ctx context.Context, mediaType string, workDir string, path string, _a4 hooks.Hooks) (specs_gov1.Descriptor, error) { - ret := _m.Called(ctx, mediaType, workDir, path, _a4) +// BuildLayer provides a mock function with given fields: ctx, mediaType, workDir, path, destPath, _a5 +func (_m *Builder) BuildLayer(ctx context.Context, mediaType string, workDir string, path string, destPath string, _a5 hooks.Hooks) (specs_gov1.Descriptor, error) { + ret := _m.Called(ctx, mediaType, workDir, path, destPath, _a5) if len(ret) == 0 { panic("no return value specified for BuildLayer") @@ -110,17 +110,17 @@ func (_m *Builder) BuildLayer(ctx context.Context, mediaType string, workDir str var r0 specs_gov1.Descriptor var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, string, hooks.Hooks) (specs_gov1.Descriptor, error)); ok { - return rf(ctx, mediaType, workDir, path, _a4) + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string, hooks.Hooks) (specs_gov1.Descriptor, error)); ok { + return rf(ctx, mediaType, workDir, path, destPath, _a5) } - if rf, ok := ret.Get(0).(func(context.Context, string, string, string, hooks.Hooks) specs_gov1.Descriptor); ok { - r0 = rf(ctx, mediaType, workDir, path, _a4) + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string, hooks.Hooks) specs_gov1.Descriptor); ok { + r0 = rf(ctx, mediaType, workDir, path, destPath, _a5) } else { r0 = ret.Get(0).(specs_gov1.Descriptor) } - if rf, ok := ret.Get(1).(func(context.Context, string, string, string, hooks.Hooks) error); ok { - r1 = rf(ctx, mediaType, workDir, path, _a4) + if rf, ok := ret.Get(1).(func(context.Context, string, string, string, string, hooks.Hooks) error); ok { + r1 = rf(ctx, mediaType, workDir, path, destPath, _a5) } else { r1 = ret.Error(1) } @@ -138,14 +138,15 @@ type Builder_BuildLayer_Call struct { // - mediaType string // - workDir string // - path string -// - _a4 hooks.Hooks -func (_e *Builder_Expecter) BuildLayer(ctx interface{}, mediaType interface{}, workDir interface{}, path interface{}, _a4 interface{}) *Builder_BuildLayer_Call { - return &Builder_BuildLayer_Call{Call: _e.mock.On("BuildLayer", ctx, mediaType, workDir, path, _a4)} +// - destPath string +// - _a5 hooks.Hooks +func (_e *Builder_Expecter) BuildLayer(ctx interface{}, mediaType interface{}, workDir interface{}, path interface{}, destPath interface{}, _a5 interface{}) *Builder_BuildLayer_Call { + return &Builder_BuildLayer_Call{Call: _e.mock.On("BuildLayer", ctx, mediaType, workDir, path, destPath, _a5)} } -func (_c *Builder_BuildLayer_Call) Run(run func(ctx context.Context, mediaType string, workDir string, path string, _a4 hooks.Hooks)) *Builder_BuildLayer_Call { +func (_c *Builder_BuildLayer_Call) Run(run func(ctx context.Context, mediaType string, workDir string, path string, destPath string, _a5 hooks.Hooks)) *Builder_BuildLayer_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(hooks.Hooks)) + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(string), args[5].(hooks.Hooks)) }) return _c } @@ -155,7 +156,7 @@ func (_c *Builder_BuildLayer_Call) Return(_a0 specs_gov1.Descriptor, _a1 error) return _c } -func (_c *Builder_BuildLayer_Call) RunAndReturn(run func(context.Context, string, string, string, hooks.Hooks) (specs_gov1.Descriptor, error)) *Builder_BuildLayer_Call { +func (_c *Builder_BuildLayer_Call) RunAndReturn(run func(context.Context, string, string, string, string, hooks.Hooks) (specs_gov1.Descriptor, error)) *Builder_BuildLayer_Call { _c.Call.Return(run) return _c } diff --git a/test/mocks/backend/build/output_strategy.go b/test/mocks/backend/build/output_strategy.go index abeafaec..282ddff7 100644 --- a/test/mocks/backend/build/output_strategy.go +++ b/test/mocks/backend/build/output_strategy.go @@ -103,9 +103,9 @@ func (_c *OutputStrategy_OutputConfig_Call) RunAndReturn(run func(context.Contex return _c } -// OutputLayer provides a mock function with given fields: ctx, mediaType, relPath, digest, size, reader, _a6 -func (_m *OutputStrategy) OutputLayer(ctx context.Context, mediaType string, relPath string, digest string, size int64, reader io.Reader, _a6 hooks.Hooks) (v1.Descriptor, error) { - ret := _m.Called(ctx, mediaType, relPath, digest, size, reader, _a6) +// OutputLayer provides a mock function with given fields: ctx, mediaType, relPath, destPath, digest, size, reader, _a7 +func (_m *OutputStrategy) OutputLayer(ctx context.Context, mediaType string, relPath string, destPath string, digest string, size int64, reader io.Reader, _a7 hooks.Hooks) (v1.Descriptor, error) { + ret := _m.Called(ctx, mediaType, relPath, destPath, digest, size, reader, _a7) if len(ret) == 0 { panic("no return value specified for OutputLayer") @@ -113,17 +113,17 @@ func (_m *OutputStrategy) OutputLayer(ctx context.Context, mediaType string, rel var r0 v1.Descriptor var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, string, int64, io.Reader, hooks.Hooks) (v1.Descriptor, error)); ok { - return rf(ctx, mediaType, relPath, digest, size, reader, _a6) + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string, int64, io.Reader, hooks.Hooks) (v1.Descriptor, error)); ok { + return rf(ctx, mediaType, relPath, destPath, digest, size, reader, _a7) } - if rf, ok := ret.Get(0).(func(context.Context, string, string, string, int64, io.Reader, hooks.Hooks) v1.Descriptor); ok { - r0 = rf(ctx, mediaType, relPath, digest, size, reader, _a6) + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string, int64, io.Reader, hooks.Hooks) v1.Descriptor); ok { + r0 = rf(ctx, mediaType, relPath, destPath, digest, size, reader, _a7) } else { r0 = ret.Get(0).(v1.Descriptor) } - if rf, ok := ret.Get(1).(func(context.Context, string, string, string, int64, io.Reader, hooks.Hooks) error); ok { - r1 = rf(ctx, mediaType, relPath, digest, size, reader, _a6) + if rf, ok := ret.Get(1).(func(context.Context, string, string, string, string, int64, io.Reader, hooks.Hooks) error); ok { + r1 = rf(ctx, mediaType, relPath, destPath, digest, size, reader, _a7) } else { r1 = ret.Error(1) } @@ -140,17 +140,18 @@ type OutputStrategy_OutputLayer_Call struct { // - ctx context.Context // - mediaType string // - relPath string +// - destPath string // - digest string // - size int64 // - reader io.Reader -// - _a6 hooks.Hooks -func (_e *OutputStrategy_Expecter) OutputLayer(ctx interface{}, mediaType interface{}, relPath interface{}, digest interface{}, size interface{}, reader interface{}, _a6 interface{}) *OutputStrategy_OutputLayer_Call { - return &OutputStrategy_OutputLayer_Call{Call: _e.mock.On("OutputLayer", ctx, mediaType, relPath, digest, size, reader, _a6)} +// - _a7 hooks.Hooks +func (_e *OutputStrategy_Expecter) OutputLayer(ctx interface{}, mediaType interface{}, relPath interface{}, destPath interface{}, digest interface{}, size interface{}, reader interface{}, _a7 interface{}) *OutputStrategy_OutputLayer_Call { + return &OutputStrategy_OutputLayer_Call{Call: _e.mock.On("OutputLayer", ctx, mediaType, relPath, destPath, digest, size, reader, _a7)} } -func (_c *OutputStrategy_OutputLayer_Call) Run(run func(ctx context.Context, mediaType string, relPath string, digest string, size int64, reader io.Reader, _a6 hooks.Hooks)) *OutputStrategy_OutputLayer_Call { +func (_c *OutputStrategy_OutputLayer_Call) Run(run func(ctx context.Context, mediaType string, relPath string, destPath string, digest string, size int64, reader io.Reader, _a7 hooks.Hooks)) *OutputStrategy_OutputLayer_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(int64), args[5].(io.Reader), args[6].(hooks.Hooks)) + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(string), args[5].(int64), args[6].(io.Reader), args[7].(hooks.Hooks)) }) return _c } @@ -160,7 +161,7 @@ func (_c *OutputStrategy_OutputLayer_Call) Return(_a0 v1.Descriptor, _a1 error) return _c } -func (_c *OutputStrategy_OutputLayer_Call) RunAndReturn(run func(context.Context, string, string, string, int64, io.Reader, hooks.Hooks) (v1.Descriptor, error)) *OutputStrategy_OutputLayer_Call { +func (_c *OutputStrategy_OutputLayer_Call) RunAndReturn(run func(context.Context, string, string, string, string, int64, io.Reader, hooks.Hooks) (v1.Descriptor, error)) *OutputStrategy_OutputLayer_Call { _c.Call.Return(run) return _c }