From 10351c00e9b855105da2dded177207f1ceeafdf2 Mon Sep 17 00:00:00 2001 From: Sri Tikkireddy Date: Wed, 24 May 2023 08:54:03 -0400 Subject: [PATCH 1/2] skip translating job task for jobs with a git source --- bundle/config/mutator/translate_paths.go | 5 ++ bundle/config/mutator/translate_paths_test.go | 66 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 56afdeb1ea..bc40f90a20 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -154,6 +154,11 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) ([]bundle.Mu return nil, fmt.Errorf("unable to determine directory for job %s: %w", key, err) } + // Do not translate job task paths if using git source + if job.GitSource != nil { + continue + } + for i := 0; i < len(job.Tasks); i++ { err := m.translateJobTask(dir, b, &job.Tasks[i]) if err != nil { diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 2cedda2c6e..aa476c522c 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -31,6 +31,72 @@ func touchEmptyFile(t *testing.T, path string) { f.Close() } +func TestTranslatePathsSkippedWithGitSource(t *testing.T) { + dir := t.TempDir() + bundle := &bundle.Bundle{ + Config: config.Root{ + Path: dir, + Workspace: config.Workspace{ + FilesPath: "/bundle", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + + Paths: resources.Paths{ + ConfigFilePath: filepath.Join(dir, "resource.yml"), + }, + JobSettings: &jobs.JobSettings{ + GitSource: &jobs.GitSource{ + GitBranch: "somebranch", + GitCommit: "somecommit", + GitProvider: "github", + GitTag: "sometag", + GitUrl: "https://github.com/someuser/somerepo", + }, + Tasks: []jobs.JobTaskSettings{ + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "my_job_notebook.py", + }, + }, + { + PythonWheelTask: &jobs.PythonWheelTask{ + PackageName: "foo", + }, + }, + { + SparkPythonTask: &jobs.SparkPythonTask{ + PythonFile: "my_python_file.py", + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err := mutator.TranslatePaths().Apply(context.Background(), bundle) + require.NoError(t, err) + + assert.Equal( + t, + "my_job_notebook.py", + bundle.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath, + ) + assert.Equal( + t, + "foo", + bundle.Config.Resources.Jobs["job"].Tasks[1].PythonWheelTask.PackageName, + ) + assert.Equal( + t, + "my_python_file.py", + bundle.Config.Resources.Jobs["job"].Tasks[2].SparkPythonTask.PythonFile, + ) +} + func TestTranslatePaths(t *testing.T) { dir := t.TempDir() touchNotebookFile(t, filepath.Join(dir, "my_job_notebook.py")) From 546d84604317a0a281032974c8629b8b1de69be8 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 7 Jun 2023 12:30:06 +0200 Subject: [PATCH 2/2] Update translate_paths_test.go --- bundle/config/mutator/translate_paths_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 325c3b229d..df99bc1917 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -77,7 +77,8 @@ func TestTranslatePathsSkippedWithGitSource(t *testing.T) { }, }, } - _, err := mutator.TranslatePaths().Apply(context.Background(), bundle) + + err := mutator.TranslatePaths().Apply(context.Background(), bundle) require.NoError(t, err) assert.Equal(