From 4d0c74b751555b6f76833136faacf1d06a733c39 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Mon, 20 Nov 2023 18:26:58 +0100 Subject: [PATCH] Work around ES-947370: PYTHONPATH not being set correctly --- .../resources/{{.project_name}}_pipeline.yml.tmpl | 3 +++ .../template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl | 1 + 2 files changed, 4 insertions(+) diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl index 498604f6b6..4b8f74d173 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl @@ -7,3 +7,6 @@ resources: libraries: - notebook: path: ../src/dlt_pipeline.ipynb + + configuration: + bundle.sourcePath: /Workspace/${workspace.file_path}/src diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl index 8c85e97ef6..4f50294f6a 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl @@ -35,6 +35,7 @@ "# Import DLT and src/{{.project_name}}\n", "import dlt\n", "import sys\n", + "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", "from pyspark.sql.functions import expr\n", "from {{.project_name}} import main" {{else}}