diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index a19c6cbd93023..ac1e2a8240872 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -71,6 +71,9 @@ spec: {{- if and $stateful .Values.scheduler.updateStrategy }} updateStrategy: {{- toYaml .Values.scheduler.updateStrategy | nindent 4 }} {{- end }} + {{- if and $stateful .Values.workers.persistence.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.workers.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} + {{- end }} {{- if and (not $stateful) .Values.scheduler.strategy }} strategy: {{- toYaml .Values.scheduler.strategy | nindent 4 }} {{- end }} diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 42e4ddf7a17fc..da109c0a26977 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -73,6 +73,9 @@ spec: {{- if and (not $persistence) (.Values.triggerer.strategy) }} strategy: {{- toYaml .Values.triggerer.strategy | nindent 4 }} {{- end }} + {{- if and $persistence .Values.triggerer.persistence.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.triggerer.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} + {{- end }} template: metadata: labels: diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 0cf3ecb9f6e2f..a32c86c289a9b 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -66,6 +66,9 @@ spec: {{- if $revisionHistoryLimit }} revisionHistoryLimit: {{ $revisionHistoryLimit }} {{- end }} + {{- if and $persistence .Values.workers.persistence.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.workers.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} + {{- end }} selector: matchLabels: tier: airflow diff --git a/chart/values.schema.json b/chart/values.schema.json index fed2168ea76a0..ac5cf02e6ec6e 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1601,6 +1601,10 @@ "type": "boolean", "default": true }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", + "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet" + }, "size": { "description": "Volume size for worker StatefulSet.", "type": "string", @@ -2946,6 +2950,10 @@ "type": "string", "default": "100Gi" }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", + "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet" + }, "storageClassName": { "description": "If using a custom StorageClass, pass name ref to all StatefulSets here.", "type": [ @@ -11835,6 +11843,27 @@ } } } + }, + "persistentVolumeClaimRetentionPolicy": { + "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet", + "type": [ + "object", + "null" + ], + "default": null, + "additionalProperties": false, + "properties": { + "whenDeleted": { + "description": "Whether to retain the PVC when the StatefulSet is deleted.", + "type": "string", + "default": "Retain" + }, + "whenScaled": { + "description": "Whether to retain the PVC when the StatefulSet is scaled.", + "type": "string", + "default": "Retain" + } + } } } } diff --git a/chart/values.yaml b/chart/values.yaml index c623f3e9dd939..bcdf71ae05020 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -619,6 +619,11 @@ workers: persistence: # Enable persistent volumes enabled: true + # This policy determines whether PVCs should be deleted when StatefulSet is scaled down or removed. + persistentVolumeClaimRetentionPolicy: ~ + # persistentVolumeClaimRetentionPolicy: + # whenDeleted: Delete + # whenScaled: Delete # Volume size for worker StatefulSet size: 100Gi # If using a custom storageClass, pass name ref to all statefulSets here @@ -1532,6 +1537,8 @@ triggerer: persistence: # Enable persistent volumes enabled: true + # This policy determines whether PVCs should be deleted when StatefulSet is scaled down or removed. + persistentVolumeClaimRetentionPolicy: ~ # Volume size for triggerer StatefulSet size: 100Gi # If using a custom storageClass, pass name ref to all statefulSets here diff --git a/helm_tests/airflow_core/test_scheduler.py b/helm_tests/airflow_core/test_scheduler.py index 3fb7f6754d8d5..a329b833a45e9 100644 --- a/helm_tests/airflow_core/test_scheduler.py +++ b/helm_tests/airflow_core/test_scheduler.py @@ -839,6 +839,24 @@ def test_scheduler_template_storage_class_name(self): "spec.volumeClaimTemplates[0].spec.storageClassName", docs[0] ) + def test_persistent_volume_claim_retention_policy(self): + docs = render_chart( + values={ + "executor": "LocalExecutor", + "workers": { + "persistence": { + "enabled": True, + "persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}, + } + }, + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + ) + + assert { + "whenDeleted": "Delete", + } == jmespath.search("spec.persistentVolumeClaimRetentionPolicy", docs[0]) + class TestSchedulerNetworkPolicy: """Tests scheduler network policy.""" diff --git a/helm_tests/airflow_core/test_triggerer.py b/helm_tests/airflow_core/test_triggerer.py index 0d8c4dd8206bc..9ef19b38df239 100644 --- a/helm_tests/airflow_core/test_triggerer.py +++ b/helm_tests/airflow_core/test_triggerer.py @@ -647,6 +647,24 @@ def test_triggerer_template_storage_class_name(self): "spec.volumeClaimTemplates[0].spec.storageClassName", docs[0] ) + def test_persistent_volume_claim_retention_policy(self): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "triggerer": { + "persistence": { + "enabled": True, + "persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}, + } + }, + }, + show_only=["templates/triggerer/triggerer-deployment.yaml"], + ) + + assert { + "whenDeleted": "Delete", + } == jmespath.search("spec.persistentVolumeClaimRetentionPolicy", docs[0]) + class TestTriggererServiceAccount: """Tests triggerer service account.""" diff --git a/helm_tests/airflow_core/test_worker.py b/helm_tests/airflow_core/test_worker.py index 1fcc88caa6809..85384f176c682 100644 --- a/helm_tests/airflow_core/test_worker.py +++ b/helm_tests/airflow_core/test_worker.py @@ -82,6 +82,24 @@ def test_should_add_extra_containers(self): "image": "test-registry/test-repo:test-tag", } == jmespath.search("spec.template.spec.containers[-1]", docs[0]) + def test_persistent_volume_claim_retention_policy(self): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": { + "persistence": { + "enabled": True, + "persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}, + } + }, + }, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + assert { + "whenDeleted": "Delete", + } == jmespath.search("spec.persistentVolumeClaimRetentionPolicy", docs[0]) + def test_should_template_extra_containers(self): docs = render_chart( values={