feat:fs s3 integration and tilesDeletionStartegy(MAPCO-9806)

.gitignore

@@ -83,6 +83,9 @@ helm/local.yaml
 
 # Local documentation (can be regenerated)
 ai-docs/implementation-plan.md
+.claude-flow
+.claude
+
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache
@@ -118,3 +121,6 @@ dist
 # Jest
 jest_html_reporters.html
 reports
+
+# tmp files
+tmp

Dockerfile

@@ -27,7 +27,7 @@ RUN npm ci --only=production
 
 COPY --chown=node:node --from=build /tmp/buildApp/dist .
 COPY --chown=node:node ./config ./config
-
+COPY --chown=node:node ./scripts ./scripts
 
 USER node
 EXPOSE 8080

config/custom-environment-variables.json

@@ -11,6 +11,17 @@
       "prettyPrint": {
         "__name": "LOG_PRETTY_PRINT_ENABLED",
         "__format": "boolean"
+      },
+      "opentelemetryOptions": {
+        "enabled": {
+          "__name": "TELEMETRY_LOGGER_OTEL_ENABLED",
+          "__format": "boolean"
+        },
+        "url": "TELEMETRY_LOGGER_OTEL_URL",
+        "resourceAttributes": {
+          "__name": "TELEMETRY_LOGGER_OTEL_RESOURCE_ATTRIBUTES",
+          "__format": "json"
+        }
       }
     }
   },
@@ -57,5 +68,37 @@
       "__name": "HTTP_RETRY_SHOULD_RESET_TIMEOUT",
       "__format": "boolean"
     }
+  },
+  "s3": {
+    "endpoint": "S3_ENDPOINT",
+    "accessKeyId": "S3_ACCESS_KEY_ID",
+    "secretAccessKey": "S3_SECRET_ACCESS_KEY",
+    "sslEnabled": {
+      "__name": "S3_SSL_ENABLED",
+      "__format": "boolean"
+    },
+    "forcePathStyle": {
+      "__name": "S3_FORCE_PATH_STYLE",
+      "__format": "boolean"
+    },
+    "region": "S3_REGION"
+  },
+  "strategies": {
+    "tilesDeletion": {
+      "s3Bucket": "TILES_DELETION_S3_BUCKET",
+      "fsBasePath": "TILES_DELETION_FS_BASE_PATH",
+      "batchSize": {
+        "__name": "TILES_DELETION_BATCH_SIZE",
+        "__format": "number"
+      },
+      "concurrency": {
+        "__name": "TILES_DELETION_CONCURRENCY",
+        "__format": "number"
+      },
+      "failureSampleSize": {
+        "__name": "TILES_DELETION_FAILURE_SAMPLE_SIZE",
+        "__format": "number"
+      }
+    }
   }
 }

config/default.json

@@ -7,7 +7,12 @@
     "shared": {},
     "logger": {
       "level": "info",
-      "prettyPrint": false
+      "prettyPrint": false,
+      "opentelemetryOptions": {
+        "enabled": false,
+        "url": "",
+        "resourceAttributes": {}
+      }
     }
   },
   "server": {
@@ -53,5 +58,22 @@
     "attempts": 3,
     "delay": "exponential",
     "shouldResetTimeout": true
+  },
+  "s3": {
+    "endpoint": "http://localhost:9000",
+    "accessKeyId": "minioadmin",
+    "secretAccessKey": "minioadmin",
+    "sslEnabled": false,
+    "forcePathStyle": true,
+    "region": "us-east-1"
+  },
+  "strategies": {
+    "tilesDeletion": {
+      "s3Bucket": "",
+      "fsBasePath": "/tiles",
+      "batchSize": 1000,
+      "concurrency": 10,
+      "failureSampleSize": 3
+    }
   }
 }

helm/templates/_helpers.tpl

@@ -107,12 +107,12 @@ Returns the tracing url from global if exists or from the chart's values
 {{- end -}}
 
 {{/*
-Returns the tracing url from global if exists or from the chart's values
+Returns the metrics url from global if exists or from the chart's values
 */}}
 {{- define "cleaner.metricsUrl" -}}
 {{- if .Values.global.metrics.url }}
     {{- .Values.global.metrics.url -}}
 {{- else -}}
-    {{- .Values.env.metrics.url -}}
+    {{- .Values.env.metrics.url | default "" -}}
 {{- end -}}
 {{- end -}}

helm/templates/_tplValues.tpl

@@ -48,3 +48,15 @@ Custom definitions
 {{- define "common.jobDefinitions.merged" -}}
 {{- include "common.tplvalues.merge" ( dict "values" ( list .Values.jobDefinitions .Values.global.jobDefinitions ) "context" . ) }}
 {{- end -}}
+
+{{- define "common.serviceUrls.merged" -}}
+{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceUrls .Values.global.serviceUrls ) "context" . ) }}
+{{- end -}}
+
+{{- define "common.storage.merged" -}}
+{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.storage .Values.global.storage ) "context" . ) }}
+{{- end -}}
+
+{{- define "common.ca.merged" -}}
+{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.ca .Values.global.ca ) "context" . ) }}
+{{- end -}}

helm/templates/configmap.yaml

@@ -1,32 +1,43 @@
 {{- $tracingUrl := include "cleaner.tracingUrl" . -}}
-{{- $metricsUrl := include "cleaner.metricsUrl" . -}}
+{{- $serviceUrls := fromYaml (include "common.serviceUrls.merged" .) -}}
+{{- $storage := fromYaml (include "common.storage.merged" .) -}}
+{{- $s3 := ($storage.s3) | default dict -}}
+{{- $internalPvc := (($storage.fs).internalPvc) | default dict -}}
+{{- $tilesFSBasePath :=  clean (printf "/%s/%s" $internalPvc.mountPath ($internalPvc.tilesSubPath | default "tiles")) -}}
 {{- if .Values.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ printf "%s-configmap" (include "cleaner.fullname" .) }}
 data:
   LOG_LEVEL: {{ .Values.env.logLevel | quote }}
-  LOG_PRETTY_PRINT_ENABLED: {{ .Values.env.logPrettyPrintEnabled | quote }}
-  {{ if .Values.env.tracing.enabled }}
+  LOG_PRETTY_PRINT_ENABLED: {{ .Values.env.prettyPrintEnabled | quote }}
+  {{- with .Values.env.openTelemetryOptions }}
+  TELEMETRY_LOGGER_OTEL_ENABLED: {{ .enabled | default false | quote }}
+  {{- if .enabled }}
+  TELEMETRY_LOGGER_OTEL_URL: {{ .url | default "" | quote }}
+  TELEMETRY_LOGGER_OTEL_RESOURCE_ATTRIBUTES: {{ .resourceAttributes | default dict | toJson | quote }}
+  {{- end}}
+  {{- end }}
+  {{- if .Values.env.tracing.enabled }}
   TELEMETRY_TRACING_ENABLED: 'true'
   TELEMETRY_TRACING_URL: {{ $tracingUrl }}
-  {{ end }}
+  {{- end }}
   npm_config_cache: /tmp/
   {{- with .Values.configManagement }}
-  CONFIG_NAME: {{ .name| quote }}
+  CONFIG_NAME: {{ .name | quote }}
   CONFIG_VERSION: {{ .version | quote }}
   CONFIG_OFFLINE_MODE: {{ .offlineMode | quote }}
   CONFIG_SERVER_URL: {{ .serverUrl | quote }}
   {{- end }}
   {{- with .Values.env.worker.capabilities }}
   WORKER_CAPABILITY_PAIRS: {{ .pairs | toJson | quote }}
   {{- end }}
+  QUEUE_JOB_MANAGER_BASE_URL: {{ $serviceUrls.jobManager | default "" | quote }}
+  QUEUE_HEARTBEAT_BASE_URL: {{ $serviceUrls.heartbeatManager | default "" | quote }}
   {{- with .Values.env.queue }}
-  QUEUE_JOB_MANAGER_BASE_URL: {{ .jobManagerBaseUrl | quote }}
-  QUEUE_HEARTBEAT_BASE_URL: {{ .heartbeatBaseUrl | quote }}
-  QUEUE_HEARTBEAT_INTERVAL_MS: {{ .heartbeatIntervalMs | quote }}
-  QUEUE_DEQUEUE_INTERVAL_MS: {{ .dequeueIntervalMs | quote }}
+  QUEUE_HEARTBEAT_INTERVAL_MS: {{ .heartbeatIntervalMs | default 1000 | quote }}
+  QUEUE_DEQUEUE_INTERVAL_MS: {{ .dequeueIntervalMs | default 3000 | quote }}
   {{- end }}
   {{- $jobDefinitions := fromYaml (include "common.jobDefinitions.merged" .) }}
   {{- if $jobDefinitions }}
@@ -36,5 +47,20 @@ data:
   HTTP_RETRY_ATTEMPTS: {{ .attempts | quote }}
   HTTP_RETRY_DELAY: {{ .delay | quote }}
   HTTP_RETRY_SHOULD_RESET_TIMEOUT: {{ .shouldResetTimeout | quote }}
-  {{- end -}}
+  {{- end }}
+  JOBNIK_SDK_BASE_URL: {{ $serviceUrls.jobManager | default "" | quote }}
+  {{- with .Values.env.jobnik.worker }}
+  JOBNIK_WORKER_CONCURRENCY: {{ .concurrency | default 1 | quote }}
+  {{- end }}
+  S3_ENDPOINT: {{ if $s3.endpointUrl }}{{ printf "%s://%s" ($s3.sslEnabled | ternary "https" "http") $s3.endpointUrl | quote }}{{ else }}{{ "" | quote }}{{ end }}
+  S3_FORCE_PATH_STYLE: {{ $s3.forcePathStyle | default false | quote }}
+  S3_SSL_ENABLED: {{ $s3.sslEnabled | default false | quote }}
+  S3_REGION: {{ $s3.region | default "us-east-1" | quote }}
+  TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }}
+  TILES_DELETION_FS_BASE_PATH: {{ $tilesFSBasePath | quote }}
+  {{- with .Values.env.strategies.tilesDeletion }}
+  TILES_DELETION_BATCH_SIZE: {{ .batchSize | default 1000 | quote }}
+  TILES_DELETION_CONCURRENCY: {{ .concurrency | default 10 | quote }}
+  TILES_DELETION_FAILURE_SAMPLE_SIZE: {{ .failureSampleSize | default 3 | quote }}
+  {{- end }}
 {{- end }}

helm/templates/deployment.yaml

@@ -4,6 +4,10 @@
 {{- $cloudProviderDockerRegistryUrl := include "cleaner.cloudProviderDockerRegistryUrl" . -}}
 {{- $cloudProviderImagePullSecretName := include "cleaner.cloudProviderImagePullSecretName" . -}}
 {{- $imageTag := include "cleaner.tag" . -}}
+{{- $ca := fromYaml (include "common.ca.merged" .) -}}
+{{- $storage := fromYaml (include "common.storage.merged" .) -}}
+{{- $s3 := ($storage.s3) | default dict -}}
+{{- $internalPvc := (($storage.fs).internalPvc) | default dict -}}
 {{- if .Values.enabled -}}
 apiVersion: apps/v1
 kind: Deployment
@@ -58,26 +62,46 @@ spec:
           {{- end }}
           {{- end }}
           volumeMounts:
-            {{- if .Values.caSecretName }}
-            - mountPath: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }}
+            {{- if $ca.secretName }}
+            - mountPath: {{ printf "%s/%s" $ca.path $ca.key | quote }}
               name: root-ca
-              subPath: {{ quote .Values.caKey }}
+              subPath: {{ quote $ca.key }}
             {{- end }}
             {{- if .Values.extraVolumeMounts -}}
               {{ toYaml .Values.extraVolumeMounts | nindent 12 }}
             {{- end }}
+            {{- if $internalPvc.enabled }}
+            - name: internal-pvc
+              mountPath: {{ $internalPvc.mountPath | quote }}
+            {{- end }}
           env:
+            - name: K8S_POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
             - name: SERVER_PORT
               value: {{ .Values.env.targetPort | quote }}
-            {{- if .Values.caSecretName }}
+            {{- if $ca.secretName }}
             - name: REQUESTS_CA_BUNDLE
-              value: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }}
+              value: {{ printf "%s/%s" $ca.path $ca.key | quote }}
             - name: NODE_EXTRA_CA_CERTS
-              value: {{ printf "[%s/%s]" .Values.caPath .Values.caKey | quote }}
+              value: {{ printf "%s/%s" $ca.path $ca.key | quote }}
+            {{- end }}
+            {{- if $s3.secretName }}
+            - name: S3_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $s3.secretName }}
+                  key: accessKeyId
+            - name: S3_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $s3.secretName }}
+                  key: secretAccessKey
             {{- end }}
             {{- if .Values.extraEnvVars }}
             {{- toYaml .Values.extraEnvVars | nindent 12 }}
-            {{- end }}            
+            {{- end }}
           envFrom:
             - configMapRef:
                 name: {{ printf "%s-configmap" (include "cleaner.fullname" .) }}
@@ -107,12 +131,17 @@ spec:
         {{ tpl (toYaml .Values.sidecars) . | nindent 8 }}
         {{- end }}
       volumes:
-        {{- if .Values.caSecretName }}
+        {{- if $ca.secretName }}
         - name: root-ca
           secret:
-            secretName: {{ .Values.caSecretName }}
+            secretName: {{ $ca.secretName }}
         {{- end }}
         {{- if .Values.extraVolumes -}}
           {{ tpl (toYaml .Values.extraVolumes) . | nindent 8 }}
-        {{- end }}    
+        {{- end }}
+        {{- if $internalPvc.enabled }}
+        - name: internal-pvc
+          persistentVolumeClaim:
+            claimName: {{ $internalPvc.name }}
+        {{- end }}
 {{- end -}}

helm/values.yaml

@@ -3,6 +3,28 @@ global:
   tracing: {}
   metrics: {}
   jobDefinitions: {}
+  storage: {}
+  serviceUrls: {}
+  ca: {}
+
+serviceUrls:
+  jobManager: ""
+  heartbeatManager: ""
+
+storage:
+  s3:
+    endpointUrl: ""
+    forcePathStyle: true
+    secretName: ""
+    sslEnabled: false
+    region: "us-east-1"
+    tilesBucket: ""
+  fs:
+    internalPvc:
+      enabled: false
+      name: ""
+      mountPath: ""
+      tilesSubPath: "tiles"
 
 
 mclabels:
@@ -63,9 +85,10 @@ cloudProvider:
   imagePullSecretName: ''
   flavor: openshift
 
-caSecretName: ''
-caPath: '/usr/local/share/ca-certificates'
-caKey: 'ca.crt'
+ca:
+  secretName: ''
+  path: '/usr/local/share/ca-certificates'
+  key: 'ca.crt'
 
 image:
   repository: cleaner
@@ -76,13 +99,15 @@ image:
 env:
   port: 8080
   logLevel: info
-  logPrettyPrintEnabled: false
+  prettyPrintEnabled: false
+  openTelemetryOptions:
+    enabled: false
+    url: ""
+    resourceAttributes: {}
   tracing:
     enabled: false
     url: http://localhost:55681/v1/trace
   queue:
-    jobManagerBaseUrl: "http//localhost:8080"
-    heartbeatBaseUrl: "http//localhost:8081"
     heartbeatIntervalMs: 1000
     dequeueIntervalMs: 3000
   worker:
@@ -91,11 +116,16 @@ env:
         - job: "Ingestion_Update"
           task: "tiles-deletion"
         - job: "Ingestion_Swap_Update"
-          task: "tiles-deletion"    
+          task: "tiles-deletion"
   httpRetry:
     attempts: 3
     delay: "exponential"
     shouldResetTimeout: true
+  strategies:
+    tilesDeletion:
+      batchSize: 1000
+      concurrency: 10
+      failureSampleSize: 3
 
 resources:
   enabled: true