From dbc2e4c589ba9d336293eb8e5aca886085844001 Mon Sep 17 00:00:00 2001 From: sjulti Date: Thu, 25 Sep 2025 13:54:24 +0200 Subject: [PATCH 1/4] Enable stability test workflow for proxy. --- .github/workflows/proxy_stability_testing.yml | 142 +++++++++++++ testing/proxy-stability/helm-values.yaml | 197 ++++++++++++++++++ .../proxy-stability/persistent-volumes.yaml | 42 ++++ 3 files changed, 381 insertions(+) create mode 100644 .github/workflows/proxy_stability_testing.yml create mode 100644 testing/proxy-stability/helm-values.yaml create mode 100644 testing/proxy-stability/persistent-volumes.yaml diff --git a/.github/workflows/proxy_stability_testing.yml b/.github/workflows/proxy_stability_testing.yml new file mode 100644 index 000000000..b00c1cbd6 --- /dev/null +++ b/.github/workflows/proxy_stability_testing.yml @@ -0,0 +1,142 @@ +name: Proxy Stability Testing AWS + +on: + workflow_call: + inputs: + env_name: + required: true + type: string + docker_image_tag: + required: true + type: string + docker_registry: + required: true + type: string + helm_values: + required: true + type: string + secrets: + ghcr_token: + required: true + docker_registry_username: + required: true + docker_registry_password: + required: true + registry_token: + required: true + aws_access_key_id: + required: true + aws_secret_access_key: + required: true + uh_license: + required: true + monitoring_token: + required: true + +jobs: + deploy-and-test: + runs-on: lightweight-generic-jobs + steps: + + - uses: actions/checkout@v4 + with: + token: ${{ secrets.ghcr_token }} + + - name: Install AWS CLI + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2 + arch: arm64 + + - name: Setup the AWS credentials + run: | + echo "AWS_ACCESS_KEY_ID=${{ secrets.aws_access_key_id}}" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=${{ secrets.aws_secret_access_key}}" >> $GITHUB_ENV + echo "AWS_DEFAULT_REGION=eu-central-1" >> $GITHUB_ENV + + - uses: azure/setup-kubectl@v4 + + - name: Download Kubeconfig + run: | + mkdir ~/.kube + aws eks update-kubeconfig --name aws-core + + - uses: azure/setup-helm@v4.3.0 + + - name: Generate Helm values + uses: danielr1996/envsubst-action@1.1.0 + env: + CORE_IMAGE: registry.ultihash.io/dev/core:${{ inputs.docker_image_tag }} + DATABASE_INIT_IMAGE: registry.ultihash.io/dev/database-init:${{ inputs.docker_image_tag }} + with: + input: ${{ inputs.helm_values }} + output: helm-values.yaml + + - name: Delete the testing environment if exists + run: | + helm uninstall ${{ inputs.env_name }} -n ${{ inputs.env_name }} --wait --ignore-not-found + kubectl delete pods -n ${{ inputs.env_name }} --all --force + kubectl delete pvc -n ${{ inputs.env_name }} --all + kubectl delete pv stability-test-input --force + kubectl delete secrets -n ${{ inputs.env_name }} --all --force + kubectl delete ns ${{ inputs.env_name }} + continue-on-error: true + + - name: Configure S3 bucket for testing + run: | + aws s3api create-bucket --region eu-central-1 --bucket ulticache-test-${{ inputs.env_name }} --create-bucket-configuration 'LocationConstraint=eu-central-1' + + - name: Install the Helm chart + run: | + kubectl create ns ${{ inputs.env_name }} + kubectl create secret docker-registry registry-credentials -n ${{ inputs.env_name }} --docker-server='${{ inputs.docker_registry }}' --docker-username='${{ secrets.docker_registry_username }}' --docker-password='${{ secrets.docker_registry_password }}' + kubectl create secret generic ultihash -n ${{ inputs.env_name }} --from-literal=license='${{ secrets.uh_license }}' --from-literal=token='${{ secrets.monitoring_token }}' + helm install ${{ inputs.env_name }} oci://registry.ultihash.io/stable/ulticache -n ${{ inputs.env_name }} --values helm-values.yaml --wait --timeout 60m + + - name: Provision persistent volumes for testing + run: | + kubectl apply -f testing/stability/persistent-volumes.yaml -n ${{ inputs.env_name }} + + - name: Run stability test + run: | + helm test ${{ inputs.env_name }} -n ${{ inputs.env_name }} --filter name=${{ inputs.env_name }}-stability-test --logs --timeout 3h + + - name: Delete S3 bucket + run: | + aws s3 rb --region eu-central-1 --bucket ulticache-test-${{ inputs.env_name }} --force + + cleanup: + runs-on: lightweight-generic-jobs + needs: deploy-and-test + steps: + + - name: Install AWS CLI + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2 + arch: arm64 + + - name: Setup the AWS credentials + run: | + echo "AWS_ACCESS_KEY_ID=${{ secrets.aws_access_key_id}}" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=${{ secrets.aws_secret_access_key}}" >> $GITHUB_ENV + echo "AWS_DEFAULT_REGION=eu-central-1" >> $GITHUB_ENV + + - uses: azure/setup-kubectl@v4 + + - name: Download Kubeconfig + run: | + mkdir ~/.kube + aws eks update-kubeconfig --name aws-core + + - uses: azure/setup-helm@v4.3.0 + + - name: Environment cleanup + run: | + helm uninstall ${{ inputs.env_name }} -n ${{ inputs.env_name }} --wait + kubectl delete pods -n ${{ inputs.env_name }} --all --force + kubectl delete pvc -n ${{ inputs.env_name }} --all + kubectl delete pv stability-test-input --force + kubectl delete secrets -n ${{ inputs.env_name }} --all --force + kubectl delete ns ${{ inputs.env_name }} + continue-on-error: true diff --git a/testing/proxy-stability/helm-values.yaml b/testing/proxy-stability/helm-values.yaml new file mode 100644 index 000000000..97e8c2273 --- /dev/null +++ b/testing/proxy-stability/helm-values.yaml @@ -0,0 +1,197 @@ +global: + image: $CORE_IMAGE + imagePullPolicy: Always + imagePullSecrets: + - registry-credentials + ultihashSecret: + name: ultihash + key: license + logLevel: DEBUG + telemetryExportInterval: 30000 + securityContext: + capabilities: + add: + - SYS_PTRACE + policies: | + { + "Version": "2012-10-17", + "Statement": { + "Sid": "AllowAllForAnybody", + "Effect": "Allow", + "Action": "*", + "Principal": "*", + "Resource": "*" + } + } + +stabilityTest: + readTimeout: 600 + concurentJobs: 2 + extraEnv: + - name: AWS_DEFAULT_REGION + value: "eu-central-1" + nodeSelector: + workload-type: tester + purpose: performance-and-stability-testing + tolerations: + - key: "purpose" + operator: "Equal" + value: "testing" + effect: "NoSchedule" + inputVolume: + type: S3 + mountPath: /data + pvcName: stability-test-input + outputVolume: + pvcName: stability-test-output + mountPath: /testing/test + +coordinator: + nodeSelector: + workload-type: directory + purpose: performance-and-stability-testing + tolerations: + - key: "purpose" + operator: "Equal" + value: "testing" + effect: "NoSchedule" + +etcd: + replicaCount: 1 + nodeSelector: + workload-type: directory + purpose: performance-and-stability-testing + tolerations: + - key: "purpose" + operator: "Equal" + value: "testing" + effect: "NoSchedule" + persistence: + size: 5Gi + storageClass: local-path + +storage: + groups: + - id: 0 + type: ROUND_ROBIN + storages: 2 + storageClass: local-path + size: 50Gi + nodeSelector: + workload-type: storage + purpose: performance-and-stability-testing + mountPath: /var/lib/uh/storage + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: role + operator: In + values: + - storage + topologyKey: kubernetes.io/hostname + tolerations: + - key: "purpose" + operator: "Equal" + value: "testing" + effect: "NoSchedule" + +database: + primary: + nodeSelector: + workload-type: directory + purpose: performance-and-stability-testing + tolerations: + - key: "purpose" + operator: "Equal" + value: "testing" + effect: "NoSchedule" + + persistence: + storageClass: local-path + size: 4Gi + +databaseInit: + image: $DATABASE_INIT_IMAGE + nodeSelector: + workload-type: directory + purpose: performance-and-stability-testing + tolerations: + - key: "purpose" + operator: "Equal" + value: "testing" + effect: "NoSchedule" + +proxy: + replicas: 1 + nodeSelector: + workload-type: storage + purpose: performance-and-stability-testing + tolerations: + - key: "purpose" + operator: "Equal" + value: "testing" + effect: "NoSchedule" + resources: {} + affinity: {} + nodeSelector: {} + tolerations: [] + securityContext: {} + extraEnv: [] + extraVolumeMounts: [] + extraVolumes: [] + + backend: + host: s3.eu-central-1.amazonaws.com + port: 443 + + service: + type: ClusterIP # Could be set to ClusterIP or NodePort depending on the environment + port: 8088 + + ingress: + enabled: false + +collector: + nodeSelector: + purpose: performance-and-stability-testing + tolerations: + - key: "purpose" + operator: "Equal" + value: "testing" + effect: "NoSchedule" + extraEnvs: + - name: UH_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: UH_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: UH_EXPORTER_TOKEN + valueFrom: + secretKeyRef: + name: ultihash + key: token + presets: + kubeletMetrics: + enabled: false + config: + exporters: + otlphttp/uptrace: + endpoint: http://uptrace-backend.uptrace.svc.cluster.local:14318 + headers: + uptrace-dsn: http://${env:UH_EXPORTER_TOKEN}@uptrace-backend.uptrace.svc.cluster.local:14318 + receivers: + prometheus: null + kubeletstats: null + service: + pipelines: + metrics: + receivers: + - otlp + +exporter: + enabled: false diff --git a/testing/proxy-stability/persistent-volumes.yaml b/testing/proxy-stability/persistent-volumes.yaml new file mode 100644 index 000000000..34a118490 --- /dev/null +++ b/testing/proxy-stability/persistent-volumes.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: stability-test-output +spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: stability-test-input +spec: + capacity: + storage: 1Gi # Ignored + accessModes: + - ReadOnlyMany + mountOptions: + - allow-delete + - region eu-central-1 + csi: + driver: s3.csi.aws.com + volumeHandle: s3-csi-driver-volume + volumeAttributes: + bucketName: ultihash-stability-test-corpora +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: stability-test-input +spec: + accessModes: + - ReadOnlyMany + storageClassName: "" + resources: + requests: + storage: 1Gi # Ignored + volumeName: stability-test-input \ No newline at end of file From 3ee5f9af8c953bc40ede166516ef46434470a96b Mon Sep 17 00:00:00 2001 From: sjulti Date: Thu, 25 Sep 2025 14:00:57 +0200 Subject: [PATCH 2/4] Add proxy-stability-test to workflow selection. --- .github/workflows/nightly_tests_executor.yml | 25 +++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly_tests_executor.yml b/.github/workflows/nightly_tests_executor.yml index a3f2eb52c..f437312f0 100644 --- a/.github/workflows/nightly_tests_executor.yml +++ b/.github/workflows/nightly_tests_executor.yml @@ -10,6 +10,7 @@ on: description: Choose the test to execute options: - feature-test + - proxy-stability-test - pyspark-test - s3a-test - stability-test @@ -102,6 +103,29 @@ jobs: uh_license: ${{ secrets.UH_LICENSE_JSON }} monitoring_token: ${{ secrets.MONITORING_TOKEN_AWS }} + proxy-stability-test: + needs: + - build-core + - build-database-init + if: | + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && github.event.inputs.test-to-execute == 'proxy-stability-test') + uses: ./.github/workflows/proxy_stability_testing.yml + with: + env_name: proxy-stability-testing + docker_image_tag: nightly-build + docker_registry: ${{ vars.DOCKER_REGISTRY }} + helm_values: testing/proxy-stability/helm-values.yaml + secrets: + ghcr_token: ${{ secrets.PAT }} + docker_registry_username: ${{ secrets.DOCKER_REGISTRY_USERNAME }} + docker_registry_password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + registry_token: ${{ secrets.REGISTRY_USER }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY}} + aws_secret_access_key: ${{ secrets.AWS_SECRET_KEY}} + uh_license: ${{ secrets.UH_LICENSE_JSON }} + monitoring_token: ${{ secrets.MONITORING_TOKEN_ON_PREMISE }} + pyspark-test-rr: needs: - build-core @@ -347,4 +371,3 @@ jobs: secrets: registry_token: ${{ secrets.REGISTRY_USER }} ghcr_token: ${{ secrets.PAT }} - \ No newline at end of file From 1407df7bca486fe907665597de66ed0807febbe6 Mon Sep 17 00:00:00 2001 From: sjulti Date: Thu, 25 Sep 2025 14:05:02 +0200 Subject: [PATCH 3/4] Remove PR template. --- .github/pull_request_template.md | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 1d48415d3..000000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,22 +0,0 @@ - -## Solving issue: - - -## Description - - -## Type of change - -- [ ] Fix (non-breaking change that fixes an issue) -- [ ] Feature (non-breaking change that adds a functionality) -- [ ] Breaking change (change that causes existing functionality to not work as expected) -- [ ] Repository management (changes that are not related to the code) - -## Checklist: - -- [ ] My change requires a change to the documentation. -- [ ] I have updated the documentation and architecture diagram accordingly. -- [ ] I have added an entry about the changes introduced by this PR to the changelog. - -## What reviewers should especially pay attention to (if applicable): -* From 0c8b06c34752f1ae4373c237ab9f90386e705666 Mon Sep 17 00:00:00 2001 From: sjulti Date: Thu, 25 Sep 2025 14:06:05 +0200 Subject: [PATCH 4/4] Only run proxy-stability-test when requested. --- .github/workflows/nightly_tests_executor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nightly_tests_executor.yml b/.github/workflows/nightly_tests_executor.yml index f437312f0..a5ede76fd 100644 --- a/.github/workflows/nightly_tests_executor.yml +++ b/.github/workflows/nightly_tests_executor.yml @@ -108,7 +108,6 @@ jobs: - build-core - build-database-init if: | - github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test-to-execute == 'proxy-stability-test') uses: ./.github/workflows/proxy_stability_testing.yml with: