From d11319bca614a21c5414cfc6b92b2c7825901f87 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 5 Mar 2024 11:42:38 +0100 Subject: [PATCH 01/79] Restore branch before the aborted release --- commons-datastore/commons-datastore-core/pom.xml | 2 +- commons-datastore/commons-datastore-mongodb/pom.xml | 3 ++- commons-datastore/commons-datastore-solr/pom.xml | 4 +++- commons-datastore/pom.xml | 2 +- commons-lib/pom.xml | 3 ++- pom.xml | 12 +++++------- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/commons-datastore/commons-datastore-core/pom.xml b/commons-datastore/commons-datastore-core/pom.xml index aa880ed32..c78e28999 100644 --- a/commons-datastore/commons-datastore-core/pom.xml +++ b/commons-datastore/commons-datastore-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons-datastore - 5.0.0 + 4.12.0 ../pom.xml diff --git a/commons-datastore/commons-datastore-mongodb/pom.xml b/commons-datastore/commons-datastore-mongodb/pom.xml index 81b7ca56b..ccfdf5440 100644 --- a/commons-datastore/commons-datastore-mongodb/pom.xml +++ b/commons-datastore/commons-datastore-mongodb/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons-datastore - 5.0.0 + 4.12.0 ../pom.xml @@ -50,6 +50,7 @@ org.hamcrest hamcrest-core + test junit diff --git a/commons-datastore/commons-datastore-solr/pom.xml b/commons-datastore/commons-datastore-solr/pom.xml index d50e0ec67..c89280165 100644 --- a/commons-datastore/commons-datastore-solr/pom.xml +++ b/commons-datastore/commons-datastore-solr/pom.xml @@ -22,7 +22,7 @@ org.opencb.commons commons-datastore - 5.0.0 + 4.12.0 ../pom.xml @@ -34,6 +34,7 @@ org.opencb.commons commons-datastore-core + org.apache.solr solr-solrj @@ -51,4 +52,5 @@ slf4j-api + diff --git a/commons-datastore/pom.xml b/commons-datastore/pom.xml index 2519bb24d..338658a66 100644 --- a/commons-datastore/pom.xml +++ b/commons-datastore/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.0.0 + 4.12.0 ../pom.xml diff --git a/commons-lib/pom.xml b/commons-lib/pom.xml index 7fbeb7c42..c28f2b56b 100644 --- a/commons-lib/pom.xml +++ b/commons-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.0.0 + 4.12.0 ../pom.xml @@ -67,6 +67,7 @@ com.fasterxml.jackson.core jackson-databind + test diff --git a/pom.xml b/pom.xml index 50c9ed941..2d6414baa 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.0.0 + 4.12.0 pom OpenCB commons project @@ -20,9 +20,9 @@ - 2.14.3 - 3.14.0 - 1.7.36 + 2.11.4 + 3.12.0 + 1.7.32 1.7.7 4.8.2 8.8.2 @@ -32,6 +32,7 @@ 1.3 4.13.2 + opencb https://sonarcloud.io 1.8 @@ -296,9 +297,6 @@ * - - * - From 195503565003989345370d88dcd4eb83ff8a2733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 3 Jun 2024 10:22:22 +0200 Subject: [PATCH 02/79] utils: add the method FileUtils.copyFile, #TASK-6297, #TASK-6255 On branch TASK-6255 Changes to be committed: modified: commons-lib/pom.xml modified: commons-lib/src/main/java/org/opencb/commons/utils/FileUtils.java --- commons-lib/pom.xml | 6 +++++ .../org/opencb/commons/utils/FileUtils.java | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/commons-lib/pom.xml b/commons-lib/pom.xml index dccbe7ec9..0e057979e 100644 --- a/commons-lib/pom.xml +++ b/commons-lib/pom.xml @@ -68,5 +68,11 @@ com.fasterxml.jackson.core jackson-databind + + commons-io + commons-io + 2.8.0 + compile + diff --git a/commons-lib/src/main/java/org/opencb/commons/utils/FileUtils.java b/commons-lib/src/main/java/org/opencb/commons/utils/FileUtils.java index 93fba0f0d..1ae5ba87d 100644 --- a/commons-lib/src/main/java/org/opencb/commons/utils/FileUtils.java +++ b/commons-lib/src/main/java/org/opencb/commons/utils/FileUtils.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.exec.Command; +import org.slf4j.LoggerFactory; import java.io.*; import java.nio.charset.Charset; @@ -188,6 +189,27 @@ public static String[] getUserAndGroup(Path path, boolean numericId) throws IOEx return new String[]{split[2], split[3]}; } + + public static void copyFile(File src, File dest) throws IOException { + try { + org.apache.commons.io.FileUtils.copyFile(src, dest); + } catch (IOException e) { + try { + if (src.length() == dest.length()) { + LoggerFactory.getLogger(FileUtils.class).warn(e.getMessage()); + return; + } + throw e; + } catch (Exception e1) { + throw e; + } + } + } + + //------------------------------------------------------------------------- + // P R I V A T E M E T H O D S + //------------------------------------------------------------------------- + private static String getLsOutput(Path path, boolean numericId) throws IOException { FileUtils.checkPath(path); From df70872e5d941c6a76327332103ebd9ac3044dc5 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 20 Jun 2024 12:07:42 +0200 Subject: [PATCH 03/79] test-xetabase-workflow.yml: New workflow to test xetabase on pullrequest approve of the libraries #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 93 ++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/test-xetabase-workflow.yml diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml new file mode 100644 index 000000000..49c4c2af0 --- /dev/null +++ b/.github/workflows/test-xetabase-workflow.yml @@ -0,0 +1,93 @@ +name: Pull request approve workflow + +on: + workflow_call: + inputs: + task: + type: string + required: true + branch: + type: string + required: true + +jobs: + test: + name: Execute JUnit and Jacoco tests + runs-on: ubuntu-22.04 + steps: + - name: Clone OpenCGA Enterprise branch '${{ github.event.inputs.branch }}' + uses: actions/checkout@v4 + with: + repository: zetta-genomics/opencga-enterprise + ref: ${{ github.event.inputs.branch }} + token: ${{ secrets.GH_PRIVATE_TOKEN }} + path: opencga-enterprise + fetch-depth: "10" + - id: get_opencga_branch + name: Get OpenCGA branch from 'pom.xml' property + run: | + pwd + chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh ${{ github.event.inputs.task }}) + echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 + with: + repository: opencb/opencga + ref: ${{ steps.get_opencga_branch.outputs.opencga_branch }} + path: opencga + fetch-depth: '10' + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8' + cache: 'maven' + - name: Install Samtools + run: sudo apt-get install samtools python3-deeptools + - name: Start MongoDB v6.0 + uses: supercharge/mongodb-github-action@1.8.0 + with: + mongodb-version: 6.0 + mongodb-replica-set: rs-test + - name: K8s Tunnel MongoDB + run: | + wget https://dl.k8s.io/release/v1.28.2/bin/linux/amd64/kubectl + chmod +x ./kubectl + echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf + ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27018:27017 --kubeconfig ./admin.conf & + - name: Install Azure AZCOPY + uses: kheiakiyama/install-azcopy-action@v1 + with: + version: 'v10' + - name: DockerHub login + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise + run: | + ln -s opencga opencga-enterprise/opencga-home + cd opencga-enterprise + ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ github.event.inputs.task }} -c localhost:27018 -H hdp3.1 + - name: Upload reports results to Github + uses: actions/upload-artifact@v4 + with: + name: report-test + path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/reports/test + - name: Upload log + uses: actions/upload-artifact@v4 + with: + name: build-log + path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/build.log + - name: Upload junit reports to a remote scp server + uses: garygrossgarten/github-action-scp@release + with: + local: opencga-enterprise/reports/test + remote: /var/www/html/reports/xetabase/${{ github.event.inputs.task }}/ + host: ${{ secrets.SSH_TESTING_SERVER_HOST}} + port: ${{ secrets.SSH_TESTING_SERVER_PORT}} + username: ${{ secrets.SSH_TESTING_SERVER_USER }} + password: ${{ secrets.SSH_TESTING_SERVER_PASSWORD }} + concurrency: 2 + + From 3f2f4eb710dab9238ead39aee8d47a0a4213d652 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 20 Jun 2024 12:08:14 +0200 Subject: [PATCH 04/79] test-xetabase-workflow.yml: New workflow to test xetabase on pullrequest approve of the libraries #TASK-6399 --- .github/workflows/scripts/xetabase-branch.sh | 42 ++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/scripts/xetabase-branch.sh diff --git a/.github/workflows/scripts/xetabase-branch.sh b/.github/workflows/scripts/xetabase-branch.sh new file mode 100644 index 000000000..af17f7f12 --- /dev/null +++ b/.github/workflows/scripts/xetabase-branch.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Function to calculate the corresponding branch of Xetabase project +get_xetabase_branch() { + # Input parameter (branch name) + input_branch="$1" + + # Check if the branch name is "develop" in that case return the same branch name + if [[ "$input_branch" == "develop" ]]; then + echo "develop" + return 0 + fi + + # Check if the branch name starts with "release-" and follows the patterns "release-a.b.x" or "release-a.b.c.x" + if [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.([0-9]+)\.x$ ]]; then + # Extract the MAJOR part of the branch name + MAJOR=${BASH_REMATCH[1]} + # Calculate the XETABASE_MAJOR by subtracting 3 from MAJOR + XETABASE_MAJOR=$((MAJOR - 3)) + # Check if the XETABASE_MAJOR is negative + if (( XETABASE_MAJOR < 0 )); then + echo "Error: 'MAJOR' digit after subtraction results in a negative number." + return 1 + fi + # Construct and echo the new branch name + echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}" + return 0 + fi + + # If the branch name does not match any of the expected patterns + echo "Error: The branch name is not correct." + return 1 +} + +# Check if the script receives exactly one argument +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Call the function with the input branch name +get_xetabase_branch "$1" From 37bfcd14022f33c5f23b6b449148709935f409bd Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 20 Jun 2024 12:09:42 +0200 Subject: [PATCH 05/79] test-xetabase-workflow.yml: Modify pull_request approve workflow to test all Xetabase instead only jcl #TASK-6399 --- .github/workflows/pull-request-approved.yml | 22 ++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 329df1be1..c20e61c90 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -5,11 +5,19 @@ on: types: [ submitted ] jobs: - build: - uses: ./.github/workflows/build-java-app-workflow.yml - test: - name: "Run all tests before merging" - uses: ./.github/workflows/test-analysis.yml - needs: build - secrets: inherit \ No newline at end of file + name: Execute JUnit and Jacoco tests + runs-on: ubuntu-22.04 + steps: + - id: get_xetabase_branch + name: Get current branch for Xetabase from 'pom.xml' property + run: | + chmod +x ./.github/workflows/scripts/xetabase-branch.sh + xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }} + echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} + echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT + - name: "Run all tests before merging" + uses: ./.github/workflows/test-xetabase-workflow.yml + with: + branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} + task: ${{ github.event.pull_request.head.ref }} \ No newline at end of file From f930213af02064275854e38a3665f4167f0644bf Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 20 Jun 2024 12:31:42 +0200 Subject: [PATCH 06/79] cicd:clone jcl to exec script in pull request approve #TASK-6399 --- .github/workflows/pull-request-approved.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index c20e61c90..ecc007801 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -9,6 +9,8 @@ jobs: name: Execute JUnit and Jacoco tests runs-on: ubuntu-22.04 steps: + - name: Clone java-common-libs + uses: actions/checkout@v4 - id: get_xetabase_branch name: Get current branch for Xetabase from 'pom.xml' property run: | From 9b2e5cf79cc6a5263e05a88da1235d0d0125420f Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 20 Jun 2024 12:34:25 +0200 Subject: [PATCH 07/79] cicd: fix parenthesi #TASK-6399 --- .github/workflows/pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index ecc007801..7f80bfed0 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -15,7 +15,7 @@ jobs: name: Get current branch for Xetabase from 'pom.xml' property run: | chmod +x ./.github/workflows/scripts/xetabase-branch.sh - xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }} + xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - name: "Run all tests before merging" From b5149297841333bdab23748264999675dbf72df9 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 20 Jun 2024 13:00:11 +0200 Subject: [PATCH 08/79] cicd: debug ./.github/workflows/ #TASK-6399 --- .github/workflows/pull-request-approved.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 7f80bfed0..36defafab 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -11,10 +11,14 @@ jobs: steps: - name: Clone java-common-libs uses: actions/checkout@v4 + with: + fetch-depth: '10' - id: get_xetabase_branch - name: Get current branch for Xetabase from 'pom.xml' property + name: "Get current branch for Xetabase from 'pom.xml' property" run: | chmod +x ./.github/workflows/scripts/xetabase-branch.sh + ls ./.github/workflows/scripts/ + ls ./.github/workflows/ xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From f8be19c4ec6154c7fe8ecc03394fb5acf9414531 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 09:42:26 +0200 Subject: [PATCH 09/79] CICD: Added ZETTA_REPO_ACCESS_TOKEN to co xetabase #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 49c4c2af0..fa78b2e24 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -1,4 +1,4 @@ -name: Pull request approve workflow +name: TEST Xetabase and publish report workflow on: workflow_call: @@ -20,7 +20,7 @@ jobs: with: repository: zetta-genomics/opencga-enterprise ref: ${{ github.event.inputs.branch }} - token: ${{ secrets.GH_PRIVATE_TOKEN }} + token: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} path: opencga-enterprise fetch-depth: "10" - id: get_opencga_branch From 6cca2db321899bfe48e857b437de5b242a9d3a32 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 09:57:54 +0200 Subject: [PATCH 10/79] cicd: Temporal workflow_dispatch for PR to tests #TASK-6399 --- .github/workflows/pull-request-approved.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 36defafab..de45af750 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -1,6 +1,7 @@ name: Pull request approve workflow on: + workflow_dispatch: pull_request_review: types: [ submitted ] From 1aa53aa3a12df7d065c2574b246e2b91db435ae7 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 09:59:05 +0200 Subject: [PATCH 11/79] cicd: Delete Temporal workflow_dispatch for PR to tests #TASK-6399 --- .github/workflows/pull-request-approved.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index de45af750..36defafab 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -1,7 +1,6 @@ name: Pull request approve workflow on: - workflow_dispatch: pull_request_review: types: [ submitted ] From e29adf1e6c06ad520b56c947e4fa6308a84f822d Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 10:07:45 +0200 Subject: [PATCH 12/79] cicd: Temporal call to TASK-6399 yml #TASK-6399 --- .github/workflows/pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 36defafab..1ee5a1419 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -23,7 +23,7 @@ jobs: echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - name: "Run all tests before merging" - uses: ./.github/workflows/test-xetabase-workflow.yml + uses: ./.github/workflows/test-xetabase-workflow.yml@TASK-6399 with: branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} task: ${{ github.event.pull_request.head.ref }} \ No newline at end of file From 55e0c2f25e0c418453b718a1e42e3de62a1f9919 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 10:11:00 +0200 Subject: [PATCH 13/79] cicd: Temporal call to TASK-6399 yml #TASK-6399 --- .github/workflows/pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 1ee5a1419..4169383cf 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -23,7 +23,7 @@ jobs: echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - name: "Run all tests before merging" - uses: ./.github/workflows/test-xetabase-workflow.yml@TASK-6399 + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 with: branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} task: ${{ github.event.pull_request.head.ref }} \ No newline at end of file From 2083020f3b2ed2035d264a8dcff845d6127b47a2 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 10:43:16 +0200 Subject: [PATCH 14/79] create workflow in develop to test xetabase #TASK-6399 --- .github/workflows/pull-request-approved.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 4169383cf..609bec4ac 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -1,4 +1,5 @@ name: Pull request approve workflow +run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}' on: pull_request_review: From e0d260146f9c5f8e655d1c7be7c8151ccf190121 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 10:52:58 +0200 Subject: [PATCH 15/79] Separate in two jobs to run the tests #TASK-6399 --- .github/workflows/pull-request-approved.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 609bec4ac..f5c6d8a6d 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -6,7 +6,7 @@ on: types: [ submitted ] jobs: - test: + calculate-xetabase-branch: name: Execute JUnit and Jacoco tests runs-on: ubuntu-22.04 steps: @@ -15,7 +15,7 @@ jobs: with: fetch-depth: '10' - id: get_xetabase_branch - name: "Get current branch for Xetabase from 'pom.xml' property" + name: "Get current branch for Xetabase from target branch" run: | chmod +x ./.github/workflows/scripts/xetabase-branch.sh ls ./.github/workflows/scripts/ @@ -23,8 +23,11 @@ jobs: xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - - name: "Run all tests before merging" - uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 - with: - branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} - task: ${{ github.event.pull_request.head.ref }} \ No newline at end of file + + test: + name: "Run all tests before merging" + needs: calculate-xetabase-branch + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 + with: + branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} + task: ${{ github.event.pull_request.head.ref }} \ No newline at end of file From 606b93ef03e0d71e161e07f80f8cc509ba18b6a1 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 11:01:43 +0200 Subject: [PATCH 16/79] cicd secrets: inherit #TASK-6399 --- .github/workflows/pull-request-approved.yml | 3 ++- .github/workflows/test-xetabase-workflow.yml | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index f5c6d8a6d..e02607698 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -30,4 +30,5 @@ jobs: uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 with: branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} - task: ${{ github.event.pull_request.head.ref }} \ No newline at end of file + task: ${{ github.event.pull_request.head.ref }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index fa78b2e24..12e8504ab 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -15,6 +15,10 @@ jobs: name: Execute JUnit and Jacoco tests runs-on: ubuntu-22.04 steps: + - name: Log inputs + run: | + echo "__OpenCGA-enterprise branch:__ \"${{ github.event.inputs.branch }}\"" | tee -a $GITHUB_STEP_SUMMARY + echo "__Task to test:__ \"${{ github.event.inputs.task }}\"" | tee -a $GITHUB_STEP_SUMMARY - name: Clone OpenCGA Enterprise branch '${{ github.event.inputs.branch }}' uses: actions/checkout@v4 with: From 8014992836d93e82c259478a4c4301f2983627a6 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 12:34:23 +0200 Subject: [PATCH 17/79] cicd create outputs #TASK-6399 --- .github/workflows/pull-request-approved.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index e02607698..36baa148d 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -4,11 +4,17 @@ run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref on: pull_request_review: types: [ submitted ] + outputs: + version: + description: "XetaBase branch to test" + value: ${{ jobs.calculate-xetabase-branch.outputs.xetabase_branch }} jobs: calculate-xetabase-branch: - name: Execute JUnit and Jacoco tests + name: Calculate Xetabase branch runs-on: ubuntu-22.04 + outputs: + xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} steps: - name: Clone java-common-libs uses: actions/checkout@v4 @@ -23,7 +29,7 @@ jobs: xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - + test: name: "Run all tests before merging" needs: calculate-xetabase-branch From 8c8cf8c95c09c06fbc93a0eeb521db94add3716c Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 12:38:14 +0200 Subject: [PATCH 18/79] cicd check bash --version #TASK-6399 --- .github/workflows/pull-request-approved.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 36baa148d..8a32022fa 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -26,6 +26,7 @@ jobs: chmod +x ./.github/workflows/scripts/xetabase-branch.sh ls ./.github/workflows/scripts/ ls ./.github/workflows/ + bash --version xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From f6719a7a5ff185936fbdbeb759c0c25f19b67683 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 12:39:55 +0200 Subject: [PATCH 19/79] cicd check bash --version #TASK-6399 --- .github/workflows/pull-request-approved.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 8a32022fa..93efbc861 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -4,10 +4,6 @@ run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref on: pull_request_review: types: [ submitted ] - outputs: - version: - description: "XetaBase branch to test" - value: ${{ jobs.calculate-xetabase-branch.outputs.xetabase_branch }} jobs: calculate-xetabase-branch: From 26914c71e137a412432514661fc15abde6291476 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 13:25:31 +0200 Subject: [PATCH 20/79] cicd: remove github.event. in inputs #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 12e8504ab..6fceeaa54 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -17,13 +17,13 @@ jobs: steps: - name: Log inputs run: | - echo "__OpenCGA-enterprise branch:__ \"${{ github.event.inputs.branch }}\"" | tee -a $GITHUB_STEP_SUMMARY - echo "__Task to test:__ \"${{ github.event.inputs.task }}\"" | tee -a $GITHUB_STEP_SUMMARY - - name: Clone OpenCGA Enterprise branch '${{ github.event.inputs.branch }}' + echo "__OpenCGA-enterprise branch:__ \"${{ inputs.branch }}\"" | tee -a $GITHUB_STEP_SUMMARY + echo "__Task to test:__ \"${{ inputs.task }}\"" | tee -a $GITHUB_STEP_SUMMARY + - name: Clone OpenCGA Enterprise branch '${{ inputs.branch }}' uses: actions/checkout@v4 with: repository: zetta-genomics/opencga-enterprise - ref: ${{ github.event.inputs.branch }} + ref: ${{ inputs.branch }} token: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} path: opencga-enterprise fetch-depth: "10" @@ -32,7 +32,7 @@ jobs: run: | pwd chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh - opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh ${{ github.event.inputs.task }}) + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh ${{ inputs.task }}) echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 with: @@ -72,7 +72,7 @@ jobs: run: | ln -s opencga opencga-enterprise/opencga-home cd opencga-enterprise - ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ github.event.inputs.task }} -c localhost:27018 -H hdp3.1 + ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: @@ -87,7 +87,7 @@ jobs: uses: garygrossgarten/github-action-scp@release with: local: opencga-enterprise/reports/test - remote: /var/www/html/reports/xetabase/${{ github.event.inputs.task }}/ + remote: /var/www/html/reports/xetabase/${{ inputs.task }}/ host: ${{ secrets.SSH_TESTING_SERVER_HOST}} port: ${{ secrets.SSH_TESTING_SERVER_PORT}} username: ${{ secrets.SSH_TESTING_SERVER_USER }} From b4b7a47f23852753aa7e82559227091d232a6c8b Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 13:41:16 +0200 Subject: [PATCH 21/79] cicd: workflow dispatch to test#TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 6fceeaa54..94782ae08 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -9,6 +9,16 @@ on: branch: type: string required: true + workflow_dispatch: + inputs: + task: + type: string + description: 'Task ID to be tested.' + required: true + branch: + type: string + description: 'Branch of opencga-enterprise to be tested and built.' + required: true jobs: test: From 39ec574b58d5b6b8d0023dd8fcea2ce499a33b3c Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 13:44:09 +0200 Subject: [PATCH 22/79] cicd: debug logs added #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 94782ae08..3cf699528 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -80,6 +80,11 @@ jobs: password: ${{ secrets.DOCKER_HUB_PASSWORD }} - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise run: | + pwd + ls -lrtha + echo "------------------------------------" + ls -lrtha /home/runner/work/java-common-libs/java-common-libs + echo "------------------------------------" ln -s opencga opencga-enterprise/opencga-home cd opencga-enterprise ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 From 3bea1075a79d7b0c305c3e6cb1212988282cc7b1 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 13:49:39 +0200 Subject: [PATCH 23/79] cicd: debug logs added #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 3cf699528..7b14f4829 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -83,10 +83,13 @@ jobs: pwd ls -lrtha echo "------------------------------------" - ls -lrtha /home/runner/work/java-common-libs/java-common-libs - echo "------------------------------------" - ln -s opencga opencga-enterprise/opencga-home cd opencga-enterprise + pwd + ls -lrtha + echo "------------------------------------" + ln -s ../opencga opencga-home + echo "------------------------------------" + ls -lrtha ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 - name: Upload reports results to Github uses: actions/upload-artifact@v4 From ae6267643cf5cc16b6e5fd4953f74af94b565e67 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 13:53:17 +0200 Subject: [PATCH 24/79] cicd: debug logs added #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 7b14f4829..54e684b5b 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -9,16 +9,16 @@ on: branch: type: string required: true - workflow_dispatch: - inputs: - task: - type: string - description: 'Task ID to be tested.' - required: true - branch: - type: string - description: 'Branch of opencga-enterprise to be tested and built.' - required: true +# workflow_dispatch: +# inputs: +# task: +# type: string +# description: 'Task ID to be tested.' +# required: true +# branch: +# type: string +# description: 'Branch of opencga-enterprise to be tested and built.' +# required: true jobs: test: From 6a7eee58003057ef8f4658639533008cf15be168 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 28 Jun 2024 10:04:47 +0200 Subject: [PATCH 25/79] cicd: Test level short to fast test upload to report server #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 54e684b5b..6f9636314 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -90,7 +90,8 @@ jobs: ln -s ../opencga opencga-home echo "------------------------------------" ls -lrtha - ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 + #./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 + ./build.sh -t -l runShortTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: From d5b4f38f6164c9684b87157149b972f304efffe3 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 28 Jun 2024 10:19:20 +0200 Subject: [PATCH 26/79] cicd: Uncomment workflow_dispatch Test level short to fast test upload to report server #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 6f9636314..f0f56f3f7 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -9,16 +9,16 @@ on: branch: type: string required: true -# workflow_dispatch: -# inputs: -# task: -# type: string -# description: 'Task ID to be tested.' -# required: true -# branch: -# type: string -# description: 'Branch of opencga-enterprise to be tested and built.' -# required: true + workflow_dispatch: + inputs: + task: + type: string + description: 'Task ID to be tested.' + required: true + branch: + type: string + description: 'Branch of opencga-enterprise to be tested and built.' + required: true jobs: test: From 5bb1d4c55600a2c0ccd564e3bbda50812c254036 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 28 Jun 2024 12:17:23 +0200 Subject: [PATCH 27/79] cicd: fix paths to upload artifacts #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index eab1f53ee..f0e5ad1df 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -103,12 +103,12 @@ jobs: uses: actions/upload-artifact@v4 with: name: report-test - path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/reports/test + path: /home/runner/work/java-common-libs/java-common-libs/opencga-enterprise/reports/test - name: Upload log uses: actions/upload-artifact@v4 with: name: build-log - path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/build.log + path: /home/runner/work/java-common-libs/java-common-libs/opencga-enterprise/build.log - name: Upload junit reports to a remote scp server uses: garygrossgarten/github-action-scp@release with: From d4ebf31452cd2c8caf33a0acb28ed96f79a38c71 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 28 Jun 2024 12:45:13 +0200 Subject: [PATCH 28/79] Added keeper passwords #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 34 ++++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index f0e5ad1df..57294ee6a 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -21,10 +21,7 @@ on: required: true env: - AZCOPY_SPA_CLIENT_SECRET: ${{ secrets.AZCOPY_SPA_CLIENT_SECRET }} AZCOPY_AUTO_LOGIN_TYPE: "SPN" - AZCOPY_SPA_APPLICATION_ID: ${{ secrets.AZCOPY_SPA_APPLICATION_ID }} - AZCOPY_TENANT_ID: ${{ secrets.AZCOPY_TENANT_ID }} jobs: @@ -32,6 +29,23 @@ jobs: name: Execute JUnit and Jacoco tests runs-on: ubuntu-22.04 steps: + - name: Retrieve secrets from Keeper + id: ksecrets + uses: Keeper-Security/ksm-action@master + with: + keeper-secret-config: ${{ secrets.KEEPER_SM_GH_OPENCB }} + secrets: | + AZCOPY_SPA_CLIENT_SECRET/field/Secret Value > env:AZCOPY_SPA_CLIENT_SECRET + AZCOPY_SPA_APPLICATION_ID/field/Secret Value > env:AZCOPY_SPA_APPLICATION_ID + AZCOPY_TENANT_ID/field/Secret Value > env:AZCOPY_TENANT_ID + #ZETTA_REPO_ACCESS_TOKEN/field/Secret Value > env:ZETTA_REPO_ACCESS_TOKEN + AZURE_KUBE_CONFIG/field/Secret Value > env:AZURE_KUBE_CONFIG + DOCKER_HUB_USER/field/Secret Value > env:DOCKER_HUB_USER + DOCKER_HUB_PASSWORD/field/Secret Value > env:DOCKER_HUB_PASSWORD + SSH_TESTING_SERVER_HOST/field/Secret Value > env:SSH_TESTING_SERVER_HOST + SSH_TESTING_SERVER_PORT/field/Secret Value > env:SSH_TESTING_SERVER_PORT + SSH_TESTING_SERVER_USER/field/Secret Value > env:SSH_TESTING_SERVER_USER + SSH_TESTING_SERVER_PASSWORD/field/Secret Value > env:SSH_TESTING_SERVER_PASSWORD - name: Log inputs run: | echo "__OpenCGA-enterprise branch:__ \"${{ inputs.branch }}\"" | tee -a $GITHUB_STEP_SUMMARY @@ -74,7 +88,7 @@ jobs: run: | wget https://dl.k8s.io/release/v1.28.2/bin/linux/amd64/kubectl chmod +x ./kubectl - echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf + echo "${{ env.AZURE_KUBE_CONFIG }}" > admin.conf ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27018:27017 --kubeconfig ./admin.conf & - name: Install Azure AZCOPY uses: kheiakiyama/install-azcopy-action@v1 @@ -83,8 +97,8 @@ jobs: - name: DockerHub login uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ env.DOCKER_HUB_PASSWORD }} - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise run: | pwd @@ -114,10 +128,10 @@ jobs: with: local: opencga-enterprise/reports/test remote: /var/www/html/reports/xetabase/${{ inputs.task }}/ - host: ${{ secrets.SSH_TESTING_SERVER_HOST}} - port: ${{ secrets.SSH_TESTING_SERVER_PORT}} - username: ${{ secrets.SSH_TESTING_SERVER_USER }} - password: ${{ secrets.SSH_TESTING_SERVER_PASSWORD }} + host: ${{ env.SSH_TESTING_SERVER_HOST}} + port: ${{ env.SSH_TESTING_SERVER_PORT}} + username: ${{ env.SSH_TESTING_SERVER_USER }} + password: ${{ env.SSH_TESTING_SERVER_PASSWORD }} concurrency: 2 From df6f77b2ffe82780c8ccf33bdbf74fd128bc3676 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 28 Jun 2024 12:50:53 +0200 Subject: [PATCH 29/79] Added keeper passwords #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 57294ee6a..afa045211 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -38,7 +38,6 @@ jobs: AZCOPY_SPA_CLIENT_SECRET/field/Secret Value > env:AZCOPY_SPA_CLIENT_SECRET AZCOPY_SPA_APPLICATION_ID/field/Secret Value > env:AZCOPY_SPA_APPLICATION_ID AZCOPY_TENANT_ID/field/Secret Value > env:AZCOPY_TENANT_ID - #ZETTA_REPO_ACCESS_TOKEN/field/Secret Value > env:ZETTA_REPO_ACCESS_TOKEN AZURE_KUBE_CONFIG/field/Secret Value > env:AZURE_KUBE_CONFIG DOCKER_HUB_USER/field/Secret Value > env:DOCKER_HUB_USER DOCKER_HUB_PASSWORD/field/Secret Value > env:DOCKER_HUB_PASSWORD From 8c9f95802cdcd658edcc091ef7146212db0f8237 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 11:31:44 +0200 Subject: [PATCH 30/79] cicd: Remove absolute references in Path #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index afa045211..459768a62 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -116,12 +116,12 @@ jobs: uses: actions/upload-artifact@v4 with: name: report-test - path: /home/runner/work/java-common-libs/java-common-libs/opencga-enterprise/reports/test + path: ./opencga-enterprise/reports/test - name: Upload log uses: actions/upload-artifact@v4 with: name: build-log - path: /home/runner/work/java-common-libs/java-common-libs/opencga-enterprise/build.log + path: ./opencga-enterprise/build.log - name: Upload junit reports to a remote scp server uses: garygrossgarten/github-action-scp@release with: From 2b5ccecda260326ae715ad249f64dc244a1c4d08 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 17:15:39 +0200 Subject: [PATCH 31/79] Added get-xetabase-branch.sh #TASK-6399 --- .github/workflows/pull-request-approved.yml | 2 +- .../workflows/scripts/get-opencga-branch.sh | 44 +++++++++++++++++++ ...abase-branch.sh => get-xetabase-branch.sh} | 8 ++++ .github/workflows/test-xetabase-workflow.yml | 5 ++- 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/scripts/get-opencga-branch.sh rename .github/workflows/scripts/{xetabase-branch.sh => get-xetabase-branch.sh} (82%) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 93efbc861..5fdf66da6 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -23,7 +23,7 @@ jobs: ls ./.github/workflows/scripts/ ls ./.github/workflows/ bash --version - xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/get-opencga-branch.sh b/.github/workflows/scripts/get-opencga-branch.sh new file mode 100644 index 000000000..29aa68712 --- /dev/null +++ b/.github/workflows/scripts/get-opencga-branch.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +## Navigate to the root folder where the pom.xml is +cd "$(dirname "$0")"/../../../ || exit 2 + +## Use the first argument passed as the branch. If it is not passed, I exit +if [[ -n $1 ]]; then + GIT_BRANCH=$1 +else + exit 2 +fi + +# If the branch exists in the opencga repository, I return it +if [ "$(git ls-remote https://github.com/opencb/opencga.git "$GIT_BRANCH" )" ] ; then + echo "$GIT_BRANCH"; + exit 0; +fi + +## Read the opencga version from the pom.xml +BUILD_VERSION=$(mvn help:evaluate -Dexpression=opencga.version -q -DforceStdout) + +## We remove the -SNAPSHOT if it exists +CLEAN_BUILD_VERSION=$(echo "$BUILD_VERSION" | cut -d "-" -f 1) + +## Read the numbers separately to compose the name of the branch +MAJOR=$(echo "$CLEAN_BUILD_VERSION" | cut -d "." -f 1) +MINOR=$(echo "$CLEAN_BUILD_VERSION" | cut -d "." -f 2) +PATCH=$(echo "$CLEAN_BUILD_VERSION" | cut -d "." -f 3) + +## it's a HOTFIX. Count the number of points to know if it is a hotfix +COUNT=$(echo "$CLEAN_BUILD_VERSION" | grep -o '\.' | wc -l ) +if [ "$COUNT" -gt 2 ]; then + echo "release-$MAJOR.$MINOR.$PATCH.x" + exit 0 +fi + +## It's develop branch +if [[ "$PATCH" == "0" ]]; then + echo "develop" + exit 0 +else #Is release branch + echo "release-$MAJOR.$MINOR.x" + exit 0 +fi \ No newline at end of file diff --git a/.github/workflows/scripts/xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh similarity index 82% rename from .github/workflows/scripts/xetabase-branch.sh rename to .github/workflows/scripts/get-xetabase-branch.sh index af17f7f12..0be5cb32d 100644 --- a/.github/workflows/scripts/xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -5,6 +5,14 @@ get_xetabase_branch() { # Input parameter (branch name) input_branch="$1" + # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it + if [[ $input_branch == TASK* ]]; then + if [ "$(git ls-remote https://github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then + echo "$GIT_BRANCH"; + exit 0; + fi + fi + # Check if the branch name is "develop" in that case return the same branch name if [[ "$input_branch" == "develop" ]]; then echo "develop" diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 459768a62..d332ec409 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -62,9 +62,10 @@ jobs: run: | pwd chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh - opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh ${{ inputs.task }}) + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-branch.sh ${{ inputs.task }}) echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - - uses: actions/checkout@v4 + - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}' + uses: actions/checkout@v4 with: repository: opencb/opencga ref: ${{ steps.get_opencga_branch.outputs.opencga_branch }} From 7d6d051929a10ab8306d52494d9b617bdd7e0ef3 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 17:24:30 +0200 Subject: [PATCH 32/79] Added remove get-opencga-branch.sh #TASK-6399 --- .../workflows/scripts/get-opencga-branch.sh | 44 ------------------- .github/workflows/test-xetabase-workflow.yml | 4 +- 2 files changed, 2 insertions(+), 46 deletions(-) delete mode 100644 .github/workflows/scripts/get-opencga-branch.sh diff --git a/.github/workflows/scripts/get-opencga-branch.sh b/.github/workflows/scripts/get-opencga-branch.sh deleted file mode 100644 index 29aa68712..000000000 --- a/.github/workflows/scripts/get-opencga-branch.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -## Navigate to the root folder where the pom.xml is -cd "$(dirname "$0")"/../../../ || exit 2 - -## Use the first argument passed as the branch. If it is not passed, I exit -if [[ -n $1 ]]; then - GIT_BRANCH=$1 -else - exit 2 -fi - -# If the branch exists in the opencga repository, I return it -if [ "$(git ls-remote https://github.com/opencb/opencga.git "$GIT_BRANCH" )" ] ; then - echo "$GIT_BRANCH"; - exit 0; -fi - -## Read the opencga version from the pom.xml -BUILD_VERSION=$(mvn help:evaluate -Dexpression=opencga.version -q -DforceStdout) - -## We remove the -SNAPSHOT if it exists -CLEAN_BUILD_VERSION=$(echo "$BUILD_VERSION" | cut -d "-" -f 1) - -## Read the numbers separately to compose the name of the branch -MAJOR=$(echo "$CLEAN_BUILD_VERSION" | cut -d "." -f 1) -MINOR=$(echo "$CLEAN_BUILD_VERSION" | cut -d "." -f 2) -PATCH=$(echo "$CLEAN_BUILD_VERSION" | cut -d "." -f 3) - -## it's a HOTFIX. Count the number of points to know if it is a hotfix -COUNT=$(echo "$CLEAN_BUILD_VERSION" | grep -o '\.' | wc -l ) -if [ "$COUNT" -gt 2 ]; then - echo "release-$MAJOR.$MINOR.$PATCH.x" - exit 0 -fi - -## It's develop branch -if [[ "$PATCH" == "0" ]]; then - echo "develop" - exit 0 -else #Is release branch - echo "release-$MAJOR.$MINOR.x" - exit 0 -fi \ No newline at end of file diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index d332ec409..411c0ee82 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -61,8 +61,8 @@ jobs: name: Get OpenCGA branch from 'pom.xml' property run: | pwd - chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh - opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-branch.sh ${{ inputs.task }}) + chmod +x ./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh false) echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}' uses: actions/checkout@v4 From 61b806a7df651eea18c891477ed97d5e4f3b83fd Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 17:46:35 +0200 Subject: [PATCH 33/79] Added log summary step #TASK-6399 --- .github/workflows/pull-request-approved.yml | 5 +---- .github/workflows/test-xetabase-workflow.yml | 5 +++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 5fdf66da6..d96c95627 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -19,10 +19,7 @@ jobs: - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | - chmod +x ./.github/workflows/scripts/xetabase-branch.sh - ls ./.github/workflows/scripts/ - ls ./.github/workflows/ - bash --version + chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 411c0ee82..1dbfed69f 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -133,5 +133,10 @@ jobs: username: ${{ env.SSH_TESTING_SERVER_USER }} password: ${{ env.SSH_TESTING_SERVER_PASSWORD }} concurrency: 2 + - name: Log summary + run: | + cat ./opencga-enterprise/build.log | tee -a $GITHUB_STEP_SUMMARY + + From 1f896dcdcc3535a58951e414a421a717ccad45d9 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 17:47:53 +0200 Subject: [PATCH 34/79] cicd: Removed test step in develop.yml #TASK-6399 --- .github/workflows/develop.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 196fc122b..f4d59ed60 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -10,11 +10,6 @@ jobs: build: uses: ./.github/workflows/build-java-app-workflow.yml - test: - uses: ./.github/workflows/test-analysis.yml - needs: build - secrets: inherit - deploy-maven: uses: ./.github/workflows/deploy-maven-repository-workflow.yml needs: test From af9e16d2ee192734fdc48769439c789e7e3ebab6 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 21:12:50 +0200 Subject: [PATCH 35/79] cicd: Add debug outputs #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 1dbfed69f..3d01cf422 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -61,6 +61,8 @@ jobs: name: Get OpenCGA branch from 'pom.xml' property run: | pwd + ls -lrtha + ls -lrtha ./opencga-enterprise chmod +x ./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh false) echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT From a0861c29c8afefd1665ca364f496efdef562f9cf Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 21:15:24 +0200 Subject: [PATCH 36/79] cicd: delete param to opencga_branch #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 3d01cf422..0cc013038 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -64,7 +64,7 @@ jobs: ls -lrtha ls -lrtha ./opencga-enterprise chmod +x ./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh - opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh false) + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh) echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}' uses: actions/checkout@v4 From 9de5d3c998b7cf23d5e5a3777fc0d08d6ec885bf Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 21:19:05 +0200 Subject: [PATCH 37/79] cicd: delete param to opencga_branch #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 0cc013038..c4e3e099d 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -65,6 +65,7 @@ jobs: ls -lrtha ./opencga-enterprise chmod +x ./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh) + echo "opencga_branch=${opencga_branch}" echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}' uses: actions/checkout@v4 From 98d385234221b50c4d732c16ed13cc78bf3dcbc7 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 21:24:32 +0200 Subject: [PATCH 38/79] cicd:call to xetabase-branch.sh #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index c4e3e099d..950c0b2f6 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -63,8 +63,8 @@ jobs: pwd ls -lrtha ls -lrtha ./opencga-enterprise - chmod +x ./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh - opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/opencga_branch.sh) + chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh) echo "opencga_branch=${opencga_branch}" echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}' From 75536daaf8c52917cd82cd521221211441742c5c Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 2 Jul 2024 10:39:31 +0200 Subject: [PATCH 39/79] cicd: Change AZURE_KUBE_CONFIG to secrets #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 950c0b2f6..eab8d1c05 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -91,7 +91,7 @@ jobs: run: | wget https://dl.k8s.io/release/v1.28.2/bin/linux/amd64/kubectl chmod +x ./kubectl - echo "${{ env.AZURE_KUBE_CONFIG }}" > admin.conf + echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27018:27017 --kubeconfig ./admin.conf & - name: Install Azure AZCOPY uses: kheiakiyama/install-azcopy-action@v1 From 3bfbf9a51afa6d13bd71f492b23aafb082e513f3 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 9 Jul 2024 15:32:54 +0200 Subject: [PATCH 40/79] Rename get-opencga-xetabase-branch.sh reference --- .github/workflows/test-xetabase-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index eab8d1c05..1dfcdc786 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -63,8 +63,8 @@ jobs: pwd ls -lrtha ls -lrtha ./opencga-enterprise - chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh - opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh) + chmod +x ./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh) echo "opencga_branch=${opencga_branch}" echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}' From 8431eb89a25088057483dc8690c07e59e0b5a08a Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 9 Jul 2024 15:32:54 +0200 Subject: [PATCH 41/79] cicd: Rename get-opencga-xetabase-branch.sh reference #TASK-6399 --- .github/workflows/test-xetabase-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index eab8d1c05..1dfcdc786 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -63,8 +63,8 @@ jobs: pwd ls -lrtha ls -lrtha ./opencga-enterprise - chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh - opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh) + chmod +x ./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh) echo "opencga_branch=${opencga_branch}" echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}' From 17eb2d6a61c856d10331e7d0f3e8daa7f1825501 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 17 Jul 2024 10:19:29 +0200 Subject: [PATCH 42/79] Prepare new development branch release-5.2.x --- commons-datastore/commons-datastore-core/pom.xml | 2 +- commons-datastore/commons-datastore-mongodb/pom.xml | 2 +- commons-datastore/commons-datastore-solr/pom.xml | 2 +- commons-datastore/pom.xml | 2 +- commons-lib/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/commons-datastore/commons-datastore-core/pom.xml b/commons-datastore/commons-datastore-core/pom.xml index ed5c139e5..08935cc57 100644 --- a/commons-datastore/commons-datastore-core/pom.xml +++ b/commons-datastore/commons-datastore-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons-datastore - 5.3.0-SNAPSHOT + 5.2.1-SNAPSHOT ../pom.xml diff --git a/commons-datastore/commons-datastore-mongodb/pom.xml b/commons-datastore/commons-datastore-mongodb/pom.xml index 294dd6ec9..695c20e78 100644 --- a/commons-datastore/commons-datastore-mongodb/pom.xml +++ b/commons-datastore/commons-datastore-mongodb/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons-datastore - 5.3.0-SNAPSHOT + 5.2.1-SNAPSHOT ../pom.xml diff --git a/commons-datastore/commons-datastore-solr/pom.xml b/commons-datastore/commons-datastore-solr/pom.xml index a6d4bc14d..43df674d7 100644 --- a/commons-datastore/commons-datastore-solr/pom.xml +++ b/commons-datastore/commons-datastore-solr/pom.xml @@ -22,7 +22,7 @@ org.opencb.commons commons-datastore - 5.3.0-SNAPSHOT + 5.2.1-SNAPSHOT ../pom.xml diff --git a/commons-datastore/pom.xml b/commons-datastore/pom.xml index db85378f8..54e31ca6b 100644 --- a/commons-datastore/pom.xml +++ b/commons-datastore/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.3.0-SNAPSHOT + 5.2.1-SNAPSHOT ../pom.xml diff --git a/commons-lib/pom.xml b/commons-lib/pom.xml index b00de26c8..7addadf13 100644 --- a/commons-lib/pom.xml +++ b/commons-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.3.0-SNAPSHOT + 5.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 8a184d389..2028f0e0e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.3.0-SNAPSHOT + 5.2.1-SNAPSHOT pom OpenCB commons project From d18cc5a1bf6ed157a4f3c7a3baae41b508026da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 21 Aug 2024 09:07:27 +0100 Subject: [PATCH 43/79] cicd: Fix "needs" dependency at develop.yml #TASK-6753 --- .github/workflows/develop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index f4d59ed60..3029b0fdb 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -12,5 +12,5 @@ jobs: deploy-maven: uses: ./.github/workflows/deploy-maven-repository-workflow.yml - needs: test + needs: build secrets: inherit From e28046696f378bc1371b5347c0eae4bcf202d43c Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 4 Sep 2024 18:08:03 +0200 Subject: [PATCH 44/79] cicd: Upload reference to develop branch in pull-request-approve to test-xetabase-workflow #TASK-6807 --- .github/workflows/pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index d96c95627..8f41edff3 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -27,7 +27,7 @@ jobs: test: name: "Run all tests before merging" needs: calculate-xetabase-branch - uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop with: branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} task: ${{ github.event.pull_request.head.ref }} From e8552edc847bf0393e007825b5a79e9f99b95bd9 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 10:35:53 +0200 Subject: [PATCH 45/79] cicd: Refactor and fix SSH env values #TASK-6807 --- .github/workflows/test-xetabase-workflow.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 1dfcdc786..4e14074c6 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -41,10 +41,10 @@ jobs: AZURE_KUBE_CONFIG/field/Secret Value > env:AZURE_KUBE_CONFIG DOCKER_HUB_USER/field/Secret Value > env:DOCKER_HUB_USER DOCKER_HUB_PASSWORD/field/Secret Value > env:DOCKER_HUB_PASSWORD - SSH_TESTING_SERVER_HOST/field/Secret Value > env:SSH_TESTING_SERVER_HOST - SSH_TESTING_SERVER_PORT/field/Secret Value > env:SSH_TESTING_SERVER_PORT - SSH_TESTING_SERVER_USER/field/Secret Value > env:SSH_TESTING_SERVER_USER - SSH_TESTING_SERVER_PASSWORD/field/Secret Value > env:SSH_TESTING_SERVER_PASSWORD + SSH_TESTING_SERVER_HOST/field/Secret Value > env:SSH_HOST + SSH_TESTING_SERVER_PORT/field/Secret Value > env:SSH_PORT + SSH_TESTING_SERVER_USER/field/Secret Value > env:SSH_USER + SSH_TESTING_SERVER_PASSWORD/field/Secret Value > env:SSH_PASS - name: Log inputs run: | echo "__OpenCGA-enterprise branch:__ \"${{ inputs.branch }}\"" | tee -a $GITHUB_STEP_SUMMARY @@ -114,8 +114,8 @@ jobs: ln -s ../opencga opencga-home echo "------------------------------------" ls -lrtha - #./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 ./build.sh -t -l runShortTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 + - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: @@ -126,16 +126,6 @@ jobs: with: name: build-log path: ./opencga-enterprise/build.log - - name: Upload junit reports to a remote scp server - uses: garygrossgarten/github-action-scp@release - with: - local: opencga-enterprise/reports/test - remote: /var/www/html/reports/xetabase/${{ inputs.task }}/ - host: ${{ env.SSH_TESTING_SERVER_HOST}} - port: ${{ env.SSH_TESTING_SERVER_PORT}} - username: ${{ env.SSH_TESTING_SERVER_USER }} - password: ${{ env.SSH_TESTING_SERVER_PASSWORD }} - concurrency: 2 - name: Log summary run: | cat ./opencga-enterprise/build.log | tee -a $GITHUB_STEP_SUMMARY From 59ad86b10ed4f8757207378a64d4d3579a27141e Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 11:21:10 +0200 Subject: [PATCH 46/79] Env variables to FIX error #TASK-6807 --- .github/workflows/test-xetabase-workflow.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 4e14074c6..c37de7847 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -115,7 +115,11 @@ jobs: echo "------------------------------------" ls -lrtha ./build.sh -t -l runShortTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 - + env: + SSH_HOST: ${{ env.SSH_HOST }} + SSH_PORT: ${{ env.SSH_PORT }} + SSH_USER: ${{ env.SSH_USER }} + SSH_PASS: ${{ env.SSH_PASS }} - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: From 85e297a925ae39cdadfb78b52c343b33b54013da Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 11:22:45 +0200 Subject: [PATCH 47/79] Env variables to FIX error #TASK-6807 --- .github/workflows/test-xetabase-workflow.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index c37de7847..eb441cfe3 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -104,16 +104,7 @@ jobs: password: ${{ env.DOCKER_HUB_PASSWORD }} - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise run: | - pwd - ls -lrtha - echo "------------------------------------" - cd opencga-enterprise - pwd - ls -lrtha - echo "------------------------------------" - ln -s ../opencga opencga-home - echo "------------------------------------" - ls -lrtha + printenv ./build.sh -t -l runShortTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 env: SSH_HOST: ${{ env.SSH_HOST }} From abb3b6b368747ca7949d1d55f37e1cbc65a8596d Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 11:42:43 +0200 Subject: [PATCH 48/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/pull-request-approved.yml | 1 + .github/workflows/scripts/get-xetabase-branch.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 8f41edff3..0cd0fdaac 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -20,6 +20,7 @@ jobs: name: "Get current branch for Xetabase from target branch" run: | chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh + echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index 0be5cb32d..bb1b18d64 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -8,8 +8,8 @@ get_xetabase_branch() { # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then if [ "$(git ls-remote https://github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then - echo "$GIT_BRANCH"; - exit 0; + echo "$input_branch"; + return 0; fi fi From 12a515909947bc0f997460d1b38c64e5cb6cfb46 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 11:52:10 +0200 Subject: [PATCH 49/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/pull-request-approved.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 0cd0fdaac..e6b13f3ca 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -21,7 +21,8 @@ jobs: run: | chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From 9182e1ea9349605fd2f9018a2c6cd27a7dd9f6f4 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 12:06:36 +0200 Subject: [PATCH 50/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/pull-request-approved.yml | 3 +++ .github/workflows/scripts/get-xetabase-branch.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index e6b13f3ca..d33577a4b 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -5,6 +5,9 @@ on: pull_request_review: types: [ submitted ] +env: + ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} + jobs: calculate-xetabase-branch: name: Calculate Xetabase branch diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index bb1b18d64..9e25d9aa0 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -7,7 +7,7 @@ get_xetabase_branch() { # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then - if [ "$(git ls-remote https://github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then + if [ "$(git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then echo "$input_branch"; return 0; fi From 2d940e334d34e940e2494cb29cbc3508f456cd76 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 12:11:37 +0200 Subject: [PATCH 51/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/pull-request-approved.yml | 3 ++- .github/workflows/scripts/get-xetabase-branch.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index d33577a4b..64792e5e4 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -25,7 +25,8 @@ jobs: chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }}) + echo "ZETTA_REPO_ACCESS_TOKEN: ${{ env.ZETTA_REPO_ACCESS_TOKEN }}" + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ env.ZETTA_REPO_ACCESS_TOKEN }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index 9e25d9aa0..ab86f9b3e 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -7,7 +7,7 @@ get_xetabase_branch() { # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then - if [ "$(git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then + if [ "$(git ls-remote https://$2@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then echo "$input_branch"; return 0; fi From 230010e4dbb585b544f5cb9c36cc0dd0fa2cb2ca Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 12:19:46 +0200 Subject: [PATCH 52/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/pull-request-approved.yml | 6 +----- .github/workflows/scripts/get-xetabase-branch.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 64792e5e4..1d7686951 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -5,9 +5,6 @@ on: pull_request_review: types: [ submitted ] -env: - ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} - jobs: calculate-xetabase-branch: name: Calculate Xetabase branch @@ -25,8 +22,7 @@ jobs: chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" - echo "ZETTA_REPO_ACCESS_TOKEN: ${{ env.ZETTA_REPO_ACCESS_TOKEN }}" - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ env.ZETTA_REPO_ACCESS_TOKEN }}) + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index ab86f9b3e..2c6aaad93 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -7,7 +7,7 @@ get_xetabase_branch() { # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then - if [ "$(git ls-remote https://$2@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then + if [ "$(git ls-remote "https://$2@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then echo "$input_branch"; return 0; fi From a4d2fbb30c92e27c6c814a49ff212d35dc7d248f Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 12:33:44 +0200 Subject: [PATCH 53/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/scripts/get-xetabase-branch.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index 2c6aaad93..4cdfe5fc2 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -4,10 +4,10 @@ get_xetabase_branch() { # Input parameter (branch name) input_branch="$1" - + ZETTA_REPO_ACCESS_TOKEN="$2" # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then - if [ "$(git ls-remote "https://$2@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then + if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then echo "$input_branch"; return 0; fi @@ -47,4 +47,4 @@ if [ "$#" -ne 1 ]; then fi # Call the function with the input branch name -get_xetabase_branch "$1" +get_xetabase_branch "$1" "$2" From 8e3cc8832c12f74aa06d4416d5c773f4f1458614 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 12:52:59 +0200 Subject: [PATCH 54/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 1d7686951..517f159d9 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -22,7 +22,7 @@ jobs: chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}) + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ secrets.ZETTA_TOKEN }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From f84ebf318d8c72ffd8d8259a5641b6a88cae79f8 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 13:02:21 +0200 Subject: [PATCH 55/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/pull-request-approved.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 517f159d9..5341b5f7c 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -22,9 +22,10 @@ jobs: chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ secrets.ZETTA_TOKEN }}) + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT + test: name: "Run all tests before merging" From 6ca7c8c4c7985e050af072c335c2755e6059caff Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 13:09:41 +0200 Subject: [PATCH 56/79] Fix get-xetabase-branch #TASK-6807 --- .github/workflows/pull-request-approved.yml | 3 ++- .github/workflows/scripts/get-xetabase-branch.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 5341b5f7c..cfc9110c3 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -22,10 +22,11 @@ jobs: chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" + echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - + test: name: "Run all tests before merging" diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index 4cdfe5fc2..559e686aa 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -1,5 +1,6 @@ #!/bin/bash - +set -e +set -x # Function to calculate the corresponding branch of Xetabase project get_xetabase_branch() { # Input parameter (branch name) From 9272829d16ce36f6a443a2e8a8fd46ec903df2b0 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 13:18:37 +0200 Subject: [PATCH 57/79] TEST workflow for pull-request #TASK-6807 --- .../workflows/test-pull-request-approved.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/test-pull-request-approved.yml diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml new file mode 100644 index 000000000..131230742 --- /dev/null +++ b/.github/workflows/test-pull-request-approved.yml @@ -0,0 +1,38 @@ +name: Pull request approve workflow +run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}' + +on: + + push: + branches: + - TASK-* + +jobs: + calculate-xetabase-branch: + name: Calculate Xetabase branch + runs-on: ubuntu-22.04 + outputs: + xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} + steps: + - name: Clone java-common-libs + uses: actions/checkout@v4 + with: + fetch-depth: '10' + - id: get_xetabase_branch + name: "Get current branch for Xetabase from target branch" + run: | + chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh + echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}) + echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} + echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT + + + test: + name: "Run all tests before merging" + needs: calculate-xetabase-branch + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop + with: + branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} + task: TASK-6807 + secrets: inherit \ No newline at end of file From 5d473dd6b8d81863ef47930f7631748f7941ec9a Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 13:20:22 +0200 Subject: [PATCH 58/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/scripts/get-xetabase-branch.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index 559e686aa..b2b6f998a 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -41,11 +41,6 @@ get_xetabase_branch() { return 1 } -# Check if the script receives exactly one argument -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi # Call the function with the input branch name get_xetabase_branch "$1" "$2" From 7b2a8b095ef20a894291bc1bab956293947cb443 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 13:23:27 +0200 Subject: [PATCH 59/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index 131230742..04bd9de91 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -23,7 +23,7 @@ jobs: run: | chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}) + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" "${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" ) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From c3fbf526ea87a690f3f09cf89801b465bdecf40d Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 14:25:26 +0200 Subject: [PATCH 60/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index 04bd9de91..473185bd9 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -22,7 +22,7 @@ jobs: name: "Get current branch for Xetabase from target branch" run: | chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh - echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" + echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" | tee -a ${GITHUB_STEP_SUMMARY} xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" "${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" ) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From 546ea977fa23d18aa1bd1e275175dd52c879d687 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 14:35:55 +0200 Subject: [PATCH 61/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/scripts/get-xetabase-branch.sh | 4 ++-- .github/workflows/test-pull-request-approved.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index b2b6f998a..68b19832a 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -5,7 +5,7 @@ set -x get_xetabase_branch() { # Input parameter (branch name) input_branch="$1" - ZETTA_REPO_ACCESS_TOKEN="$2" + # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then @@ -43,4 +43,4 @@ get_xetabase_branch() { # Call the function with the input branch name -get_xetabase_branch "$1" "$2" +get_xetabase_branch "$1" diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index 473185bd9..d1f628982 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -23,7 +23,7 @@ jobs: run: | chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" | tee -a ${GITHUB_STEP_SUMMARY} - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" "${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" ) + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" ) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From b870aa047e5c3b0416a505a70e1c70c686eeae67 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 14:40:11 +0200 Subject: [PATCH 62/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index d1f628982..5e4e3943d 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -2,11 +2,12 @@ name: Pull request approve workflow run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}' on: - push: branches: - TASK-* - +env: + ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} + jobs: calculate-xetabase-branch: name: Calculate Xetabase branch From ba5a19e2c75f3003764a38402969a68a09ddd508 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 14:42:53 +0200 Subject: [PATCH 63/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index 5e4e3943d..ee9609b3e 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -7,7 +7,7 @@ on: - TASK-* env: ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} - + jobs: calculate-xetabase-branch: name: Calculate Xetabase branch @@ -22,13 +22,15 @@ jobs: - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | + if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then + echo "Here it is TASK-6807 branch"; + fi chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" | tee -a ${GITHUB_STEP_SUMMARY} xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" ) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - - + test: name: "Run all tests before merging" needs: calculate-xetabase-branch From ca2ede13502012c5715478b64940361cf42aaae4 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 14:44:14 +0200 Subject: [PATCH 64/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index ee9609b3e..f23215fea 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -5,8 +5,7 @@ on: push: branches: - TASK-* -env: - ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} + jobs: calculate-xetabase-branch: @@ -30,7 +29,8 @@ jobs: xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" ) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - + env: + ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} test: name: "Run all tests before merging" needs: calculate-xetabase-branch From 9241eb2f82f028b421687901cd7fdc867f65d511 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 15:11:01 +0200 Subject: [PATCH 65/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index f23215fea..2377cba9d 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -21,6 +21,10 @@ jobs: - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | + if [ "$( git ls-remote https://github.com/opencb/opencga.git "TASK-6807" )" ] ; then + echo "OPENCGA TASK-6807 branch"; + fi + if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then echo "Here it is TASK-6807 branch"; fi From ca6debc69032831a527d5258ffdbaac290d6eef0 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 15:12:44 +0200 Subject: [PATCH 66/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index 2377cba9d..e4b3c33dd 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -21,7 +21,7 @@ jobs: - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | - if [ "$( git ls-remote https://github.com/opencb/opencga.git "TASK-6807" )" ] ; then + if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/opencb/opencga.git "TASK-6807" )" ] ; then echo "OPENCGA TASK-6807 branch"; fi From 47688af142d78e73e694a17e5c53b8e8923c6d8f Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 15:14:46 +0200 Subject: [PATCH 67/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index e4b3c33dd..9b378be12 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -25,7 +25,7 @@ jobs: echo "OPENCGA TASK-6807 branch"; fi - if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then + if [ "$(git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then echo "Here it is TASK-6807 branch"; fi chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh From 3efd0b513c3ad382bb0184864c03f42ca7583ebf Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 15:16:38 +0200 Subject: [PATCH 68/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index 9b378be12..912542865 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -25,7 +25,7 @@ jobs: echo "OPENCGA TASK-6807 branch"; fi - if [ "$(git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then + if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then echo "Here it is TASK-6807 branch"; fi chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh From 45e3902e7265f2ead52c6d0a89ccad95245b26bd Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 15:22:16 +0200 Subject: [PATCH 69/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index 912542865..afc6e1460 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -35,6 +35,8 @@ jobs: echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT env: ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} + with: + persist-credentials: false test: name: "Run all tests before merging" needs: calculate-xetabase-branch From f4a286fde616d14813bcf6589a58dfff9d3cbf53 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 15:24:50 +0200 Subject: [PATCH 70/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/test-pull-request-approved.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index afc6e1460..c1c9e56e0 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -18,6 +18,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: '10' + persist-credentials: false - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | @@ -35,8 +36,7 @@ jobs: echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT env: ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} - with: - persist-credentials: false + test: name: "Run all tests before merging" needs: calculate-xetabase-branch From 534106c700301117719f6ee03a5aaba3ffc879da Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 5 Sep 2024 15:31:53 +0200 Subject: [PATCH 71/79] TEST workflow for pull-request #TASK-6807 --- .github/workflows/pull-request-approved.yml | 7 +++++-- .github/workflows/scripts/get-xetabase-branch.sh | 2 +- .github/workflows/test-pull-request-approved.yml | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index cfc9110c3..de0bbc576 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -16,6 +16,8 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: '10' + ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found" + persist-credentials: false - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | @@ -23,10 +25,11 @@ jobs: echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }} ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}) + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - + env: + ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} test: name: "Run all tests before merging" diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index 68b19832a..139e6a0a7 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -9,7 +9,7 @@ get_xetabase_branch() { # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then - echo "$input_branch"; + echo $input_branch; return 0; fi fi diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml index c1c9e56e0..41f878cf7 100644 --- a/.github/workflows/test-pull-request-approved.yml +++ b/.github/workflows/test-pull-request-approved.yml @@ -18,6 +18,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: '10' + ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found" persist-credentials: false - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" From 039c4fd69277b38d20aba926e0497aae7f3a2bdb Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 9 Sep 2024 12:04:41 +0200 Subject: [PATCH 72/79] Fix ssh to report server #TASK-6807 --- .github/workflows/test-xetabase-workflow.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index eb441cfe3..a2dde507b 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -93,15 +93,20 @@ jobs: chmod +x ./kubectl echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27018:27017 --kubeconfig ./admin.conf & - - name: Install Azure AZCOPY - uses: kheiakiyama/install-azcopy-action@v1 - with: - version: 'v10' - name: DockerHub login uses: docker/login-action@v3 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ env.DOCKER_HUB_PASSWORD }} + - name: Install sshpass + run: sudo apt-get install sshpass + - name: Add SSH Host to known_hosts + run: | + mkdir -p ~/.ssh + ssh-keyscan -p ${{ env.SSH_PORT }} ${{ env.SSH_HOST }} >> ~/.ssh/known_hosts + env: + SSH_HOST: ${{ env.SSH_HOST }} + SSH_PORT: ${{ env.SSH_PORT }} - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise run: | printenv From c07c90248b7143b29b6483e507ccbfd3719442f0 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 12 Sep 2024 09:53:08 +0200 Subject: [PATCH 73/79] pull-request-approve Remove fail-never #TASK-6807 --- .github/workflows/test-xetabase-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index a2dde507b..4793f828e 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -110,7 +110,7 @@ jobs: - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise run: | printenv - ./build.sh -t -l runShortTests -b -s -f -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 + ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 env: SSH_HOST: ${{ env.SSH_HOST }} SSH_PORT: ${{ env.SSH_PORT }} From 502537f26a379001e56f68799bed0e9c168294ff Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 12 Sep 2024 09:54:54 +0200 Subject: [PATCH 74/79] pull-request-approve Remove fail-never #TASK-6807 --- .github/workflows/test-xetabase-workflow.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 4793f828e..ed802879d 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -20,10 +20,6 @@ on: description: 'Branch of opencga-enterprise to be tested and built.' required: true -env: - AZCOPY_AUTO_LOGIN_TYPE: "SPN" - - jobs: test: name: Execute JUnit and Jacoco tests @@ -35,9 +31,6 @@ jobs: with: keeper-secret-config: ${{ secrets.KEEPER_SM_GH_OPENCB }} secrets: | - AZCOPY_SPA_CLIENT_SECRET/field/Secret Value > env:AZCOPY_SPA_CLIENT_SECRET - AZCOPY_SPA_APPLICATION_ID/field/Secret Value > env:AZCOPY_SPA_APPLICATION_ID - AZCOPY_TENANT_ID/field/Secret Value > env:AZCOPY_TENANT_ID AZURE_KUBE_CONFIG/field/Secret Value > env:AZURE_KUBE_CONFIG DOCKER_HUB_USER/field/Secret Value > env:DOCKER_HUB_USER DOCKER_HUB_PASSWORD/field/Secret Value > env:DOCKER_HUB_PASSWORD @@ -109,13 +102,9 @@ jobs: SSH_PORT: ${{ env.SSH_PORT }} - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise run: | - printenv + cd opencga-enterprise + ln -s ../opencga opencga-home ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 - env: - SSH_HOST: ${{ env.SSH_HOST }} - SSH_PORT: ${{ env.SSH_PORT }} - SSH_USER: ${{ env.SSH_USER }} - SSH_PASS: ${{ env.SSH_PASS }} - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: From 113c1f23877afafdf7485addcc1ece1213afd625 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 13 Sep 2024 11:50:03 +0200 Subject: [PATCH 75/79] Prepare release 5.2.1 --- commons-datastore/commons-datastore-core/pom.xml | 2 +- commons-datastore/commons-datastore-mongodb/pom.xml | 2 +- commons-datastore/commons-datastore-solr/pom.xml | 2 +- commons-datastore/pom.xml | 2 +- commons-lib/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/commons-datastore/commons-datastore-core/pom.xml b/commons-datastore/commons-datastore-core/pom.xml index 08935cc57..b07eb2807 100644 --- a/commons-datastore/commons-datastore-core/pom.xml +++ b/commons-datastore/commons-datastore-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons-datastore - 5.2.1-SNAPSHOT + 5.2.1 ../pom.xml diff --git a/commons-datastore/commons-datastore-mongodb/pom.xml b/commons-datastore/commons-datastore-mongodb/pom.xml index 695c20e78..b668b8440 100644 --- a/commons-datastore/commons-datastore-mongodb/pom.xml +++ b/commons-datastore/commons-datastore-mongodb/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons-datastore - 5.2.1-SNAPSHOT + 5.2.1 ../pom.xml diff --git a/commons-datastore/commons-datastore-solr/pom.xml b/commons-datastore/commons-datastore-solr/pom.xml index 43df674d7..3f5198401 100644 --- a/commons-datastore/commons-datastore-solr/pom.xml +++ b/commons-datastore/commons-datastore-solr/pom.xml @@ -22,7 +22,7 @@ org.opencb.commons commons-datastore - 5.2.1-SNAPSHOT + 5.2.1 ../pom.xml diff --git a/commons-datastore/pom.xml b/commons-datastore/pom.xml index 54e31ca6b..7e1766efd 100644 --- a/commons-datastore/pom.xml +++ b/commons-datastore/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.2.1-SNAPSHOT + 5.2.1 ../pom.xml diff --git a/commons-lib/pom.xml b/commons-lib/pom.xml index 30da55ed5..f6845f0f4 100644 --- a/commons-lib/pom.xml +++ b/commons-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.2.1-SNAPSHOT + 5.2.1 ../pom.xml diff --git a/pom.xml b/pom.xml index 2028f0e0e..16820bf8f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.2.1-SNAPSHOT + 5.2.1 pom OpenCB commons project From b41110c5467f015c76a7f5213ec909004c9102b6 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 3 Oct 2024 11:10:59 +0200 Subject: [PATCH 76/79] Prepare Port Patch 5.2.1 -> 6.0.0 XB 2.2.1 -> 3.0.0 #TASK-6780 --- commons-datastore/commons-datastore-core/pom.xml | 2 +- commons-datastore/commons-datastore-mongodb/pom.xml | 2 +- commons-datastore/commons-datastore-solr/pom.xml | 2 +- commons-datastore/pom.xml | 2 +- commons-lib/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/commons-datastore/commons-datastore-core/pom.xml b/commons-datastore/commons-datastore-core/pom.xml index b07eb2807..f75e11a92 100644 --- a/commons-datastore/commons-datastore-core/pom.xml +++ b/commons-datastore/commons-datastore-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons-datastore - 5.2.1 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/commons-datastore/commons-datastore-mongodb/pom.xml b/commons-datastore/commons-datastore-mongodb/pom.xml index e22c83815..202d066c5 100644 --- a/commons-datastore/commons-datastore-mongodb/pom.xml +++ b/commons-datastore/commons-datastore-mongodb/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons-datastore - 5.2.1 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/commons-datastore/commons-datastore-solr/pom.xml b/commons-datastore/commons-datastore-solr/pom.xml index 3485cfd59..789a9a915 100644 --- a/commons-datastore/commons-datastore-solr/pom.xml +++ b/commons-datastore/commons-datastore-solr/pom.xml @@ -22,7 +22,7 @@ org.opencb.commons commons-datastore - 5.2.1 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/commons-datastore/pom.xml b/commons-datastore/pom.xml index 7e1766efd..1e503e66c 100644 --- a/commons-datastore/pom.xml +++ b/commons-datastore/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.2.1 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/commons-lib/pom.xml b/commons-lib/pom.xml index b4a7323c0..84a6c880b 100644 --- a/commons-lib/pom.xml +++ b/commons-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.2.1 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 1cd2bb2bd..3542c53b4 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.commons commons - 5.2.1 + 6.0.0-SNAPSHOT pom OpenCB commons project From b1a19c63bc3923fb18c549b28a301604f1e7d43e Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 3 Oct 2024 16:11:57 +0200 Subject: [PATCH 77/79] CICD: deleted tmp yml file #TASK-6780 --- .../workflows/tmp-test-xetabase-branch.yml | 48 ------------------- 1 file changed, 48 deletions(-) delete mode 100644 .github/workflows/tmp-test-xetabase-branch.yml diff --git a/.github/workflows/tmp-test-xetabase-branch.yml b/.github/workflows/tmp-test-xetabase-branch.yml deleted file mode 100644 index c10d99ec5..000000000 --- a/.github/workflows/tmp-test-xetabase-branch.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: TMP test-xetabase-branch -run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}' - -on: - push: - branches: - - TASK-* - - -jobs: - calculate-xetabase-branch: - name: Calculate Xetabase branch - runs-on: ubuntu-22.04 - outputs: - xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} - steps: - - name: Clone java-common-libs - uses: actions/checkout@v4 - with: - fetch-depth: '10' - ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found" - persist-credentials: false - - id: get_xetabase_branch - name: "Get current branch for Xetabase from target branch" - run: | - if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/opencb/opencga.git "TASK-6879" )" ] ; then - echo "OPENCGA TASK-6879 branch"; - fi - - if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then - echo "Here it is TASK-6807 branch"; - fi - chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh - echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" | tee -a ${GITHUB_STEP_SUMMARY} - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6879" ) - echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} - echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - env: - ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} - - test: - name: "Run all tests before merging" - needs: calculate-xetabase-branch - uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop - with: - branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} - task: TASK-6879 - secrets: inherit \ No newline at end of file From 1918805ec569687cfb7ad7936176dba4bd8d00e1 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 3 Oct 2024 16:13:49 +0200 Subject: [PATCH 78/79] CICD: deleted tmp yml file #TASK-6780 --- .../workflows/test-pull-request-approved.yml | 48 ------------------- 1 file changed, 48 deletions(-) delete mode 100644 .github/workflows/test-pull-request-approved.yml diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml deleted file mode 100644 index 41f878cf7..000000000 --- a/.github/workflows/test-pull-request-approved.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Pull request approve workflow -run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}' - -on: - push: - branches: - - TASK-* - - -jobs: - calculate-xetabase-branch: - name: Calculate Xetabase branch - runs-on: ubuntu-22.04 - outputs: - xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} - steps: - - name: Clone java-common-libs - uses: actions/checkout@v4 - with: - fetch-depth: '10' - ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found" - persist-credentials: false - - id: get_xetabase_branch - name: "Get current branch for Xetabase from target branch" - run: | - if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/opencb/opencga.git "TASK-6807" )" ] ; then - echo "OPENCGA TASK-6807 branch"; - fi - - if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then - echo "Here it is TASK-6807 branch"; - fi - chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh - echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" | tee -a ${GITHUB_STEP_SUMMARY} - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" ) - echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} - echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT - env: - ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} - - test: - name: "Run all tests before merging" - needs: calculate-xetabase-branch - uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop - with: - branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} - task: TASK-6807 - secrets: inherit \ No newline at end of file From 2b1d96c8f0db69e7b9437fb07eac0035d34abf9c Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 14 Oct 2024 11:54:24 +0200 Subject: [PATCH 79/79] Fix downgrade versions in merge #TASK-6780 --- commons-datastore/commons-datastore-mongodb/pom.xml | 1 - commons-datastore/commons-datastore-solr/pom.xml | 2 -- pom.xml | 10 ++++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/commons-datastore/commons-datastore-mongodb/pom.xml b/commons-datastore/commons-datastore-mongodb/pom.xml index 202d066c5..842daef6b 100644 --- a/commons-datastore/commons-datastore-mongodb/pom.xml +++ b/commons-datastore/commons-datastore-mongodb/pom.xml @@ -50,7 +50,6 @@ org.hamcrest hamcrest-core - test junit diff --git a/commons-datastore/commons-datastore-solr/pom.xml b/commons-datastore/commons-datastore-solr/pom.xml index 789a9a915..e178396a7 100644 --- a/commons-datastore/commons-datastore-solr/pom.xml +++ b/commons-datastore/commons-datastore-solr/pom.xml @@ -34,7 +34,6 @@ org.opencb.commons commons-datastore-core - org.apache.solr solr-solrj @@ -52,5 +51,4 @@ slf4j-api - diff --git a/pom.xml b/pom.xml index 3542c53b4..34df5f277 100644 --- a/pom.xml +++ b/pom.xml @@ -19,9 +19,9 @@ - 2.11.4 - 3.12.0 - 1.7.32 + 2.14.3 + 3.14.0 + 1.7.36 1.7.7 4.8.2 8.8.2 @@ -31,7 +31,6 @@ 1.3 4.13.2 - opencb https://sonarcloud.io 1.8 @@ -296,6 +295,9 @@ * + + * +