e2e: updated compatibility tests to support running against all unreleased versions (#2680)

chatton · web-flow · commit 65b3ddfa1d5a · 2022-11-15T14:07:06.000Z
diff --git a/.github/compatibility-test-matrices/unreleased/client.json b/.github/compatibility-test-matrices/unreleased/client.json
@@ -0,0 +1,10 @@
+{
+  "chain-a": ["release-v6.0.x", "release-v5.1.x", "release-v5.0.x", "release-v4.2.x", "release-v3.4.x", "release-v2.5.x"],
+  "chain-b": ["release-v6.0.x", "release-v5.1.x", "release-v5.0.x", "release-v4.2.x", "release-v3.4.x", "release-v2.5.x"],
+  "entrypoint": ["TestClientTestSuite"],
+  "test": [
+    "TestClientUpdateProposal_Succeeds"
+  ],
+  "chain-binary": ["simd"],
+  "chain-image": ["ghcr.io/cosmos/ibc-go-simd"]
+}
diff --git a/.github/compatibility-test-matrices/unreleased/connection.json b/.github/compatibility-test-matrices/unreleased/connection.json
@@ -0,0 +1,10 @@
+{
+  "chain-a": ["release-v6.0.x", "release-v5.1.x", "release-v5.0.x", "release-v4.2.x", "release-v3.4.x", "release-v2.5.x"],
+  "chain-b": ["release-v6.0.x", "release-v5.1.x", "release-v5.0.x", "release-v4.2.x", "release-v3.4.x", "release-v2.5.x"],
+  "entrypoint": ["TestConnectionTestSuite"],
+  "test": [
+    "TestMaxExpectedTimePerBlockParam"
+  ],
+  "chain-binary": ["simd"],
+  "chain-image": ["ghcr.io/cosmos/ibc-go-simd"]
+}
diff --git a/.github/compatibility-test-matrices/unreleased/incentivized-transfer.json b/.github/compatibility-test-matrices/unreleased/incentivized-transfer.json
@@ -0,0 +1,15 @@
+{
+  "chain-a": ["release-v6.0.x", "release-v5.1.x", "release-v5.0.x", "release-v4.2.x"],
+  "chain-b": ["release-v6.0.x", "release-v5.1.x", "release-v5.0.x", "release-v4.2.x"],
+  "entrypoint": ["TestIncentivizedTransferTestSuite"],
+  "test": [
+    "TestMsgPayPacketFee_AsyncSingleSender_Succeeds",
+    "TestMsgPayPacketFee_InvalidReceiverAccount",
+    "TestMultiMsg_MsgPayPacketFeeSingleSender",
+    "TestMsgPayPacketFee_SingleSender_TimesOut",
+    "TestPayPacketFeeAsync_SingleSender_NoCounterPartyAddress",
+    "TestMsgPayPacketFee_AsyncMultipleSenders_Succeeds"
+  ],
+  "chain-binary": ["simd"],
+  "chain-image": ["ghcr.io/cosmos/ibc-go-simd"]
+}
diff --git a/.github/compatibility-test-matrices/unreleased/transfer.json b/.github/compatibility-test-matrices/unreleased/transfer.json
@@ -0,0 +1,15 @@
+{
+  "chain-a": ["release-v6.0.x", "release-v5.1.x", "release-v4.2.x", "release-v3.4.x", "release-v2.5.x"],
+  "chain-b": ["release-v6.0.x", "release-v5.1.x", "release-v4.2.x", "release-v3.4.x", "release-v2.5.x"],
+  "entrypoint": ["TestTransferTestSuite"],
+  "test": [
+    "TestMsgTransfer_Succeeds_Nonincentivized",
+    "TestMsgTransfer_Fails_InvalidAddress",
+    "TestMsgTransfer_Timeout_Nonincentivized",
+    "TestMsgTransfer_WithMemo",
+    "TestSendEnabledParam",
+    "TestReceiveEnabledParam"
+  ],
+  "chain-binary": ["simd"],
+  "chain-image": ["ghcr.io/cosmos/ibc-go-simd"]
+}
diff --git a/.github/workflows/e2e-compatibility-workflow-call.yaml b/.github/workflows/e2e-compatibility-workflow-call.yaml
@@ -1,8 +1,8 @@
 on:
   workflow_call:
     inputs:
-      docker-tag:
-        description: 'Docker tag being used'
+      test-file-directory:
+        description: 'Directory containing compatibility matrices'
         required: true
         type: string
       test-suite:
@@ -21,7 +21,7 @@ jobs:
       - run: |
           # use jq -c to put the full json contents on a single line. This is required when using the json body
           # to create the matrix in the following job.
-          test_matrix="$(cat .github/compatibility-test-matrices/${{ inputs.docker-tag }}/${{ inputs.test-suite }}.json | jq -c)"
+          test_matrix="$(cat .github/compatibility-test-matrices/${{ inputs.test-file-directory }}/${{ inputs.test-suite }}.json | jq -c)"
           echo $test_matrix
           echo "::set-output name=test-matrix::$test_matrix"
         id: set-test-matrix
diff --git a/.github/workflows/e2e-compatibility.yaml b/.github/workflows/e2e-compatibility.yaml
@@ -3,7 +3,7 @@ on:
   workflow_dispatch:
     inputs:
       release-branch:
-        description: 'Release branch to test'
+        description: 'Release branch to test (unreleased to run all release branches against each other)'
         required: true
         type: choice
         options:
@@ -13,6 +13,7 @@ on:
           - release/v5.0.x
           - release/v5.1.x
           - release/v6.0.x
+          - unreleased
 
 env:
   REGISTRY: ghcr.io
@@ -21,49 +22,66 @@ env:
   RELEASE_BRANCH: '${{ inputs.release-branch }}'
 
 jobs:
-  determine-docker-tag:
+  determine-test-directory:
     runs-on: ubuntu-latest
     outputs:
-      docker-tag: ${{ steps.set-docker-tag.outputs.docker-tag }}
+      test-directory: ${{ steps.set-test-dir.outputs.test-directory }}
     steps:
       - run: |
-          docker_tag="$(echo $RELEASE_BRANCH | sed 's/\//-/')"
-          echo $docker_tag
-          echo "::set-output name=docker-tag::$docker_tag"
-        id: set-docker-tag
+          test_dir="$(echo $RELEASE_BRANCH | sed 's/\//-/')"
+          echo $test_dir
+          echo "::set-output name=test-directory::$test_dir"
+        id: set-test-dir
 
-  build-release-image:
+  # build-release-images builds all docker images that are relevant for the compatibility tests. If a single release
+  # branch is specified, only that image will be built, e.g. release-v6.0.x but if 'unreleased' is specified
+  # every image will be built.
+  build-release-images:
     runs-on: ubuntu-latest
-    needs: determine-docker-tag
+    strategy:
+      matrix:
+        release-branch:
+          - release/v2.5.x
+          - release/v3.4.x
+          - release/v4.2.x
+          - release/v5.0.x
+          - release/v5.1.x
+          - release/v6.0.x
     steps:
       - uses: actions/checkout@v3
+        if: env.RELEASE_BRANCH == 'unreleased' || env.RELEASE_BRANCH == matrix.release-branch
         with:
-          ref: "${{ env.RELEASE_BRANCH }}"
+          ref: "${{ matrix.release-branch }}"
           fetch-depth: 0
       - name: Log in to the Container registry
+        if: env.RELEASE_BRANCH == 'unreleased' || env.RELEASE_BRANCH == matrix.release-branch
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Fetch Makefile dependencies
+        if: env.RELEASE_BRANCH == 'unreleased' || env.RELEASE_BRANCH == matrix.release-branch
         run: |
           mkdir -p contrib/devtools
           curl https://raw.githubusercontent.com/cosmos/ibc-go/main/contrib/devtools/Makefile -o contrib/devtools/Makefile
       - name: Fetch latest Dockerfile
+        if: env.RELEASE_BRANCH == 'unreleased' || env.RELEASE_BRANCH == matrix.release-branch
         run: curl https://raw.githubusercontent.com/cosmos/ibc-go/main/Dockerfile -o Dockerfile
       - name: Build image
+        if: env.RELEASE_BRANCH == 'unreleased' || env.RELEASE_BRANCH == matrix.release-branch
         run: |
-          docker build . -t "${REGISTRY}/${ORG}/${IMAGE_NAME}:${{ needs.determine-docker-tag.outputs.docker-tag }}"
-          docker push "${REGISTRY}/${ORG}/${IMAGE_NAME}:${{ needs.determine-docker-tag.outputs.docker-tag }}"
+          docker_tag="$(echo ${{ matrix.release-branch }} | sed 's/\//-/')"
+          docker build . -t "${REGISTRY}/${ORG}/${IMAGE_NAME}:$docker_tag"
+          docker push "${REGISTRY}/${ORG}/${IMAGE_NAME}:$docker_tag"
 
   transfer:
     needs:
-      - build-release-image
-      - determine-docker-tag
+      - build-release-images
+      - determine-test-directory
     uses: ./.github/workflows/e2e-compatibility-workflow-call.yaml
     with:
-      docker-tag: "${{ needs.determine-docker-tag.outputs.docker-tag }}"
+      test-file-directory: "${{ needs.determine-test-directory.outputs.test-directory }}"
       test-suite: "transfer"
 
   transfer-params:
@@ -86,18 +104,18 @@ jobs:
 
   client:
     needs:
-      - build-release-image
-      - determine-docker-tag
+      - build-release-images
+      - determine-test-directory
     uses: ./.github/workflows/e2e-compatibility-workflow-call.yaml
     with:
-      docker-tag: "${{ needs.determine-docker-tag.outputs.docker-tag }}"
+      test-file-directory: "${{ needs.determine-test-directory.outputs.test-directory }}"
       test-suite: "client"
 
   incentivized-transfer:
     needs:
-      - build-release-image
-      - determine-docker-tag
+      - build-release-images
+      - determine-test-directory
     uses: ./.github/workflows/e2e-compatibility-workflow-call.yaml
     with:
-      docker-tag: "${{ needs.determine-docker-tag.outputs.docker-tag }}"
+      test-file-directory: "${{ needs.determine-test-directory.outputs.test-directory }}"
       test-suite: "incentivized-transfer"
