From 0bef4857a8b6cce6fe5285f4b9f4d76b68ac71a3 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 11 Apr 2024 17:32:05 -0700 Subject: [PATCH 01/11] Run LlamaDemo app on AWS Device Farm --- .github/workflows/android.yml | 58 +++++++++++++++++++ .../example/executorchllamademo/PerfTest.java | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 0d8931cf102..628d12b0a79 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -33,6 +33,7 @@ jobs: submodules: 'true' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 + upload-artifact: android-apps script: | set -eux @@ -45,3 +46,60 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" # Build Android demo app bash build/test_android_ci.sh + + # Copy the app and its test suite to S3 + cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/debug/*.apk "${RUNNER_TEMP}/artifacts" + cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/androidTest/debug/*.apk "${RUNNER_TEMP}/artifacts" + + sleep 300 + + # Upload the app and its test suite to S3 so that they can be downloaded by the test job + upload-artifacts: + needs: test-demo-android + runs-on: linux.2xlarge + steps: + - name: Download the artifacts + uses: actions/download-artifact@v3 + with: + name: android-apps + path: ${{ runner.temp }}/artifacts/ + + - name: Verify the artifacts + shell: bash + working-directory: ${{ runner.temp }}/artifacts/ + run: | + ls -lah ./ + + - name: Upload the artifacts to S3 + uses: seemethere/upload-artifact-s3@v5 + working-directory: ${{ runner.temp }}/artifacts/ + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact + retention-days: 14 + if-no-files-found: ignore + path: *.apk + + # Let's see how expensive this job is, we might want to tone it down by running it periodically + test-llama-app: + needs: upload-to-s3 + permissions: + id-token: write + contents: read + uses: pytorch/test-infra/.github/workflows/mobile_job.yml@add-android-device-farm-support + with: + device-type: android + runner: ubuntu-latest + # This is the ARN of ExecuTorch project on AWS + project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6 + # This is the custom Android device pool that only includes Samsung Galaxy S2x + device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa + # Uploaded to S3 from the previous job + android-app-archive: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/8a7276b7-03ac-4053-8b09-1305c429f59d + android-test-archive: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/528113d6-2003-420b-aedb-4242b6d14254 + # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml + test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/414cb54d-4d83-4576-8317-93244e4dc50e + # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. + # Among the input, this is the biggest file and uploading it to AWS beforehand makes the test run much faster + extra-data: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/bd15825b-ddab-4e47-9fef-a9c8935778dd diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java index 4aa4bc4fd55..b8988d1f4ba 100644 --- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java +++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java @@ -28,7 +28,7 @@ public class PerfTest implements LlamaCallback { private static final String TOKENIZER_BIN = "tokenizer.bin"; // From https://github.com/pytorch/executorch/blob/main/examples/models/llama2/README.md - private static final Float EXPECTED_TPS = 7.0F; + private static final Float EXPECTED_TPS = 10.0F; private final List results = new ArrayList<>(); private final List tokensPerSecond = new ArrayList<>(); From 9cc7c191ba2e824f608e358301f2f8d6f6c86f10 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 11 Apr 2024 17:34:21 -0700 Subject: [PATCH 02/11] Fix workflow syntax --- .github/workflows/android.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 628d12b0a79..feb3cae6b90 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -72,14 +72,13 @@ jobs: - name: Upload the artifacts to S3 uses: seemethere/upload-artifact-s3@v5 - working-directory: ${{ runner.temp }}/artifacts/ with: s3-bucket: gha-artifacts s3-prefix: | ${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact retention-days: 14 if-no-files-found: ignore - path: *.apk + path: ${{ runner.temp }}/artifacts/**/*.apk # Let's see how expensive this job is, we might want to tone it down by running it periodically test-llama-app: From ada8eacb3b8feabb78dd66009292d71024696def Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 11 Apr 2024 17:49:21 -0700 Subject: [PATCH 03/11] Fix workflow syntax --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index feb3cae6b90..784c8fd386f 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -82,7 +82,7 @@ jobs: # Let's see how expensive this job is, we might want to tone it down by running it periodically test-llama-app: - needs: upload-to-s3 + needs: upload-artifacts permissions: id-token: write contents: read From ab3a8fbf691f5591bb780d7c5c15229ca9ab74ce Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 11 Apr 2024 19:03:01 -0700 Subject: [PATCH 04/11] Upload custom artifacts --- .github/workflows/android.yml | 10 +++++----- build/test_android_ci.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 784c8fd386f..ffafdfe4cfc 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -21,7 +21,7 @@ concurrency: jobs: test-demo-android: name: test-demo-android - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job.yml@support-custom-artifact-upload strategy: matrix: include: @@ -47,11 +47,10 @@ jobs: # Build Android demo app bash build/test_android_ci.sh + mkdir -p artifacts-to-be-uploaded # Copy the app and its test suite to S3 - cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/debug/*.apk "${RUNNER_TEMP}/artifacts" - cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/androidTest/debug/*.apk "${RUNNER_TEMP}/artifacts" - - sleep 300 + cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/debug/*.apk artifacts-to-be-uploaded/ + cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/androidTest/debug/*.apk artifacts-to-be-uploaded/ # Upload the app and its test suite to S3 so that they can be downloaded by the test job upload-artifacts: @@ -61,6 +60,7 @@ jobs: - name: Download the artifacts uses: actions/download-artifact@v3 with: + # The name here needs to match the name of the upload-artifact parameter name: android-apps path: ${{ runner.temp }}/artifacts/ diff --git a/build/test_android_ci.sh b/build/test_android_ci.sh index acc853727fa..5c855680ad4 100755 --- a/build/test_android_ci.sh +++ b/build/test_android_ci.sh @@ -31,7 +31,7 @@ build_android_demo_app() { build_android_llama_demo_app() { pushd examples/demo-apps/android/LlamaDemo ANDROID_NDK=/opt/ndk ANDROID_ABI=arm64-v8a ./gradlew setup - ANDROID_HOME=/opt/android/sdk ./gradlew build + ANDROID_HOME=/opt/android/sdk ./gradlew build && ./gradlew assembleAndroidTest popd } From 574967c3aa889941530b54a4ebbd7823202a4e33 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 11 Apr 2024 20:55:28 -0700 Subject: [PATCH 05/11] Fix build androidTest command --- build/test_android_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/test_android_ci.sh b/build/test_android_ci.sh index 5c855680ad4..2b019a1cd6c 100755 --- a/build/test_android_ci.sh +++ b/build/test_android_ci.sh @@ -31,7 +31,8 @@ build_android_demo_app() { build_android_llama_demo_app() { pushd examples/demo-apps/android/LlamaDemo ANDROID_NDK=/opt/ndk ANDROID_ABI=arm64-v8a ./gradlew setup - ANDROID_HOME=/opt/android/sdk ./gradlew build && ./gradlew assembleAndroidTest + ANDROID_HOME=/opt/android/sdk ./gradlew build + ANDROID_HOME=/opt/android/sdk ./gradlew assembleAndroidTest popd } From 4f8a77b4e2fff72da3339fe42fbbefe4a746ed01 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 10:35:47 -0700 Subject: [PATCH 06/11] Fix app upload part --- .github/workflows/android.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index ffafdfe4cfc..152f0d5558f 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -90,13 +90,17 @@ jobs: with: device-type: android runner: ubuntu-latest + # TODO: FOR TESTING + test-infra-ref: add-android-device-farm-support # This is the ARN of ExecuTorch project on AWS project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6 # This is the custom Android device pool that only includes Samsung Galaxy S2x device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa - # Uploaded to S3 from the previous job - android-app-archive: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/8a7276b7-03ac-4053-8b09-1305c429f59d - android-test-archive: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/528113d6-2003-420b-aedb-4242b6d14254 + # Uploaded to S3 from the previous job, the name of the app comes from the project itself + android-app-archive: | + https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact/app-debug.apk + android-test-archive: | + https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/414cb54d-4d83-4576-8317-93244e4dc50e # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. From e2762debe5d419cd9abe4bda5a7e0bb81d5d6623 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 11:35:51 -0700 Subject: [PATCH 07/11] Don't use newline --- .github/workflows/android.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 152f0d5558f..18da4baf57b 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -97,10 +97,8 @@ jobs: # This is the custom Android device pool that only includes Samsung Galaxy S2x device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa # Uploaded to S3 from the previous job, the name of the app comes from the project itself - android-app-archive: | - https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact/app-debug.apk - android-test-archive: | - https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact/app-debug-androidTest.apk + android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact/app-debug.apk + android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/414cb54d-4d83-4576-8317-93244e4dc50e # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. From 156bef2427b582727e3d5e6650d97e87fe660063 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 12:24:01 -0700 Subject: [PATCH 08/11] Upload the share libraries --- .github/workflows/android.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 18da4baf57b..bb2da1ebb92 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -21,7 +21,7 @@ concurrency: jobs: test-demo-android: name: test-demo-android - uses: pytorch/test-infra/.github/workflows/linux_job.yml@support-custom-artifact-upload + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main strategy: matrix: include: @@ -51,6 +51,8 @@ jobs: # Copy the app and its test suite to S3 cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/debug/*.apk artifacts-to-be-uploaded/ cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/androidTest/debug/*.apk artifacts-to-be-uploaded/ + # Also copy the share libraries + cp cmake-android-out/lib/*.a artifacts-to-be-uploaded/ # Upload the app and its test suite to S3 so that they can be downloaded by the test job upload-artifacts: @@ -75,10 +77,10 @@ jobs: with: s3-bucket: gha-artifacts s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact + ${{ github.repository }}/${{ github.run_id }}/artifact retention-days: 14 if-no-files-found: ignore - path: ${{ runner.temp }}/artifacts/**/*.apk + path: ${{ runner.temp }}/artifacts/ # Let's see how expensive this job is, we might want to tone it down by running it periodically test-llama-app: @@ -97,8 +99,8 @@ jobs: # This is the custom Android device pool that only includes Samsung Galaxy S2x device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa # Uploaded to S3 from the previous job, the name of the app comes from the project itself - android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact/app-debug.apk - android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/artifact/app-debug-androidTest.apk + android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug.apk + android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/414cb54d-4d83-4576-8317-93244e4dc50e # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. From 225b754ce451c1a18681b9199c53668841095c00 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 12:58:24 -0700 Subject: [PATCH 09/11] It's cmake-out-android --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index bb2da1ebb92..372c60c2c73 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -52,7 +52,7 @@ jobs: cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/debug/*.apk artifacts-to-be-uploaded/ cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/androidTest/debug/*.apk artifacts-to-be-uploaded/ # Also copy the share libraries - cp cmake-android-out/lib/*.a artifacts-to-be-uploaded/ + cp cmake-out-android/lib/*.a artifacts-to-be-uploaded/ # Upload the app and its test suite to S3 so that they can be downloaded by the test job upload-artifacts: From 4f6a0214c0618ee5bc5a629920a8b19a9724253d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 14:40:53 -0700 Subject: [PATCH 10/11] Prepare for landing --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 372c60c2c73..2f5f2e04b20 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -88,7 +88,7 @@ jobs: permissions: id-token: write contents: read - uses: pytorch/test-infra/.github/workflows/mobile_job.yml@add-android-device-farm-support + uses: pytorch/test-infra/.github/workflows/mobile_job.yml@main with: device-type: android runner: ubuntu-latest From 77c60e0a265ab7d93343acb23f65b90e8fa9559d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 14:45:13 -0700 Subject: [PATCH 11/11] Minor tweak to test-infra branch (now main) --- .github/workflows/android.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 2f5f2e04b20..6948f9fd1a0 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -92,8 +92,7 @@ jobs: with: device-type: android runner: ubuntu-latest - # TODO: FOR TESTING - test-infra-ref: add-android-device-farm-support + test-infra-ref: '' # This is the ARN of ExecuTorch project on AWS project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6 # This is the custom Android device pool that only includes Samsung Galaxy S2x