From 710e765d3c0896ba49cb3c83e56de9e50d673d02 Mon Sep 17 00:00:00 2001 From: Jeremy Massel <1123407+jkmassel@users.noreply.github.com> Date: Thu, 23 Apr 2026 15:43:35 -0600 Subject: [PATCH 1/6] ci: add Android instrumented test step on mac-metal queue Runs :app:connectedDebugAndroidTest against an ephemeral AVD booted via the start-android-emulator helper on the mac-metal agent. See: https://appsinfrap2.wordpress.com/2025/10/10/using-the-mac-metal-queue-for-android-instrumented-tests/ --- .buildkite/pipeline.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8729445ce..dd7a809ae 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -74,3 +74,18 @@ steps: tar -xzf dist.tar.gz make test-ios-e2e plugins: *plugins + + - label: ':android: Test Android E2E' + depends_on: build-react + command: | + buildkite-agent artifact download dist.tar.gz . + tar -xzf dist.tar.gz + echo "--- :robot_face: Starting Android emulator" + start-android-emulator \ + --system-image "system-images;android-34;google_apis;arm64-v8a" \ + --device "pixel_5" + echo "--- :android: Running Android instrumented tests" + make test-android-e2e + agents: + queue: mac-metal + plugins: *plugins From 414393b1c8efa1a12266250a16e7a885395f3177 Mon Sep 17 00:00:00 2001 From: Jeremy Massel <1123407+jkmassel@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:23:47 -0600 Subject: [PATCH 2/6] fix(android): restore Compose UI test and Espresso Web dependencies These were lost during a merge conflict resolution in #339 and the gap was invisible until the mac-metal CI step started exercising the androidTest sources. Re-adds espresso-web, compose-ui-test-junit4, and compose-ui-test-manifest to the version catalog and wires them into the app module so EditorInteractionTest compiles. --- android/app/build.gradle.kts | 4 ++++ android/gradle/libs.versions.toml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index bc18059cf..dcd336268 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -102,4 +102,8 @@ dependencies { testImplementation(libs.junit) androidTestImplementation(libs.androidx.junit) androidTestImplementation(libs.androidx.espresso.core) + androidTestImplementation(libs.androidx.espresso.web) + androidTestImplementation(platform(libs.androidx.compose.bom)) + androidTestImplementation(libs.androidx.compose.ui.test.junit4) + debugImplementation(libs.androidx.compose.ui.test.manifest) } diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index ed8aedc4f..99e7be089 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -6,6 +6,7 @@ coreKtx = "1.13.1" junit = "4.13.2" junitVersion = "1.2.1" espressoCore = "3.6.1" +espressoWeb = "3.6.1" appcompat = "1.7.0" material = "1.12.0" activity = "1.9.0" @@ -28,6 +29,7 @@ androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = junit = { group = "junit", name = "junit", version.ref = "junit" } androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" } androidx-espresso-core = { group = "androidx.test.espresso", name = "espresso-core", version.ref = "espressoCore" } +androidx-espresso-web = { group = "androidx.test.espresso", name = "espresso-web", version.ref = "espressoWeb" } androidx-appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" } material = { group = "com.google.android.material", name = "material", version.ref = "material" } androidx-activity = { group = "androidx.activity", name = "activity", version.ref = "activity" } @@ -47,6 +49,8 @@ androidx-compose-ui = { group = "androidx.compose.ui", name = "ui" } androidx-compose-material3 = { group = "androidx.compose.material3", name = "material3" } androidx-compose-material-icons-extended = { group = "androidx.compose.material", name = "material-icons-extended" } androidx-compose-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview" } +androidx-compose-ui-test-junit4 = { group = "androidx.compose.ui", name = "ui-test-junit4" } +androidx-compose-ui-test-manifest = { group = "androidx.compose.ui", name = "ui-test-manifest" } androidx-activity-compose = { group = "androidx.activity", name = "activity-compose", version.ref = "activityCompose" } jsoup = { group = "org.jsoup", name = "jsoup", version.ref = "jsoup" } okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" } From a2760ca22e839364e7e7e2cef45fd70c2087e137 Mon Sep 17 00:00:00 2001 From: Jeremy Massel <1123407+jkmassel@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:58:48 -0600 Subject: [PATCH 3/6] ci(android): wire up Buildkite Test Engine collector Registers the stock InstrumentedTestCollector as a test listener when BUILDKITE_ANALYTICS_TOKEN_ANDROID is present in the environment. The listener is only attached when the token is set, so local runs and unauthenticated CI stay fully inert. The Android Test Engine suite + agent secret need to be set up separately for data to start flowing. --- android/app/build.gradle.kts | 9 +++++++++ android/gradle/libs.versions.toml | 2 ++ 2 files changed, 11 insertions(+) diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index dcd336268..9ffafc3fa 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -51,6 +51,14 @@ android { testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + // Buildkite Test Engine: upload per-test results when a token is present in the env. + // Absent locally → no listener is registered and the collector is fully inert. + System.getenv("BUILDKITE_ANALYTICS_TOKEN_ANDROID")?.takeIf { it.isNotBlank() }?.let { token -> + testInstrumentationRunnerArguments["listener"] = + "com.buildkite.test.collector.android.InstrumentedTestCollector" + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = token + } + buildConfigField("String", "WP_ENV_SITE_URL", "\"${wpEnvCredentials["siteUrl"] ?: ""}\"") buildConfigField("String", "WP_ENV_SITE_API_ROOT", "\"${wpEnvCredentials["siteApiRoot"] ?: ""}\"") buildConfigField("String", "WP_ENV_AUTH_HEADER", "\"${wpEnvCredentials["authHeader"] ?: ""}\"") @@ -105,5 +113,6 @@ dependencies { androidTestImplementation(libs.androidx.espresso.web) androidTestImplementation(platform(libs.androidx.compose.bom)) androidTestImplementation(libs.androidx.compose.ui.test.junit4) + androidTestImplementation(libs.buildkite.test.collector.instrumented) debugImplementation(libs.androidx.compose.ui.test.manifest) } diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index 99e7be089..27536b9a6 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -23,6 +23,7 @@ activityCompose = "1.9.3" jsoup = "1.18.1" okhttp = "4.12.0" detekt = "1.23.8" +buildkite-test-collector = "0.4.0" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } @@ -55,6 +56,7 @@ androidx-activity-compose = { group = "androidx.activity", name = "activity-comp jsoup = { group = "org.jsoup", name = "jsoup", version.ref = "jsoup" } okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" } okhttp-mockwebserver = { group = "com.squareup.okhttp3", name = "mockwebserver", version.ref = "okhttp" } +buildkite-test-collector-instrumented = { group = "com.buildkite.test-collector-android", name = "instrumented-test-collector", version.ref = "buildkite-test-collector" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } From 0e056395601968d948246aeea3c7ce075dc4dd10 Mon Sep 17 00:00:00 2001 From: Jeremy Massel <1123407+jkmassel@users.noreply.github.com> Date: Thu, 23 Apr 2026 17:02:07 -0600 Subject: [PATCH 4/6] ci(android): read test collector token from BUILDKITE_ANALYTICS_TOKEN_ANDROID_E2E Matches the env var name already provisioned on Automattic agents rather than inventing a new one. --- android/app/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 9ffafc3fa..b5ca664f0 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -53,7 +53,7 @@ android { // Buildkite Test Engine: upload per-test results when a token is present in the env. // Absent locally → no listener is registered and the collector is fully inert. - System.getenv("BUILDKITE_ANALYTICS_TOKEN_ANDROID")?.takeIf { it.isNotBlank() }?.let { token -> + System.getenv("BUILDKITE_ANALYTICS_TOKEN_ANDROID_E2E")?.takeIf { it.isNotBlank() }?.let { token -> testInstrumentationRunnerArguments["listener"] = "com.buildkite.test.collector.android.InstrumentedTestCollector" testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = token From 9b4809fd4bfff87af0f23371861177322136b836 Mon Sep 17 00:00:00 2001 From: Jeremy Massel <1123407+jkmassel@users.noreply.github.com> Date: Fri, 24 Apr 2026 09:50:24 -0600 Subject: [PATCH 5/6] ci(android): forward Buildkite build metadata to test collector Pass BUILDKITE_BUILD_ID/URL/BRANCH/COMMIT/BUILD_NUMBER/JOB_ID/MESSAGE through to the collector so uploaded test results link back to the Buildkite job. Per buildkite/test-collector-android CI_CONFIGURATION.md. --- android/app/build.gradle.kts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index b5ca664f0..4cb01731a 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -53,10 +53,24 @@ android { // Buildkite Test Engine: upload per-test results when a token is present in the env. // Absent locally → no listener is registered and the collector is fully inert. + // Build metadata env vars are forwarded so uploaded results link back to the CI job: + // https://github.com/buildkite/test-collector-android/blob/main/CI_CONFIGURATION.md#buildkite System.getenv("BUILDKITE_ANALYTICS_TOKEN_ANDROID_E2E")?.takeIf { it.isNotBlank() }?.let { token -> testInstrumentationRunnerArguments["listener"] = "com.buildkite.test.collector.android.InstrumentedTestCollector" testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = token + + listOf( + "BUILDKITE_BUILD_ID", + "BUILDKITE_BUILD_URL", + "BUILDKITE_BRANCH", + "BUILDKITE_COMMIT", + "BUILDKITE_BUILD_NUMBER", + "BUILDKITE_JOB_ID", + "BUILDKITE_MESSAGE", + ).forEach { key -> + System.getenv(key)?.let { value -> testInstrumentationRunnerArguments[key] = value } + } } buildConfigField("String", "WP_ENV_SITE_URL", "\"${wpEnvCredentials["siteUrl"] ?: ""}\"") From d9329d24d757fd535e8b700677b503bfaf33bf6a Mon Sep 17 00:00:00 2001 From: Jeremy Massel <1123407+jkmassel@users.noreply.github.com> Date: Fri, 24 Apr 2026 09:54:49 -0600 Subject: [PATCH 6/6] ci(android): upload test reports as artifacts for debugging Pick up Gradle's JUnit XML, HTML reports, and any additional emulator outputs (logcat captures, screenshots) from connectedDebugAndroidTest. Artifacts upload on both success and failure, so the reports are available directly from the Buildkite UI when a test run goes wrong. --- .buildkite/pipeline.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index dd7a809ae..710747f24 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -89,3 +89,7 @@ steps: agents: queue: mac-metal plugins: *plugins + artifact_paths: + - 'android/app/build/reports/androidTests/connected/**/*' + - 'android/app/build/outputs/androidTest-results/connected/**/*' + - 'android/app/build/outputs/connected_android_test_additional_output/**/*'