Issue #467: iOS support globs for test-targets

Author: csessa

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -6,7 +6,6 @@ import ftl.args.ArgsHelper.createGcsBucket
 import ftl.args.ArgsHelper.createJunitBucket
 import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.mergeYmlMaps
-import ftl.args.ArgsHelper.validateTestMethods
 import ftl.args.ArgsHelper.yamlMapper
 import ftl.args.ArgsToString.devicesToString
 import ftl.args.ArgsToString.listToString
@@ -62,13 +61,10 @@ class IosArgs(
 
     // computed properties not specified in yaml
     override val testShardChunks: List<List<String>> by lazy {
+        if (disableSharding) return@lazy listOf(emptyList<String>())
+
         val validTestMethods = Xctestrun.findTestNames(xctestrunFile)
-        validateTestMethods(testTargets, validTestMethods, "xctest binary")
-        val testsToShard = if (testTargets.isEmpty()) {
-            validTestMethods
-        } else {
-            testTargets
-        }.distinct()
+        val testsToShard = filterTests(validTestMethods, testTargets).distinct()
 
         ArgsHelper.calculateShards(testsToShard, this)
     }
@@ -162,3 +158,19 @@ ${listToString(testTargets)}
         }
     }
 }
+
+fun filterTests(validTestMethods: List<String>, testTargets: List<String>): List<String> {
+    if (testTargets.isEmpty()) {
+        return validTestMethods
+    }
+
+    return validTestMethods.filter { test ->
+        testTargets.forEach { target ->
+            if (test.matches(target.toRegex())) {
+                return@filter true
+            }
+        }
+
+        return@filter false
+    }
+}

test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt

@@ -40,15 +40,16 @@ object GcIosTestMatrix {
         val gcsBucket = args.resultsBucket
         val matrixGcsSuffix = join(runGcsPath, shardCounter.next())
         val matrixGcsPath = join(gcsBucket, matrixGcsSuffix)
-        val methods = args.testShardChunks.elementAt(testShardsIndex)
 
         // Parameterized tests on iOS don't shard correctly.
         // Avoid changing Xctestrun file when disableSharding is on.
         val generatedXctestrun = if (args.disableSharding) {
             xcTestParsed.toByteArray()
         } else {
+            val methods = args.testShardChunks.elementAt(testShardsIndex)
             Xctestrun.rewrite(xcTestParsed, methods)
         }
+
         val xctestrunFileGcsPath = GcStorage.uploadXCTestFile(args, gcsBucket, matrixGcsSuffix, generatedXctestrun)
 
         val iOSXCTest = IosXcTest()

test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt

@@ -27,7 +27,7 @@ class IosArgsFileTest {
     private val xctestrunZip = getPath("src/test/kotlin/ftl/fixtures/tmp/EarlGreyExample.zip")
     private val xctestrunFile =
         getPath("src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun")
-    private val testName = "EarlGreyExampleMixedTests/testBasicSelection"
+    private val testName = "EarlGreyExampleSwiftTests/testLayout"
     // NOTE: Change working dir to '%MODULE_WORKING_DIR%' in IntelliJ to match gradle for this test to pass.
     @Test
     fun configLoadsSuccessfully() {
@@ -69,17 +69,16 @@ class IosArgsFileTest {
         val config = IosArgs.load(yamlFile2)
 
         val chunk0 = arrayListOf(
-            "EarlGreyExampleMixedTests/testGrantCameraPermission",
-            "EarlGreyExampleMixedTests/testGrantMicrophonePermission",
-            "EarlGreyExampleMixedTests/testBasicSelection1",
-            "EarlGreyExampleMixedTests/testBasicSelection4"
+            "EarlGreyExampleSwiftTests/testWithGreyAssertions",
+            "EarlGreyExampleSwiftTests/testWithInRoot",
+            "EarlGreyExampleSwiftTests/testWithCondition",
+            "EarlGreyExampleSwiftTests/testWithCustomFailureHandler"
         )
 
         val chunk1 = arrayListOf(
-            "EarlGreyExampleMixedTests/testGrantCameraPermission",
-            "EarlGreyExampleMixedTests/testGrantMicrophonePermission",
-            "EarlGreyExampleMixedTests/testBasicSelection2",
-            "EarlGreyExampleMixedTests/testBasicSelection3"
+            "EarlGreyExampleSwiftTests/testWithGreyAssertions",
+            "EarlGreyExampleSwiftTests/testWithCustomMatcher",
+            "EarlGreyExampleSwiftTests/testWithCustomAssertion"
         )
 
         val testShardChunks = config.testShardChunks

test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt

@@ -26,6 +26,7 @@ class IosArgsTest {
     private val testPath = "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExample.zip"
     private val xctestrunFile =
         "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun"
+    private val invalidApp = "../test_app/apks/invalid.apk"
     private val xctestrunFileAbsolutePath = xctestrunFile.absolutePath()
     private val testAbsolutePath = testPath.absolutePath()
     private val iosNonDefault = """
@@ -91,10 +92,10 @@ class IosArgsTest {
     fun iosArgs_invalidXcodeExits() {
         exceptionRule.expectMessage("Xcode 99.9 is not a supported Xcode version")
         IosArgs(
-                GcloudYml(),
-                IosGcloudYml(IosGcloudYmlParams(test = testPath, xctestrunFile = xctestrunFile, xcodeVersion = "99.9")),
-                FlankYml(),
-                IosFlankYml(),
+            GcloudYml(),
+            IosGcloudYml(IosGcloudYmlParams(test = testPath, xctestrunFile = xctestrunFile, xcodeVersion = "99.9")),
+            FlankYml(),
+            IosFlankYml(),
             ""
         )
     }
@@ -256,6 +257,30 @@ IosArgs
         }
     }
 
+    @Test
+    fun `disableSharding allows using invalid app`() {
+        val yaml = """
+        gcloud:
+          test: $invalidApp
+          xctestrun-file: $invalidApp
+        flank:
+          disableSharding: true
+      """
+        IosArgs.load(yaml).testShardChunks
+    }
+
+    @Test(expected = RuntimeException::class)
+    fun `Invalid app throws`() {
+        val yaml = """
+        gcloud:
+          test: $invalidApp
+          xctestrun-file: $invalidApp
+        flank:
+          disableSharding: false
+      """
+        IosArgs.load(yaml).testShardChunks
+    }
+
     // gcloudYml
 
     @Test
@@ -591,4 +616,60 @@ IosArgs
         val androidArgs = IosArgs.load(yaml, cli)
         assertThat(androidArgs.flakyTestAttempts).isEqualTo(3)
     }
+
+    private fun getValidTestsSample() = listOf(
+        "ClassOneTest/testOne",
+        "ClassOneTest/testTwo",
+        "ClassOneScreenshots/testOne",
+        "ClassTwoScreenshots/testTwo",
+        "ClassThreeTest/testName",
+        "ClassFourTest/testFour"
+    )
+
+    @Test
+    fun filterTests_emptyFilter() {
+        val tests = getValidTestsSample()
+        val actual = filterTests(tests, emptyList())
+
+        assertThat(actual).containsExactlyElementsIn(tests)
+    }
+
+    @Test
+    fun filterTests_regularFilter() {
+        val tests = getValidTestsSample()
+        val filter = listOf("ClassOneTest/testOne", "ClassFourTest/testFour")
+        val actual = filterTests(tests, filter)
+
+        val expected = listOf("ClassOneTest/testOne", "ClassFourTest/testFour")
+
+        assertThat(actual).containsExactlyElementsIn(expected)
+    }
+
+    @Test
+    fun filterTests_starFilter() {
+        val tests = getValidTestsSample()
+        val filter = listOf(".*?Test/testOne", ".*?/testFour")
+        val actual = filterTests(tests, filter)
+
+        val expected = listOf(
+            "ClassOneTest/testOne",
+            "ClassFourTest/testFour"
+        )
+
+        assertThat(actual).containsExactlyElementsIn(expected)
+    }
+
+    @Test
+    fun filterTests_starAndRegularFilter() {
+        val tests = getValidTestsSample()
+        val filter = listOf(".*?Screenshots/testTwo", "ClassOneTest/testOne")
+        val actual = filterTests(tests, filter)
+
+        val expected = listOf(
+            "ClassTwoScreenshots/testTwo",
+            "ClassOneTest/testOne"
+        )
+
+        assertThat(actual).containsExactlyElementsIn(expected)
+    }
 }

test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml

@@ -16,6 +16,6 @@ gcloud:
 
 flank:
   test-targets:
-  - EarlGreyExampleMixedTests/testBasicSelection
+  - EarlGreyExampleSwiftTests/testLayout
   testShards: 1
   repeatTests: 1

test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml

@@ -13,6 +13,6 @@ gcloud:
 
 flank:
   test-targets:
-  - EarlGreyExampleMixedTests/testBasicSelection
+  - EarlGreyExampleSwiftTests/testLayout
   testShards: 1
   repeatTests: 1

test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml

@@ -15,12 +15,6 @@ flank:
   testShards: 2
   repeatTests: 1
   test-targets:
-  - EarlGreyExampleMixedTests/testGrantCameraPermission
-  - EarlGreyExampleMixedTests/testGrantMicrophonePermission
-  - EarlGreyExampleMixedTests/testBasicSelection1
-  - EarlGreyExampleMixedTests/testBasicSelection2
-  - EarlGreyExampleMixedTests/testBasicSelection3
-  - EarlGreyExampleMixedTests/testBasicSelection4
+    - EarlGreyExampleSwiftTests/testWith.*$
   test-targets-always-run:
-  - EarlGreyExampleMixedTests/testGrantCameraPermission
-  - EarlGreyExampleMixedTests/testGrantMicrophonePermission
+  - EarlGreyExampleSwiftTests/testWithGreyAssertions