Skip to content

Commit 4134fa7

Browse files
jan-goraljan-goral
authored
Fix test filters when using both notPackage and notClass (#654)
* Fix test filters when using both notPackage and notClass * Improve TestFilters description readability * Check if inclusion filter overrides exclusion filter
1 parent 8f11b59 commit 4134fa7

7 files changed

Lines changed: 123 additions & 14 deletions

File tree

‎test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@ import androidx.test.ext.junit.runners.AndroidJUnit4
44
import org.junit.Test
55
import org.junit.runner.RunWith
66

7+
annotation class Annotation
8+
79
@RunWith(AndroidJUnit4::class)
810
class InstrumentedTest : BaseInstrumentedTest() {
911

1012
@Test
13+
@Annotation
1114
fun test0() = testMethod()
1215

1316
@Test

‎test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package com.example.test_app.bar
2+
3+
import androidx.test.ext.junit.runners.AndroidJUnit4
4+
import com.example.test_app.BaseInstrumentedTest
5+
import org.junit.Test
6+
import org.junit.runner.RunWith
7+
8+
@RunWith(AndroidJUnit4::class)
9+
class BarInstrumentedTest : BaseInstrumentedTest() {
10+
11+
@Test
12+
fun testBar() = testMethod()
13+
}

‎test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package com.example.test_app.foo
2+
3+
import androidx.test.ext.junit.runners.AndroidJUnit4
4+
import com.example.test_app.BaseInstrumentedTest
5+
import org.junit.Test
6+
import org.junit.runner.RunWith
7+
8+
@RunWith(AndroidJUnit4::class)
9+
class FooInstrumentedTest : BaseInstrumentedTest() {
10+
11+
@Test
12+
fun testFoo() = testMethod()
13+
}

‎test_app/bash/test_filters.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
3+
# Helper functions for testing AndroidJUnitRunner in action.
4+
# Can be helpful for checking if flank test filters match AndroidJUnitRunner policy.
5+
6+
PACKAGE="com.example.test_app"
7+
PACKAGE_FOO="com.example.test_app.foo"
8+
PACKAGE_BAR="com.example.test_app.bar"
9+
10+
ANNOTATION="${PACKAGE}.Annotation"
11+
12+
CLASS="${PACKAGE}.InstrumentedTest"
13+
CLASS_FOO="${PACKAGE_FOO}.FooInstrumentedTest"
14+
CLASS_BAR="${PACKAGE_BAR}.BarInstrumentedTest"
15+
16+
METHOD_1="${CLASS}#test1"
17+
METHOD_FOO="${CLASS_FOO}#testFoo"
18+
METHOD_BAR="${CLASS_BAR}#testBar"
19+
20+
RUNNER=com.example.test_app.test/androidx.test.runner.AndroidJUnitRunner
21+
22+
set -euxo pipefail
23+
24+
# should run all tests
25+
function run_instrument {
26+
adb shell am instrument -r -w $@ ${RUNNER}
27+
}
28+
29+
# should run only method 1, last inclusion filter overrides the rest
30+
function filter_package_bar_class_foo_method_1 {
31+
run_instrument -e package ${PACKAGE_BAR} -e class ${CLASS_FOO} -e class ${METHOD_1}
32+
}
33+
34+
# should run nothing because of annotation intersect with other filters
35+
function filter_annotation_method_foo {
36+
run_instrument -e annotation ${ANNOTATION} -e class ${METHOD_FOO}
37+
}
38+
39+
# should exclude both
40+
function filter_notPackage_foo_notClass_bar {
41+
run_instrument -e notPackage ${PACKAGE_FOO} -e notClass ${CLASS_BAR}
42+
}

‎test_runner/src/main/kotlin/ftl/filter/TestFilters.kt‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,11 @@ object TestFilters {
8181

8282
// select test method name filters and short circuit if they match ex: class a.b#c
8383
val annotationFilters = parsedFilters.filter { it.isAnnotation }.toTypedArray()
84-
val otherFilters = parsedFilters.filterNot { it.isAnnotation }.toTypedArray()
84+
val otherFilters = parsedFilters.filterNot { it.isAnnotation }
85+
val exclude = otherFilters.filter { it.describe.startsWith("not") }.toTypedArray()
86+
val include = otherFilters.filterNot { it.describe.startsWith("not") }.toTypedArray()
8587

86-
val result = allOf(*annotationFilters, anyOf(*otherFilters))
88+
val result = allOf(*annotationFilters, *exclude, anyOf(*include))
8789
if (FtlConstants.useMock) println(result.describe)
8890
return result
8991
}
@@ -128,29 +130,31 @@ object TestFilters {
128130
}
129131

130132
private fun withPackageName(packageNames: List<String>): TestFilter = TestFilter(
131-
describe = "withPackageName ${packageNames.joinToString(", ")}",
133+
describe = "withPackageName (${packageNames.joinToString(", ")})",
132134
shouldRun = { testMethod ->
133-
packageNames.any { packageName -> testMethod.testName.startsWith(packageName) }
135+
packageNames.any { packageName ->
136+
testMethod.testName.startsWith(packageName)
137+
}
134138
}
135139
)
136140

137141
private fun withClassName(classNames: List<String>): TestFilter = TestFilter(
138-
describe = "withClassName ${classNames.joinToString(", ")}",
142+
describe = "withClassName (${classNames.joinToString(", ")})",
139143
shouldRun = { testMethod ->
140144
withPackageName(classNames).shouldRun(testMethod)
141145
}
142146
)
143147

144148
private fun withAnnotation(annotations: List<String>): TestFilter = TestFilter(
145-
describe = "withAnnotation ${annotations.joinToString(", ")}",
149+
describe = "withAnnotation (${annotations.joinToString(", ")})",
146150
shouldRun = { testMethod ->
147151
testMethod.annotationNames.any { annotations.contains(it) }
148152
},
149153
isAnnotation = true
150154
)
151155

152156
private fun not(filter: TestFilter): TestFilter = TestFilter(
153-
describe = "not ${filter.describe}",
157+
describe = "not (${filter.describe})",
154158
shouldRun = { testMethod ->
155159
filter.shouldRun(testMethod).not()
156160
},
@@ -169,9 +173,9 @@ object TestFilters {
169173
shouldRun = { testMethod ->
170174
if (FtlConstants.useMock) println(":: ${testMethod.testName} @${testMethod.annotations.firstOrNull()}")
171175
filters.isEmpty() || filters.all { filter ->
172-
val result = filter.shouldRun(testMethod)
173-
if (FtlConstants.useMock) println(" $result ${filter.describe}")
174-
result
176+
filter.shouldRun(testMethod).also { result ->
177+
if (FtlConstants.useMock) println(" $result ${filter.describe}")
178+
}
175179
}
176180
}
177181
)

‎test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ val WITHOUT_LARGE_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList())
2929
val WITHOUT_MEDIUM_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList())
3030
val WITHOUT_SMALL_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList())
3131
const val TEST_FILE = "src/test/kotlin/ftl/filter/fixtures/dummy-tests-file.txt"
32+
const val TEST_FILE_2 = "src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt"
3233
private const val IGNORE_ANNOTATION = "org.junit.Ignore"
3334

35+
@Suppress("TooManyFunctions")
3436
@RunWith(FlankTestRunner::class)
3537
class TestFiltersTest {
3638

@@ -249,10 +251,10 @@ class TestFiltersTest {
249251
}.map { "class ${it.testName}" }.toList()
250252

251253
val expected = listOf(
252-
"false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]",
253-
"false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]",
254-
"true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]",
255-
"false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]"
254+
"false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]",
255+
"false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]",
256+
"true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]",
257+
"false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]"
256258
)
257259

258260
assertThat(output).isEqualTo(expected)
@@ -322,6 +324,36 @@ class TestFiltersTest {
322324

323325
assertEquals(byNotAnnotation, expected)
324326
}
327+
328+
@Test
329+
fun testFilteringClassAndPackageNegative() {
330+
val filter = fromTestTargets(listOf("notPackage foo", "notClass whatever.Bar"))
331+
332+
assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse()
333+
assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse()
334+
assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue()
335+
}
336+
337+
@Test
338+
fun testFilteringClassAndPackageNegativeFromFile() {
339+
val file = TestHelper.getPath(TEST_FILE_2) // contents: foo whatever.Bar
340+
val filePath = file.toString()
341+
342+
val filter = fromTestTargets(listOf("notTestFile $filePath"))
343+
344+
assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse()
345+
assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse()
346+
assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue()
347+
}
348+
349+
@Test
350+
fun `inclusion filter should override exclusion filter`() {
351+
val filter = fromTestTargets(listOf("notPackage foo", "class whatever.Bar"))
352+
353+
assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse()
354+
assertThat(filter.shouldRun(BAR_PACKAGE)).isFalse()
355+
assertThat(filter.shouldRun(BAR_CLASSNAME)).isTrue()
356+
}
325357
}
326358

327359
private fun getDefaultTestMethod(testName: String, annotation: String) =

‎test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
foo
2+
whatever.Bar

0 commit comments

Comments
 (0)