1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.testutils
18 
19 import androidx.test.ext.junit.runners.AndroidJUnit4
20 import com.android.net.module.util.LinkPropertiesUtils.CompareOrUpdateResult
21 import com.android.testutils.DevSdkIgnoreRule.IgnoreAfter
22 import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo
23 import java.lang.reflect.Modifier
24 import org.junit.runner.Description
25 import org.junit.runner.Runner
26 import org.junit.runner.manipulation.Filter
27 import org.junit.runner.manipulation.Filterable
28 import org.junit.runner.manipulation.NoTestsRemainException
29 import org.junit.runner.manipulation.Sortable
30 import org.junit.runner.manipulation.Sorter
31 import org.junit.runner.notification.Failure
32 import org.junit.runner.notification.RunNotifier
33 import org.junit.runners.Parameterized
34 import org.mockito.Mockito
35 
36 /**
37  * A runner that can skip tests based on the development SDK as defined in [DevSdkIgnoreRule].
38  *
39  * Generally [DevSdkIgnoreRule] should be used for that purpose (using rules is preferable over
40  * replacing the test runner), however JUnit runners inspect all methods in the test class before
41  * processing test rules. This may cause issues if the test methods are referencing classes that do
42  * not exist on the SDK of the device the test is run on.
43  *
44  * This runner inspects [IgnoreAfter] and [IgnoreUpTo] annotations on the test class, and will skip
45  * the whole class if they do not match the development SDK as defined in [DevSdkIgnoreRule].
46  * Otherwise, it will delegate to [AndroidJUnit4] to run the test as usual.
47  *
48  * This class automatically uses the Parameterized runner as its base runner when needed, so the
49  * @Parameterized.Parameters annotation and its friends can be used in tests using this runner.
50  *
51  * Example usage:
52  *
53  *     @RunWith(DevSdkIgnoreRunner::class)
54  *     @IgnoreUpTo(Build.VERSION_CODES.Q)
55  *     class MyTestClass { ... }
56  */
57 class DevSdkIgnoreRunner(private val klass: Class<*>) : Runner(), Filterable, Sortable {
58     private val leakMonitorDesc = Description.createTestDescription(klass, "ThreadLeakMonitor")
59     private val shouldThreadLeakFailTest = klass.isAnnotationPresent(MonitorThreadLeak::class.java)
60 
61     // Inference correctly infers Runner & Filterable & Sortable for |baseRunner|, but the
62     // Java bytecode doesn't have a way to express this. Give this type a name by wrapping it.
63     private class RunnerWrapper<T>(private val wrapped: T) :
64             Runner(), Filterable by wrapped, Sortable by wrapped
65             where T : Runner, T : Filterable, T : Sortable {
getDescriptionnull66         override fun getDescription(): Description = wrapped.description
67         override fun run(notifier: RunNotifier?) = wrapped.run(notifier)
68     }
69 
70     // Annotation for test classes to indicate the test runner should monitor thread leak.
71     // TODO(b/307693729): Remove this annotation and monitor thread leak by default.
72     annotation class MonitorThreadLeak
73 
74     private val baseRunner: RunnerWrapper<*>? = klass.let {
75         val ignoreAfter = it.getAnnotation(IgnoreAfter::class.java)
76         val ignoreUpTo = it.getAnnotation(IgnoreUpTo::class.java)
77 
78         if (!isDevSdkInRange(ignoreUpTo, ignoreAfter)) {
79             null
80         } else if (it.hasParameterizedMethod()) {
81             // Parameterized throws if there is no static method annotated with @Parameters, which
82             // isn't too useful. Use it if there are, otherwise use its base AndroidJUnit4 runner.
83             RunnerWrapper(Parameterized(klass))
84         } else {
85             RunnerWrapper(AndroidJUnit4(klass))
86         }
87     }
88 
<lambda>null89     private fun <T> Class<T>.hasParameterizedMethod(): Boolean = methods.any {
90         Modifier.isStatic(it.modifiers) &&
91                 it.isAnnotationPresent(Parameterized.Parameters::class.java) }
92 
runnull93     override fun run(notifier: RunNotifier) {
94         if (baseRunner == null) {
95             // Report a single, skipped placeholder test for this class, as the class is expected to
96             // report results when run. In practice runners that apply the Filterable implementation
97             // would see a NoTestsRemainException and not call the run method.
98             notifier.fireTestIgnored(
99                     Description.createTestDescription(klass, "skippedClassForDevSdkMismatch"))
100             return
101         }
102         if (!shouldThreadLeakFailTest) {
103             baseRunner.run(notifier)
104             return
105         }
106 
107         // Dump threads as a baseline to monitor thread leaks.
108         val threadCountsBeforeTest = getAllThreadNameCounts()
109 
110         baseRunner.run(notifier)
111 
112         notifier.fireTestStarted(leakMonitorDesc)
113         val threadCountsAfterTest = getAllThreadNameCounts()
114         // TODO : move CompareOrUpdateResult to its own util instead of LinkProperties.
115         val threadsDiff = CompareOrUpdateResult(
116                 threadCountsBeforeTest.entries,
117                 threadCountsAfterTest.entries
118         ) { it.key }
119         // Ignore removed threads, which typically are generated by previous tests.
120         // Because this is in the threadsDiff.updated member, for sure there is a
121         // corresponding key in threadCountsBeforeTest.
122         val increasedThreads = threadsDiff.updated
123                 .filter { threadCountsBeforeTest[it.key]!! < it.value }
124         if (threadsDiff.added.isNotEmpty() || increasedThreads.isNotEmpty()) {
125             notifier.fireTestFailure(Failure(leakMonitorDesc,
126                     IllegalStateException("Unexpected thread changes: $threadsDiff")))
127         }
128         // Clears up internal state of all inline mocks.
129         // TODO: Call clearInlineMocks() at the end of each test.
130         Mockito.framework().clearInlineMocks()
131         notifier.fireTestFinished(leakMonitorDesc)
132     }
133 
getAllThreadNameCountsnull134     private fun getAllThreadNameCounts(): Map<String, Int> {
135         // Get the counts of threads in the group per name.
136         // Filter system thread groups.
137         // Also ignore threads with 1 count, this effectively filtered out threads created by the
138         // test runner or other system components. e.g. hwuiTask*, queued-work-looper,
139         // SurfaceSyncGroupTimer, RenderThread, Time-limited test, etc.
140         return Thread.getAllStackTraces().keys
141                 .filter { it.threadGroup?.name != "system" }
142                 .groupingBy { it.name }.eachCount()
143                 .filter { it.value != 1 }
144     }
145 
getDescriptionnull146     override fun getDescription(): Description {
147         if (baseRunner == null) {
148             return Description.createSuiteDescription(klass)
149         }
150 
151         return baseRunner.description.also {
152             if (shouldThreadLeakFailTest) {
153                 it.addChild(leakMonitorDesc)
154             }
155         }
156     }
157 
158     /**
159      * Get the test count before applying the [Filterable] implementation.
160      */
testCountnull161     override fun testCount(): Int {
162         // When ignoring the tests, a skipped placeholder test is reported, so test count is 1.
163         if (baseRunner == null) return 1
164 
165         return baseRunner.testCount() + if (shouldThreadLeakFailTest) 1 else 0
166     }
167 
168     @Throws(NoTestsRemainException::class)
filternull169     override fun filter(filter: Filter?) {
170         baseRunner?.filter(filter) ?: throw NoTestsRemainException()
171     }
172 
sortnull173     override fun sort(sorter: Sorter?) {
174         baseRunner?.sort(sorter)
175     }
176 }
177