1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package android.platform.test.rule
17 
18 import android.os.Build
19 import android.platform.test.rule.DeviceProduct.CF_PHONE
20 import android.platform.test.rule.DeviceProduct.CF_TABLET
21 import android.util.Log
22 import androidx.test.platform.app.InstrumentationRegistry
23 import kotlin.annotation.AnnotationRetention.RUNTIME
24 import kotlin.annotation.AnnotationTarget.CLASS
25 import kotlin.annotation.AnnotationTarget.FUNCTION
26 import org.junit.AssumptionViolatedException
27 import org.junit.rules.TestRule
28 import org.junit.runner.Description
29 import org.junit.runners.model.Statement
30 
31 /** Limits the test to run on devices specified by [allowed], */
32 @Retention(RUNTIME)
33 @Target(FUNCTION, CLASS)
34 annotation class AllowedDevices(vararg val allowed: DeviceProduct)
35 
36 /** Does not run the test on device specified by [denied], */
37 @Retention(RUNTIME)
38 @Target(FUNCTION, CLASS)
39 annotation class DeniedDevices(vararg val denied: DeviceProduct)
40 
41 /** Limits the test on default screenshot devices, or [allowed] devices if specified. */
42 @Retention(RUNTIME)
43 @Target(FUNCTION, CLASS)
44 annotation class ScreenshotTestDevices(vararg val allowed: DeviceProduct = [CF_PHONE, CF_TABLET])
45 
46 /**
47  * Only runs the test on [flakyProducts] if this configuration is running flaky tests (see
48  * runningFlakyTests parameter on [LimitDevicesRule] constructor Runs it normally on all other
49  * devices.
50  */
51 @Retention(RUNTIME)
52 @Target(FUNCTION, CLASS)
53 annotation class FlakyDevices(vararg val flaky: DeviceProduct)
54 
55 /**
56  * Ignore LimitDevicesRule constraints when [ignoreLimit] is true. Main use case is to allow local
57  * builds to bypass [LimitDevicesRule] and be able to run on any devices.
58  */
59 @Retention(RUNTIME) @Target(FUNCTION, CLASS) annotation class IgnoreLimit(val ignoreLimit: Boolean)
60 
61 /**
62  * Limits a test to run specified devices.
63  *
64  * Devices are specified by [AllowedDevices], [DeniedDevices], [ScreenshotTestDevices], and
65  * [FlakyDevices] annotations. Only one annotation on class or one per test is supported. Values are
66  * matched against [thisDevice].
67  *
68  * To read the instrumentation args to determine whether to run on [FlakyDevices] (recommended), use
69  * [readParamsFromInstrumentation] to construct the rule
70  *
71  * NOTE: It's not encouraged to use this to filter if it's possible to filter based on other device
72  * characteristics. For example, to run a test only only on large screens or foldable,
73  * [DeviceTypeRule] is encouraged. This rule should **never** be used to avoid running a test on a
74  * tablet when the test is broken.
75  */
76 class LimitDevicesRule(
77     private val thisDevice: String = Build.PRODUCT,
78     private val runningFlakyTests: Boolean = false
79 ) : TestRule {
applynull80     override fun apply(base: Statement, description: Description): Statement {
81         if (description.ignoreLimit()) {
82             return base
83         }
84 
85         val limitDevicesAnnotations = description.limitDevicesAnnotation()
86         if (limitDevicesAnnotations.count() > 1) {
87             return makeAssumptionViolatedStatement(
88                 "Only one LimitDeviceRule annotation is supported. Found $limitDevicesAnnotations"
89             )
90         }
91         val deniedDevices = description.deniedDevices()
92         if (thisDevice in deniedDevices) {
93             return makeAssumptionViolatedStatement(
94                 "Skipping test as $thisDevice is in $deniedDevices"
95             )
96         }
97 
98         val flakyDevices = description.flakyDevices()
99         if (thisDevice in flakyDevices) {
100             if (!runningFlakyTests) {
101                 return makeAssumptionViolatedStatement(
102                     "Skipping test as $thisDevice is flaky and this config excludes fakes"
103                 )
104             }
105         }
106 
107         val allowedDevices = description.allowedDevices()
108         if (allowedDevices.isEmpty() || thisDevice in allowedDevices) {
109             return base
110         }
111         return makeAssumptionViolatedStatement(
112             "Skipping test as $thisDevice in not in $allowedDevices"
113         )
114     }
115 
allowedDevicesnull116     private fun Description.allowedDevices(): List<String> =
117         listOf(
118                 getMostSpecificAnnotation<AllowedDevices>()?.allowed,
119                 getMostSpecificAnnotation<ScreenshotTestDevices>()?.allowed
120             )
121             .collectProducts()
122 
123     private fun Description.deniedDevices(): List<String> =
124         listOf(getMostSpecificAnnotation<DeniedDevices>()?.denied).collectProducts()
125 
126     private fun Description.flakyDevices(): List<String> =
127         listOf(getMostSpecificAnnotation<FlakyDevices>()?.flaky).collectProducts()
128 
129     private fun Description.limitDevicesAnnotation(): Set<Annotation> =
130         listOfNotNull(
131                 getMostSpecificAnnotation<AllowedDevices>(),
132                 getMostSpecificAnnotation<DeniedDevices>(),
133                 getMostSpecificAnnotation<ScreenshotTestDevices>(),
134                 getMostSpecificAnnotation<FlakyDevices>()
135             )
136             .toSet()
137 
138     private fun Description.ignoreLimit(): Boolean =
139         getAnnotation(IgnoreLimit::class.java)?.ignoreLimit == true ||
140             testClass?.getClassAnnotation<IgnoreLimit>()?.ignoreLimit == true
141 
142     private inline fun <reified T : Annotation> Description.getMostSpecificAnnotation(): T? {
143         getAnnotation(T::class.java)?.let {
144             return it
145         }
146         return testClass?.getClassAnnotation<T>()
147     }
148 
getClassAnnotationnull149     private inline fun <reified T : Annotation> Class<*>.getClassAnnotation() =
150         getLowestAncestorClassAnnotation(this, T::class.java)
151 
152     private fun List<Array<out DeviceProduct>?>.collectProducts() =
153         filterNotNull().flatMap { it.toList() }.map { it.product }
154 
155     companion object {
isRunningFlakyTestsnull156         private fun isRunningFlakyTests(): Boolean {
157             val args = InstrumentationRegistry.getArguments()
158             val isRunning = args.getString(RUNNING_FLAKY_TESTS_KEY, "false").toBoolean()
159             if (isRunning) {
160                 Log.d(TAG, "Running on flaky devices, due to $RUNNING_FLAKY_TESTS_KEY param.")
161             }
162             return isRunning
163         }
164 
readParamsFromInstrumentationnull165         fun readParamsFromInstrumentation(thisDevice: String = Build.PRODUCT) =
166             LimitDevicesRule(thisDevice, isRunningFlakyTests())
167 
168         private const val RUNNING_FLAKY_TESTS_KEY = "running-flaky-tests"
169         private const val TAG = "LimitDevicesRule"
170     }
171 }
172 
173 enum class DeviceProduct(val product: String) {
174     CF_PHONE("cf_x86_64_phone"),
175     CF_TABLET("cf_x86_64_tablet"),
176     CF_FOLDABLE("cf_x86_64_foldable"),
177     CF_AUTO("cf_x86_64_auto"),
178     TANGORPRO("tangorpro"),
179     FELIX("felix"),
180     ROBOLECTRIC("robolectric"),
181 }
182 
makeAssumptionViolatedStatementnull183 private fun makeAssumptionViolatedStatement(errorMessage: String): Statement =
184     object : Statement() {
185         override fun evaluate() {
186             throw AssumptionViolatedException(errorMessage)
187         }
188     }
189