1 /*
<lambda>null2  * Copyright (C) 2024 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.tools.flicker
18 
19 import android.app.Instrumentation
20 import android.device.collectors.BaseMetricListener
21 import android.device.collectors.DataRecord
22 import android.tools.FLICKER_TAG
23 import android.tools.Scenario
24 import android.tools.ScenarioBuilder
25 import android.tools.flicker.assertions.AssertionResult
26 import android.tools.flicker.config.FlickerServiceConfig
27 import android.tools.flicker.config.ScenarioId
28 import android.tools.io.Reader
29 import android.tools.io.RunStatus
30 import android.tools.traces.getDefaultFlickerOutputDir
31 import android.util.Log
32 import androidx.test.platform.app.InstrumentationRegistry
33 import com.android.internal.annotations.VisibleForTesting
34 import java.io.File
35 import org.junit.runner.Description
36 import org.junit.runner.Result
37 import org.junit.runner.notification.Failure
38 
39 /**
40  * Collects all the Flicker Service's metrics which are then uploaded for analysis and monitoring to
41  * the CrystalBall database.
42  */
43 class FlickerServiceResultsCollector(
44     private val tracesCollector: TracesCollector,
45     private val flickerService: FlickerService =
46         FlickerService(FlickerConfig().use(FlickerServiceConfig.DEFAULT)),
47     instrumentation: Instrumentation = InstrumentationRegistry.getInstrumentation(),
48     private val collectMetricsPerTest: Boolean = true,
49     private val reportOnlyForPassingTests: Boolean = true
50 ) : BaseMetricListener(), IFlickerServiceResultsCollector {
51     private var hasFailedTest = false
52     private var testSkipped = false
53 
54     private val _executionErrors = mutableListOf<Throwable>()
55     override val executionErrors
56         get() = _executionErrors
57 
58     @VisibleForTesting val assertionResults = mutableListOf<AssertionResult>()
59 
60     @VisibleForTesting
61     val assertionResultsByTest = mutableMapOf<Description, Collection<AssertionResult>>()
62 
63     @VisibleForTesting
64     val detectedScenariosByTest = mutableMapOf<Description, Collection<ScenarioId>>()
65 
66     private var testRunScenario: Scenario? = null
67     private var testScenario: Scenario? = null
68 
69     init {
70         setInstrumentation(instrumentation)
71     }
72 
73     override fun onTestRunStart(runData: DataRecord, description: Description) {
74         errorReportingBlock {
75             tracesCollector.cleanup() // Cleanup any trace archives from previous runs
76 
77             Log.i(LOG_TAG, "onTestRunStart :: collectMetricsPerTest = $collectMetricsPerTest")
78             if (!collectMetricsPerTest) {
79                 hasFailedTest = false
80                 val scenario =
81                     ScenarioBuilder()
82                         .forClass(description.testClass.canonicalName)
83                         .withDescriptionOverride("")
84                         .build()
85                 testRunScenario = scenario
86                 tracesCollector.start(scenario)
87             }
88         }
89     }
90 
91     override fun onTestStart(testData: DataRecord, description: Description) {
92         errorReportingBlock {
93             Log.i(LOG_TAG, "onTestStart :: collectMetricsPerTest = $collectMetricsPerTest")
94             if (collectMetricsPerTest) {
95                 hasFailedTest = false
96                 val scenario =
97                     ScenarioBuilder()
98                         .forClass(
99                             "${description.testClass.canonicalName}#${description.methodName}"
100                         )
101                         .withDescriptionOverride("")
102                         .build()
103                 testScenario = scenario
104                 tracesCollector.start(scenario)
105             }
106             testSkipped = false
107         }
108     }
109 
110     override fun onTestFail(testData: DataRecord, description: Description, failure: Failure) {
111         errorReportingBlock {
112             Log.i(LOG_TAG, "onTestFail")
113             hasFailedTest = true
114         }
115     }
116 
117     override fun testAssumptionFailure(failure: Failure?) {
118         errorReportingBlock {
119             Log.i(LOG_TAG, "testAssumptionFailure")
120             testSkipped = true
121         }
122     }
123 
124     override fun testSkipped(description: Description) {
125         errorReportingBlock {
126             Log.i(LOG_TAG, "testSkipped")
127             testSkipped = true
128         }
129     }
130 
131     override fun onTestEnd(testData: DataRecord, description: Description) {
132         Log.i(LOG_TAG, "onTestEnd :: collectMetricsPerTest = $collectMetricsPerTest")
133         if (collectMetricsPerTest) {
134             val results = errorReportingBlock {
135                 Log.i(LOG_TAG, "Stopping trace collection")
136                 val reader = tracesCollector.stop()
137                 Log.i(LOG_TAG, "Stopped trace collection")
138 
139                 if (reportOnlyForPassingTests && hasFailedTest) {
140                     return@errorReportingBlock null
141                 }
142 
143                 if (testSkipped) {
144                     return@errorReportingBlock null
145                 }
146 
147                 return@errorReportingBlock collectFlickerMetrics(testData, reader, description)
148             }
149 
150             reportFlickerServiceStatus(
151                 testData,
152                 results,
153                 testScenario ?: error("Test scenario should not be null"),
154                 testData
155             )
156         }
157     }
158 
159     override fun onTestRunEnd(runData: DataRecord, result: Result) {
160         Log.i(LOG_TAG, "onTestRunEnd :: collectMetricsPerTest = $collectMetricsPerTest")
161         if (!collectMetricsPerTest) {
162             val results = errorReportingBlock {
163                 Log.i(LOG_TAG, "Stopping trace collection")
164                 val reader = tracesCollector.stop()
165                 Log.i(LOG_TAG, "Stopped trace collection")
166 
167                 if (reportOnlyForPassingTests && hasFailedTest) {
168                     return@errorReportingBlock null
169                 }
170 
171                 return@errorReportingBlock collectFlickerMetrics(runData, reader)
172             }
173 
174             reportFlickerServiceStatus(
175                 runData,
176                 results,
177                 testRunScenario ?: error("Test run scenario should not be null"),
178                 runData
179             )
180         }
181     }
182 
183     private fun collectFlickerMetrics(
184         dataRecord: DataRecord,
185         reader: Reader,
186         description: Description? = null
187     ): Collection<AssertionResult>? {
188         return errorReportingBlock {
189             return@errorReportingBlock try {
190                 Log.i(LOG_TAG, "Processing traces")
191                 val scenarios = flickerService.detectScenarios(reader)
192                 val results = scenarios.flatMap { it.generateAssertions() }.map { it.execute() }
193                 reader.artifact.updateStatus(RunStatus.RUN_EXECUTED)
194                 Log.i(LOG_TAG, "Got ${results.size} results")
195                 assertionResults.addAll(results)
196                 if (description != null) {
197                     require(assertionResultsByTest[description] == null) {
198                         "Test description ($description) already contains flicker assertion results"
199                     }
200                     require(detectedScenariosByTest[description] == null) {
201                         "Test description ($description) already contains detected scenarios"
202                     }
203                     assertionResultsByTest[description] = results
204                     detectedScenariosByTest[description] = scenarios.map { it.type }.distinct()
205                 }
206                 if (results.any { it.status == AssertionResult.Status.FAIL }) {
207                     reader.artifact.updateStatus(RunStatus.ASSERTION_FAILED)
208                 } else {
209                     reader.artifact.updateStatus(RunStatus.ASSERTION_SUCCESS)
210                 }
211 
212                 Log.v(LOG_TAG, "Adding metric $FLICKER_ASSERTIONS_COUNT_KEY = ${results.size}")
213                 dataRecord.addStringMetric(FLICKER_ASSERTIONS_COUNT_KEY, "${results.size}")
214 
215                 val aggregatedResults = processFlickerResults(results)
216                 collectMetrics(dataRecord, aggregatedResults)
217 
218                 results
219             } finally {
220                 Log.v(LOG_TAG, "Adding metric $WINSCOPE_FILE_PATH_KEY = ${reader.artifactPath}")
221                 dataRecord.addStringMetric(WINSCOPE_FILE_PATH_KEY, reader.artifactPath)
222             }
223         }
224     }
225 
226     private fun processFlickerResults(
227         results: Collection<AssertionResult>
228     ): Map<String, AggregatedFlickerResult> {
229         val aggregatedResults = mutableMapOf<String, AggregatedFlickerResult>()
230         for (result in results) {
231             val key = getKeyForAssertionResult(result)
232             if (!aggregatedResults.containsKey(key)) {
233                 aggregatedResults[key] = AggregatedFlickerResult()
234             }
235             aggregatedResults[key]!!.addResult(result)
236         }
237         return aggregatedResults
238     }
239 
240     private fun collectMetrics(
241         data: DataRecord,
242         aggregatedResults: Map<String, AggregatedFlickerResult>
243     ) {
244         val it = aggregatedResults.entries.iterator()
245 
246         while (it.hasNext()) {
247             val (key, aggregatedResult) = it.next()
248             aggregatedResult.results.forEachIndexed { index, result ->
249                 if (result.status == AssertionResult.Status.ASSUMPTION_VIOLATION) {
250                     // skip
251                     return@forEachIndexed
252                 }
253 
254                 val resultStatus = if (result.status == AssertionResult.Status.PASS) 0 else 1
255                 Log.v(LOG_TAG, "Adding metric ${key}_$index = $resultStatus")
256                 data.addStringMetric("${key}_$index", "$resultStatus")
257             }
258         }
259     }
260 
261     private fun <T> errorReportingBlock(function: () -> T): T? {
262         return try {
263             function()
264         } catch (e: Throwable) {
265             Log.e(FLICKER_TAG, "Error executing in FlickerServiceResultsCollector", e)
266             _executionErrors.add(e)
267             null
268         }
269     }
270 
271     override fun resultsForTest(description: Description): Collection<AssertionResult> {
272         val resultsForTest = assertionResultsByTest[description]
273         requireNotNull(resultsForTest) { "No results set for test $description" }
274         return resultsForTest
275     }
276 
277     override fun detectedScenariosForTest(description: Description): Collection<ScenarioId> {
278         val scenariosForTest = detectedScenariosByTest[description]
279         requireNotNull(scenariosForTest) { "No detected scenarios set for test $description" }
280         return scenariosForTest
281     }
282 
283     private fun reportFlickerServiceStatus(
284         record: DataRecord,
285         results: Collection<AssertionResult>?,
286         scenario: Scenario,
287         dataRecord: DataRecord
288     ) {
289         val status = if (executionErrors.isEmpty()) OK_STATUS_CODE else EXECUTION_ERROR_STATUS_CODE
290         record.addStringMetric(FAAS_STATUS_KEY, status.toString())
291 
292         val maxLineLength = 120
293         val statusFile = createFlickerServiceStatusFile(scenario)
294         val flickerResultString = buildString {
295             appendLine(
296                 "FAAS_STATUS: ${if (executionErrors.isEmpty()) "OK" else "EXECUTION_ERROR"}\n"
297             )
298 
299             appendLine("EXECUTION ERRORS:\n")
300             if (executionErrors.isEmpty()) {
301                 appendLine("None".prependIndent())
302             } else {
303                 appendLine(
304                     executionErrors
305                         .joinToString("\n\n${"-".repeat(maxLineLength / 2)}\n\n") {
306                             it.stackTraceToString()
307                         }
308                         .prependIndent()
309                 )
310             }
311 
312             appendLine()
313             appendLine("FLICKER RESULTS:\n")
314             val executionErrorsString =
315                 buildString {
316                         results?.forEach {
317                             append("${it.name} (${it.stabilityGroup}) :: ")
318                             append("${it.status}\n")
319                             appendLine(
320                                 it.assertionErrors
321                                     .joinToString("\n${"-".repeat(maxLineLength / 2)}\n\n") { error
322                                         ->
323                                         error.message
324                                     }
325                                     .prependIndent()
326                             )
327                         }
328                     }
329                     .prependIndent()
330             appendLine(executionErrorsString)
331         }
332 
333         statusFile.writeText(flickerResultString.replace(Regex("(.{$maxLineLength})"), "$1\n"))
334 
335         Log.v(LOG_TAG, "Adding metric $FAAS_RESULTS_FILE_PATH_KEY = ${statusFile.absolutePath}")
336         dataRecord.addStringMetric(FAAS_RESULTS_FILE_PATH_KEY, statusFile.absolutePath)
337     }
338 
339     private fun createFlickerServiceStatusFile(scenario: Scenario): File {
340         val fileName = "FAAS_RESULTS_$scenario"
341 
342         val outputDir = getDefaultFlickerOutputDir()
343         // Ensure output directory exists
344         outputDir.mkdirs()
345         return outputDir.resolve(fileName)
346     }
347 
348     companion object {
349         // Unique prefix to add to all FaaS metrics to identify them
350         const val FAAS_METRICS_PREFIX = "FAAS"
351         private const val LOG_TAG = "$FLICKER_TAG-Collector"
352         const val FAAS_STATUS_KEY = "${FAAS_METRICS_PREFIX}_STATUS"
353         const val WINSCOPE_FILE_PATH_KEY = "winscope_file_path"
354         const val FAAS_RESULTS_FILE_PATH_KEY = "faas_results_file_path"
355         const val FLICKER_ASSERTIONS_COUNT_KEY = "flicker_assertions_count"
356         const val OK_STATUS_CODE = 0
357         const val EXECUTION_ERROR_STATUS_CODE = 1
358 
359         fun getKeyForAssertionResult(result: AssertionResult): String {
360             return "$FAAS_METRICS_PREFIX::${result.name}"
361         }
362 
363         class AggregatedFlickerResult {
364             val results = mutableListOf<AssertionResult>()
365             var failures = 0
366             var passes = 0
367             var assumptionViolations = 0
368             val errors = mutableListOf<String>()
369             var invocationGroup: AssertionInvocationGroup? = null
370 
371             fun addResult(result: AssertionResult) {
372                 results.add(result)
373 
374                 when (result.status) {
375                     AssertionResult.Status.PASS -> passes++
376                     AssertionResult.Status.FAIL -> {
377                         failures++
378                         result.assertionErrors.forEach { errors.add(it.message) }
379                     }
380                     AssertionResult.Status.ASSUMPTION_VIOLATION -> assumptionViolations++
381                 }
382 
383                 if (invocationGroup == null) {
384                     invocationGroup = result.stabilityGroup
385                 }
386 
387                 if (invocationGroup != result.stabilityGroup) {
388                     error("Unexpected assertion group mismatch")
389                 }
390             }
391         }
392     }
393 }
394