1 /*
2  * Copyright (C) 2024 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tools.metalava.testing
18 
19 import org.junit.AssumptionViolatedException
20 import org.junit.rules.TestRule
21 import org.junit.runner.Description
22 import org.junit.runners.model.Statement
23 
24 private const val GRADLEW_UPDATE_MODEL_TEST_SUITE_BASELINE =
25     "`scripts/refresh-baselines.py` to update the baseline"
26 
27 /**
28  * A JUnit [TestRule] that uses information from the [ModelTestSuiteBaseline] to ignore tests.
29  *
30  * @param baselineOwner the name of the owner of the baseline, used for error reporting.
31  */
32 class BaselineTestRule(
33     private val baselineOwner: String,
34     resourcePath: String,
35 ) : TestRule {
36 
37     /**
38      * The [ModelTestSuiteBaseline] that indicates whether the tests are expected to fail or not.
39      */
40     private val baseline = BaselineFile.fromResource(resourcePath)
41 
applynull42     override fun apply(base: Statement, description: Description): Statement {
43         return object : Statement() {
44             override fun evaluate() {
45                 val expectedFailure =
46                     baseline.isExpectedFailure(description.className, description.methodName)
47                 try {
48                     // Run the test even if it is expected to fail as a change that fixes one test
49                     // may fix more. Instead, this will just discard any failure.
50                     base.evaluate()
51                 } catch (e: AssumptionViolatedException) {
52                     // Catch and rethrow but do not treat as a failure.
53                     throw e
54                 } catch (e: Throwable) {
55                     if (expectedFailure) {
56                         // If this was expected to fail then throw an AssumptionViolatedException
57                         // that way it is not treated as either a pass or fail. Indent the exception
58                         // output and include it in the message instead of chaining the exception as
59                         // that reads better than the default formatting of chained exceptions.
60                         val actualErrorStackTrace = e.stackTraceToString().prependIndent("    ")
61                         throw AssumptionViolatedException(
62                             "Test skipped since it is listed in the baseline file for $baselineOwner.\n$actualErrorStackTrace"
63                         )
64                     } else {
65                         // Inform the developer on how to ignore this failing test.
66                         System.err.println(
67                             "Failing tests can be ignored by running $GRADLEW_UPDATE_MODEL_TEST_SUITE_BASELINE"
68                         )
69 
70                         // Rethrow the error
71                         throw e
72                     }
73                 }
74 
75                 // Perform this check outside the try...catch block otherwise the exception gets
76                 // caught, making it look like an actual failing test.
77                 if (expectedFailure) {
78                     // If a test that was expected to fail passes then updating the baseline
79                     // will remove that test from the expected test failures. Fail the test so
80                     // that the developer will be forced to clean it up.
81                     throw IllegalStateException(
82                         """
83                             **************************************************************************************************
84                                 Test was listed in the baseline file as it was expected to fail but it passed, please run:
85                                     $GRADLEW_UPDATE_MODEL_TEST_SUITE_BASELINE
86                             **************************************************************************************************
87 
88                         """
89                             .trimIndent()
90                     )
91                 }
92             }
93         }
94     }
95 }
96