1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.result;
17 
18 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
19 import com.android.tradefed.result.skipped.SkipReason;
20 import com.android.tradefed.util.proto.TfMetricProtoUtil;
21 
22 import java.util.HashMap;
23 import java.util.Map;
24 
25 /**
26  * Receives event notifications during instrumentation test runs.
27  *
28  * <p>Patterned after org.junit.runner.notification.RunListener
29  *
30  * <p>The sequence of calls will be:
31  *
32  * <ul>
33  *   <li> testRunStarted
34  *   <li> testStarted
35  *   <li> [testFailed]
36  *   <li> [testAssumptionFailure]
37  *   <li> [testIgnored]
38  *   <li> testEnded
39  *   <li> ....
40  *   <li> [testRunFailed]
41  *   <li> testRunEnded
42  * </ul>
43  */
44 public interface ITestLifeCycleReceiver {
45 
46     /**
47      * Reports the start of a test run.
48      *
49      * @param runName the test run name
50      * @param testCount total number of tests in test run
51      */
testRunStarted(String runName, int testCount)52     public default void testRunStarted(String runName, int testCount) {}
53 
54     /**
55      * Reports the start of a test run.
56      *
57      * @param runName the test run name
58      * @param testCount total number of tests in test run
59      * @param attemptNumber order number, identifying the different attempts of the same runName
60      *     that run multiple times. The attemptNumber is 0-indexed and should increment everytime
61      *     a new run happens. e.g. A test is granular retried 3 times, it should have 4 total
62      *     runs under the same runName and the attemptNumber is from 0 to 3.
63      */
testRunStarted(String runName, int testCount, int attemptNumber)64     public default void testRunStarted(String runName, int testCount, int attemptNumber) {
65         testRunStarted(runName, testCount);
66     }
67 
68     /**
69      * Reports the start of a test run.
70      *
71      * @param runName the test run name
72      * @param testCount total number of tests in test run
73      * @param attemptNumber order number, identifying the different attempts of the same runName
74      *     that run multiple times. The attemptNumber is 0-indexed and should increment everytime a
75      *     new run happens. e.g. A test is granular retried 3 times, it should have 4 total runs
76      *     under the same runName and the attemptNumber is from 0 to 3.
77      * @param startTime the time the run started, measured via {@link System#currentTimeMillis()}
78      */
testRunStarted( String runName, int testCount, int attemptNumber, long startTime)79     public default void testRunStarted(
80             String runName, int testCount, int attemptNumber, long startTime) {
81         testRunStarted(runName, testCount);
82     }
83 
84     /**
85      * Reports test run failed to complete due to a fatal error.
86      *
87      * @param errorMessage {@link String} describing reason for run failure.
88      */
testRunFailed(String errorMessage)89     public default void testRunFailed(String errorMessage) {}
90 
91     /**
92      * Reports test run failed to complete due to a failure described by {@link FailureDescription}.
93      *
94      * @param failure {@link FailureDescription} describing the failure and its context.
95      */
testRunFailed(FailureDescription failure)96     public default void testRunFailed(FailureDescription failure) {
97         testRunFailed(failure.toString());
98     }
99 
100     /**
101      * Reports end of test run.
102      *
103      * @param elapsedTimeMillis device reported elapsed time, in milliseconds
104      * @param runMetrics key-value pairs reported at the end of a test run
105      */
testRunEnded(long elapsedTimeMillis, Map<String, String> runMetrics)106     public default void testRunEnded(long elapsedTimeMillis, Map<String, String> runMetrics) {
107         testRunEnded(elapsedTimeMillis, TfMetricProtoUtil.upgradeConvert(runMetrics));
108     }
109 
110     /**
111      * Reports end of test run. FIXME: We cannot have two Map<> interfaces with different type, so
112      * we have to use HashMap here.
113      *
114      * @param elapsedTimeMillis device reported elapsed time, in milliseconds
115      * @param runMetrics key-value pairs reported at the end of a test run with {@link Metric}.
116      */
testRunEnded(long elapsedTimeMillis, HashMap<String, Metric> runMetrics)117     public default void testRunEnded(long elapsedTimeMillis, HashMap<String, Metric> runMetrics) {}
118 
119     /**
120      * Reports test run stopped before completion due to a user request.
121      *
122      * <p>TODO: currently unused, consider removing
123      *
124      * @param elapsedTime device reported elapsed time, in milliseconds
125      */
testRunStopped(long elapsedTime)126     public default void testRunStopped(long elapsedTime) {}
127 
128     /**
129      * Reports the start of an individual test case. Older interface, should use {@link
130      * #testStarted(TestDescription)} whenever possible.
131      *
132      * @param test identifies the test
133      */
testStarted(TestDescription test)134     public default void testStarted(TestDescription test) {}
135 
136     /**
137      * Alternative to {@link #testStarted(TestDescription)} where we also specify when the test was
138      * started, combined with {@link #testEnded(TestDescription, long, Map)} for accurate measure.
139      *
140      * @param test identifies the test
141      * @param startTime the time the test started, measured via {@link System#currentTimeMillis()}
142      */
testStarted(TestDescription test, long startTime)143     default void testStarted(TestDescription test, long startTime) {
144         testStarted(test);
145     }
146 
147     /**
148      * Reports the failure of a individual test case.
149      *
150      * <p>Will be called between testStarted and testEnded.
151      *
152      * @param test identifies the test
153      * @param trace stack trace of failure
154      */
testFailed(TestDescription test, String trace)155     public default void testFailed(TestDescription test, String trace) {}
156 
157     /**
158      * Reports the failure of a individual test case.
159      *
160      * <p>Will be called between testStarted and testEnded.
161      *
162      * @param test identifies the test
163      * @param failure {@link FailureDescription} describing the failure and its context.
164      */
testFailed(TestDescription test, FailureDescription failure)165     public default void testFailed(TestDescription test, FailureDescription failure) {
166         testFailed(test, failure.toString());
167     }
168 
169     /**
170      * Called when an atomic test flags that it assumes a condition that is false
171      *
172      * @param test identifies the test
173      * @param trace stack trace of failure
174      */
testAssumptionFailure(TestDescription test, String trace)175     public default void testAssumptionFailure(TestDescription test, String trace) {}
176 
177     /**
178      * Called when an atomic test flags that it assumes a condition that is false
179      *
180      * @param test identifies the test
181      * @param failure {@link FailureDescription} describing the failure and its context.
182      */
testAssumptionFailure(TestDescription test, FailureDescription failure)183     public default void testAssumptionFailure(TestDescription test, FailureDescription failure) {
184         testAssumptionFailure(test, failure.toString());
185     }
186 
187     /**
188      * Called when a test will not be run, generally because a test method is annotated with
189      * org.junit.Ignore.
190      *
191      * @param test identifies the test
192      */
testIgnored(TestDescription test)193     public default void testIgnored(TestDescription test) {}
194 
195     /**
196      * Called when a test is skipped and did not execute for a reason that is not usually expected.
197      * These tests will be attempted to be retried to attempt to get a proper execution.
198      *
199      * @param test identifies the test
200      * @param reason {@link SkipReason}
201      */
testSkipped(TestDescription test, SkipReason reason)202     public default void testSkipped(TestDescription test, SkipReason reason) {}
203 
204     /**
205      * Reports the execution end of an individual test case.
206      *
207      * <p>If {@link #testFailed} was not invoked, this test passed. Also returns any key/value
208      * metrics which may have been emitted during the test case's execution.
209      *
210      * @param test identifies the test
211      * @param testMetrics a {@link Map} of the metrics emitted
212      */
testEnded(TestDescription test, Map<String, String> testMetrics)213     public default void testEnded(TestDescription test, Map<String, String> testMetrics) {
214         testEnded(test, TfMetricProtoUtil.upgradeConvert(testMetrics));
215     }
216 
217     /**
218      * Reports the execution end of an individual test case.
219      *
220      * <p>If {@link #testFailed} was not invoked, this test passed. Also returns any key/value
221      * metrics which may have been emitted during the test case's execution.
222      *
223      * @param test identifies the test
224      * @param testMetrics a {@link Map} of the metrics emitted
225      */
testEnded(TestDescription test, HashMap<String, Metric> testMetrics)226     public default void testEnded(TestDescription test, HashMap<String, Metric> testMetrics) {}
227 
228     /**
229      * Alternative to {@link #testEnded(TestDescription, Map)} where we can specify the end time
230      * directly. Combine with {@link #testStarted(TestDescription, long)} for accurate measure.
231      *
232      * @param test identifies the test
233      * @param endTime the time the test ended, measured via {@link System#currentTimeMillis()}
234      * @param testMetrics a {@link Map} of the metrics emitted
235      */
testEnded( TestDescription test, long endTime, Map<String, String> testMetrics)236     public default void testEnded(
237             TestDescription test, long endTime, Map<String, String> testMetrics) {
238         testEnded(test, endTime, TfMetricProtoUtil.upgradeConvert(testMetrics));
239     }
240 
241     /**
242      * Alternative to {@link #testEnded(TestDescription, Map)} where we can specify the end time
243      * directly. Combine with {@link #testStarted(TestDescription, long)} for accurate measure.
244      *
245      * @param test identifies the test
246      * @param endTime the time the test ended, measured via {@link System#currentTimeMillis()}
247      * @param testMetrics a {@link Map} of the metrics emitted
248      */
testEnded( TestDescription test, long endTime, HashMap<String, Metric> testMetrics)249     public default void testEnded(
250             TestDescription test, long endTime, HashMap<String, Metric> testMetrics) {
251         testEnded(test, testMetrics);
252     }
253 }
254