1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.result;
17 
18 import com.android.tradefed.metrics.proto.MetricMeasurement.Measurements;
19 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
20 import com.android.tradefed.result.skipped.SkipReason;
21 import com.android.tradefed.retry.MergeStrategy;
22 
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.LinkedHashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Objects;
30 
31 /** Container for a result of a single test. */
32 public class TestResult {
33     // Key that mark that an aggregation is hiding a failure.
34     public static final String IS_FLAKY = "is_flaky";
35 
36     private TestStatus mStatus;
37     private FailureDescription mFailureDescription;
38     private SkipReason mSkipReason;
39     private Map<String, String> mMetrics;
40     private HashMap<String, Metric> mProtoMetrics;
41     private Map<String, LogFile> mLoggedFiles;
42     // the start and end time of the test, measured via {@link System#currentTimeMillis()}
43     private long mStartTime = 0;
44     private long mEndTime = 0;
45 
TestResult()46     public TestResult() {
47         mStatus = TestStatus.INCOMPLETE;
48         mStartTime = System.currentTimeMillis();
49         mLoggedFiles = new LinkedHashMap<String, LogFile>();
50         mMetrics = new HashMap<>();
51         mProtoMetrics = new HashMap<>();
52     }
53 
54     /** Get the {@link TestStatus} result of the test. */
getStatus()55     public com.android.ddmlib.testrunner.TestResult.TestStatus getStatus() {
56         return TestStatus.convertToDdmlibType(mStatus);
57     }
58 
59     /** Get the {@link TestStatus} result of the test. */
getResultStatus()60     public TestStatus getResultStatus() {
61         return mStatus;
62     }
63 
64     /**
65      * Get the associated {@link String} stack trace. Should be <code>null</code> if {@link
66      * #getStatus()} is {@link TestStatus#PASSED}.
67      */
getStackTrace()68     public String getStackTrace() {
69         if (mFailureDescription == null) {
70             return null;
71         }
72         return mFailureDescription.toString();
73     }
74 
75     /**
76      * Get the associated {@link FailureDescription}. Should be <code>null</code> if {@link
77      * #getStatus()} is {@link TestStatus#PASSED}.
78      */
getFailure()79     public FailureDescription getFailure() {
80         return mFailureDescription;
81     }
82 
getSkipReason()83     public SkipReason getSkipReason() {
84         return mSkipReason;
85     }
86 
87     /** Get the associated test metrics. */
getMetrics()88     public Map<String, String> getMetrics() {
89         return mMetrics;
90     }
91 
92     /** Get the associated test metrics in proto format. */
getProtoMetrics()93     public HashMap<String, Metric> getProtoMetrics() {
94         return mProtoMetrics;
95     }
96 
97     /** Set the test metrics, overriding any previous values. */
setMetrics(Map<String, String> metrics)98     public void setMetrics(Map<String, String> metrics) {
99         mMetrics = metrics;
100     }
101 
102     /** Set the test proto metrics format, overriding any previous values. */
setProtoMetrics(HashMap<String, Metric> metrics)103     public void setProtoMetrics(HashMap<String, Metric> metrics) {
104         mProtoMetrics = metrics;
105     }
106 
107     /** Add a logged file tracking associated with that test case */
addLoggedFile(String dataName, LogFile loggedFile)108     public void addLoggedFile(String dataName, LogFile loggedFile) {
109         mLoggedFiles.put(dataName, loggedFile);
110     }
111 
112     /** Returns a copy of the map containing all the logged file associated with that test case. */
getLoggedFiles()113     public Map<String, LogFile> getLoggedFiles() {
114         return new LinkedHashMap<>(mLoggedFiles);
115     }
116 
117     /**
118      * Return the {@link System#currentTimeMillis()} time that the {@link
119      * ITestInvocationListener#testStarted(TestDescription)} event was received.
120      */
getStartTime()121     public long getStartTime() {
122         return mStartTime;
123     }
124 
125     /**
126      * Allows to set the time when the test was started, to be used with {@link
127      * ITestInvocationListener#testStarted(TestDescription, long)}.
128      */
setStartTime(long startTime)129     public void setStartTime(long startTime) {
130         mStartTime = startTime;
131     }
132 
133     /**
134      * Return the {@link System#currentTimeMillis()} time that the {@link
135      * ITestInvocationListener#testEnded(TestDescription, Map)} event was received.
136      */
getEndTime()137     public long getEndTime() {
138         return mEndTime;
139     }
140 
setStatus(com.android.ddmlib.testrunner.TestResult.TestStatus ddmlibStatus)141     public TestResult setStatus(com.android.ddmlib.testrunner.TestResult.TestStatus ddmlibStatus) {
142         mStatus = TestStatus.convertFromDdmlibType(ddmlibStatus);
143         return this;
144     }
145 
146     /** Set the {@link TestStatus}. */
setStatus(TestStatus status)147     public TestResult setStatus(TestStatus status) {
148         mStatus = status;
149         return this;
150     }
151 
152     /** Set the stack trace. */
setStackTrace(String stackTrace)153     public void setStackTrace(String stackTrace) {
154         mFailureDescription = FailureDescription.create(stackTrace);
155     }
156 
157     /** Set the stack trace. */
setFailure(FailureDescription failureDescription)158     public void setFailure(FailureDescription failureDescription) {
159         mFailureDescription = failureDescription;
160     }
161 
setSkipReason(SkipReason reason)162     public void setSkipReason(SkipReason reason) {
163         mSkipReason = reason;
164     }
165 
166     /** Sets the end time */
setEndTime(long currentTimeMillis)167     public void setEndTime(long currentTimeMillis) {
168         mEndTime = currentTimeMillis;
169     }
170 
171     @Override
hashCode()172     public int hashCode() {
173         return Arrays.hashCode(new Object[] {mMetrics, mFailureDescription, mStatus});
174     }
175 
176     @Override
equals(Object obj)177     public boolean equals(Object obj) {
178         if (this == obj) {
179             return true;
180         }
181         if (obj == null) {
182             return false;
183         }
184         if (getClass() != obj.getClass()) {
185             return false;
186         }
187         TestResult other = (TestResult) obj;
188         return Objects.equals(mMetrics, other.mMetrics)
189                 && Objects.equals(
190                         String.valueOf(mFailureDescription),
191                         String.valueOf(other.mFailureDescription))
192                 && Objects.equals(mStatus, other.mStatus);
193     }
194 
markFlaky()195     private void markFlaky() {
196         mProtoMetrics.put(
197                 IS_FLAKY,
198                 Metric.newBuilder()
199                         .setMeasurements(Measurements.newBuilder().setSingleString("true").build())
200                         .build());
201     }
202 
203     /**
204      * Merge the attempts for a same test case based on the merging strategy.
205      *
206      * @param results List of {@link TestResult} that will be merged
207      * @param strategy the {@link MergeStrategy} to be used to determine the merging outcome.
208      * @return the merged {@link TestResult} or null if there is nothing to merge.
209      */
merge(List<TestResult> results, MergeStrategy strategy)210     public static TestResult merge(List<TestResult> results, MergeStrategy strategy) {
211         if (results.isEmpty()) {
212             return null;
213         }
214         if (MergeStrategy.NO_MERGE.equals(strategy)) {
215             throw new IllegalArgumentException(
216                     "TestResult#merge cannot be called with NO_MERGE strategy.");
217         }
218         TestResult mergedResult = new TestResult();
219 
220         long earliestStartTime = Long.MAX_VALUE;
221         long latestEndTime = Long.MIN_VALUE;
222 
223         List<FailureDescription> errors = new ArrayList<>();
224         List<SkipReason> skipReasons = new ArrayList<>();
225         int pass = 0;
226         int fail = 0;
227         int assumption_failure = 0;
228         int ignored = 0;
229         int incomplete = 0;
230 
231         TestStatus lastStatus = null;
232         for (TestResult attempt : results) {
233             mergedResult.mProtoMetrics.putAll(attempt.getProtoMetrics());
234             mergedResult.mMetrics.putAll(attempt.getMetrics());
235             mergedResult.mLoggedFiles.putAll(attempt.getLoggedFiles());
236             earliestStartTime = Math.min(attempt.getStartTime(), earliestStartTime);
237             latestEndTime = Math.max(attempt.getEndTime(), latestEndTime);
238             switch (attempt.getResultStatus()) {
239                 case PASSED:
240                     pass++;
241                     break;
242                 case FAILURE:
243                     fail++;
244                     if (attempt.getFailure() != null) {
245                         errors.add(attempt.getFailure());
246                     }
247                     break;
248                 case INCOMPLETE:
249                     incomplete++;
250                     errors.add(FailureDescription.create("incomplete test case result."));
251                     break;
252                 case ASSUMPTION_FAILURE:
253                     assumption_failure++;
254                     if (attempt.getFailure() != null) {
255                         errors.add(attempt.getFailure());
256                     }
257                     break;
258                 case IGNORED:
259                     ignored++;
260                     break;
261                 case SKIPPED:
262                     skipReasons.add(attempt.getSkipReason());
263                     break;
264             }
265             lastStatus = attempt.mStatus;
266         }
267 
268         switch (strategy) {
269             case ANY_PASS_IS_PASS:
270             case ONE_TESTCASE_PASS_IS_PASS:
271                 // We prioritize passing the test due to the merging strategy.
272                 if (pass > 0) {
273                     mergedResult.setStatus(TestStatus.PASSED);
274                     if (fail > 0) {
275                         mergedResult.markFlaky();
276                     }
277                 } else if (fail == 0) {
278                     if (ignored > 0) {
279                         mergedResult.setStatus(TestStatus.IGNORED);
280                     } else if (assumption_failure > 0) {
281                         mergedResult.setStatus(TestStatus.ASSUMPTION_FAILURE);
282                     } else if (incomplete > 0) {
283                         mergedResult.setStatus(TestStatus.INCOMPLETE);
284                     } else if (!skipReasons.isEmpty()) {
285                         mergedResult.setStatus(TestStatus.SKIPPED);
286                         mergedResult.setSkipReason(skipReasons.get(0));
287                     }
288                 } else {
289                     if (TestStatus.ASSUMPTION_FAILURE.equals(lastStatus)) {
290                         mergedResult.setStatus(TestStatus.ASSUMPTION_FAILURE);
291                     } else if (TestStatus.IGNORED.equals(lastStatus)) {
292                         mergedResult.setStatus(TestStatus.IGNORED);
293                     } else {
294                         mergedResult.setStatus(TestStatus.FAILURE);
295                     }
296                 }
297                 break;
298             default:
299                 // We keep a default of one failure is a failure that should be reported.
300                 if (fail > 0) {
301                     mergedResult.setStatus(TestStatus.FAILURE);
302                 } else {
303                     if (ignored > 0) {
304                         mergedResult.setStatus(TestStatus.IGNORED);
305                     } else if (assumption_failure > 0) {
306                         mergedResult.setStatus(TestStatus.ASSUMPTION_FAILURE);
307                     } else if (incomplete > 0) {
308                         mergedResult.setStatus(TestStatus.INCOMPLETE);
309                     } else if (!skipReasons.isEmpty()) {
310                         mergedResult.setStatus(TestStatus.SKIPPED);
311                         mergedResult.setSkipReason(skipReasons.get(0));
312                     } else {
313                         mergedResult.setStatus(TestStatus.PASSED);
314                     }
315                 }
316                 break;
317         }
318         if (errors.isEmpty()) {
319             mergedResult.mFailureDescription = null;
320         } else if (errors.size() == 1) {
321             mergedResult.mFailureDescription = errors.get(0);
322         } else {
323             mergedResult.mFailureDescription = new MultiFailureDescription(errors);
324         }
325         mergedResult.setStartTime(earliestStartTime);
326         mergedResult.setEndTime(latestEndTime);
327         return mergedResult;
328     }
329 }
330