1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite.retry;
17 
18 import com.android.annotations.VisibleForTesting;
19 import com.android.ddmlib.Log.LogLevel;
20 import com.android.tradefed.config.IConfiguration;
21 import com.android.tradefed.config.IConfigurationReceiver;
22 import com.android.tradefed.device.DeviceNotAvailableException;
23 import com.android.tradefed.device.ITestDevice;
24 import com.android.tradefed.device.StubDevice;
25 import com.android.tradefed.invoker.IInvocationContext;
26 import com.android.tradefed.invoker.TestInformation;
27 import com.android.tradefed.log.LogUtil.CLog;
28 import com.android.tradefed.metrics.proto.MetricMeasurement.Measurements;
29 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
30 import com.android.tradefed.result.ILogSaverListener;
31 import com.android.tradefed.result.ITestInvocationListener;
32 import com.android.tradefed.result.LogFile;
33 import com.android.tradefed.result.TestDescription;
34 import com.android.tradefed.result.TestResult;
35 import com.android.tradefed.result.TestRunResult;
36 import com.android.tradefed.testtype.IRemoteTest;
37 import com.android.tradefed.util.TimeUtil;
38 
39 import java.util.ArrayList;
40 import java.util.Collection;
41 import java.util.HashMap;
42 import java.util.LinkedHashMap;
43 import java.util.LinkedHashSet;
44 import java.util.List;
45 import java.util.Map;
46 import java.util.Map.Entry;
47 import java.util.Set;
48 
49 /** Special runner that replays the results given to it. */
50 public final class ResultsPlayer implements IRemoteTest, IConfigurationReceiver {
51 
52     public static final String REPLAY_DONE = "REPLAY_DONE";
53 
54     private class ReplayModuleHolder {
55         public IInvocationContext mModuleContext;
56         public List<Entry<TestDescription, TestResult>> mResults = new ArrayList<>();
57     }
58 
59     private Map<TestRunResult, ReplayModuleHolder> mModuleResult;
60     private IConfiguration mConfiguration;
61     private boolean mCompleted;
62 
63     /** Ctor. */
ResultsPlayer()64     public ResultsPlayer() {
65         mModuleResult = new LinkedHashMap<>();
66     }
67 
68     @VisibleForTesting
ResultsPlayer(boolean completed)69     public ResultsPlayer(boolean completed) {
70         mCompleted = completed;
71     }
72 
73     @Override
run(TestInformation testInfo, ITestInvocationListener listener)74     public void run(TestInformation testInfo, ITestInvocationListener listener)
75             throws DeviceNotAvailableException {
76         // Very first thing of the retry is to check whether all devices are available, this avoids
77         // use wasting time replaying result for an invocation that will fail right after during
78         // the re-run.
79         for (ITestDevice device : testInfo.getContext().getDevices()) {
80             if (device.getIDevice() instanceof StubDevice) {
81                 continue;
82             }
83             device.waitForDeviceAvailable();
84         }
85         testInfo.getContext().getBuildInfos().get(0).addBuildAttribute(REPLAY_DONE, "false");
86 
87         long startReplay = System.currentTimeMillis();
88         CLog.logAndDisplay(
89                 LogLevel.DEBUG,
90                 "Start replaying the previous results. Please wait this can take a few minutes.");
91         // Change the logging level to avoid too much logs from the replay.
92         LogLevel originalLevel = mConfiguration.getLogOutput().getLogLevel();
93         mConfiguration.getLogOutput().setLogLevel(LogLevel.WARN);
94 
95         Set<Entry<TestRunResult, ReplayModuleHolder>> entries = mModuleResult.entrySet();
96         for (Entry<TestRunResult, ReplayModuleHolder> e : new LinkedHashSet<>(entries)) {
97             TestRunResult module = e.getKey();
98             ReplayModuleHolder holder = e.getValue();
99             // Remove tracking to free memory
100             mModuleResult.remove(module);
101 
102             IInvocationContext moduleContext = holder.mModuleContext;
103             if (moduleContext != null) {
104                 for (String deviceName : testInfo.getContext().getDeviceConfigNames()) {
105                     moduleContext.addAllocatedDevice(
106                             deviceName, testInfo.getContext().getDevice(deviceName));
107                     moduleContext.addDeviceBuildInfo(
108                             deviceName, testInfo.getContext().getBuildInfo(deviceName));
109                 }
110                 listener.testModuleStarted(moduleContext);
111             }
112 
113             // Replay full or partial results
114             Collection<Entry<TestDescription, TestResult>> testSet = holder.mResults;
115             if (testSet.isEmpty()) {
116                 testSet = module.getTestResults().entrySet();
117             }
118 
119             forwardTestResults(module, testSet, listener);
120 
121             if (moduleContext != null) {
122                 listener.testModuleEnded();
123             }
124 
125             // Clean up the memory: IRemoteTest object are kept in memory until the command finish
126             // So we need to clean up the entries when we are done with them to free up the
127             // memory early
128             holder.mResults.clear();
129             module.getTestResults().clear();
130         }
131         // Restore the original log level to continue execution with the requested log level.
132         mConfiguration.getLogOutput().setLogLevel(originalLevel);
133         CLog.logAndDisplay(
134                 LogLevel.DEBUG,
135                 "Done replaying results in %s",
136                 TimeUtil.formatElapsedTime(System.currentTimeMillis() - startReplay));
137         mModuleResult.clear();
138         mCompleted = true;
139 
140         testInfo.getContext().getBuildInfos().get(0).removeBuildAttribute(REPLAY_DONE);
141         testInfo.getContext().getBuildInfos().get(0).addBuildAttribute(REPLAY_DONE, "true");
142     }
143 
144     /**
145      * Register a module to be replayed.
146      *
147      * @param moduleContext The Context of the module. Or null if it's a simple test run.
148      * @param module The results of the test run or module.
149      * @param testResult The particular test and its result to replay. Can be null if the full
150      *     module should be replayed.
151      */
addToReplay( IInvocationContext moduleContext, TestRunResult module, Entry<TestDescription, TestResult> testResult)152     void addToReplay(
153             IInvocationContext moduleContext,
154             TestRunResult module,
155             Entry<TestDescription, TestResult> testResult) {
156         ReplayModuleHolder holder = mModuleResult.get(module);
157         if (holder == null) {
158             holder = new ReplayModuleHolder();
159             holder.mModuleContext = moduleContext;
160             mModuleResult.put(module, holder);
161         }
162         if (testResult != null) {
163             holder.mResults.add(testResult);
164         }
165     }
166 
167     /** {@inheritDoc} */
168     @Override
setConfiguration(IConfiguration configuration)169     public void setConfiguration(IConfiguration configuration) {
170         mConfiguration = configuration;
171     }
172 
173     /** Returns whether or not the ResultsReplayer is done replaying the results. */
completed()174     public boolean completed() {
175         return mCompleted;
176     }
177 
forwardTestResults( TestRunResult module, Collection<Entry<TestDescription, TestResult>> testSet, ITestInvocationListener listener)178     private void forwardTestResults(
179             TestRunResult module,
180             Collection<Entry<TestDescription, TestResult>> testSet,
181             ITestInvocationListener listener) {
182         listener.testRunStarted(module.getName(), testSet.size());
183         for (Map.Entry<TestDescription, TestResult> testEntry : testSet) {
184             listener.testStarted(testEntry.getKey(), testEntry.getValue().getStartTime());
185             switch (testEntry.getValue().getResultStatus()) {
186                 case FAILURE:
187                     listener.testFailed(testEntry.getKey(), testEntry.getValue().getStackTrace());
188                     break;
189                 case ASSUMPTION_FAILURE:
190                     listener.testAssumptionFailure(
191                             testEntry.getKey(), testEntry.getValue().getStackTrace());
192                     break;
193                 case IGNORED:
194                     listener.testIgnored(testEntry.getKey());
195                     break;
196                 case SKIPPED:
197                     listener.testSkipped(testEntry.getKey(), testEntry.getValue().getSkipReason());
198                     break;
199                 case INCOMPLETE:
200                     listener.testFailed(
201                             testEntry.getKey(), "Test did not complete due to exception.");
202                     break;
203                 default:
204                     break;
205             }
206             // Provide a strong association of the test to its logs.
207             for (Entry<String, LogFile> logFile :
208                     testEntry.getValue().getLoggedFiles().entrySet()) {
209                 if (listener instanceof ILogSaverListener) {
210                     ((ILogSaverListener) listener)
211                             .logAssociation(logFile.getKey(), logFile.getValue());
212                 }
213             }
214             HashMap<String, Metric> metrics = testEntry.getValue().getProtoMetrics();
215             // Mark result as cached
216             metrics.put(
217                     "cached",
218                     Metric.newBuilder()
219                             .setMeasurements(Measurements.newBuilder().setSingleString("true"))
220                             .build());
221             listener.testEnded(testEntry.getKey(), testEntry.getValue().getEndTime(), metrics);
222         }
223         if (module.isRunFailure()) {
224             listener.testRunFailed(module.getRunFailureMessage());
225         }
226         listener.testRunEnded(module.getElapsedTime(), module.getRunProtoMetrics());
227     }
228 }
229