1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.invoker;
17 
18 import com.android.ddmlib.IDevice;
19 import com.android.ddmlib.Log.LogLevel;
20 import com.android.tradefed.build.BuildInfo;
21 import com.android.tradefed.build.BuildInfoKey.BuildInfoFileKey;
22 import com.android.tradefed.build.BuildRetrievalError;
23 import com.android.tradefed.build.IBuildInfo;
24 import com.android.tradefed.build.IBuildInfo.BuildInfoProperties;
25 import com.android.tradefed.build.IBuildProvider;
26 import com.android.tradefed.build.IDeviceBuildInfo;
27 import com.android.tradefed.build.IDeviceBuildProvider;
28 import com.android.tradefed.command.ICommandOptions;
29 import com.android.tradefed.config.GlobalConfiguration;
30 import com.android.tradefed.config.IConfiguration;
31 import com.android.tradefed.config.IConfigurationReceiver;
32 import com.android.tradefed.config.IDeviceConfiguration;
33 import com.android.tradefed.config.filter.GetPreviousPassedHelper;
34 import com.android.tradefed.device.DeviceNotAvailableException;
35 import com.android.tradefed.device.ITestDevice;
36 import com.android.tradefed.device.NativeDevice;
37 import com.android.tradefed.device.StubDevice;
38 import com.android.tradefed.device.cloud.GceAvdInfo;
39 import com.android.tradefed.device.cloud.GceManager;
40 import com.android.tradefed.device.cloud.OxygenUtil;
41 import com.android.tradefed.device.metric.AutoLogCollector;
42 import com.android.tradefed.device.metric.CollectorHelper;
43 import com.android.tradefed.device.metric.CountTestCasesCollector;
44 import com.android.tradefed.device.metric.IMetricCollector;
45 import com.android.tradefed.device.metric.IMetricCollectorReceiver;
46 import com.android.tradefed.error.HarnessRuntimeException;
47 import com.android.tradefed.invoker.ExecutionFiles.FilesKey;
48 import com.android.tradefed.invoker.TestInvocation.Stage;
49 import com.android.tradefed.invoker.logger.CurrentInvocation;
50 import com.android.tradefed.invoker.logger.CurrentInvocation.IsolationGrade;
51 import com.android.tradefed.invoker.logger.InvocationMetricLogger;
52 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationGroupMetricKey;
53 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
54 import com.android.tradefed.invoker.logger.TfObjectTracker;
55 import com.android.tradefed.invoker.shard.IShardHelper;
56 import com.android.tradefed.invoker.shard.TestsPoolPoller;
57 import com.android.tradefed.invoker.tracing.CloseableTraceScope;
58 import com.android.tradefed.log.ITestLogger;
59 import com.android.tradefed.log.LogUtil.CLog;
60 import com.android.tradefed.result.ByteArrayInputStreamSource;
61 import com.android.tradefed.result.ITestInvocationListener;
62 import com.android.tradefed.result.ITestLoggerReceiver;
63 import com.android.tradefed.result.InputStreamSource;
64 import com.android.tradefed.result.LogDataType;
65 import com.android.tradefed.result.error.TestErrorIdentifier;
66 import com.android.tradefed.retry.IRetryDecision;
67 import com.android.tradefed.retry.RetryLogSaverResultForwarder;
68 import com.android.tradefed.retry.RetryStatistics;
69 import com.android.tradefed.retry.RetryStrategy;
70 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver;
71 import com.android.tradefed.targetprep.BuildError;
72 import com.android.tradefed.targetprep.IHostCleaner;
73 import com.android.tradefed.targetprep.ILabPreparer;
74 import com.android.tradefed.targetprep.ITargetPreparer;
75 import com.android.tradefed.targetprep.TargetSetupError;
76 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer;
77 import com.android.tradefed.testtype.IBuildReceiver;
78 import com.android.tradefed.testtype.IDeviceTest;
79 import com.android.tradefed.testtype.IInvocationContextReceiver;
80 import com.android.tradefed.testtype.IRemoteTest;
81 import com.android.tradefed.testtype.ITestFilterReceiver;
82 import com.android.tradefed.testtype.retry.IAutoRetriableTest;
83 import com.android.tradefed.testtype.suite.BaseTestSuite;
84 import com.android.tradefed.testtype.suite.ITestSuite;
85 import com.android.tradefed.testtype.suite.ModuleListener;
86 import com.android.tradefed.util.CommandResult;
87 import com.android.tradefed.util.CommandStatus;
88 import com.android.tradefed.util.FileUtil;
89 import com.android.tradefed.util.PrettyPrintDelimiter;
90 import com.android.tradefed.util.RunInterruptedException;
91 import com.android.tradefed.util.RunUtil;
92 import com.android.tradefed.util.SystemUtil;
93 import com.android.tradefed.util.SystemUtil.EnvVariable;
94 import com.android.tradefed.util.TimeUtil;
95 import com.android.tradefed.util.executor.ParallelDeviceExecutor;
96 
97 import com.google.common.annotations.VisibleForTesting;
98 import com.google.common.base.Strings;
99 
100 import java.io.File;
101 import java.io.IOException;
102 import java.time.Duration;
103 import java.util.ArrayList;
104 import java.util.HashSet;
105 import java.util.List;
106 import java.util.ListIterator;
107 import java.util.Map;
108 import java.util.Set;
109 import java.util.Timer;
110 import java.util.TimerTask;
111 import java.util.concurrent.Callable;
112 import java.util.concurrent.ConcurrentHashMap;
113 import java.util.concurrent.TimeUnit;
114 
115 /**
116  * Class that describes all the invocation steps: build download, target_prep, run tests, clean up.
117  * Can be extended to override the default behavior of some steps. Order of the steps is driven by
118  * {@link TestInvocation}.
119  */
120 public class InvocationExecution implements IInvocationExecution {
121 
122     public static final String ADB_VERSION_KEY = "adb_version";
123     public static final String JAVA_VERSION_KEY = "java_version";
124     public static final String JAVA_CLASSPATH_KEY = "java_classpath";
125     // Track which preparer ran in setup to ensure we don't trigger tearDown if setup was skipped.
126     private Set<IMultiTargetPreparer> mTrackMultiPreparers = null;
127     private Map<String, Set<ITargetPreparer>> mTrackLabPreparers = null;
128     private Map<String, Set<ITargetPreparer>> mTrackTargetPreparers = null;
129     // GceManager for multi-device leasing. It's needed for releasing the devices.
130     private GceManager mMultiDeviceRequester = null;
131 
132     /** Timer to make sure Test Phase does not run for too long. */
133     private class TestPhaseMonitor extends TimerTask {
134 
135         private TestThread mTestThread;
136 
TestPhaseMonitor(TestThread toMonitor)137         public TestPhaseMonitor(TestThread toMonitor) {
138             mTestThread = toMonitor;
139         }
140 
141         @Override
run()142         public void run() {
143             if (mTestThread != null) {
144                 mTestThread.stopTestThread();
145             }
146         }
147     }
148 
149     /** A Thread to execute {@link IRemoteTest} */
150     private class TestThread extends Thread {
151         private TestInformation mTestInfo;
152         private ITestInvocationListener mTestListener;
153         private IRemoteTest mTest;
154         private Throwable lastThrownException;
155 
TestThread( TestInformation info, ITestInvocationListener listener, IRemoteTest test)156         public TestThread(
157                 TestInformation info, ITestInvocationListener listener, IRemoteTest test) {
158             mTestInfo = info;
159             mTestListener = listener;
160             mTest = test;
161         }
162 
163         @Override
run()164         public void run() {
165             try {
166                 mTest.run(mTestInfo, mTestListener);
167             } catch (Exception e) {
168                 lastThrownException = e;
169             }
170         }
171 
getLastThrownException()172         public Throwable getLastThrownException() {
173             return lastThrownException;
174         }
175 
stopTestThread()176         public void stopTestThread() {
177             this.interrupt();
178             mTestInfo.notifyTimeout();
179             // record this interrupt as an exception so that TestInvocation thread can throw this.
180             lastThrownException =
181                     new RunInterruptedException(
182                             "Test Phase Timeout Reached.",
183                             TestErrorIdentifier.TEST_PHASE_TIMED_OUT);
184         }
185     }
186 
187     @Override
fetchBuild( TestInformation testInfo, IConfiguration config, IRescheduler rescheduler, ITestInvocationListener listener)188     public boolean fetchBuild(
189             TestInformation testInfo,
190             IConfiguration config,
191             IRescheduler rescheduler,
192             ITestInvocationListener listener)
193             throws DeviceNotAvailableException, BuildRetrievalError {
194         String currentDeviceName = null;
195         IBuildInfo buildReplicat = null;
196         try {
197             // TODO: evaluate fetching build in parallel
198             for (int i = 0; i < testInfo.getContext().getDeviceConfigNames().size(); i++) {
199                 currentDeviceName = testInfo.getContext().getDeviceConfigNames().get(i);
200                 if (buildReplicat != null) {
201                     // TODO: evaluate if cloning the build is needed
202                     testInfo.getContext().addDeviceBuildInfo(currentDeviceName, buildReplicat);
203                     continue;
204                 }
205                 IBuildInfo info = null;
206                 ITestDevice device = testInfo.getContext().getDevice(currentDeviceName);
207                 IDeviceConfiguration deviceConfig = config.getDeviceConfigByName(currentDeviceName);
208                 IBuildProvider provider = deviceConfig.getBuildProvider();
209                 TfObjectTracker.countWithParents(provider.getClass());
210                 // Inject the context to the provider if it can receive it
211                 if (provider instanceof IInvocationContextReceiver) {
212                     ((IInvocationContextReceiver) provider)
213                             .setInvocationContext(testInfo.getContext());
214                 }
215                 if (provider instanceof ITestLoggerReceiver) {
216                     ((ITestLoggerReceiver) provider).setTestLogger(listener);
217                 }
218                 // Get the build
219                 if (provider instanceof IDeviceBuildProvider) {
220                     // Download a device build if the provider can handle it.
221                     info = ((IDeviceBuildProvider) provider).getBuild(device);
222                 } else {
223                     info = provider.getBuild();
224                 }
225                 if (info != null) {
226                     info.setDeviceSerial(device.getSerialNumber());
227                     testInfo.getContext().addDeviceBuildInfo(currentDeviceName, info);
228                     device.setRecovery(deviceConfig.getDeviceRecovery());
229                 } else {
230                     CLog.logAndDisplay(
231                             LogLevel.WARN,
232                             "No build found to test for device: %s",
233                             device.getSerialNumber());
234                     IBuildInfo notFoundStub = new BuildInfo();
235                     updateBuild(notFoundStub, config);
236                     testInfo.getContext().addDeviceBuildInfo(currentDeviceName, notFoundStub);
237                     return false;
238                 }
239                 // TODO: remove build update when reporting is done on context
240                 updateBuild(info, config);
241                 linkExternalDirs(info, testInfo);
242 
243                 if (config.getCommandOptions().shouldUseReplicateSetup()) {
244                     buildReplicat = info;
245                 }
246             }
247         } catch (BuildRetrievalError e) {
248             CLog.e(e);
249             if (currentDeviceName != null) {
250                 IBuildInfo errorBuild = e.getBuildInfo();
251                 updateBuild(errorBuild, config);
252                 testInfo.getContext().addDeviceBuildInfo(currentDeviceName, errorBuild);
253             }
254             throw e;
255         } catch (RuntimeException re) {
256             if (currentDeviceName != null) {
257                 IBuildInfo errorBuild =
258                         TestInvocation.backFillBuildInfoForReporting(config.getCommandLine());
259                 updateBuild(errorBuild, config);
260                 testInfo.getContext().addDeviceBuildInfo(currentDeviceName, errorBuild);
261             }
262             throw re;
263         }
264         setBinariesVersion(testInfo.getContext());
265         copyRemoteFiles(config.getCommandOptions(), testInfo.getBuildInfo());
266         return true;
267     }
268 
269     @Override
cleanUpBuilds(IInvocationContext context, IConfiguration config)270     public void cleanUpBuilds(IInvocationContext context, IConfiguration config) {
271         // Ensure build infos are always cleaned up at the end of invocation.
272         for (String cleanUpDevice : context.getDeviceConfigNames()) {
273             if (context.getBuildInfo(cleanUpDevice) != null) {
274                 try {
275                     config.getDeviceConfigByName(cleanUpDevice)
276                             .getBuildProvider()
277                             .cleanUp(context.getBuildInfo(cleanUpDevice));
278                 } catch (RuntimeException e) {
279                     // We catch an simply log exception in cleanUp to avoid missing any final
280                     // step of the invocation.
281                     CLog.e(e);
282                 }
283             }
284         }
285     }
286 
287     @Override
shardConfig( IConfiguration config, TestInformation testInfo, IRescheduler rescheduler, ITestLogger logger)288     public boolean shardConfig(
289             IConfiguration config,
290             TestInformation testInfo,
291             IRescheduler rescheduler,
292             ITestLogger logger) {
293         IShardHelper helper = createShardHelper();
294         CLog.d("IShardHelper selected: %s", helper);
295         return helper.shardConfig(config, testInfo, rescheduler, logger);
296     }
297 
298     /** Create an return the {@link IShardHelper} to be used. */
299     @VisibleForTesting
createShardHelper()300     protected IShardHelper createShardHelper() {
301         return GlobalConfiguration.getInstance().getShardingStrategy();
302     }
303 
304     /**
305      * Retrieve a list of target preparers to run on this device.
306      *
307      * <p>Overridden in sandbox classes to restrict lab preparers from being run inside the sandbox
308      * child
309      */
getTargetPreparersToRun( IConfiguration config, String deviceName)310     protected List<ITargetPreparer> getTargetPreparersToRun(
311             IConfiguration config, String deviceName) {
312         List<ITargetPreparer> preparersToRun = new ArrayList<>();
313         preparersToRun.addAll(config.getDeviceConfigByName(deviceName).getTargetPreparers());
314         return preparersToRun;
315     }
316 
317     /**
318      * Retrieve a list of lab preparers to run on this device.
319      *
320      * <p>Overridden in sandbox classes to restrict lab preparers from being run inside the sandbox
321      * child
322      */
getLabPreparersToRun(IConfiguration config, String deviceName)323     protected List<ITargetPreparer> getLabPreparersToRun(IConfiguration config, String deviceName) {
324         List<ITargetPreparer> preparersToRun = new ArrayList<>();
325         preparersToRun.addAll(config.getDeviceConfigByName(deviceName).getLabPreparers());
326         return preparersToRun;
327     }
328 
329     @Override
doSetup(TestInformation testInfo, IConfiguration config, final ITestLogger listener)330     public void doSetup(TestInformation testInfo, IConfiguration config, final ITestLogger listener)
331             throws TargetSetupError, BuildError, DeviceNotAvailableException {
332         long start = System.currentTimeMillis();
333         mTrackLabPreparers = new ConcurrentHashMap<>();
334         mTrackTargetPreparers = new ConcurrentHashMap<>();
335         InvocationMetricLogger.addInvocationMetrics(InvocationMetricKey.SETUP_START, start);
336 
337         for (String deviceName : testInfo.getContext().getDeviceConfigNames()) {
338             ITestDevice device = testInfo.getContext().getDevice(deviceName);
339             CLog.d("Starting setup for device: '%s'", device.getSerialNumber());
340             if (device instanceof ITestLoggerReceiver) {
341                 ((ITestLoggerReceiver) testInfo.getContext().getDevice(deviceName))
342                         .setTestLogger(listener);
343             }
344             mTrackLabPreparers.put(deviceName, new HashSet<>());
345             mTrackTargetPreparers.put(deviceName, new HashSet<>());
346         }
347         try {
348             try (CloseableTraceScope ignored =
349                     new CloseableTraceScope(InvocationMetricKey.pre_multi_preparer.name())) {
350                 // Before all the individual setup, make the multi-pre-target-preparer devices setup
351                 runMultiTargetPreparers(
352                         config.getMultiPreTargetPreparers(),
353                         listener,
354                         testInfo,
355                         "multi pre target preparer setup");
356             } finally {
357                 long end = System.currentTimeMillis();
358                 // Pre-multi-preparer are test specific and account toward test setup
359                 InvocationMetricLogger.addInvocationPairMetrics(
360                         InvocationMetricKey.TEST_SETUP_PAIR, start, end);
361             }
362             start = System.currentTimeMillis();
363             try (CloseableTraceScope ignored =
364                     new CloseableTraceScope(InvocationMetricKey.lab_setup.name())) {
365                 runLabPreparersSetup(testInfo, config, listener);
366             } finally {
367                 long end = System.currentTimeMillis();
368                 InvocationMetricLogger.addInvocationMetrics(InvocationMetricKey.SETUP_END, end);
369                 InvocationMetricLogger.addInvocationPairMetrics(
370                         InvocationMetricKey.SETUP_PAIR, start, end);
371             }
372             long startPreparer = System.currentTimeMillis();
373             try (CloseableTraceScope ignored =
374                     new CloseableTraceScope(InvocationMetricKey.test_setup.name())) {
375                 runPreparersSetup(testInfo, config, listener);
376 
377                 // After all the individual setup, make the multi-devices setup
378                 runMultiTargetPreparers(
379                         config.getMultiTargetPreparers(),
380                         listener,
381                         testInfo,
382                         "multi target preparer setup");
383                 // Collect some info automatically after setup
384                 collectAutoInfo(config, testInfo);
385             } finally {
386                 // Note: These metrics are handled in a try in case of a kernel reset or device
387                 // issue.
388                 // Setup timing metric. It does not include flashing time on boot tests.
389                 long end = System.currentTimeMillis();
390                 InvocationMetricLogger.addInvocationPairMetrics(
391                         InvocationMetricKey.TEST_SETUP_PAIR, startPreparer, end);
392                 long setupDuration = end - start;
393                 InvocationMetricLogger.addInvocationMetrics(
394                         InvocationMetricKey.SETUP, setupDuration);
395                 CLog.d("Total setup duration: %s'", TimeUtil.formatElapsedTime(setupDuration));
396             }
397         } finally {
398             // Upload the setup logcat after setup is complete.
399             for (ITestDevice device : testInfo.getDevices()) {
400                 reportLogs(device, listener, Stage.SETUP);
401             }
402         }
403     }
404 
runLabPreparersSetup( TestInformation testInfo, IConfiguration config, ITestLogger listener)405     private void runLabPreparersSetup(
406             TestInformation testInfo, IConfiguration config, ITestLogger listener)
407             throws TargetSetupError, BuildError, DeviceNotAvailableException {
408         int index = 0;
409         if ((config.getCommandOptions().shouldUseParallelSetup()
410                         || config.getCommandOptions().shouldUseReplicateSetup())
411                 && config.getDeviceConfig().size() > 1) {
412             CLog.d("Using parallel setup.");
413             ParallelDeviceExecutor<Boolean> executor =
414                     new ParallelDeviceExecutor<>(testInfo.getContext().getDevices().size());
415             List<Callable<Boolean>> callableTasks = new ArrayList<>();
416             for (String deviceName : testInfo.getContext().getDeviceConfigNames()) {
417                 final int deviceIndex = index;
418                 // Replicate TestInfo
419                 TestInformation replicated =
420                         TestInformation.createModuleTestInfo(testInfo, testInfo.getContext());
421                 Callable<Boolean> callableTask =
422                         () -> {
423                             // Lab preparer then target preparer
424                             runLabPreparationOnDevice(
425                                     replicated,
426                                     deviceName,
427                                     deviceIndex,
428                                     getLabPreparersToRun(config, deviceName),
429                                     mTrackLabPreparers.get(deviceName),
430                                     listener);
431                             return true;
432                         };
433                 callableTasks.add(callableTask);
434                 index++;
435             }
436             Duration timeout = config.getCommandOptions().getParallelSetupTimeout();
437             executor.invokeAll(callableTasks, timeout.toMillis(), TimeUnit.MILLISECONDS);
438             if (executor.hasErrors()) {
439                 List<Throwable> errors = executor.getErrors();
440                 // TODO: Handle throwing multi-exceptions, right now throw the first one.
441                 for (Throwable error : errors) {
442                     if (error instanceof TargetSetupError) {
443                         throw (TargetSetupError) error;
444                     }
445                     if (error instanceof BuildError) {
446                         throw (BuildError) error;
447                     }
448                     if (error instanceof DeviceNotAvailableException) {
449                         throw (DeviceNotAvailableException) error;
450                     }
451                     if (error instanceof HarnessRuntimeException) {
452                         throw (HarnessRuntimeException) error;
453                     }
454                     throw new RuntimeException(error);
455                 }
456             }
457         } else {
458             for (String deviceName : testInfo.getContext().getDeviceConfigNames()) {
459                 // Lab preparer then target preparer
460                 runLabPreparationOnDevice(
461                         testInfo,
462                         deviceName,
463                         index,
464                         getLabPreparersToRun(config, deviceName),
465                         mTrackLabPreparers.get(deviceName),
466                         listener);
467                 index++;
468             }
469         }
470     }
471 
runPreparersSetup( TestInformation testInfo, IConfiguration config, ITestLogger listener)472     private void runPreparersSetup(
473             TestInformation testInfo, IConfiguration config, ITestLogger listener)
474             throws TargetSetupError, BuildError, DeviceNotAvailableException {
475         int index = 0;
476         if ((config.getCommandOptions().shouldUseParallelSetup()
477                         || config.getCommandOptions().shouldUseReplicateSetup())
478                 && config.getDeviceConfig().size() > 1) {
479             CLog.d("Using parallel setup.");
480             ParallelDeviceExecutor<Boolean> executor =
481                     new ParallelDeviceExecutor<>(testInfo.getContext().getDevices().size());
482             List<Callable<Boolean>> callableTasks = new ArrayList<>();
483             for (String deviceName : testInfo.getContext().getDeviceConfigNames()) {
484                 final int deviceIndex = index;
485                 // Replicate TestInfo
486                 TestInformation replicated =
487                         TestInformation.createModuleTestInfo(testInfo, testInfo.getContext());
488                 Callable<Boolean> callableTask =
489                         () -> {
490                             runPreparationOnDevice(
491                                     replicated,
492                                     deviceName,
493                                     deviceIndex,
494                                     getTargetPreparersToRun(config, deviceName),
495                                     mTrackTargetPreparers.get(deviceName),
496                                     listener);
497                             return true;
498                         };
499                 callableTasks.add(callableTask);
500                 index++;
501             }
502             Duration timeout = config.getCommandOptions().getParallelSetupTimeout();
503             executor.invokeAll(callableTasks, timeout.toMillis(), TimeUnit.MILLISECONDS);
504             if (executor.hasErrors()) {
505                 List<Throwable> errors = executor.getErrors();
506                 // TODO: Handle throwing multi-exceptions, right now throw the first one.
507                 for (Throwable error : errors) {
508                     if (error instanceof TargetSetupError) {
509                         throw (TargetSetupError) error;
510                     }
511                     if (error instanceof BuildError) {
512                         throw (BuildError) error;
513                     }
514                     if (error instanceof DeviceNotAvailableException) {
515                         throw (DeviceNotAvailableException) error;
516                     }
517                     throw new RuntimeException(error);
518                 }
519             }
520         } else {
521             for (String deviceName : testInfo.getContext().getDeviceConfigNames()) {
522                 runPreparationOnDevice(
523                         testInfo,
524                         deviceName,
525                         index,
526                         getTargetPreparersToRun(config, deviceName),
527                         mTrackTargetPreparers.get(deviceName),
528                         listener);
529                 index++;
530             }
531         }
532     }
533 
runLabPreparationOnDevice( TestInformation testInfo, String deviceName, int index, List<ITargetPreparer> labPreparersToRun, Set<ITargetPreparer> trackLabPreparers, ITestLogger logger)534     private void runLabPreparationOnDevice(
535             TestInformation testInfo,
536             String deviceName,
537             int index,
538             List<ITargetPreparer> labPreparersToRun,
539             Set<ITargetPreparer> trackLabPreparers,
540             ITestLogger logger)
541             throws TargetSetupError, BuildError, DeviceNotAvailableException {
542         ITestDevice device = testInfo.getContext().getDevice(deviceName);
543 
544         // Run lab preparers on the device
545         for (ITargetPreparer preparer : labPreparersToRun) {
546             if (preparer.isDisabled()) {
547                 CLog.d("%s has been disabled. skipping.", preparer);
548                 continue;
549             }
550             // Track object invoked as lab_preparer that are not ILabPreparer
551             if (!(preparer instanceof ILabPreparer)) {
552                 InvocationMetricLogger.addInvocationMetrics(
553                         InvocationMetricKey.LAB_PREPARER_NOT_ILAB,
554                         preparer.getClass().getCanonicalName());
555             }
556 
557             TfObjectTracker.countWithParents(preparer.getClass());
558             if (preparer instanceof ITestLoggerReceiver) {
559                 ((ITestLoggerReceiver) preparer).setTestLogger(logger);
560             }
561 
562             long startTime = System.currentTimeMillis();
563             CLog.d(
564                     "starting lab preparer '%s' on device: '%s'",
565                     preparer, device.getSerialNumber());
566             try (CloseableTraceScope ignore =
567                     new CloseableTraceScope(preparer.getClass().getSimpleName())) {
568                 testInfo.setActiveDeviceIndex(index);
569                 preparer.setUp(testInfo);
570             } finally {
571                 testInfo.setActiveDeviceIndex(0);
572                 long elapsedTime = System.currentTimeMillis() - startTime;
573 
574                 CLog.d(
575                         "done with lab preparer '%s' on device: '%s' in %s",
576                         preparer,
577                         device.getSerialNumber(),
578                         TimeUtil.formatElapsedTime(elapsedTime));
579 
580                 InvocationMetricLogger.addInvocationMetrics(
581                         InvocationGroupMetricKey.LAB_PREPARER_SETUP_LATENCY,
582                         preparer.getClass().getName(),
583                         elapsedTime);
584             }
585             // Track which lab preparers were executed separately from the target preparers
586             trackLabPreparers.add(preparer);
587         }
588     }
589 
runPreparationOnDevice( TestInformation testInfo, String deviceName, int index, List<ITargetPreparer> targetPreparersToRun, Set<ITargetPreparer> trackPreparers, ITestLogger logger)590     private void runPreparationOnDevice(
591             TestInformation testInfo,
592             String deviceName,
593             int index,
594             List<ITargetPreparer> targetPreparersToRun,
595             Set<ITargetPreparer> trackPreparers,
596             ITestLogger logger)
597             throws TargetSetupError, BuildError, DeviceNotAvailableException {
598         ITestDevice device = testInfo.getContext().getDevice(deviceName);
599         for (ITargetPreparer preparer : targetPreparersToRun) {
600             if (preparer.isDisabled()) {
601                 CLog.d("%s has been disabled. skipping.", preparer);
602                 continue;
603             }
604             // Track object invoked as target_preparer but is ILabPreparer
605             if (preparer instanceof ILabPreparer) {
606                 InvocationMetricLogger.addInvocationMetrics(
607                         InvocationMetricKey.TARGET_PREPARER_IS_ILAB,
608                         preparer.getClass().getCanonicalName());
609             }
610 
611             TfObjectTracker.countWithParents(preparer.getClass());
612             if (preparer instanceof ITestLoggerReceiver) {
613                 ((ITestLoggerReceiver) preparer).setTestLogger(logger);
614             }
615 
616             long startTime = System.currentTimeMillis();
617             CLog.d("starting preparer '%s' on device: '%s'", preparer, device.getSerialNumber());
618             try (CloseableTraceScope ignore =
619                     new CloseableTraceScope(preparer.getClass().getSimpleName())) {
620                 testInfo.setActiveDeviceIndex(index);
621                 preparer.setUp(testInfo);
622             } finally {
623                 testInfo.setActiveDeviceIndex(0);
624             }
625 
626             trackPreparers.add(preparer);
627             long elapsedTime = System.currentTimeMillis() - startTime;
628 
629             CLog.d(
630                     "done with preparer '%s' on device: '%s' in %s",
631                     preparer, device.getSerialNumber(), TimeUtil.formatElapsedTime(elapsedTime));
632 
633             InvocationMetricLogger.addInvocationMetrics(
634                     InvocationGroupMetricKey.TARGET_PREPARER_SETUP_LATENCY,
635                     preparer.getClass().getName(),
636                     elapsedTime);
637         }
638 
639         CLog.d("Done with setup of device: '%s'", device.getSerialNumber());
640     }
641 
642     /** {@inheritDoc} */
643     @Override
runDevicePreInvocationSetup( IInvocationContext context, IConfiguration config, ITestLogger logger)644     public void runDevicePreInvocationSetup(
645             IInvocationContext context, IConfiguration config, ITestLogger logger)
646             throws DeviceNotAvailableException, TargetSetupError {
647         if (config.getCommandOptions().shouldDisableInvocationSetupAndTeardown()) {
648             CLog.i("--disable-invocation-setup-and-teardown, skipping pre-invocation setup.");
649             return;
650         }
651         long start = System.currentTimeMillis();
652         customizeDevicePreInvocation(config, context);
653 
654         // Multi-device test scenario
655         Integer multiDeviceCount = config.getCommandOptions().getMultiDeviceCount();
656         boolean allVirtualDevices = true;
657         for (IDeviceConfiguration deviceConfig : config.getDeviceConfig()) {
658             if (!deviceConfig.getDeviceRequirements().gceDeviceRequested()) {
659                 allVirtualDevices = false;
660                 break;
661             }
662         }
663         if (multiDeviceCount != null && multiDeviceCount != 1 && allVirtualDevices) {
664             try (CloseableTraceScope ignore =
665                     new CloseableTraceScope("runMultiVirtualDevicesPreInvocationSetup")) {
666                 runMultiVirtualDevicesPreInvocationSetup(context, config, logger);
667             } catch (TargetSetupError e) {
668                 OxygenUtil util = new OxygenUtil();
669                 util.downloadLaunchFailureLogs(e, logger);
670                 throw e;
671             }
672         } else {
673             try (CloseableTraceScope ignore =
674                     new CloseableTraceScope("device_pre_invocation_setup")) {
675                 List<String> deviceNames = context.getDeviceConfigNames();
676                 if (config.getCommandOptions().shouldUseParallelPreInvocationSetup()
677                         && deviceNames.size() > 1) {
678                     CLog.d("Using parallel preInvocationSetup.");
679                     List<Callable<Void>> callableTasks = new ArrayList<>();
680                     for (String deviceName : deviceNames) {
681                         callableTasks.add(
682                                 () -> {
683                                     runSingleDevicePreInvocationSetup(
684                                             deviceName, context, config, logger);
685                                     return null;
686                                 });
687                     }
688                     // The threads are also controlled by
689                     // host_options:concurrent-virtual-device-startup-limit.
690                     ParallelDeviceExecutor<Void> executor =
691                             new ParallelDeviceExecutor<>(callableTasks.size());
692                     executor.invokeAll(
693                             callableTasks,
694                             config.getCommandOptions()
695                                     .getParallelPreInvocationSetupTimeout()
696                                     .toMillis(),
697                             TimeUnit.MILLISECONDS);
698                     // TODO: Handle throwing multi-exceptions, right now throw the first one.
699                     for (Throwable error : executor.getErrors()) {
700                         if (error instanceof DeviceNotAvailableException) {
701                             throw (DeviceNotAvailableException) error;
702                         }
703                         if (error instanceof TargetSetupError) {
704                             throw (TargetSetupError) error;
705                         }
706                         throw new RuntimeException(error);
707                     }
708                 } else {
709                     if (config.getCommandOptions().shouldUseParallelPreInvocationSetup()) {
710                         CLog.w("Parallel pre-invocation setup is enabled but device count <= 1.");
711                     }
712                     for (String deviceName : deviceNames) {
713                         runSingleDevicePreInvocationSetup(deviceName, context, config, logger);
714                     }
715                 }
716             }
717         }
718         // Also report device pre invocation into setup
719         InvocationMetricLogger.addInvocationPairMetrics(
720                 InvocationMetricKey.SETUP_PAIR, start, System.currentTimeMillis());
721     }
722 
723     /**
724      * Launch multiple virtual devices together, then invoke the {@link
725      * ITestDevice#preInvocationSetup(IBuildInfo)} for each device part of the invocation with
726      * setting the GceAvdInfo of the device beforehand.
727      *
728      * @param context the {@link IInvocationContext} of the invocation.
729      * @param config the {@link IConfiguration} of this test run.
730      * @param logger the {@link ITestLogger} to report logs.
731      * @throws DeviceNotAvailableException
732      * @throws TargetSetupError
733      */
runMultiVirtualDevicesPreInvocationSetup( IInvocationContext context, IConfiguration config, ITestLogger logger)734     private void runMultiVirtualDevicesPreInvocationSetup(
735             IInvocationContext context, IConfiguration config, ITestLogger logger)
736             throws TargetSetupError, DeviceNotAvailableException {
737         // One GceManager is needed to lease the whole device group
738         String firstDeviceName = context.getDeviceConfigNames().get(0);
739         ITestDevice firstDevice = context.getDevice(firstDeviceName);
740         mMultiDeviceRequester =
741                 new GceManager(
742                         firstDevice.getDeviceDescriptor(),
743                         firstDevice.getOptions(),
744                         context.getBuildInfo(firstDeviceName));
745 
746         List<ITestDevice> devices = context.getDevices();
747         List<IBuildInfo> buildInfos = context.getBuildInfos();
748         // Set logger on all devices first
749         for (ITestDevice device : devices) {
750             if (device instanceof ITestLoggerReceiver) {
751                 ((ITestLoggerReceiver) device).setTestLogger(logger);
752             }
753         }
754 
755         // Start multiple devices in a group
756         List<GceAvdInfo> gceAvdInfoList =
757                 mMultiDeviceRequester.startMultiDevicesGce(buildInfos, context.getAttributes());
758         for (int i = 0; i < devices.size(); i++) {
759             // For each device, do setup with its GceAvdInfo
760             CLog.d(
761                     "Starting device pre invocation launched device setup with GceAvdInfo %s"
762                             + " for : '%s'",
763                     gceAvdInfoList.get(i), devices.get(i).getSerialNumber());
764             // Use the most common basic interface for device connection setup
765             NativeDevice device = (NativeDevice) devices.get(i);
766 
767             device.setConnectionAvdInfo(gceAvdInfoList.get(i));
768             device.preInvocationSetup(buildInfos.get(i), context.getAttributes());
769 
770             // Last device in the group is responsible for releasing the whole device group
771             if (i != devices.size() - 1) {
772                 CLog.d(
773                         "Set device %s to skip tear down because only the last device in the"
774                                 + " device group will be responsible for tearing down the whole"
775                                 + " device group",
776                         device.getSerialNumber());
777                 device.getOptions().setSkipTearDown(true);
778             }
779         }
780     }
781 
782     /**
783      * Run preInvocationSetup for one device.
784      *
785      * @param deviceName the name of the device to be set up.
786      * @param context the {@link IInvocationContext} of the invocation.
787      * @param config the {@link IConfiguration} of this test run.
788      * @param logger the {@link ITestLogger} to report logs.
789      * @throws DeviceNotAvailableException
790      * @throws TargetSetupError
791      */
runSingleDevicePreInvocationSetup( String deviceName, IInvocationContext context, IConfiguration config, ITestLogger logger)792     private void runSingleDevicePreInvocationSetup(
793             String deviceName,
794             IInvocationContext context,
795             IConfiguration config,
796             ITestLogger logger)
797             throws DeviceNotAvailableException, TargetSetupError {
798         ITestDevice device = context.getDevice(deviceName);
799         CLog.d("Starting device pre invocation setup for : '%s'", device.getSerialNumber());
800         if (device instanceof ITestLoggerReceiver) {
801             ((ITestLoggerReceiver) context.getDevice(deviceName)).setTestLogger(logger);
802         }
803         IDeviceConfiguration deviceConfig = config.getDeviceConfigByName(deviceName);
804         if (deviceConfig != null && deviceConfig.isFake()) {
805             CLog.d("Skip preInvocationSetup on fake device %s", device);
806         } else {
807             device.preInvocationSetup(context.getBuildInfo(deviceName), context.getAttributes());
808         }
809     }
810 
811     /**
812      * Give a chance to customize some of the device before preInvocationSetup.
813      *
814      * @param config The config of the invocation.
815      * @param context The current invocation context.
816      */
customizeDevicePreInvocation(IConfiguration config, IInvocationContext context)817     protected void customizeDevicePreInvocation(IConfiguration config, IInvocationContext context) {
818         // Empty by default
819     }
820 
821     /** {@inheritDoc} */
822     @Override
runDevicePostInvocationTearDown( IInvocationContext context, IConfiguration config, Throwable exception)823     public void runDevicePostInvocationTearDown(
824             IInvocationContext context, IConfiguration config, Throwable exception) {
825         // Extra tear down step for the device
826         if (config.getCommandOptions().shouldDisableInvocationSetupAndTeardown()) {
827             CLog.i("--disable-invocation-setup-and-teardown, skipping post-invocation teardown.");
828             return;
829         }
830         // Check if device tear down is needed for multi-device tests.
831         boolean shouldTearDown = false;
832         for (String deviceName : context.getDeviceConfigNames()) {
833             ITestDevice device = context.getDevice(deviceName);
834             IDeviceConfiguration deviceConfig = config.getDeviceConfigByName(deviceName);
835             if (deviceConfig != null && deviceConfig.isFake()) {
836                 CLog.d("Skip postInvocationTearDown on fake device %s", device);
837                 continue;
838             }
839             // For multi-device tests, only the last device is flagged to be tear down if needed.
840             shouldTearDown |= !device.getOptions().shouldSkipTearDown();
841             device.postInvocationTearDown(exception);
842         }
843         if (mMultiDeviceRequester != null && shouldTearDown) {
844             mMultiDeviceRequester.shutdownGce();
845         }
846     }
847 
848     /** Runs the {@link IMultiTargetPreparer} specified. */
runMultiTargetPreparers( List<IMultiTargetPreparer> multiPreparers, ITestLogger logger, TestInformation testInfo, String description)849     private void runMultiTargetPreparers(
850             List<IMultiTargetPreparer> multiPreparers,
851             ITestLogger logger,
852             TestInformation testInfo,
853             String description)
854             throws TargetSetupError, BuildError, DeviceNotAvailableException {
855         if (mTrackMultiPreparers == null) {
856             mTrackMultiPreparers = new HashSet<>();
857         }
858         for (IMultiTargetPreparer multiPreparer : multiPreparers) {
859             // do not call the preparer if it was disabled
860             if (multiPreparer.isDisabled()) {
861                 CLog.d("%s has been disabled. skipping.", multiPreparer);
862                 continue;
863             }
864             TfObjectTracker.countWithParents(multiPreparer.getClass());
865             if (multiPreparer instanceof ITestLoggerReceiver) {
866                 ((ITestLoggerReceiver) multiPreparer).setTestLogger(logger);
867             }
868             long startTime = System.currentTimeMillis();
869             try (CloseableTraceScope ignore =
870                     new CloseableTraceScope(multiPreparer.getClass().getSimpleName())) {
871                 CLog.d("Starting %s '%s'", description, multiPreparer);
872                 multiPreparer.setUp(testInfo);
873                 mTrackMultiPreparers.add(multiPreparer);
874                 long elapsedTime = System.currentTimeMillis() - startTime;
875                 CLog.d(
876                         "Done with %s '%s' in %s",
877                         description, multiPreparer, TimeUtil.formatElapsedTime(elapsedTime));
878             }
879         }
880     }
881 
882     /** Runs the {@link IMultiTargetPreparer} specified tearDown. */
runMultiTargetPreparersTearDown( List<IMultiTargetPreparer> multiPreparers, TestInformation testInfo, ITestLogger logger, Throwable throwable, String description)883     private Throwable runMultiTargetPreparersTearDown(
884             List<IMultiTargetPreparer> multiPreparers,
885             TestInformation testInfo,
886             ITestLogger logger,
887             Throwable throwable,
888             String description)
889             throws Throwable {
890         ListIterator<IMultiTargetPreparer> iterator =
891                 multiPreparers.listIterator(multiPreparers.size());
892         Throwable deferredThrowable = null;
893 
894         while (iterator.hasPrevious()) {
895             IMultiTargetPreparer multipreparer = iterator.previous();
896             if (multipreparer.isDisabled() || multipreparer.isTearDownDisabled()) {
897                 CLog.d("%s has been disabled. skipping.", multipreparer);
898                 continue;
899             }
900             if (mTrackMultiPreparers == null || !mTrackMultiPreparers.contains(multipreparer)) {
901                 CLog.d("%s didn't run setUp, skipping tearDown.", multipreparer);
902                 continue;
903             }
904             if (multipreparer instanceof ITestLoggerReceiver) {
905                 ((ITestLoggerReceiver) multipreparer).setTestLogger(logger);
906             }
907             long startTime = System.currentTimeMillis();
908             CLog.d("Starting %s '%s'", description, multipreparer);
909             try (CloseableTraceScope ignore =
910                     new CloseableTraceScope(multipreparer.getClass().getSimpleName())) {
911                 multipreparer.tearDown(testInfo, throwable);
912             } catch (Throwable t) {
913                 // We catch it and rethrow later to allow each multi_targetprep to be attempted.
914                 // Only the first one will be thrown but all should be logged.
915                 CLog.e("Deferring throw for:");
916                 CLog.e(t);
917                 if (deferredThrowable == null) {
918                     deferredThrowable = t;
919                 }
920             }
921             long elapsedTime = System.currentTimeMillis() - startTime;
922 
923             CLog.d(
924                     "Done with %s '%s' in %s",
925                     description, multipreparer, TimeUtil.formatElapsedTime(elapsedTime));
926             InvocationMetricLogger.addInvocationMetrics(
927                     InvocationGroupMetricKey.MULTI_TARGET_PREPARER_TEARDOWN_LATENCY,
928                     multipreparer.getClass().getName(),
929                     elapsedTime);
930         }
931 
932         return deferredThrowable;
933     }
934 
935     @Override
doTeardown( TestInformation testInfo, IConfiguration config, ITestLogger logger, Throwable exception)936     public void doTeardown(
937             TestInformation testInfo,
938             IConfiguration config,
939             ITestLogger logger,
940             Throwable exception)
941             throws Throwable {
942         IInvocationContext context = testInfo.getContext();
943         Throwable deferredThrowable;
944         long start = System.currentTimeMillis();
945         InvocationMetricLogger.addInvocationMetrics(InvocationMetricKey.TEARDOWN_START, start);
946         try {
947             int deviceIndex = 0;
948             try {
949                 List<IMultiTargetPreparer> multiPreparers = config.getMultiTargetPreparers();
950                 deferredThrowable =
951                         runMultiTargetPreparersTearDown(
952                                 multiPreparers,
953                                 testInfo,
954                                 logger,
955                                 exception,
956                                 "multi target preparer teardown");
957 
958                 for (String deviceName : context.getDeviceConfigNames()) {
959                     ITestDevice device = context.getDevice(deviceName);
960                     device.clearLastConnectedWifiNetwork();
961 
962                     List<ITargetPreparer> targetPreparersToRun =
963                             getTargetPreparersToRun(config, deviceName);
964                     Throwable firstLocalThrowable =
965                             runPreparersTearDown(
966                                     testInfo,
967                                     device,
968                                     deviceName,
969                                     deviceIndex,
970                                     logger,
971                                     exception,
972                                     targetPreparersToRun,
973                                     mTrackTargetPreparers);
974                     if (deferredThrowable == null) {
975                         deferredThrowable = firstLocalThrowable;
976                     }
977 
978                     deviceIndex++;
979                 }
980 
981                 if (exception == null) {
982                     exception = deferredThrowable;
983                 }
984             } finally {
985                 InvocationMetricLogger.addInvocationPairMetrics(
986                         InvocationMetricKey.TEST_TEARDOWN_PAIR, start, System.currentTimeMillis());
987             }
988 
989             start = System.currentTimeMillis();
990             try {
991                 deviceIndex = 0;
992                 for (String deviceName : context.getDeviceConfigNames()) {
993                     ITestDevice device = context.getDevice(deviceName);
994                     List<ITargetPreparer> labPreparersToRun =
995                             getLabPreparersToRun(config, deviceName);
996                     Throwable secondLocalThrowable =
997                             runPreparersTearDown(
998                                     testInfo,
999                                     device,
1000                                     deviceName,
1001                                     deviceIndex,
1002                                     logger,
1003                                     exception,
1004                                     labPreparersToRun,
1005                                     mTrackLabPreparers);
1006                     if (deferredThrowable == null) {
1007                         deferredThrowable = secondLocalThrowable;
1008                     }
1009 
1010                     deviceIndex++;
1011                 }
1012 
1013                 if (exception == null) {
1014                     exception = deferredThrowable;
1015                 }
1016                 // Extra tear down step for the device
1017                 runDevicePostInvocationTearDown(context, config, exception);
1018 
1019                 // After all, run the multi_pre_target_preparer tearDown.
1020                 List<IMultiTargetPreparer> multiPrePreparers = config.getMultiPreTargetPreparers();
1021                 Throwable preTargetTearDownException =
1022                         runMultiTargetPreparersTearDown(
1023                                 multiPrePreparers,
1024                                 testInfo,
1025                                 logger,
1026                                 exception,
1027                                 "multi pre target preparer teardown");
1028                 if (deferredThrowable == null) {
1029                     deferredThrowable = preTargetTearDownException;
1030                 }
1031             } finally {
1032                 InvocationMetricLogger.addInvocationPairMetrics(
1033                         InvocationMetricKey.TEARDOWN_PAIR, start, System.currentTimeMillis());
1034             }
1035         } finally {
1036             // Collect adb logs.
1037             logHostAdb(config, logger);
1038             InvocationMetricLogger.addInvocationMetrics(
1039                     InvocationMetricKey.TEARDOWN_END, System.currentTimeMillis());
1040         }
1041 
1042         if (deferredThrowable != null) {
1043             throw deferredThrowable;
1044         }
1045     }
1046 
runPreparersTearDown( TestInformation testInfo, ITestDevice device, String deviceName, int deviceIndex, ITestLogger logger, Throwable exception, List<ITargetPreparer> preparersToRun, Map<String, Set<ITargetPreparer>> trackPreparersMap)1047     protected Throwable runPreparersTearDown(
1048             TestInformation testInfo,
1049             ITestDevice device,
1050             String deviceName,
1051             int deviceIndex,
1052             ITestLogger logger,
1053             Throwable exception,
1054             List<ITargetPreparer> preparersToRun,
1055             Map<String, Set<ITargetPreparer>> trackPreparersMap) {
1056         Throwable deferredThrowable = null;
1057         ListIterator<ITargetPreparer> itr = preparersToRun.listIterator(preparersToRun.size());
1058         while (itr.hasPrevious()) {
1059             ITargetPreparer preparer = itr.previous();
1060             // do not call the cleaner if it was disabled
1061             if (preparer.isDisabled() || preparer.isTearDownDisabled()) {
1062                 CLog.d("%s has been disabled. skipping.", preparer);
1063                 continue;
1064             }
1065             if (trackPreparersMap == null
1066                     || !trackPreparersMap.containsKey(deviceName)
1067                     || !trackPreparersMap.get(deviceName).contains(preparer)) {
1068                 CLog.d("%s didn't run setUp, skipping tearDown.", preparer);
1069                 continue;
1070             }
1071             // If setup hit a targetSetupError, the setUp() and setTestLogger might not have
1072             // run, ensure we still have the logger.
1073             if (preparer instanceof ITestLoggerReceiver) {
1074                 ((ITestLoggerReceiver) preparer).setTestLogger(logger);
1075             }
1076             long startTime = System.currentTimeMillis();
1077             try (CloseableTraceScope remoteTest =
1078                     new CloseableTraceScope(preparer.getClass().getSimpleName())) {
1079                 CLog.d(
1080                         "starting tearDown '%s' on device: '%s'",
1081                         preparer, device.getSerialNumber());
1082                 testInfo.setActiveDeviceIndex(deviceIndex);
1083                 Throwable tearDownException = exception;
1084                 // If a previous teardown fail, still notify following ones.
1085                 if (exception == null && deferredThrowable != null) {
1086                     tearDownException = deferredThrowable;
1087                 }
1088                 preparer.tearDown(testInfo, tearDownException);
1089             } catch (Throwable e) {
1090                 // We catch it and rethrow later to allow each targetprep to be attempted.
1091                 // Only the first one will be thrown but all should be logged.
1092                 CLog.e("Deferring throw for:");
1093                 CLog.e(e);
1094                 if (deferredThrowable == null) {
1095                     deferredThrowable = e;
1096                 }
1097             } finally {
1098                 testInfo.setActiveDeviceIndex(0);
1099                 long elapsedTime = System.currentTimeMillis() - startTime;
1100                 CLog.d(
1101                         "done with tearDown '%s' on device: '%s' in %s",
1102                         preparer,
1103                         device.getSerialNumber(),
1104                         TimeUtil.formatElapsedTime(elapsedTime));
1105                 InvocationMetricLogger.addInvocationMetrics(
1106                         InvocationGroupMetricKey.TARGET_PREPARER_TEARDOWN_LATENCY,
1107                         preparer.getClass().getName(),
1108                         elapsedTime);
1109             }
1110         }
1111         return deferredThrowable;
1112     }
1113 
1114     @Override
doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception)1115     public void doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception) {
1116         for (String deviceName : context.getDeviceConfigNames()) {
1117 
1118             List<ITargetPreparer> targetPreparers = getTargetPreparersToRun(config, deviceName);
1119 
1120             ListIterator<ITargetPreparer> itr =
1121                     targetPreparers.listIterator(targetPreparers.size());
1122             while (itr.hasPrevious()) {
1123                 ITargetPreparer preparer = itr.previous();
1124                 if (preparer instanceof IHostCleaner) {
1125                     IHostCleaner cleaner = (IHostCleaner) preparer;
1126                     if (preparer.isDisabled() || preparer.isTearDownDisabled()) {
1127                         CLog.d("%s has been disabled. skipping.", cleaner);
1128                         continue;
1129                     }
1130                     cleaner.cleanUp(context.getBuildInfo(deviceName), exception);
1131                 }
1132             }
1133 
1134             List<ITargetPreparer> labPreparers = getLabPreparersToRun(config, deviceName);
1135 
1136             // Yes this ends up very redundant to the above stanza, but 8 lines isn't really worth
1137             // extracting to a helper method.
1138             itr = labPreparers.listIterator(labPreparers.size());
1139             while (itr.hasPrevious()) {
1140                 ITargetPreparer preparer = itr.previous();
1141                 if (preparer instanceof IHostCleaner) {
1142                     IHostCleaner cleaner = (IHostCleaner) preparer;
1143                     if (preparer.isDisabled() || preparer.isTearDownDisabled()) {
1144                         CLog.d("%s has been disabled. skipping.", cleaner);
1145                         continue;
1146                     }
1147                     cleaner.cleanUp(context.getBuildInfo(deviceName), exception);
1148                 }
1149             }
1150         }
1151     }
1152 
1153     @Override
runTests( TestInformation info, IConfiguration config, ITestInvocationListener listener)1154     public void runTests(
1155             TestInformation info, IConfiguration config, ITestInvocationListener listener)
1156             throws Throwable {
1157         Timer testPhaseTimer = new Timer(true);
1158         long remainingTestPhaseTime =
1159                 GlobalConfiguration.getInstance().getHostOptions().getTestPhaseTimeout();
1160         boolean testPhaseTimeoutNeeded = remainingTestPhaseTime > 0;
1161         // Make sure Test Phase timeout is less than or equal to invocation timeout
1162         long invocationTimeout = config.getCommandOptions().getInvocationTimeout();
1163         if (testPhaseTimeoutNeeded && invocationTimeout > 0) {
1164             remainingTestPhaseTime = Math.min(remainingTestPhaseTime, invocationTimeout);
1165         }
1166 
1167         List<IRemoteTest> remainingTests = new ArrayList<>(config.getTests());
1168         UnexecutedTestReporterThread reporterThread =
1169                 new UnexecutedTestReporterThread(listener, remainingTests);
1170         Runtime.getRuntime().addShutdownHook(reporterThread);
1171         TestInvocation.printStageDelimiter(Stage.TEST, false);
1172         long start = System.currentTimeMillis();
1173         try (CloseableTraceScope ignored =
1174                 new CloseableTraceScope(InvocationMetricKey.test_execution.name())) {
1175             GetPreviousPassedHelper previousPassHelper = new GetPreviousPassedHelper();
1176             // Add new exclude filters to global filters
1177             Set<String> previousPassedFilters = previousPassHelper.getPreviousPassedFilters(config);
1178             // TODO: Ensure global filters are cloned for local sharding
1179             config.getGlobalFilters().addPreviousPassedTests(previousPassedFilters);
1180             for (IRemoteTest test : config.getTests()) {
1181                 try (CloseableTraceScope remoteTest =
1182                         new CloseableTraceScope(test.getClass().getSimpleName())) {
1183                     TfObjectTracker.countWithParents(test.getClass());
1184                     // For compatibility of those receivers, they are assumed to be single device
1185                     // alloc.
1186                     if (test instanceof IDeviceTest) {
1187                         ((IDeviceTest) test).setDevice(info.getDevice());
1188                     }
1189                     if (test instanceof IBuildReceiver) {
1190                         ((IBuildReceiver) test).setBuild(info.getBuildInfo());
1191                     }
1192                     if (test instanceof ISystemStatusCheckerReceiver) {
1193                         ((ISystemStatusCheckerReceiver) test)
1194                                 .setSystemStatusChecker(config.getSystemStatusCheckers());
1195                     }
1196                     if (test instanceof IInvocationContextReceiver) {
1197                         ((IInvocationContextReceiver) test).setInvocationContext(info.getContext());
1198                     }
1199 
1200                     updateAutoCollectors(config);
1201 
1202                     IRetryDecision decision = config.getRetryDecision();
1203                     // Apply the filters
1204                     if (test instanceof ITestFilterReceiver) {
1205                         config.getGlobalFilters().applyFiltersToTest((ITestFilterReceiver) test);
1206                     } else if (test instanceof BaseTestSuite) {
1207                         config.getGlobalFilters().applyFiltersToTest((BaseTestSuite) test);
1208                     }
1209                     // Handle the no-retry use case
1210                     if (!decision.isAutoRetryEnabled()
1211                             || RetryStrategy.NO_RETRY.equals(decision.getRetryStrategy())
1212                             || test instanceof ITestSuite
1213                             // TODO: Handle auto-retry in local-sharding for non-suite
1214                             || test instanceof TestsPoolPoller
1215                             // If test doesn't support auto-retry
1216                             || (!(test instanceof ITestFilterReceiver)
1217                                     && !(test instanceof IAutoRetriableTest)
1218                                     && !RetryStrategy.ITERATIONS.equals(
1219                                             decision.getRetryStrategy()))) {
1220                         try {
1221                             long timeSpentOnTest =
1222                                     runTest(
1223                                             config,
1224                                             info,
1225                                             listener,
1226                                             test,
1227                                             testPhaseTimer,
1228                                             remainingTestPhaseTime,
1229                                             testPhaseTimeoutNeeded);
1230                             remainingTestPhaseTime -= timeSpentOnTest;
1231                         } finally {
1232                             CurrentInvocation.setRunIsolation(IsolationGrade.NOT_ISOLATED);
1233                             CurrentInvocation.setModuleIsolation(IsolationGrade.NOT_ISOLATED);
1234                             // Clean the suite internals once done
1235                             if (test instanceof BaseTestSuite) {
1236                                 ((BaseTestSuite) test).cleanUpSuiteSetup();
1237                             }
1238                         }
1239                         remainingTests.remove(test);
1240                         continue;
1241                     }
1242                     CLog.d("Using RetryLogSaverResultForwarder to forward results.");
1243                     ModuleListener mainGranularRunListener =
1244                             new ModuleListener(null, info.getContext());
1245                     RetryLogSaverResultForwarder runListener =
1246                             initializeListeners(config, listener, mainGranularRunListener);
1247                     mainGranularRunListener.setAttemptIsolation(
1248                             CurrentInvocation.runCurrentIsolation());
1249                     try {
1250                         long timeSpentOnTest =
1251                                 runTest(
1252                                         config,
1253                                         info,
1254                                         runListener,
1255                                         test,
1256                                         testPhaseTimer,
1257                                         remainingTestPhaseTime,
1258                                         testPhaseTimeoutNeeded);
1259                         remainingTestPhaseTime -= timeSpentOnTest;
1260                     } finally {
1261                         CurrentInvocation.setRunIsolation(IsolationGrade.NOT_ISOLATED);
1262                         CurrentInvocation.setModuleIsolation(IsolationGrade.NOT_ISOLATED);
1263                     }
1264                     remainingTests.remove(test);
1265                     runListener.incrementAttempt();
1266 
1267                     // Avoid entering the loop if no retry to be done.
1268                     if (!decision.shouldRetry(
1269                             test, 0, mainGranularRunListener.getTestRunForAttempts(0))) {
1270                         continue;
1271                     }
1272                     // Avoid rechecking the shouldRetry below the first time as it could retrigger
1273                     // reboot.
1274                     boolean firstCheck = true;
1275                     long startTime = System.currentTimeMillis();
1276                     try {
1277                         PrettyPrintDelimiter.printStageDelimiter("Starting auto-retry");
1278                         for (int attemptNumber = 1;
1279                                 attemptNumber < decision.getMaxRetryCount();
1280                                 attemptNumber++) {
1281                             if (!firstCheck) {
1282                                 boolean retry =
1283                                         decision.shouldRetry(
1284                                                 test,
1285                                                 attemptNumber - 1,
1286                                                 mainGranularRunListener.getTestRunForAttempts(
1287                                                         attemptNumber - 1));
1288                                 if (!retry) {
1289                                     continue;
1290                                 }
1291                             }
1292                             firstCheck = false;
1293                             CLog.d("auto-retry attempt number '%s'", attemptNumber);
1294                             mainGranularRunListener.setAttemptIsolation(
1295                                     CurrentInvocation.runCurrentIsolation());
1296                             try {
1297                                 // Run the tests again
1298                                 long timeSpent =
1299                                         runTest(
1300                                                 config,
1301                                                 info,
1302                                                 runListener,
1303                                                 test,
1304                                                 testPhaseTimer,
1305                                                 remainingTestPhaseTime,
1306                                                 testPhaseTimeoutNeeded);
1307                                 remainingTestPhaseTime -= timeSpent;
1308                             } finally {
1309                                 CurrentInvocation.setRunIsolation(IsolationGrade.NOT_ISOLATED);
1310                                 CurrentInvocation.setModuleIsolation(IsolationGrade.NOT_ISOLATED);
1311                             }
1312                             runListener.incrementAttempt();
1313                         }
1314                         // Feed the last attempt if we reached here.
1315                         decision.addLastAttempt(
1316                                 mainGranularRunListener.getTestRunForAttempts(
1317                                         decision.getMaxRetryCount() - 1));
1318                     } finally {
1319                         RetryStatistics retryStats = decision.getRetryStatistics();
1320                         // Track how long we spend in retry
1321                         retryStats.mRetryTime = System.currentTimeMillis() - startTime;
1322                         addRetryTime(retryStats.mRetryTime);
1323                     }
1324                 }
1325             }
1326         } finally {
1327             testPhaseTimer.cancel();
1328             TestInvocation.printStageDelimiter(Stage.TEST, true);
1329             // TODO: Look if this can be improved to DeviceNotAvailableException too.
1330             try {
1331                 Runtime.getRuntime().removeShutdownHook(reporterThread);
1332             } catch (IllegalStateException e) {
1333                 // Ignore as it would throw only if JVM shutdown is in progress.
1334             }
1335             // Only log if it was no already logged to keep the value closest to execution
1336             if (!InvocationMetricLogger.getInvocationMetrics()
1337                     .containsKey(InvocationMetricKey.TEST_PAIR.toString())) {
1338                 InvocationMetricLogger.addInvocationPairMetrics(
1339                         InvocationMetricKey.TEST_PAIR, start, System.currentTimeMillis());
1340             }
1341         }
1342     }
1343 
1344     @Override
reportLogs(ITestDevice device, ITestLogger listener, Stage stage)1345     public void reportLogs(ITestDevice device, ITestLogger listener, Stage stage) {
1346         if (device == null) {
1347             return;
1348         }
1349         IDevice idevice = device.getIDevice();
1350         try (InputStreamSource logcatSource = device.getLogcat()) {
1351             device.clearLogcat();
1352             if (logcatSource != null && logcatSource.size() > 0L) {
1353                 String name =
1354                         String.format(
1355                                 "%s_%s",
1356                                 TestInvocation.getDeviceLogName(stage), device.getSerialNumber());
1357                 listener.testLog(name, LogDataType.LOGCAT, logcatSource);
1358             }
1359         }
1360         // Emulator logs
1361         if (idevice != null && idevice.isEmulator()) {
1362             try (InputStreamSource emulatorOutput = device.getEmulatorOutput()) {
1363                 // TODO: Clear the emulator log
1364                 String name = TestInvocation.getEmulatorLogName(stage);
1365                 listener.testLog(name, LogDataType.TEXT, emulatorOutput);
1366             }
1367         }
1368     }
1369 
1370     /** Helper to create the test tag from the configuration. */
getTestTag(IConfiguration config)1371     private String getTestTag(IConfiguration config) {
1372         String testTag = config.getCommandOptions().getTestTag();
1373         if (config.getCommandOptions().getTestTagSuffix() != null) {
1374             testTag =
1375                     String.format("%s-%s", testTag, config.getCommandOptions().getTestTagSuffix());
1376         }
1377         return testTag;
1378     }
1379 
1380     /** Handle setting the test tag on the build info. */
setTestTag(IBuildInfo info, IConfiguration config)1381     protected void setTestTag(IBuildInfo info, IConfiguration config) {
1382         // When CommandOption is set, it overrides any test-tag from build_providers
1383         if (!"stub".equals(config.getCommandOptions().getTestTag())) {
1384             info.setTestTag(getTestTag(config));
1385         } else if (Strings.isNullOrEmpty(info.getTestTag())) {
1386             // We ensure that that a default test-tag is always available.
1387             info.setTestTag("stub");
1388         }
1389     }
1390 
1391     /**
1392      * Update the {@link IBuildInfo} with additional info from the {@link IConfiguration}.
1393      *
1394      * @param info the {@link IBuildInfo}
1395      * @param config the {@link IConfiguration}
1396      */
updateBuild(IBuildInfo info, IConfiguration config)1397     void updateBuild(IBuildInfo info, IConfiguration config) {
1398         setTestTag(info, config);
1399         if (config.getCommandOptions().getInvocationData().containsKey("subprocess")) {
1400             // Avoid relogging the properties in a subprocess
1401             return;
1402         }
1403         if (config.getCommandLine() != null) {
1404             // TODO: obfuscate the password if any.
1405             info.addBuildAttribute(TestInvocation.COMMAND_ARGS_KEY, config.getCommandLine());
1406         }
1407         if (config.getCommandOptions().getShardCount() != null) {
1408             info.addBuildAttribute(
1409                     "shard_count", config.getCommandOptions().getShardCount().toString());
1410         }
1411         if (config.getCommandOptions().getShardIndex() != null) {
1412             info.addBuildAttribute(
1413                     "shard_index", config.getCommandOptions().getShardIndex().toString());
1414         }
1415     }
1416 
1417     /**
1418      * Runs a test and returns the time taken to finish the test.
1419      *
1420      * <p>Tests will be run on a separate thread with a timer when test phase level timeout is
1421      * needed.
1422      */
runTest( IConfiguration config, TestInformation info, ITestInvocationListener listener, IRemoteTest test, Timer timer, long testPhaseTimeout, boolean testPhaseTimeoutNeeded)1423     private long runTest(
1424             IConfiguration config,
1425             TestInformation info,
1426             ITestInvocationListener listener,
1427             IRemoteTest test,
1428             Timer timer,
1429             long testPhaseTimeout,
1430             boolean testPhaseTimeoutNeeded)
1431             throws DeviceNotAvailableException, Throwable {
1432         // We clone the collectors for each IRemoteTest to ensure no state conflicts.
1433         List<IMetricCollector> clonedCollectors = new ArrayList<>();
1434         // Add automated collectors
1435         for (AutoLogCollector auto : config.getCommandOptions().getAutoLogCollectors()) {
1436             clonedCollectors.add(auto.getInstanceForValue());
1437         }
1438         // Add the collector from the configuration
1439         clonedCollectors.addAll(CollectorHelper.cloneCollectors(config.getMetricCollectors()));
1440         if (test instanceof IMetricCollectorReceiver) {
1441             ((IMetricCollectorReceiver) test).setMetricCollectors(clonedCollectors);
1442             // If test can receive collectors then let it handle the how to set them up
1443             if (testPhaseTimeoutNeeded) {
1444                 return runTestThread(info, listener, test, timer, testPhaseTimeout);
1445             } else {
1446                 long startTime = System.currentTimeMillis();
1447                 test.run(info, listener);
1448                 return System.currentTimeMillis() - startTime;
1449             }
1450         } else {
1451             // Wrap collectors in each other and collection will be sequential, do this in the
1452             // loop to ensure they are always initialized against the right context.
1453             ITestInvocationListener listenerWithCollectors = listener;
1454             if (config.getCommandOptions().reportTestCaseCount()) {
1455                 CountTestCasesCollector counter = new CountTestCasesCollector(test);
1456                 clonedCollectors.add(counter);
1457             }
1458             for (IMetricCollector collector : clonedCollectors) {
1459                 if (collector.isDisabled()) {
1460                     CLog.d("%s has been disabled. Skipping.", collector);
1461                 } else {
1462                     if (collector instanceof IConfigurationReceiver) {
1463                         ((IConfigurationReceiver) collector).setConfiguration(config);
1464                     }
1465                     listenerWithCollectors =
1466                             collector.init(info.getContext(), listenerWithCollectors);
1467                     TfObjectTracker.countWithParents(collector.getClass());
1468                 }
1469             }
1470             if (testPhaseTimeoutNeeded) {
1471                 return runTestThread(info, listenerWithCollectors, test, timer, testPhaseTimeout);
1472             } else {
1473                 long startTime = System.currentTimeMillis();
1474                 test.run(info, listenerWithCollectors);
1475                 return System.currentTimeMillis() - startTime;
1476             }
1477         }
1478     }
1479 
1480     /** Runs a test in a separate thread and returns the time spent on running the test. */
runTestThread( TestInformation info, ITestInvocationListener listener, IRemoteTest test, Timer timer, long testPhaseTimeout)1481     private long runTestThread(
1482             TestInformation info,
1483             ITestInvocationListener listener,
1484             IRemoteTest test,
1485             Timer timer,
1486             long testPhaseTimeout)
1487             throws Throwable {
1488         if (testPhaseTimeout <= 0) {
1489             // throw run interrupted exception so that it can be handled the same way as TestThreads
1490             // when timeout is reached.
1491             throw new RunInterruptedException(
1492                     "Test Phase Timeout Reached.", TestErrorIdentifier.TEST_PHASE_TIMED_OUT);
1493         }
1494         TestThread testThread = new TestThread(info, listener, test);
1495         TestPhaseMonitor testPhaseMonitor = new TestPhaseMonitor(testThread);
1496         timer.schedule(testPhaseMonitor, testPhaseTimeout);
1497         long startTime = System.currentTimeMillis();
1498         testThread.start();
1499         try {
1500             testThread.join();
1501         } catch (InterruptedException e) {
1502             CLog.e(e);
1503         } finally {
1504             testPhaseMonitor.cancel();
1505             long timeSpent = System.currentTimeMillis() - startTime;
1506             if (testThread.getLastThrownException() != null) {
1507                 throw testThread.getLastThrownException();
1508             }
1509             return timeSpent;
1510         }
1511     }
1512 
initializeListeners( IConfiguration config, ITestInvocationListener mainListener, ITestInvocationListener mainGranularLevelListener)1513     private RetryLogSaverResultForwarder initializeListeners(
1514             IConfiguration config,
1515             ITestInvocationListener mainListener,
1516             ITestInvocationListener mainGranularLevelListener) {
1517         List<ITestInvocationListener> currentTestListeners = new ArrayList<>();
1518         currentTestListeners.add(mainGranularLevelListener);
1519         currentTestListeners.add(mainListener);
1520         return new RetryLogSaverResultForwarder(config.getLogSaver(), currentTestListeners) {
1521             @Override
1522             public void testLog(
1523                     String dataName, LogDataType dataType, InputStreamSource dataStream) {
1524                 // We know for sure that the sub-listeners are LogSaverResultForwarder
1525                 // so we delegate to them to save and generate the logAssociation.
1526                 testLogForward(dataName, dataType, dataStream);
1527             }
1528         };
1529     }
1530 
1531     private void addRetryTime(long retryTimeMs) {
1532         // InvocationMetricLogger automatically adds the auto retry time.
1533         InvocationMetricLogger.addInvocationMetrics(
1534                 InvocationMetricKey.AUTO_RETRY_TIME, retryTimeMs);
1535     }
1536 
1537     private void linkExternalDirs(IBuildInfo info, TestInformation testInfo) {
1538         if (info.getProperties().contains(BuildInfoProperties.DO_NOT_LINK_TESTS_DIR)) {
1539             CLog.d("Skip linking external directory as FileProperty was set.");
1540             return;
1541         }
1542         // Load environment tests dir.
1543         if (info instanceof IDeviceBuildInfo) {
1544             // TODO: Use tests directory from TestInformation instead.
1545             File testsDir = ((IDeviceBuildInfo) info).getTestsDir();
1546             if (testsDir != null && testsDir.exists()) {
1547                 if (testInfo.executionFiles().get(FilesKey.TARGET_TESTS_DIRECTORY) == null) {
1548                     File targetTestCases =
1549                             handleLinkingExternalDirs(
1550                                     (IDeviceBuildInfo) info,
1551                                     testsDir,
1552                                     EnvVariable.ANDROID_TARGET_OUT_TESTCASES,
1553                                     BuildInfoFileKey.TARGET_LINKED_DIR.getFileKey());
1554                     if (targetTestCases != null) {
1555                         testInfo.executionFiles()
1556                                 .put(FilesKey.TARGET_TESTS_DIRECTORY, targetTestCases, true);
1557                     }
1558                 }
1559                 if (testInfo.executionFiles().get(FilesKey.HOST_TESTS_DIRECTORY) == null) {
1560                     File hostTestCases =
1561                             handleLinkingExternalDirs(
1562                                     (IDeviceBuildInfo) info,
1563                                     testsDir,
1564                                     EnvVariable.ANDROID_HOST_OUT_TESTCASES,
1565                                     BuildInfoFileKey.HOST_LINKED_DIR.getFileKey());
1566                     if (hostTestCases != null) {
1567                         testInfo.executionFiles()
1568                                 .put(FilesKey.HOST_TESTS_DIRECTORY, hostTestCases, true);
1569                     }
1570                 }
1571             }
1572         }
1573     }
1574 
1575     private File handleLinkingExternalDirs(
1576             IDeviceBuildInfo info, File testsDir, EnvVariable var, String baseName) {
1577         File externalDir = getExternalTestCasesDirs(var);
1578         if (externalDir == null) {
1579             String path = SystemUtil.ENV_VARIABLE_PATHS_IN_TESTS_DIR.get(var);
1580             File varDir = FileUtil.getFileForPath(testsDir, path);
1581             if (varDir.exists()) {
1582                 // If we found a dir already in the tests dir we keep track of it
1583                 info.setFile(
1584                         baseName,
1585                         varDir,
1586                         /** version */
1587                         "v1");
1588                 return varDir;
1589             }
1590             return null;
1591         }
1592         try {
1593             // Avoid conflict by creating a randomized name for the arriving symlink file.
1594             File subDir = FileUtil.createTempDir(baseName, testsDir);
1595             subDir.delete();
1596             FileUtil.symlinkFile(externalDir, subDir);
1597             // Tag the dir in the build info to be possibly cleaned.
1598             info.setFile(
1599                     baseName,
1600                     subDir,
1601                     /** version */
1602                     "v1");
1603             // Ensure we always delete the linking, no matter how the JVM exits.
1604             subDir.deleteOnExit();
1605             return subDir;
1606         } catch (IOException e) {
1607             CLog.e("Failed to load external test dir %s. Ignoring it.", externalDir);
1608             CLog.e(e);
1609         }
1610         return null;
1611     }
1612 
1613     private void setBinariesVersion(IInvocationContext context) {
1614         String version = getAdbVersion();
1615         if (version != null) {
1616             context.addInvocationAttribute(ADB_VERSION_KEY, version);
1617         }
1618         String javaVersion = System.getProperty("java.version");
1619         if (javaVersion != null && !javaVersion.isEmpty()) {
1620             context.addInvocationAttribute(JAVA_VERSION_KEY, javaVersion);
1621         }
1622         String javaClasspath = System.getProperty("java.class.path");
1623         if (javaClasspath != null && !javaClasspath.isEmpty()) {
1624             context.addInvocationAttribute(JAVA_CLASSPATH_KEY, javaClasspath);
1625         }
1626     }
1627 
1628     private void copyRemoteFiles(ICommandOptions options, IBuildInfo info) {
1629         for (String remoteFile : options.getRemoteFiles()) {
1630             info.setFile(
1631                     IBuildInfo.REMOTE_FILE_PREFIX,
1632                     new File(remoteFile),
1633                     IBuildInfo.REMOTE_FILE_VERSION);
1634         }
1635     }
1636 
1637     /** Convert the legacy *-on-failure options to the new auto-collect. */
1638     private void updateAutoCollectors(IConfiguration config) {
1639         if (config.getCommandOptions().captureScreenshotOnFailure()) {
1640             config.getCommandOptions()
1641                     .getAutoLogCollectors()
1642                     .add(AutoLogCollector.SCREENSHOT_ON_FAILURE);
1643         }
1644         if (config.getCommandOptions().captureLogcatOnFailure()) {
1645             config.getCommandOptions()
1646                     .getAutoLogCollectors()
1647                     .add(AutoLogCollector.LOGCAT_ON_FAILURE);
1648         }
1649     }
1650 
1651     /** Collect the logs from $TMPDIR/adb.$UID.log. */
1652     @VisibleForTesting
1653     protected void logHostAdb(IConfiguration config, ITestLogger logger) {
1654         if (SystemUtil.isLocalMode()) {
1655             // Skip logging host adb locally
1656             return;
1657         }
1658         if (config.getCommandOptions().getInvocationData().containsKey("subprocess")) {
1659             // Avoid relogging the adb log in a subprocess
1660             return;
1661         }
1662         String tmpDir = "/tmp";
1663         if (System.getenv("TMPDIR") != null) {
1664             tmpDir = System.getenv("TMPDIR");
1665         }
1666         CommandResult uidRes =
1667                 RunUtil.getDefault()
1668                         .runTimedCmd(60000, "id", "-u", System.getProperty("user.name"));
1669         if (!CommandStatus.SUCCESS.equals(uidRes.getStatus())) {
1670             CLog.e("Failed to collect UID for adb logs: %s", uidRes.getStderr());
1671             return;
1672         }
1673         String uid = uidRes.getStdout().trim();
1674         File adbLog = new File(tmpDir, String.format("adb.%s.log", uid));
1675         if (!adbLog.exists()) {
1676             CLog.i("Did not find adb log file: %s, upload skipped.", adbLog);
1677             return;
1678         }
1679         CommandResult truncAdb =
1680                 RunUtil.getDefault()
1681                         .runTimedCmd(60000, "tail", "-c", "10MB", adbLog.getAbsolutePath());
1682         if (!CommandStatus.SUCCESS.equals(truncAdb.getStatus())) {
1683             CLog.e("Failed to truncate the adb log: %s\n%s", adbLog, truncAdb.getStderr());
1684             return;
1685         }
1686         try (InputStreamSource source =
1687                 new ByteArrayInputStreamSource(truncAdb.getStdout().getBytes())) {
1688             logger.testLog("host_adb_log", LogDataType.ADB_HOST_LOG, source);
1689         }
1690     }
1691 
1692     /** Returns the external directory coming from the environment. */
1693     @VisibleForTesting
1694     File getExternalTestCasesDirs(EnvVariable envVar) {
1695         return SystemUtil.getExternalTestCasesDir(envVar);
1696     }
1697 
1698     /** Returns the adb version in use for the invocation. */
1699     protected String getAdbVersion() {
1700         return GlobalConfiguration.getDeviceManagerInstance().getAdbVersion();
1701     }
1702 
1703     /** Collect automatically some information on the primary device under test. */
1704     protected void collectAutoInfo(IConfiguration config, TestInformation info)
1705             throws DeviceNotAvailableException {
1706         if (SystemUtil.isLocalMode()) {
1707             // Avoid collecting for local modes since data collected in this method is used
1708             // in CI only.
1709             return;
1710         }
1711         if (config.getCommandOptions().getInvocationData().containsKey("subprocess")) {
1712             // Avoid logging in the subprocess
1713             return;
1714         }
1715         ITestDevice device = info.getDevice();
1716         if (device.getIDevice() instanceof StubDevice) {
1717             return;
1718         }
1719         try (CloseableTraceScope ignored = new CloseableTraceScope("collect_device_info")) {
1720             CommandResult kernelInfoResult = device.executeShellV2Command("uname -a");
1721             if (kernelInfoResult != null
1722                     && CommandStatus.SUCCESS.equals(kernelInfoResult.getStatus())) {
1723                 CLog.i(
1724                         "Device %s kernel information: '%s'",
1725                         device.getSerialNumber(), kernelInfoResult.getStdout().trim());
1726                 info.getBuildInfo()
1727                         .addBuildAttribute(
1728                                 "device_kernel_info", kernelInfoResult.getStdout().trim());
1729             }
1730             String system_img_info = device.getProperty("ro.system.build.fingerprint");
1731             if (system_img_info != null) {
1732                 CLog.i(
1733                         "Device %s system image build information: '%s'",
1734                         device.getSerialNumber(), system_img_info);
1735                 info.getBuildInfo().addBuildAttribute("system_img_info", system_img_info);
1736             }
1737             String vendor_img_info = device.getProperty("ro.vendor.build.fingerprint");
1738             if (vendor_img_info != null) {
1739                 CLog.i(
1740                         "Device %s vendor image build information: '%s'",
1741                         device.getSerialNumber(), vendor_img_info);
1742                 info.getBuildInfo().addBuildAttribute("vendor_img_info", vendor_img_info);
1743             }
1744         }
1745     }
1746 }
1747