1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.ddmlib.Log.LogLevel; 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.config.Configuration; 22 import com.android.tradefed.config.ConfigurationDescriptor; 23 import com.android.tradefed.config.ConfigurationException; 24 import com.android.tradefed.config.DeviceConfigurationHolder; 25 import com.android.tradefed.config.DynamicRemoteFileResolver; 26 import com.android.tradefed.config.IConfiguration; 27 import com.android.tradefed.config.IConfigurationReceiver; 28 import com.android.tradefed.config.IDeviceConfiguration; 29 import com.android.tradefed.dependencies.ExternalDependency; 30 import com.android.tradefed.dependencies.IExternalDependency; 31 import com.android.tradefed.device.DeviceNotAvailableException; 32 import com.android.tradefed.device.ITestDevice; 33 import com.android.tradefed.device.ITestDevice.RecoveryMode; 34 import com.android.tradefed.device.StubDevice; 35 import com.android.tradefed.device.connection.AdbTcpConnection; 36 import com.android.tradefed.device.metric.BugreportzOnTestCaseFailureCollector; 37 import com.android.tradefed.device.metric.IMetricCollector; 38 import com.android.tradefed.device.metric.LogcatOnFailureCollector; 39 import com.android.tradefed.device.metric.ScreenshotOnFailureCollector; 40 import com.android.tradefed.error.HarnessRuntimeException; 41 import com.android.tradefed.error.IHarnessException; 42 import com.android.tradefed.invoker.IInvocationContext; 43 import com.android.tradefed.invoker.InvocationContext; 44 import com.android.tradefed.invoker.TestInformation; 45 import com.android.tradefed.invoker.logger.CurrentInvocation; 46 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 47 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 48 import com.android.tradefed.invoker.logger.TfObjectTracker; 49 import com.android.tradefed.invoker.shard.token.TokenProperty; 50 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 51 import com.android.tradefed.log.ILogRegistry.EventType; 52 import com.android.tradefed.log.ITestLogger; 53 import com.android.tradefed.log.LogRegistry; 54 import com.android.tradefed.log.LogUtil.CLog; 55 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 56 import com.android.tradefed.result.FailureDescription; 57 import com.android.tradefed.result.ILogSaver; 58 import com.android.tradefed.result.ILogSaverListener; 59 import com.android.tradefed.result.ITestInvocationListener; 60 import com.android.tradefed.result.ITestLoggerReceiver; 61 import com.android.tradefed.result.LogFile; 62 import com.android.tradefed.result.MultiFailureDescription; 63 import com.android.tradefed.result.ResultForwarder; 64 import com.android.tradefed.result.TestDescription; 65 import com.android.tradefed.result.TestResult; 66 import com.android.tradefed.result.TestRunResult; 67 import com.android.tradefed.result.error.DeviceErrorIdentifier; 68 import com.android.tradefed.result.error.ErrorIdentifier; 69 import com.android.tradefed.result.error.InfraErrorIdentifier; 70 import com.android.tradefed.result.error.TestErrorIdentifier; 71 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus; 72 import com.android.tradefed.retry.IRetryDecision; 73 import com.android.tradefed.retry.RetryPreparationDecision; 74 import com.android.tradefed.retry.RetryStatistics; 75 import com.android.tradefed.retry.RetryStrategy; 76 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 77 import com.android.tradefed.targetprep.BuildError; 78 import com.android.tradefed.targetprep.ITargetPreparer; 79 import com.android.tradefed.targetprep.TargetSetupError; 80 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer; 81 import com.android.tradefed.testtype.IBuildReceiver; 82 import com.android.tradefed.testtype.IDeviceTest; 83 import com.android.tradefed.testtype.IInvocationContextReceiver; 84 import com.android.tradefed.testtype.IRemoteTest; 85 import com.android.tradefed.testtype.IRuntimeHintProvider; 86 import com.android.tradefed.testtype.ITestCollector; 87 import com.android.tradefed.testtype.ITestFileFilterReceiver; 88 import com.android.tradefed.testtype.ITestFilterReceiver; 89 import com.android.tradefed.testtype.suite.module.BaseModuleController; 90 import com.android.tradefed.testtype.suite.module.IModuleController.RunStrategy; 91 import com.android.tradefed.util.FileUtil; 92 import com.android.tradefed.util.MultiMap; 93 import com.android.tradefed.util.StreamUtil; 94 import com.android.tradefed.util.SystemUtil; 95 import com.android.tradefed.util.proto.TfMetricProtoUtil; 96 97 import com.google.api.client.util.Joiner; 98 import com.google.common.annotations.VisibleForTesting; 99 100 import java.io.File; 101 import java.io.IOException; 102 import java.util.ArrayList; 103 import java.util.Collection; 104 import java.util.Collections; 105 import java.util.HashMap; 106 import java.util.HashSet; 107 import java.util.LinkedHashSet; 108 import java.util.List; 109 import java.util.ListIterator; 110 import java.util.Map; 111 import java.util.Map.Entry; 112 import java.util.Set; 113 import java.util.stream.Collectors; 114 115 /** 116 * Container for the test run configuration. This class is an helper to prepare and run the tests. 117 */ 118 public class ModuleDefinition implements Comparable<ModuleDefinition>, ITestCollector { 119 120 /** key names used for saving module info into {@link IInvocationContext} */ 121 /** 122 * Module name is the base name associated with the module, usually coming from the Xml TF 123 * config file the module was loaded from. 124 */ 125 public static final String MODULE_NAME = "module-name"; 126 public static final String MODULE_ABI = "module-abi"; 127 public static final String MODULE_PARAMETERIZATION = "module-param"; 128 public static final String MODULE_EXTERNAL_DEPENDENCIES = "module-external-dependencies"; 129 /** 130 * Module ID the name that will be used to identify uniquely the module during testRunStart. It 131 * will usually be a combination of MODULE_ABI + MODULE_NAME. 132 */ 133 public static final String MODULE_ID = "module-id"; 134 /** This property is set to true if the module was running on a freshly prepared device. */ 135 public static final String MODULE_ISOLATED = "module-isolated"; 136 /** This property is set to true if the test module results were cached. */ 137 public static final String MODULE_CACHED = "module-cached"; 138 /** This property is set to true if only module level events are reported. */ 139 public static final String SPARSE_MODULE = "sparse-module"; 140 141 public static final String MODULE_CONTROLLER = "module_controller"; 142 143 public static final String PREPARATION_TIME = "PREP_TIME"; 144 public static final String TEAR_DOWN_TIME = "TEARDOWN_TIME"; 145 public static final String TEST_TIME = "TEST_TIME"; 146 public static final String MODULE_TEST_COUNT = "MODULE_TEST_COUNT"; 147 public static final String RETRY_TIME = "MODULE_RETRY_TIME"; 148 public static final String ISOLATION_COST = "ISOLATION_COST"; 149 public static final String RETRY_SUCCESS_COUNT = "MODULE_RETRY_SUCCESS"; 150 public static final String RETRY_FAIL_COUNT = "MODULE_RETRY_FAILED"; 151 152 private final IInvocationContext mModuleInvocationContext; 153 private final IConfiguration mModuleConfiguration; 154 private IConfiguration mInternalTestConfiguration; 155 private IConfiguration mInternalTargetPreparerConfiguration; 156 private ILogSaver mLogSaver; 157 private TestInformation mModuleInfo; 158 private ITestInvocationListener mInvocationListener; 159 160 private final String mId; 161 private Collection<IRemoteTest> mTests = null; 162 private Map<String, List<ITargetPreparer>> mPreparersPerDevice = null; 163 private Map<String, List<ITargetPreparer>> mSuitePreparersPerDevice = null; 164 165 private List<IMultiTargetPreparer> mMultiPreparers = new ArrayList<>(); 166 private IBuildInfo mBuild; 167 private ITestDevice mDevice; 168 private List<IMetricCollector> mRunMetricCollectors = new ArrayList<>(); 169 private boolean mCollectTestsOnly = false; 170 171 private List<TestRunResult> mTestsResults = new ArrayList<>(); 172 private List<ModuleListener> mRunListenersResults = new ArrayList<>(); 173 private int mExpectedTests = 0; 174 private boolean mIsFailedModule = false; 175 private boolean mRetriedModulePreparationSuccess = false; 176 177 // Tracking of preparers performance 178 private long mElapsedPreparation = 0L; 179 private long mElapsedTearDown = 0L; 180 181 private long mStartTestTime = 0L; 182 private Long mStartModuleRunDate = null; 183 184 // Tracking of retry performance 185 private List<RetryStatistics> mRetryStats = new ArrayList<>(); 186 private boolean mDisableAutoRetryTimeReporting = false; 187 188 private boolean mMergeAttempts = true; 189 private IRetryDecision mRetryDecision; 190 191 // Token during sharding 192 private Set<TokenProperty> mRequiredTokens = new HashSet<>(); 193 194 private boolean mEnableDynamicDownload = false; 195 private GranularRetriableTestWrapper mCurrentTestWrapper = null; 196 private int mMaxRetry = 1; 197 // TODO(b/226453043): Eventually we need to remove this. 198 private int mTargetPreparerRetryCount = 0; 199 200 private Set<TestDescription> mPassThroughFilters = new LinkedHashSet<>(); 201 202 private boolean mRecoverVirtualDevice = false; 203 204 @VisibleForTesting ModuleDefinition()205 public ModuleDefinition() { 206 mModuleInvocationContext = null; 207 mModuleConfiguration = null; 208 mId = ""; 209 } 210 211 /** 212 * Constructor 213 * 214 * @param name unique name of the test configuration. 215 * @param tests list of {@link IRemoteTest} that needs to run. 216 * @param preparersPerDevice list of {@link ITargetPreparer} to be used to setup the device. 217 * @param moduleConfig the {@link IConfiguration} of the underlying module config. 218 */ ModuleDefinition( String name, Collection<IRemoteTest> tests, Map<String, List<ITargetPreparer>> preparersPerDevice, List<IMultiTargetPreparer> multiPreparers, IConfiguration moduleConfig)219 public ModuleDefinition( 220 String name, 221 Collection<IRemoteTest> tests, 222 Map<String, List<ITargetPreparer>> preparersPerDevice, 223 List<IMultiTargetPreparer> multiPreparers, 224 IConfiguration moduleConfig) { 225 this(name, tests, preparersPerDevice, null, multiPreparers, moduleConfig); 226 } 227 228 /** 229 * Constructor 230 * 231 * @param name unique name of the test configuration. 232 * @param tests list of {@link IRemoteTest} that needs to run. 233 * @param preparersPerDevice list of {@link ITargetPreparer} to be used to setup the device. 234 * @param moduleConfig the {@link IConfiguration} of the underlying module config. 235 */ ModuleDefinition( String name, Collection<IRemoteTest> tests, Map<String, List<ITargetPreparer>> preparersPerDevice, Map<String, List<ITargetPreparer>> suitePreparersPerDevice, List<IMultiTargetPreparer> multiPreparers, IConfiguration moduleConfig)236 public ModuleDefinition( 237 String name, 238 Collection<IRemoteTest> tests, 239 Map<String, List<ITargetPreparer>> preparersPerDevice, 240 Map<String, List<ITargetPreparer>> suitePreparersPerDevice, 241 List<IMultiTargetPreparer> multiPreparers, 242 IConfiguration moduleConfig) { 243 mId = name; 244 mTests = tests; 245 mModuleConfiguration = moduleConfig; 246 ConfigurationDescriptor configDescriptor = moduleConfig.getConfigurationDescription(); 247 mModuleInvocationContext = new InvocationContext(); 248 mModuleInvocationContext.setConfigurationDescriptor(configDescriptor.clone()); 249 250 // If available in the suite, add the abi name 251 if (configDescriptor.getAbi() != null) { 252 mModuleInvocationContext.addInvocationAttribute( 253 MODULE_ABI, configDescriptor.getAbi().getName()); 254 } 255 if (configDescriptor.getModuleName() != null) { 256 mModuleInvocationContext.addInvocationAttribute( 257 MODULE_NAME, configDescriptor.getModuleName()); 258 } 259 String parameterization = 260 configDescriptor 261 .getAllMetaData() 262 .getUniqueMap() 263 .get(ConfigurationDescriptor.ACTIVE_PARAMETER_KEY); 264 if (parameterization != null) { 265 mModuleInvocationContext.addInvocationAttribute( 266 MODULE_PARAMETERIZATION, parameterization); 267 } 268 // If there is no specific abi, module-id should be module-name 269 mModuleInvocationContext.addInvocationAttribute(MODULE_ID, mId); 270 271 // Add External Dependencies of this module to the module context 272 Set<ExternalDependency> externalDependencies = new LinkedHashSet<>(); 273 for (IDeviceConfiguration deviceConfig : moduleConfig.getDeviceConfig()) { 274 for (Object obj : deviceConfig.getAllObjects()) { 275 if (obj instanceof IExternalDependency) { 276 externalDependencies.addAll(((IExternalDependency) obj).getDependencies()); 277 } 278 } 279 } 280 if (!externalDependencies.isEmpty()) { 281 final List<String> dependencyClassNames = 282 externalDependencies.stream() 283 .map(dependency -> dependency.getClass().getName()) 284 .collect(Collectors.toList()); 285 mModuleInvocationContext.addInvocationAttribute( 286 MODULE_EXTERNAL_DEPENDENCIES, String.join(", ", dependencyClassNames)); 287 } 288 289 mMultiPreparers.addAll(multiPreparers); 290 mPreparersPerDevice = preparersPerDevice; 291 mSuitePreparersPerDevice = suitePreparersPerDevice; 292 293 // Get the tokens of the module 294 List<String> tokens = configDescriptor.getMetaData(ITestSuite.TOKEN_KEY); 295 if (tokens != null) { 296 for (String token : tokens) { 297 mRequiredTokens.add(TokenProperty.valueOf(token.toUpperCase())); 298 } 299 } 300 } 301 302 /** Returns the number of devices expected to run this test. */ neededDevices()303 public int neededDevices() { 304 return mModuleConfiguration.getDeviceConfig().size(); 305 } 306 307 /** 308 * Returns the next {@link IRemoteTest} from the list of tests. The list of tests of a module 309 * may be shared with another one in case of sharding. 310 */ poll()311 IRemoteTest poll() { 312 synchronized (mTests) { 313 if (mTests.isEmpty()) { 314 return null; 315 } 316 IRemoteTest test = mTests.iterator().next(); 317 mTests.remove(test); 318 return test; 319 } 320 } 321 322 /** 323 * Add some {@link IRemoteTest} to be executed as part of the module. Used when merging two 324 * modules. 325 */ addTests(List<IRemoteTest> test)326 void addTests(List<IRemoteTest> test) { 327 synchronized (mTests) { 328 mTests.addAll(test); 329 } 330 } 331 332 /** Returns the current number of {@link IRemoteTest} waiting to be executed. */ numTests()333 public int numTests() { 334 synchronized (mTests) { 335 return mTests.size(); 336 } 337 } 338 339 /** 340 * Return True if the Module still has {@link IRemoteTest} to run in its pool. False otherwise. 341 */ hasTests()342 protected boolean hasTests() { 343 synchronized (mTests) { 344 return !mTests.isEmpty(); 345 } 346 } 347 348 /** Return the unique module name. */ getId()349 public String getId() { 350 return mId; 351 } 352 353 /** 354 * {@inheritDoc} 355 */ 356 @Override compareTo(ModuleDefinition moduleDef)357 public int compareTo(ModuleDefinition moduleDef) { 358 return getId().compareTo(moduleDef.getId()); 359 } 360 361 /** 362 * Inject the {@link IBuildInfo} to be used during the tests. 363 */ setBuild(IBuildInfo build)364 public void setBuild(IBuildInfo build) { 365 mBuild = build; 366 } 367 368 /** 369 * Inject the {@link ITestDevice} to be used during the tests. 370 */ setDevice(ITestDevice device)371 public void setDevice(ITestDevice device) { 372 mDevice = device; 373 } 374 375 /** Inject the List of {@link IMetricCollector} to be used by the module. */ setMetricCollectors(List<IMetricCollector> collectors)376 public void setMetricCollectors(List<IMetricCollector> collectors) { 377 if (collectors == null) { 378 return; 379 } 380 mRunMetricCollectors.addAll(collectors); 381 } 382 383 /** Pass the invocation log saver to the module so it can use it if necessary. */ setLogSaver(ILogSaver logSaver)384 public void setLogSaver(ILogSaver logSaver) { 385 mLogSaver = logSaver; 386 } 387 388 /** 389 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 390 * after to setup and clean the device. 391 * 392 * @param listener the {@link ITestInvocationListener} where to report results. 393 * @throws DeviceNotAvailableException in case of device going offline. 394 */ run(TestInformation moduleInfo, ITestInvocationListener listener)395 public final void run(TestInformation moduleInfo, ITestInvocationListener listener) 396 throws DeviceNotAvailableException { 397 run(moduleInfo, listener, null); 398 } 399 400 /** 401 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 402 * after to setup and clean the device. 403 * 404 * @param listener the {@link ITestInvocationListener} where to report results. 405 * @param moduleLevelListeners The list of listeners at the module level. 406 * @throws DeviceNotAvailableException in case of device going offline. 407 */ run( TestInformation moduleInfo, ITestInvocationListener listener, List<ITestInvocationListener> moduleLevelListeners)408 public final void run( 409 TestInformation moduleInfo, 410 ITestInvocationListener listener, 411 List<ITestInvocationListener> moduleLevelListeners) 412 throws DeviceNotAvailableException { 413 run(moduleInfo, listener, moduleLevelListeners, 1); 414 } 415 416 /** 417 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 418 * after to setup and clean the device. 419 * 420 * @param moduleInfo the {@link TestInformation} for the module. 421 * @param listener the {@link ITestInvocationListener} where to report results. 422 * @param moduleLevelListeners The list of listeners at the module level. 423 * @param failureListener a particular listener to collect logs on testFail. Can be null. 424 * @param maxRunLimit the max number of runs for each testcase. 425 * @throws DeviceNotAvailableException in case of device going offline. 426 */ run( TestInformation moduleInfo, ITestInvocationListener listener, List<ITestInvocationListener> moduleLevelListeners, int maxRunLimit)427 public final void run( 428 TestInformation moduleInfo, 429 ITestInvocationListener listener, 430 List<ITestInvocationListener> moduleLevelListeners, 431 int maxRunLimit) 432 throws DeviceNotAvailableException { 433 mMaxRetry = maxRunLimit; 434 mModuleInfo = moduleInfo; 435 mInvocationListener = listener; 436 437 mStartModuleRunDate = System.currentTimeMillis(); 438 // Load extra configuration for the module from module_controller 439 // TODO: make module_controller a full TF object 440 boolean skipTestCases = false; 441 RunStrategy rs = applyConfigurationControl(); 442 if (RunStrategy.FULL_MODULE_BYPASS.equals(rs)) { 443 CLog.d("module_controller applied and module %s should not run.", getId()); 444 return; 445 } else if (RunStrategy.SKIP_MODULE_TESTCASES.equals(rs)) { 446 CLog.d("All tests cases for %s will be marked skipped.", getId()); 447 skipTestCases = true; 448 } 449 450 CLog.logAndDisplay(LogLevel.DEBUG, "Running module %s", getId()); 451 // Exception generated during setUp or run of the tests 452 Throwable preparationException; 453 DeviceNotAvailableException runException = null; 454 long start = System.currentTimeMillis(); 455 // Resolve dynamic files except for the IRemoteTest ones 456 try (CloseableTraceScope ignored = new CloseableTraceScope("download_files")) { 457 preparationException = 458 invokeRemoteDynamic(moduleInfo.getDevice(), mModuleConfiguration); 459 460 if (preparationException == null) { 461 mInternalTargetPreparerConfiguration = 462 new Configuration("tmp-download", "tmp-download"); 463 mInternalTargetPreparerConfiguration 464 .getCommandOptions() 465 .getDynamicDownloadArgs() 466 .putAll(mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 467 for (String device : mPreparersPerDevice.keySet()) { 468 mInternalTargetPreparerConfiguration.setDeviceConfig( 469 new DeviceConfigurationHolder(device)); 470 for (ITargetPreparer preparer : mPreparersPerDevice.get(device)) { 471 try { 472 mInternalTargetPreparerConfiguration 473 .getDeviceConfigByName(device) 474 .addSpecificConfig(preparer); 475 } catch (ConfigurationException e) { 476 // Shouldn't happen; 477 throw new RuntimeException(e); 478 } 479 } 480 } 481 mInternalTargetPreparerConfiguration.setMultiTargetPreparers(mMultiPreparers); 482 preparationException = 483 invokeRemoteDynamic( 484 moduleInfo.getDevice(), mInternalTargetPreparerConfiguration); 485 } 486 } 487 // Setup 488 if (preparationException == null) { 489 try (CloseableTraceScope ignored = new CloseableTraceScope("module_preparation")) { 490 preparationException = runPreparation(false); 491 } 492 } 493 494 while (preparationException != null) { 495 RetryPreparationDecision retryDecision = 496 mRetryDecision.shouldRetryPreparation( 497 this, 498 mTargetPreparerRetryCount, 499 maxRunLimit); 500 boolean shouldFailRun = retryDecision.shouldFailRun(); 501 reportSetupFailure( 502 preparationException, 503 listener, 504 moduleLevelListeners, 505 mTargetPreparerRetryCount, 506 shouldFailRun); 507 if (shouldFailRun) { 508 return; 509 } 510 mTargetPreparerRetryCount++; 511 if (!retryDecision.shouldRetry()) { 512 CLog.i("Retry target preparers for module: %s successfully at the %s retry", 513 this.getId(), mTargetPreparerRetryCount); 514 // This flag is set true for any target preparation error, set it false when the 515 // retrying succeed. 516 mIsFailedModule = false; 517 mRetriedModulePreparationSuccess = true; 518 InvocationMetricLogger.addInvocationMetrics( 519 InvocationMetricKey.DEVICE_RESET_MODULES_FOR_TARGET_PREPARER, this.getId()); 520 } 521 preparationException = retryDecision.getPreviousException(); 522 } 523 524 // Total count of retried (preparation + test) shouldn't exceed maxRunLimit and mMaxRetry. 525 maxRunLimit = maxRunLimit - mTargetPreparerRetryCount; 526 mMaxRetry = mMaxRetry - mTargetPreparerRetryCount; 527 InvocationMetricLogger.addInvocationPairMetrics( 528 InvocationMetricKey.MODULE_SETUP_PAIR, start, System.currentTimeMillis()); 529 530 // Run the tests 531 try { 532 mStartTestTime = getCurrentTime(); 533 int perModuleRetryQuota = mMaxRetry; 534 while (true) { 535 IRemoteTest test = poll(); 536 if (test == null) { 537 return; 538 } 539 TfObjectTracker.countWithParents(test.getClass()); 540 if (test instanceof IBuildReceiver) { 541 ((IBuildReceiver) test).setBuild(mBuild); 542 } 543 if (test instanceof IDeviceTest) { 544 ((IDeviceTest) test).setDevice(mDevice); 545 } 546 if (test instanceof IInvocationContextReceiver) { 547 ((IInvocationContextReceiver) test) 548 .setInvocationContext(mModuleInvocationContext); 549 } 550 mInternalTestConfiguration = new Configuration("tmp-download", "tmp-download"); 551 mInternalTestConfiguration 552 .getCommandOptions() 553 .getDynamicDownloadArgs() 554 .putAll(mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 555 // We do it before the official set, otherwise the IConfiguration will not be the 556 // right one. 557 mInternalTestConfiguration.setTest(test); 558 if (test instanceof IConfigurationReceiver) { 559 ((IConfigurationReceiver) test).setConfiguration(mModuleConfiguration); 560 } 561 if (mDevice instanceof IConfigurationReceiver) { 562 ((IConfigurationReceiver) mDevice).setConfiguration(mModuleConfiguration); 563 } 564 if (test instanceof ISystemStatusCheckerReceiver) { 565 // We do not pass down Status checker because they are already running at the 566 // top level suite. 567 ((ISystemStatusCheckerReceiver) test).setSystemStatusChecker(new ArrayList<>()); 568 } 569 if (test instanceof ITestCollector) { 570 if (skipTestCases) { 571 mCollectTestsOnly = true; 572 } 573 ((ITestCollector) test).setCollectTestsOnly(mCollectTestsOnly); 574 } 575 if (!mPassThroughFilters.isEmpty()) { 576 applyFilterToTest(test, mPassThroughFilters); 577 } 578 mCurrentTestWrapper = 579 prepareGranularRetriableWrapper( 580 test, 581 listener, 582 moduleLevelListeners, 583 skipTestCases, 584 perModuleRetryQuota); 585 mCurrentTestWrapper.setCollectTestsOnly(mCollectTestsOnly); 586 // Resolve the dynamic options for that one test. 587 preparationException = 588 invokeRemoteDynamic(moduleInfo.getDevice(), mInternalTestConfiguration); 589 if (preparationException != null) { 590 reportSetupFailure( 591 preparationException, 592 listener, 593 moduleLevelListeners, 594 mTargetPreparerRetryCount, 595 true); 596 return; 597 } 598 try (CloseableTraceScope ignored = new CloseableTraceScope("module_test")) { 599 mCurrentTestWrapper.run(moduleInfo, listener); 600 } catch (DeviceNotAvailableException dnae) { 601 runException = dnae; 602 // We do special logging of some information in Context of the module for easier 603 // debugging. 604 CLog.e( 605 "Module %s threw a DeviceNotAvailableException on device %s during " 606 + "test %s", 607 getId(), mDevice.getSerialNumber(), test.getClass()); 608 CLog.e(dnae); 609 // log an events 610 logDeviceEvent( 611 EventType.MODULE_DEVICE_NOT_AVAILABLE, 612 mDevice.getSerialNumber(), 613 dnae, 614 getId()); 615 throw dnae; 616 } finally { 617 mInternalTestConfiguration.cleanConfigurationData(); 618 mInternalTestConfiguration = null; 619 if (mMergeAttempts) { 620 // A single module can generate several test runs 621 mTestsResults.addAll(mCurrentTestWrapper.getFinalTestRunResults()); 622 } else { 623 // Keep track of each listener for attempts 624 mRunListenersResults.add(mCurrentTestWrapper.getResultListener()); 625 } 626 if (mModuleConfiguration 627 .getConfigurationDescription() 628 .isNotIRemoteTestShardable()) { 629 mPassThroughFilters.addAll(mCurrentTestWrapper.getPassedTests()); 630 } 631 // Limit escalating retries across all sub-IRemoteTests 632 if (mRetryDecision != null 633 && RetryStrategy.RETRY_ANY_FAILURE.equals( 634 mRetryDecision.getRetryStrategy())) { 635 perModuleRetryQuota -= mCurrentTestWrapper.getRetryCount(); 636 } 637 638 mExpectedTests += mCurrentTestWrapper.getExpectedTestsCount(); 639 // Get information about retry 640 if (mRetryDecision != null) { 641 RetryStatistics res = mRetryDecision.getRetryStatistics(); 642 if (res != null) { 643 addRetryTime(res.mRetryTime); 644 mRetryStats.add(res); 645 } 646 } 647 } 648 // After the run, if the test failed (even after retry the final result passed) has 649 // failed, capture a bugreport. 650 if (mCurrentTestWrapper.getResultListener().hasLastAttemptFailed()) { 651 captureBugreport( 652 listener, 653 getId(), 654 mCurrentTestWrapper 655 .getResultListener() 656 .getCurrentRunResults() 657 .getRunFailureDescription()); 658 } 659 } 660 } finally { 661 mPassThroughFilters.clear(); 662 // Clean target preparers dynamic files. 663 if (mInternalTargetPreparerConfiguration != null) { 664 mInternalTargetPreparerConfiguration.cleanConfigurationData(); 665 mInternalTargetPreparerConfiguration = null; 666 } 667 long cleanStartTime = getCurrentTime(); 668 RuntimeException tearDownException = null; 669 try (CloseableTraceScope ignored = new CloseableTraceScope("module_teardown")) { 670 Throwable exception = (runException != null) ? runException : preparationException; 671 try { 672 // Tear down 673 runTearDown(moduleInfo, exception); 674 } catch (DeviceNotAvailableException dnae) { 675 if (runException == null) { 676 // Ignore the exception and attempt recovery. 677 CLog.e( 678 "Module %s failed during tearDown with: %s", 679 getId(), StreamUtil.getStackTrace(dnae)); 680 recoverDevice(moduleInfo, dnae); 681 } else { 682 throw dnae; 683 } 684 } 685 // If still available, verify that device didn't crash 686 if (runException == null) { 687 checkEndModuleDevice(moduleInfo); 688 } 689 } catch (DeviceNotAvailableException dnae) { 690 CLog.e( 691 "Module %s failed during tearDown with: %s", 692 getId(), StreamUtil.getStackTrace(dnae)); 693 throw dnae; 694 } catch (RuntimeException e) { 695 CLog.e("Exception while running tearDown:"); 696 CLog.e(e); 697 tearDownException = e; 698 } finally { 699 InvocationMetricLogger 700 .addInvocationPairMetrics(InvocationMetricKey.MODULE_TEARDOWN_PAIR, 701 cleanStartTime, getCurrentTime()); 702 mElapsedTearDown = getCurrentTime() - cleanStartTime; 703 // finalize results 704 if (preparationException == null) { 705 mModuleConfiguration.cleanConfigurationData(); 706 if (mMergeAttempts) { 707 reportFinalResults( 708 listener, mExpectedTests, mTestsResults, null, tearDownException); 709 } else { 710 boolean reported = false; 711 // Push the attempts one by one 712 for (int i = 0; i < maxRunLimit; i++) { 713 // Get all the results for the attempt 714 List<TestRunResult> runResultList = new ArrayList<TestRunResult>(); 715 int expectedCount = 0; 716 for (ModuleListener attemptListener : mRunListenersResults) { 717 for (String runName : attemptListener.getTestRunNames()) { 718 TestRunResult run = 719 attemptListener.getTestRunAtAttempt(runName, i); 720 if (run != null) { 721 runResultList.add(run); 722 expectedCount += run.getExpectedTestCount(); 723 } 724 } 725 } 726 727 if (!runResultList.isEmpty() || ( 728 !reported && mRetriedModulePreparationSuccess)) { 729 if (runResultList.isEmpty()) { 730 reported = true; 731 CLog.i("Module preparation retry pass but no test cases were " + 732 "executed. Keep reporting the result to notify it " + 733 "failed in the 1st run but passed after retrying."); 734 } 735 reportFinalResults( 736 listener, 737 expectedCount, 738 runResultList, 739 i, 740 tearDownException); 741 } else { 742 CLog.d("No results to be forwarded for attempt %s.", i); 743 } 744 } 745 } 746 } 747 } 748 } 749 } 750 751 /** 752 * Create a wrapper class for the {@link IRemoteTest} which has built-in logic to schedule 753 * multiple test runs for the same module, and have the ability to run testcases at a more 754 * granular level (a subset of testcases in the module). 755 * 756 * @param test the {@link IRemoteTest} that is being wrapped. 757 * @param failureListener a particular listener to collect logs on testFail. Can be null. 758 * @param skipTestCases A run strategy when SKIP_MODULE_TESTCASES is defined. 759 * @param maxRunLimit a rate-limiter on testcases retrying times. 760 */ 761 @VisibleForTesting prepareGranularRetriableWrapper( IRemoteTest test, ITestInvocationListener listener, List<ITestInvocationListener> moduleLevelListeners, boolean skipTestCases, int maxRunLimit)762 GranularRetriableTestWrapper prepareGranularRetriableWrapper( 763 IRemoteTest test, 764 ITestInvocationListener listener, 765 List<ITestInvocationListener> moduleLevelListeners, 766 boolean skipTestCases, 767 int maxRunLimit) { 768 GranularRetriableTestWrapper retriableTest = 769 new GranularRetriableTestWrapper( 770 test, this, listener, moduleLevelListeners, maxRunLimit); 771 retriableTest.setModuleId(getId()); 772 retriableTest.setMarkTestsSkipped(skipTestCases); 773 retriableTest.setMetricCollectors(mRunMetricCollectors); 774 retriableTest.setModuleConfig(mModuleConfiguration); 775 retriableTest.setInvocationContext(mModuleInvocationContext); 776 retriableTest.setLogSaver(mLogSaver); 777 retriableTest.setRetryDecision(mRetryDecision); 778 return retriableTest; 779 } 780 captureBugreport( ITestLogger listener, String moduleId, FailureDescription failure)781 private void captureBugreport( 782 ITestLogger listener, String moduleId, FailureDescription failure) { 783 FailureStatus status = failure.getFailureStatus(); 784 if (!FailureStatus.LOST_SYSTEM_UNDER_TEST.equals(status) 785 && !FailureStatus.SYSTEM_UNDER_TEST_CRASHED.equals(status)) { 786 return; 787 } 788 for (ITestDevice device : mModuleInvocationContext.getDevices()) { 789 if (device.getIDevice() instanceof StubDevice) { 790 continue; 791 } 792 device.logBugreport( 793 String.format( 794 "module-%s-failure-%s-bugreport", moduleId, device.getSerialNumber()), 795 listener); 796 } 797 } 798 799 /** Helper to log the device events. */ logDeviceEvent(EventType event, String serial, Throwable t, String moduleId)800 private void logDeviceEvent(EventType event, String serial, Throwable t, String moduleId) { 801 Map<String, String> args = new HashMap<>(); 802 args.put("serial", serial); 803 args.put("trace", StreamUtil.getStackTrace(t)); 804 args.put("module-id", moduleId); 805 LogRegistry.getLogRegistry().logEvent(LogLevel.DEBUG, event, args); 806 } 807 808 /** Finalize results to report them all and count if there are missing tests. */ reportFinalResults( ITestInvocationListener listener, int totalExpectedTests, List<TestRunResult> listResults, Integer attempt, RuntimeException tearDownException)809 private void reportFinalResults( 810 ITestInvocationListener listener, 811 int totalExpectedTests, 812 List<TestRunResult> listResults, 813 Integer attempt, 814 RuntimeException tearDownException) { 815 long elapsedTime = 0L; 816 HashMap<String, Metric> metricsProto = new HashMap<>(); 817 if (attempt != null) { 818 long startTime = 819 listResults.isEmpty() ? mStartTestTime : listResults.get(0).getStartTime(); 820 listener.testRunStarted( 821 getId(), totalExpectedTests, attempt + mTargetPreparerRetryCount, startTime); 822 } else { 823 listener.testRunStarted( 824 getId(), totalExpectedTests, mTargetPreparerRetryCount, mStartTestTime); 825 } 826 int numResults = 0; 827 MultiMap<String, LogFile> aggLogFiles = new MultiMap<>(); 828 List<FailureDescription> runFailureMessages = new ArrayList<>(); 829 for (TestRunResult runResult : listResults) { 830 numResults += runResult.getTestResults().size(); 831 forwardTestResults(runResult.getTestResults(), listener); 832 if (runResult.isRunFailure()) { 833 runFailureMessages.add(runResult.getRunFailureDescription()); 834 } 835 elapsedTime += runResult.getElapsedTime(); 836 // put metrics from the tests 837 metricsProto.putAll(runResult.getRunProtoMetrics()); 838 aggLogFiles.putAll(runResult.getRunLoggedFiles()); 839 } 840 // put metrics from the preparation 841 metricsProto.put( 842 PREPARATION_TIME, 843 TfMetricProtoUtil.createSingleValue(mElapsedPreparation, "milliseconds")); 844 metricsProto.put( 845 TEAR_DOWN_TIME, 846 TfMetricProtoUtil.createSingleValue(mElapsedTearDown, "milliseconds")); 847 metricsProto.put( 848 TEST_TIME, TfMetricProtoUtil.createSingleValue(elapsedTime, "milliseconds")); 849 metricsProto.put(MODULE_TEST_COUNT, TfMetricProtoUtil.createSingleValue(numResults, "int")); 850 // Report all the retry informations 851 if (!mRetryStats.isEmpty()) { 852 if (attempt != null) { 853 long cost = RetryStatistics.isolationCostPerAttempt(attempt, mRetryStats); 854 if (cost != 0L) { 855 metricsProto.put( 856 ISOLATION_COST, 857 TfMetricProtoUtil.createSingleValue(cost, "milliseconds")); 858 } 859 } else { 860 RetryStatistics agg = RetryStatistics.aggregateStatistics(mRetryStats); 861 metricsProto.put( 862 RETRY_TIME, 863 TfMetricProtoUtil.createSingleValue(agg.mRetryTime, "milliseconds")); 864 metricsProto.put( 865 RETRY_SUCCESS_COUNT, 866 TfMetricProtoUtil.createSingleValue(agg.mRetrySuccess, "")); 867 metricsProto.put( 868 RETRY_FAIL_COUNT, 869 TfMetricProtoUtil.createSingleValue(agg.mRetryFailure, "")); 870 } 871 } 872 873 // Only report the mismatch if there were no error during the run. 874 if (runFailureMessages.isEmpty() && totalExpectedTests != numResults) { 875 String error = 876 String.format( 877 "Module %s only ran %d out of %d expected tests.", 878 getId(), numResults, totalExpectedTests); 879 FailureDescription mismatch = 880 FailureDescription.create(error) 881 .setFailureStatus(FailureStatus.TEST_FAILURE) 882 .setErrorIdentifier(InfraErrorIdentifier.EXPECTED_TESTS_MISMATCH); 883 runFailureMessages.add(mismatch); 884 CLog.e(error); 885 } 886 887 if (tearDownException != null) { 888 FailureDescription failure = 889 CurrentInvocation.createFailure( 890 StreamUtil.getStackTrace(tearDownException), null) 891 .setCause(tearDownException); 892 runFailureMessages.add(failure); 893 } 894 // If there is any errors report them all at once 895 if (!runFailureMessages.isEmpty()) { 896 if (runFailureMessages.size() == 1) { 897 listener.testRunFailed(runFailureMessages.get(0)); 898 } else { 899 listener.testRunFailed(new MultiFailureDescription(runFailureMessages)); 900 } 901 mIsFailedModule = true; 902 } 903 904 // Provide a strong association of the run to its logs. 905 for (String key : aggLogFiles.keySet()) { 906 for (LogFile logFile : aggLogFiles.get(key)) { 907 if (listener instanceof ILogSaverListener) { 908 ((ILogSaverListener) listener).logAssociation(key, logFile); 909 } 910 } 911 } 912 // Allow each attempt to have its own start/end time 913 if (attempt != null) { 914 listener.testRunEnded(elapsedTime, metricsProto); 915 } else { 916 listener.testRunEnded(getCurrentTime() - mStartTestTime, metricsProto); 917 } 918 } 919 forwardTestResults( Map<TestDescription, TestResult> testResults, ITestInvocationListener listener)920 private void forwardTestResults( 921 Map<TestDescription, TestResult> testResults, ITestInvocationListener listener) { 922 for (Map.Entry<TestDescription, TestResult> testEntry : testResults.entrySet()) { 923 listener.testStarted(testEntry.getKey(), testEntry.getValue().getStartTime()); 924 switch (testEntry.getValue().getResultStatus()) { 925 case FAILURE: 926 listener.testFailed(testEntry.getKey(), testEntry.getValue().getFailure()); 927 break; 928 case ASSUMPTION_FAILURE: 929 listener.testAssumptionFailure( 930 testEntry.getKey(), testEntry.getValue().getFailure()); 931 break; 932 case IGNORED: 933 listener.testIgnored(testEntry.getKey()); 934 break; 935 case SKIPPED: 936 listener.testSkipped(testEntry.getKey(), testEntry.getValue().getSkipReason()); 937 break; 938 case INCOMPLETE: 939 listener.testFailed( 940 testEntry.getKey(), 941 FailureDescription.create( 942 "Test did not complete due to exception.", 943 FailureStatus.TEST_FAILURE)); 944 break; 945 default: 946 break; 947 } 948 // Provide a strong association of the test to its logs. 949 for (Entry<String, LogFile> logFile : 950 testEntry.getValue().getLoggedFiles().entrySet()) { 951 if (listener instanceof ILogSaverListener) { 952 ((ILogSaverListener) listener) 953 .logAssociation(logFile.getKey(), logFile.getValue()); 954 } 955 } 956 listener.testEnded( 957 testEntry.getKey(), 958 testEntry.getValue().getEndTime(), 959 testEntry.getValue().getProtoMetrics()); 960 } 961 } 962 963 /** 964 * Run preparers of the test, including suite level preparers if specified. 965 * 966 * @param includeSuitePreparers Set to {@code true} to also run suite level preparers. 967 * @return {@link Throwable} of any exception raised when running preparers. 968 */ runPreparation(boolean includeSuitePreparers)969 public Throwable runPreparation(boolean includeSuitePreparers) { 970 Throwable preparationException = null; 971 long prepStartTime = getCurrentTime(); 972 if (includeSuitePreparers) { 973 // Run suite level preparers. 974 preparationException = runTargetPreparation(mSuitePreparersPerDevice); 975 } 976 977 if (preparationException == null) { 978 preparationException = runTargetPreparation(mPreparersPerDevice); 979 } 980 // Skip multi-preparation if preparation already failed. 981 if (preparationException == null) { 982 for (IMultiTargetPreparer multiPreparer : mMultiPreparers) { 983 preparationException = runMultiPreparerSetup(multiPreparer); 984 if (preparationException != null) { 985 mIsFailedModule = true; 986 CLog.e("Some preparation step failed. failing the module %s", getId()); 987 break; 988 } 989 } 990 } 991 mElapsedPreparation = getCurrentTime() - prepStartTime; 992 return preparationException; 993 } 994 995 /** Run all the prepare steps. */ runPreparerSetup( ITargetPreparer preparer, int deviceIndex)996 private Throwable runPreparerSetup( 997 ITargetPreparer preparer, 998 int deviceIndex) { 999 if (preparer.isDisabled()) { 1000 // If disabled skip completely. 1001 return null; 1002 } 1003 TfObjectTracker.countWithParents(preparer.getClass()); 1004 CLog.d("Running setup preparer: %s", preparer.getClass().getSimpleName()); 1005 try (CloseableTraceScope ignored = 1006 new CloseableTraceScope(preparer.getClass().getName())) { 1007 if (preparer instanceof IConfigurationReceiver) { 1008 ((IConfigurationReceiver) preparer).setConfiguration(mModuleConfiguration); 1009 } 1010 // set the logger in case they need it. 1011 if (preparer instanceof ITestLoggerReceiver) { 1012 ((ITestLoggerReceiver) preparer).setTestLogger(mInvocationListener); 1013 } 1014 if (preparer instanceof IInvocationContextReceiver) { 1015 ((IInvocationContextReceiver) preparer) 1016 .setInvocationContext(mModuleInvocationContext); 1017 } 1018 mModuleInfo.setActiveDeviceIndex(deviceIndex); 1019 preparer.setUp(mModuleInfo); 1020 return null; 1021 } catch (BuildError 1022 | TargetSetupError 1023 | DeviceNotAvailableException 1024 | RuntimeException 1025 | AssertionError 1026 | LinkageError e) { 1027 // We catch all the TargetPreparer possible exception + RuntimeException to avoid 1028 // specific issues + AssertionError since it's widely used in tests and doesn't notify 1029 // something very wrong with the harness. 1030 CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName()); 1031 CLog.e(e); 1032 return e; 1033 } finally { 1034 mModuleInfo.setActiveDeviceIndex(0); 1035 } 1036 } 1037 1038 /** Run all multi target preparer step. */ runMultiPreparerSetup(IMultiTargetPreparer preparer)1039 private Throwable runMultiPreparerSetup(IMultiTargetPreparer preparer) { 1040 if (preparer.isDisabled()) { 1041 // If disabled skip completely. 1042 return null; 1043 } 1044 TfObjectTracker.countWithParents(preparer.getClass()); 1045 CLog.d("Running setup multi preparer: %s", preparer.getClass().getSimpleName()); 1046 try (CloseableTraceScope ignored = 1047 new CloseableTraceScope(preparer.getClass().getName())) { 1048 if (preparer instanceof IConfigurationReceiver) { 1049 ((IConfigurationReceiver) preparer).setConfiguration(mModuleConfiguration); 1050 } 1051 // set the logger in case they need it. 1052 if (preparer instanceof ITestLoggerReceiver) { 1053 ((ITestLoggerReceiver) preparer).setTestLogger(mInvocationListener); 1054 } 1055 if (preparer instanceof IInvocationContextReceiver) { 1056 ((IInvocationContextReceiver) preparer) 1057 .setInvocationContext(mModuleInvocationContext); 1058 } 1059 preparer.setUp(mModuleInfo); 1060 return null; 1061 } catch (BuildError 1062 | TargetSetupError 1063 | DeviceNotAvailableException 1064 | RuntimeException 1065 | AssertionError 1066 | LinkageError e) { 1067 // We catch all the MultiTargetPreparer possible exception + RuntimeException to avoid 1068 // specific issues + AssertionError since it's widely used in tests and doesn't notify 1069 // something very wrong with the harness. 1070 CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName()); 1071 CLog.e(e); 1072 return e; 1073 } 1074 } 1075 1076 /** Run all the tear down steps from preparers. */ runTearDown(TestInformation moduleInfo, Throwable exception)1077 private void runTearDown(TestInformation moduleInfo, Throwable exception) 1078 throws DeviceNotAvailableException { 1079 // Tear down 1080 List<IMultiTargetPreparer> cleanerList = new ArrayList<>(mMultiPreparers); 1081 Collections.reverse(cleanerList); 1082 for (IMultiTargetPreparer multiCleaner : cleanerList) { 1083 if (multiCleaner.isDisabled() || multiCleaner.isTearDownDisabled()) { 1084 // If disabled skip completely. 1085 continue; 1086 } 1087 CLog.d("Running teardown multi cleaner: %s", multiCleaner.getClass().getSimpleName()); 1088 multiCleaner.tearDown(moduleInfo, exception); 1089 } 1090 1091 for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) { 1092 String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i); 1093 ITestDevice device = mModuleInvocationContext.getDevice(deviceName); 1094 if (i >= mPreparersPerDevice.size()) { 1095 CLog.d( 1096 "Main configuration has more devices than the module configuration. '%s' " 1097 + "will not run any tear down.", 1098 deviceName); 1099 continue; 1100 } 1101 List<ITargetPreparer> preparers = mPreparersPerDevice.get(deviceName); 1102 if (preparers == null) { 1103 CLog.w( 1104 "Module configuration devices mismatch the main configuration " 1105 + "(Missing device '%s'), resolving preparers by index.", 1106 deviceName); 1107 String key = new ArrayList<>(mPreparersPerDevice.keySet()).get(i); 1108 preparers = mPreparersPerDevice.get(key); 1109 } 1110 ListIterator<ITargetPreparer> itr = preparers.listIterator(preparers.size()); 1111 while (itr.hasPrevious()) { 1112 ITargetPreparer preparer = itr.previous(); 1113 // do not call the cleaner if it was disabled 1114 if (preparer.isDisabled() || preparer.isTearDownDisabled()) { 1115 CLog.d("%s has been disabled. skipping.", preparer); 1116 continue; 1117 } 1118 1119 RecoveryMode origMode = null; 1120 try (CloseableTraceScope ignored = 1121 new CloseableTraceScope(preparer.getClass().getName())) { 1122 // If an exception was generated in setup with a DNAE do not attempt any 1123 // recovery again in case we hit the device not available again. 1124 if (exception != null && exception instanceof DeviceNotAvailableException) { 1125 origMode = device.getRecoveryMode(); 1126 device.setRecoveryMode(RecoveryMode.NONE); 1127 } 1128 moduleInfo.setActiveDeviceIndex(i); 1129 preparer.tearDown(moduleInfo, exception); 1130 } finally { 1131 moduleInfo.setActiveDeviceIndex(0); 1132 if (origMode != null) { 1133 device.setRecoveryMode(origMode); 1134 } 1135 } 1136 } 1137 } 1138 } 1139 1140 /** Verify that the device did not crash after the module. */ checkEndModuleDevice(TestInformation testInfo)1141 private void checkEndModuleDevice(TestInformation testInfo) throws DeviceNotAvailableException { 1142 if (SystemUtil.isLocalMode()) { 1143 CLog.d("Skipping check for device availability after end of module for local run."); 1144 return; 1145 } 1146 try (CloseableTraceScope check = new CloseableTraceScope("checkEndModuleDevice")) { 1147 for (ITestDevice device : testInfo.getDevices()) { 1148 if (device.getIDevice() instanceof StubDevice) { 1149 continue; 1150 } 1151 // Check device is still online 1152 try { 1153 device.waitForDeviceAvailable(); 1154 } catch (DeviceNotAvailableException e) { 1155 // Wrap exception for better message 1156 String error_msg = 1157 String.format("Device went offline after running module '%s'", mId); 1158 // TODO: If module is the last one, it won't need to do device recovery. 1159 if (!mRecoverVirtualDevice) { 1160 throw new DeviceNotAvailableException( 1161 error_msg, 1162 e, 1163 e.getSerial(), 1164 DeviceErrorIdentifier.DEVICE_UNAVAILABLE); 1165 } 1166 CLog.d(error_msg); 1167 String snapshotId = null; 1168 if (device.getConnection() instanceof AdbTcpConnection) { 1169 snapshotId = 1170 ((AdbTcpConnection) device.getConnection()) 1171 .getSuiteSnapshots() 1172 .get(device); 1173 } 1174 device.getConnection().recoverVirtualDevice(device, snapshotId, e); 1175 } 1176 } 1177 } 1178 } 1179 recoverDevice(TestInformation testInfo, DeviceNotAvailableException e)1180 private void recoverDevice(TestInformation testInfo, DeviceNotAvailableException e) 1181 throws DeviceNotAvailableException { 1182 if (SystemUtil.isLocalMode()) { 1183 CLog.d("Skipping device recovery for local run."); 1184 throw e; 1185 } 1186 if (!mRecoverVirtualDevice) { 1187 CLog.d("Skipping device recovery for as option recover-device-by-cvd is not enabled."); 1188 throw e; 1189 } 1190 try (CloseableTraceScope check = new CloseableTraceScope("recover_device")) { 1191 for (ITestDevice device : testInfo.getDevices()) { 1192 if (device.getIDevice() instanceof StubDevice) { 1193 continue; 1194 } 1195 String snapshotId = null; 1196 if (device.getConnection() instanceof AdbTcpConnection) { 1197 snapshotId = 1198 ((AdbTcpConnection) device.getConnection()) 1199 .getSuiteSnapshots() 1200 .get(device); 1201 } 1202 device.getConnection().recoverVirtualDevice(device, snapshotId, e); 1203 } 1204 } 1205 } 1206 1207 /** Returns the current time. */ getCurrentTime()1208 private long getCurrentTime() { 1209 return System.currentTimeMillis(); 1210 } 1211 1212 @Override setCollectTestsOnly(boolean collectTestsOnly)1213 public void setCollectTestsOnly(boolean collectTestsOnly) { 1214 mCollectTestsOnly = collectTestsOnly; 1215 } 1216 1217 /** Sets should recover virtual device. */ setRecoverVirtualDevice(boolean recoverVirtualDevice)1218 public void setRecoverVirtualDevice(boolean recoverVirtualDevice) { 1219 mRecoverVirtualDevice = recoverVirtualDevice; 1220 } 1221 1222 /** Returns if we should recover virtual device. */ shouldRecoverVirtualDevice()1223 public boolean shouldRecoverVirtualDevice() { 1224 return mRecoverVirtualDevice; 1225 } 1226 1227 /** Sets whether or not we should merge results. */ setMergeAttemps(boolean mergeAttempts)1228 public final void setMergeAttemps(boolean mergeAttempts) { 1229 mMergeAttempts = mergeAttempts; 1230 } 1231 1232 /** Sets the {@link IRetryDecision} to be used for intra-module retry. */ setRetryDecision(IRetryDecision decision)1233 public final void setRetryDecision(IRetryDecision decision) { 1234 mRetryDecision = decision; 1235 // Carry the retry decision to the module configuration 1236 mModuleConfiguration.setRetryDecision(decision); 1237 } 1238 1239 /** Returns a list of tests that ran in this module. */ getTestsResults()1240 List<TestRunResult> getTestsResults() { 1241 return mTestsResults; 1242 } 1243 1244 /** Returns the number of tests that was expected to be run */ getNumExpectedTests()1245 int getNumExpectedTests() { 1246 return mExpectedTests; 1247 } 1248 1249 /** Returns True if a testRunFailure has been called on the module * */ hasModuleFailed()1250 public boolean hasModuleFailed() { 1251 return mIsFailedModule; 1252 } 1253 getRequiredTokens(TestInformation testInfo)1254 public Set<TokenProperty> getRequiredTokens(TestInformation testInfo) { 1255 // If there are no controllers just return directly 1256 List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER); 1257 if (ctrlObjectList == null) { 1258 return mRequiredTokens; 1259 } 1260 // Clone the module context to get its metadata and then provide the device information 1261 // the same as ITestSuite would do during execution to run only the controllers 1262 InvocationContext clonedContext = 1263 InvocationContext.fromProto(mModuleInvocationContext.toProto()); 1264 for (String deviceName : testInfo.getContext().getDeviceConfigNames()) { 1265 clonedContext.addAllocatedDevice( 1266 deviceName, testInfo.getContext().getDevice(deviceName)); 1267 clonedContext.addDeviceBuildInfo( 1268 deviceName, testInfo.getContext().getBuildInfo(deviceName)); 1269 } 1270 try { 1271 if (!RunStrategy.RUN.equals(shouldRunWithController(clonedContext))) { 1272 // Bypass token since the module isn't expected to run 1273 return null; 1274 } 1275 } catch (RuntimeException | DeviceNotAvailableException e) { 1276 CLog.e(e); 1277 } 1278 return mRequiredTokens; 1279 } 1280 1281 /** {@inheritDoc} */ 1282 @Override toString()1283 public String toString() { 1284 return getId(); 1285 } 1286 1287 /** Returns the approximate time to run all the tests in the module. */ getRuntimeHint()1288 public long getRuntimeHint() { 1289 long hint = 0L; 1290 for (IRemoteTest test : mTests) { 1291 if (test instanceof IRuntimeHintProvider) { 1292 hint += ((IRuntimeHintProvider) test).getRuntimeHint(); 1293 } else { 1294 hint += 60000; 1295 } 1296 } 1297 return hint; 1298 } 1299 1300 /** Returns the list of {@link IRemoteTest} defined for this module. */ 1301 @VisibleForTesting getTests()1302 List<IRemoteTest> getTests() { 1303 return new ArrayList<>(mTests); 1304 } 1305 1306 /** Returns the list of {@link ITargetPreparer} associated with the given device name */ 1307 @VisibleForTesting getTargetPreparerForDevice(String deviceName)1308 List<ITargetPreparer> getTargetPreparerForDevice(String deviceName) { 1309 return mPreparersPerDevice.get(deviceName); 1310 } 1311 1312 /** 1313 * Returns the list of suite level {@link ITargetPreparer} associated with the given device name 1314 */ 1315 @VisibleForTesting getSuitePreparerForDevice(String deviceName)1316 List<ITargetPreparer> getSuitePreparerForDevice(String deviceName) { 1317 return mSuitePreparersPerDevice.get(deviceName); 1318 } 1319 1320 /** 1321 * When running unit tests for ModuleDefinition we don't want to unnecessarily report some auto 1322 * retry times. 1323 */ 1324 @VisibleForTesting disableAutoRetryReportingTime()1325 void disableAutoRetryReportingTime() { 1326 mDisableAutoRetryTimeReporting = true; 1327 } 1328 1329 /** Returns the {@link IInvocationContext} associated with the module. */ getModuleInvocationContext()1330 public IInvocationContext getModuleInvocationContext() { 1331 return mModuleInvocationContext; 1332 } 1333 getModuleConfiguration()1334 public IConfiguration getModuleConfiguration() { 1335 return mModuleConfiguration; 1336 } 1337 1338 /** Report completely not executed modules. */ reportNotExecuted(ITestInvocationListener listener, String message)1339 public final void reportNotExecuted(ITestInvocationListener listener, String message) { 1340 if (mStartModuleRunDate == null) { 1341 listener.testModuleStarted(getModuleInvocationContext()); 1342 } 1343 if (mCurrentTestWrapper != null) { 1344 mRunListenersResults.add(mCurrentTestWrapper.getResultListener()); 1345 HarnessRuntimeException interruptedException = 1346 new HarnessRuntimeException( 1347 message, TestErrorIdentifier.MODULE_DID_NOT_EXECUTE); 1348 for (int i = 0; i < mMaxRetry; i++) { 1349 // Get all the results for the attempt 1350 List<TestRunResult> runResultList = new ArrayList<TestRunResult>(); 1351 int expectedCount = 0; 1352 for (ModuleListener attemptListener : mRunListenersResults) { 1353 for (String runName : attemptListener.getTestRunNames()) { 1354 TestRunResult run = 1355 attemptListener.getTestRunAtAttempt(runName, i); 1356 if (run != null) { 1357 runResultList.add(run); 1358 expectedCount += run.getExpectedTestCount(); 1359 } 1360 } 1361 } 1362 1363 if (!runResultList.isEmpty()) { 1364 reportFinalResults( 1365 listener, 1366 expectedCount, 1367 runResultList, 1368 i, 1369 interruptedException); 1370 } else { 1371 CLog.d("No results to be forwarded for attempt %s.", i); 1372 } 1373 } 1374 } else { 1375 listener.testRunStarted( 1376 getId(), 0, mTargetPreparerRetryCount, System.currentTimeMillis()); 1377 FailureDescription description = 1378 FailureDescription.create(message) 1379 .setFailureStatus(FailureStatus.NOT_EXECUTED) 1380 .setErrorIdentifier(TestErrorIdentifier.MODULE_DID_NOT_EXECUTE); 1381 listener.testRunFailed(description); 1382 listener.testRunEnded(0, new HashMap<String, Metric>()); 1383 } 1384 listener.testModuleEnded(); 1385 } 1386 1387 /** Whether or not to enable dynamic download at module level. */ setEnableDynamicDownload(boolean enableDynamicDownload)1388 public void setEnableDynamicDownload(boolean enableDynamicDownload) { 1389 mEnableDynamicDownload = enableDynamicDownload; 1390 } 1391 1392 /** Copy a few of the suite level configuration */ transferSuiteLevelOptions(IConfiguration mSuiteConfiguration)1393 public void transferSuiteLevelOptions(IConfiguration mSuiteConfiguration) { 1394 mModuleConfiguration 1395 .getCommandOptions() 1396 .getDynamicDownloadArgs() 1397 .putAll(mSuiteConfiguration.getCommandOptions().getDynamicDownloadArgs()); 1398 mModuleConfiguration 1399 .getCommandOptions() 1400 .setReportTestCaseCount( 1401 mSuiteConfiguration.getCommandOptions().reportTestCaseCount()); 1402 } 1403 1404 /** 1405 * Allow to load module_controller objects to tune how should a particular module run. They will 1406 * be applied in order of appearance in the XML. 1407 * 1408 * @param failureListener The {@link TestFailureListener} taking actions on tests failures. 1409 * @return The strategy to use to run the tests. 1410 */ applyConfigurationControl()1411 private RunStrategy applyConfigurationControl() throws DeviceNotAvailableException { 1412 List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER); 1413 if (ctrlObjectList == null) { 1414 return RunStrategy.RUN; 1415 } 1416 for (Object ctrlObject : ctrlObjectList) { 1417 if (ctrlObject instanceof BaseModuleController) { 1418 BaseModuleController controller = (BaseModuleController) ctrlObject; 1419 // Track usage of the controller 1420 TfObjectTracker.countWithParents(controller.getClass()); 1421 if (!controller.shouldCaptureLogcat()) { 1422 mRunMetricCollectors.removeIf(c -> (c instanceof LogcatOnFailureCollector)); 1423 } 1424 if (!controller.shouldCaptureScreenshot()) { 1425 mRunMetricCollectors.removeIf(c -> (c instanceof ScreenshotOnFailureCollector)); 1426 } 1427 if (!controller.shouldCaptureBugreport()) { 1428 mRunMetricCollectors.removeIf( 1429 c -> (c instanceof BugreportzOnTestCaseFailureCollector)); 1430 } 1431 } 1432 } 1433 return shouldRunWithController(mModuleInvocationContext); 1434 } 1435 shouldRunWithController(IInvocationContext context)1436 private RunStrategy shouldRunWithController(IInvocationContext context) 1437 throws DeviceNotAvailableException { 1438 List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER); 1439 if (ctrlObjectList == null) { 1440 return RunStrategy.RUN; 1441 } 1442 // We keep the most stringent strategy across controllers. 1443 RunStrategy current = RunStrategy.RUN; 1444 for (Object ctrlObject : ctrlObjectList) { 1445 if (ctrlObject instanceof BaseModuleController) { 1446 BaseModuleController controller = (BaseModuleController) ctrlObject; 1447 RunStrategy strategy = controller.shouldRunModule(context); 1448 if (RunStrategy.FULL_MODULE_BYPASS.equals(strategy)) { 1449 current = strategy; 1450 } else if (RunStrategy.SKIP_MODULE_TESTCASES.equals(strategy) 1451 && RunStrategy.RUN.equals(current)) { 1452 current = strategy; 1453 } 1454 } 1455 } 1456 return current; 1457 } 1458 addRetryTime(long retryTimeMs)1459 private void addRetryTime(long retryTimeMs) { 1460 if (retryTimeMs <= 0 || mDisableAutoRetryTimeReporting) { 1461 return; 1462 } 1463 InvocationMetricLogger.addInvocationMetrics( 1464 InvocationMetricKey.AUTO_RETRY_TIME, retryTimeMs); 1465 } 1466 runTargetPreparation(Map<String, List<ITargetPreparer>> preparersPerDevice)1467 private Throwable runTargetPreparation(Map<String, List<ITargetPreparer>> preparersPerDevice) { 1468 Throwable preparationException = null; 1469 for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) { 1470 String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i); 1471 if (i >= preparersPerDevice.size()) { 1472 CLog.d( 1473 "Main configuration has more devices than the module configuration. '%s' " 1474 + "will not run any preparation.", 1475 deviceName); 1476 continue; 1477 } 1478 List<ITargetPreparer> preparers = preparersPerDevice.get(deviceName); 1479 if (preparers == null) { 1480 CLog.w( 1481 "Module configuration devices mismatch the main configuration " 1482 + "(Missing device '%s'), resolving preparers by index.", 1483 deviceName); 1484 String key = new ArrayList<>(preparersPerDevice.keySet()).get(i); 1485 preparers = preparersPerDevice.get(key); 1486 } 1487 for (ITargetPreparer preparer : preparers) { 1488 preparationException = runPreparerSetup(preparer, i); 1489 if (preparationException != null) { 1490 mIsFailedModule = true; 1491 CLog.e("Some preparation step failed. failing the module %s", getId()); 1492 // If one device errored out, we skip the remaining devices. 1493 return preparationException; 1494 } 1495 } 1496 } 1497 return null; 1498 } 1499 1500 /** 1501 * Handle calling the {@link IConfiguration#resolveDynamicOptions(DynamicRemoteFileResolver)}. 1502 */ invokeRemoteDynamic(ITestDevice device, IConfiguration moduleConfiguration)1503 private Exception invokeRemoteDynamic(ITestDevice device, IConfiguration moduleConfiguration) { 1504 if (!mEnableDynamicDownload) { 1505 return null; 1506 } 1507 // TODO: Add elapsed time tracking 1508 try { 1509 CLog.d("Attempting to resolve dynamic files from %s", getId()); 1510 DynamicRemoteFileResolver resolver = new DynamicRemoteFileResolver(); 1511 resolver.setDevice(device); 1512 resolver.addExtraArgs(moduleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 1513 moduleConfiguration.resolveDynamicOptions(resolver); 1514 return null; 1515 } catch (RuntimeException | ConfigurationException | BuildRetrievalError e) { 1516 mIsFailedModule = true; 1517 return e; 1518 } 1519 } 1520 1521 /** Report a setup exception as a run failure and notify all the listeners. */ reportSetupFailure( Throwable setupException, ITestInvocationListener invocListener, List<ITestInvocationListener> moduleListeners, int attemptNumber, boolean shouldFail)1522 private void reportSetupFailure( 1523 Throwable setupException, 1524 ITestInvocationListener invocListener, 1525 List<ITestInvocationListener> moduleListeners, 1526 int attemptNumber, 1527 boolean shouldFail) 1528 throws DeviceNotAvailableException { 1529 List<ITestInvocationListener> allListeners = new ArrayList<>(); 1530 allListeners.add(invocListener); 1531 if (moduleListeners != null) { 1532 allListeners.addAll(moduleListeners); 1533 } 1534 // Report the early module failures to the moduleListeners too in order for them 1535 // to know about it. 1536 ITestInvocationListener forwarder = new ResultForwarder(allListeners); 1537 // For reporting purpose we create a failure placeholder with the error stack 1538 // similar to InitializationError of JUnit. 1539 forwarder.testRunStarted(getId(), 1, attemptNumber, System.currentTimeMillis()); 1540 FailureDescription failureDescription = 1541 CurrentInvocation.createFailure(StreamUtil.getStackTrace(setupException), null); 1542 if (setupException instanceof IHarnessException 1543 && ((IHarnessException) setupException).getErrorId() != null) { 1544 ErrorIdentifier id = ((IHarnessException) setupException).getErrorId(); 1545 failureDescription.setErrorIdentifier(id); 1546 failureDescription.setFailureStatus(id.status()); 1547 failureDescription.setOrigin(((IHarnessException) setupException).getOrigin()); 1548 } else if (setupException instanceof RuntimeException) { 1549 // TODO: switch to customer_issue 1550 failureDescription.setFailureStatus(FailureStatus.UNSET); 1551 failureDescription.setErrorIdentifier( 1552 InfraErrorIdentifier.MODULE_SETUP_RUNTIME_EXCEPTION); 1553 } else { 1554 failureDescription.setFailureStatus(FailureStatus.UNSET); 1555 } 1556 failureDescription.setCause(setupException); 1557 forwarder.testRunFailed(failureDescription); 1558 HashMap<String, Metric> metricsProto = new HashMap<>(); 1559 metricsProto.put(TEST_TIME, TfMetricProtoUtil.createSingleValue(0L, "milliseconds")); 1560 forwarder.testRunEnded(0, metricsProto); 1561 // If it was a not available exception rethrow it to signal the new device state. 1562 if (setupException instanceof DeviceNotAvailableException) { 1563 if (!shouldFail) { 1564 CLog.i("Do not report the exception as module error, returning..."); 1565 return; 1566 } 1567 throw (DeviceNotAvailableException) setupException; 1568 } 1569 } 1570 applyFilterToTest(IRemoteTest test, Set<TestDescription> filters)1571 private void applyFilterToTest(IRemoteTest test, Set<TestDescription> filters) { 1572 Set<String> filterNames = 1573 filters.stream().map(f -> f.toString()).collect(Collectors.toSet()); 1574 if (test instanceof ITestFileFilterReceiver) { 1575 File excludeFilterFile = ((ITestFileFilterReceiver) test).getExcludeTestFile(); 1576 if (excludeFilterFile == null) { 1577 try { 1578 excludeFilterFile = FileUtil.createTempFile("exclude-filter", ".txt"); 1579 } catch (IOException e) { 1580 throw new HarnessRuntimeException( 1581 e.getMessage(), e, InfraErrorIdentifier.FAIL_TO_CREATE_FILE); 1582 } 1583 ((ITestFileFilterReceiver) test).setExcludeTestFile(excludeFilterFile); 1584 } 1585 try { 1586 FileUtil.writeToFile(Joiner.on('\n').join(filterNames), excludeFilterFile, true); 1587 } catch (IOException e) { 1588 CLog.e(e); 1589 } 1590 } else if (test instanceof ITestFilterReceiver) { 1591 ((ITestFilterReceiver) test).addAllExcludeFilters(filterNames); 1592 } 1593 } 1594 } 1595