1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.tradefed.config.ConfigurationException; 19 import com.android.tradefed.config.IConfiguration; 20 import com.android.tradefed.config.IConfigurationReceiver; 21 import com.android.tradefed.config.Option; 22 import com.android.tradefed.config.Option.Importance; 23 import com.android.tradefed.config.OptionClass; 24 import com.android.tradefed.config.OptionCopier; 25 import com.android.tradefed.device.DeviceNotAvailableException; 26 import com.android.tradefed.device.ITestDevice; 27 import com.android.tradefed.device.metric.CollectorHelper; 28 import com.android.tradefed.device.metric.IMetricCollector; 29 import com.android.tradefed.device.metric.IMetricCollectorReceiver; 30 import com.android.tradefed.error.HarnessRuntimeException; 31 import com.android.tradefed.invoker.TestInformation; 32 import com.android.tradefed.log.LogUtil.CLog; 33 import com.android.tradefed.result.ITestInvocationListener; 34 import com.android.tradefed.result.TestDescription; 35 import com.android.tradefed.result.TestRunResult; 36 import com.android.tradefed.result.TestStatus; 37 import com.android.tradefed.result.error.DeviceErrorIdentifier; 38 import com.android.tradefed.result.error.InfraErrorIdentifier; 39 import com.android.tradefed.testtype.retry.IAutoRetriableTest; 40 import com.android.tradefed.util.AbiFormatter; 41 import com.android.tradefed.util.CommandResult; 42 import com.android.tradefed.util.CommandStatus; 43 import com.android.tradefed.util.FileUtil; 44 import com.android.tradefed.util.ListInstrumentationParser; 45 import com.android.tradefed.util.ListInstrumentationParser.InstrumentationTarget; 46 47 import java.io.File; 48 import java.io.IOException; 49 import java.util.ArrayList; 50 import java.util.Arrays; 51 import java.util.Collection; 52 import java.util.HashMap; 53 import java.util.LinkedHashSet; 54 import java.util.LinkedList; 55 import java.util.List; 56 import java.util.Map; 57 import java.util.Set; 58 59 /** Runs all instrumentation found on current device. */ 60 @OptionClass(alias = "installed-instrumentation") 61 public class InstalledInstrumentationsTest 62 implements IDeviceTest, 63 IShardableTest, 64 IMetricCollectorReceiver, 65 IAutoRetriableTest, 66 IConfigurationReceiver { 67 68 private static final String PM_LIST_CMD = "pm list instrumentation"; 69 private static final String LINE_SEPARATOR = "\\r?\\n"; 70 71 private ITestDevice mDevice; 72 73 @Option(name = "shell-timeout", 74 description="The defined timeout (in milliseconds) is used as a maximum waiting time " 75 + "when expecting the command output from the device. At any time, if the " 76 + "shell command does not output anything for a period longer than defined " 77 + "timeout the TF run terminates. For no timeout, set to 0.") 78 private long mShellTimeout = 10 * 60 * 1000; // default to 10 minutes 79 80 @Option(name = "test-timeout", 81 description="Sets timeout (in milliseconds) that will be applied to each test. In the " 82 + "event of a test timeout it will log the results and proceed with executing " 83 + "the next test. For no timeout, set to 0.") 84 private long mTestTimeout = 5 * 60 * 1000; // default to 5 minutes 85 86 @Option(name = "size", 87 description = "Restrict tests to a specific test size. " + 88 "One of 'small', 'medium', 'large'", 89 importance = Importance.IF_UNSET) 90 private String mTestSize = null; 91 92 @Option(name = "runner", 93 description = "Restrict tests executed to a specific instrumentation class runner. " + 94 "Installed instrumentations that do not have this runner will be skipped.") 95 private String mRunner = null; 96 97 @Option(name = "rerun", 98 description = "Rerun unexecuted tests individually on same device if test run " + 99 "fails to complete.") 100 private boolean mIsRerunMode = true; 101 102 /** @deprecated delete when we are sure it's not used anywhere. */ 103 @Deprecated 104 @Option(name = "send-coverage", description = "Send coverage target info to test listeners.") 105 private boolean mSendCoverage = false; 106 107 /** @deprecated delete when we are sure it's not used anywhere. */ 108 @Deprecated 109 @Option(name = "screenshot-on-failure", description = "Take a screenshot on every test failure") 110 private boolean mScreenshotOnFailure = false; 111 112 /** @deprecated delete when we are sure it's not used anywhere. */ 113 @Deprecated 114 @Option(name = "logcat-on-failure", description = 115 "take a logcat snapshot on every test failure.") 116 private boolean mLogcatOnFailures = false; 117 118 @Option(name = "class", 119 description = "Only run tests in specified class") 120 private String mTestClass = null; 121 122 @Option(name = "package", 123 description = 124 "Only run tests within this specific java package. Will be ignored if --class is set.") 125 private String mTestPackageName = null; 126 127 @Option(name = "instrumentation-arg", 128 description = "Additional instrumentation arguments to provide.") 129 private Map<String, String> mInstrArgMap = new HashMap<String, String>(); 130 131 @Option( 132 name = "rerun-from-file", 133 description = 134 "Use test file instead of separate adb commands for each test " 135 + "when re-running instrumentations for tests that failed to run in " 136 + "previous attempts. " 137 ) 138 private boolean mReRunUsingTestFile = false; 139 140 @Option(name = "rerun-from-file-attempts", description = 141 "Max attempts to rerun tests from file. -1 means rerun from file infinitely.") 142 private int mReRunUsingTestFileAttempts = -1; 143 144 @Option(name = "disable", description = 145 "Disable the test by setting this flag to true.") 146 private boolean mDisable = false; 147 148 @Option( 149 name = "coverage", 150 description = 151 "Collect code coverage for this test run. Note that the build under test must be a " 152 + "coverage build or else this will fail." 153 ) 154 private boolean mCoverage = false; 155 156 @Option( 157 name = "hidden-api-checks", 158 description = 159 "If set to false, the '--no-hidden-api-checks' flag will be passed to the am " 160 + "instrument command. Only works for P or later." 161 ) 162 private boolean mHiddenApiChecks = true; 163 164 @Option( 165 name = "test-api-access", 166 description = 167 "If set to false and hidden API checks are enabled, the '--no-test-api-access'" 168 + " flag will be passed to the am instrument command." 169 + " Only works for R or later.") 170 private boolean mTestApiAccess = true; 171 172 @Option( 173 name = "isolated-storage", 174 description = 175 "If set to false, the '--no-isolated-storage' flag will be passed to the am " 176 + "instrument command. Only works for Q or later." 177 ) 178 private boolean mIsolatedStorage = true; 179 180 @Option( 181 name = "window-animation", 182 description = 183 "If set to false, the '--no-window-animation' flag will be passed to the am " 184 + "instrument command. Only works for ICS or later." 185 ) 186 private boolean mWindowAnimation = true; 187 188 @Option( 189 name = "disable-duplicate-test-check", 190 description = 191 "If set to true, it will not check that a method is only run once by a " 192 + "given instrumentation.") 193 private boolean mDisableDuplicateCheck = false; 194 195 @Option( 196 name = "create-instrumentation-tests", 197 description = 198 "Create InstrumentationTest type rather than more recent AndroidJUnitTest.") 199 private boolean mDowngradeInstrumentation = false; 200 201 @Option( 202 name = "test-storage-dir", 203 description = "The device directory path where test storage read files.") 204 private String mTestStorageInternalDir = "/sdcard/googletest/test_runfiles"; 205 206 @Option( 207 name = "use-test-storage", 208 description = 209 "If set to true, we will push filters to the test storage instead of disk.") 210 private boolean mUseTestStorage = true; 211 212 private int mTotalShards = 0; 213 private int mShardIndex = 0; 214 private List<IMetricCollector> mMetricCollectorList = new ArrayList<>(); 215 private IConfiguration mConfiguration; 216 217 private List<InstrumentationTest> mTests = null; 218 private Map<String, Set<TestDescription>> mRunTestsFailureMap = null; 219 220 @Option(name = AbiFormatter.FORCE_ABI_STRING, 221 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 222 importance = Importance.IF_UNSET) 223 private String mForceAbi = null; 224 225 @Override shouldRetry( int attemptJustExecuted, List<TestRunResult> previousResults, Set<String> skipList)226 public boolean shouldRetry( 227 int attemptJustExecuted, List<TestRunResult> previousResults, Set<String> skipList) 228 throws DeviceNotAvailableException { 229 boolean retry = false; 230 if (mRunTestsFailureMap == null) { 231 mRunTestsFailureMap = new HashMap<>(); 232 } 233 for (TestRunResult run : previousResults) { 234 if (run == null) { 235 continue; 236 } 237 if (run.isRunFailure() || run.hasFailedTests()) { 238 Set<TestDescription> excludes = 239 new LinkedHashSet<>( 240 run.getTestsInState( 241 Arrays.asList( 242 TestStatus.PASSED, 243 TestStatus.ASSUMPTION_FAILURE, 244 TestStatus.IGNORED))); 245 if (mRunTestsFailureMap.get(run.getName()) != null) { 246 excludes.addAll(mRunTestsFailureMap.get(run.getName())); 247 } 248 Set<TestDescription> skipListDescriptor = convertStringToDescription(skipList); 249 // Complete the excludes with skip list 250 excludes.addAll(skipListDescriptor); 251 // Exclude passed tests from rerunning 252 retry = shouldRetry(run, skipListDescriptor); 253 mRunTestsFailureMap.put(run.getName(), excludes); 254 } else { 255 // Set null if we should not rerun it 256 mRunTestsFailureMap.put(run.getName(), null); 257 } 258 } 259 260 if (!retry) { 261 CLog.d("No test run or test case failures. No need to retry."); 262 mRunTestsFailureMap = null; 263 } 264 return retry; 265 } 266 convertStringToDescription(Set<String> skipList)267 private Set<TestDescription> convertStringToDescription(Set<String> skipList) { 268 Set<TestDescription> descriptions = new LinkedHashSet<TestDescription>(); 269 for (String s : skipList) { 270 String[] classMethod = s.split("#", 2); 271 descriptions.add(new TestDescription(classMethod[0], classMethod[1])); 272 } 273 return descriptions; 274 } 275 shouldRetry(TestRunResult run, Set<TestDescription> skipList)276 private boolean shouldRetry(TestRunResult run, Set<TestDescription> skipList) { 277 if (run.isRunFailure()) { 278 return true; 279 } 280 Set<TestDescription> failedTests = run.getFailedTests(); 281 failedTests.removeAll(skipList); 282 if (failedTests.isEmpty()) { 283 return false; 284 } 285 return true; 286 } 287 288 @Override setConfiguration(IConfiguration configuration)289 public void setConfiguration(IConfiguration configuration) { 290 mConfiguration = configuration; 291 } 292 293 /** 294 * {@inheritDoc} 295 */ 296 @Override getDevice()297 public ITestDevice getDevice() { 298 return mDevice; 299 } 300 301 /** 302 * {@inheritDoc} 303 */ 304 @Override setDevice(ITestDevice device)305 public void setDevice(ITestDevice device) { 306 mDevice = device; 307 } 308 309 /** 310 * Gets the list of {@link InstrumentationTest}s contained within. 311 * <p/> 312 * Exposed for unit testing. 313 */ getTests()314 List<InstrumentationTest> getTests() { 315 return mTests; 316 } 317 318 /** 319 * Sets the number of total shards this test should be split into. 320 * <p/> 321 * Exposed for unit testing. 322 */ setTotalShards(int totalShards)323 void setTotalShards(int totalShards) { 324 mTotalShards = totalShards; 325 } 326 327 /** 328 * Sets the shard index number of this test. 329 * <p/> 330 * Exposed for unit testing. 331 */ setShardIndex(int shardIndex)332 void setShardIndex(int shardIndex) { 333 mShardIndex = shardIndex; 334 } 335 336 /** {@inheritDoc} */ 337 @Override run(TestInformation testInfo, ITestInvocationListener listener)338 public void run(TestInformation testInfo, ITestInvocationListener listener) 339 throws DeviceNotAvailableException { 340 if (getDevice() == null) { 341 throw new IllegalArgumentException("Device has not been set"); 342 } 343 344 if (mDisable) { 345 return; 346 } 347 buildTests(); 348 doRun(testInfo, listener); 349 } 350 351 /** 352 * Build the list of tests to run from the device, if not done already. Note: Can be called 353 * multiple times in case of resumed runs. 354 * @throws DeviceNotAvailableException 355 */ buildTests()356 private void buildTests() throws DeviceNotAvailableException { 357 if (mTests == null) { 358 359 ListInstrumentationParser parser = new ListInstrumentationParser(); 360 CommandResult pmListResult = getDevice().executeShellV2Command(PM_LIST_CMD); 361 if (!CommandStatus.SUCCESS.equals(pmListResult.getStatus())) { 362 throw new HarnessRuntimeException( 363 String.format( 364 "Failed to execute '%s'." + "stdout: %s\nstderr: %s", 365 PM_LIST_CMD, pmListResult.getStdout(), pmListResult.getStderr()), 366 DeviceErrorIdentifier.DEVICE_UNEXPECTED_RESPONSE); 367 } 368 String pmListOutput = pmListResult.getStdout(); 369 String pmListLines[] = pmListOutput.split(LINE_SEPARATOR); 370 parser.processNewLines(pmListLines); 371 372 if (parser.getInstrumentationTargets().isEmpty()) { 373 throw new HarnessRuntimeException( 374 String.format( 375 "No instrumentations were found on device %s - <%s>", 376 getDevice().getSerialNumber(), pmListOutput), 377 DeviceErrorIdentifier.DEVICE_UNEXPECTED_RESPONSE); 378 } 379 380 int numUnshardedTests = 0; 381 mTests = new LinkedList<InstrumentationTest>(); 382 for (InstrumentationTarget target : parser.getInstrumentationTargets()) { 383 if (mRunner == null || mRunner.equals(target.runnerName)) { 384 // Some older instrumentations are not shardable. As a result, we should try to 385 // shard these instrumentations by APKs, rather than by test methods. 386 if (mTotalShards > 0 && !target.isShardable()) { 387 numUnshardedTests += 1; 388 if ((numUnshardedTests - 1) % mTotalShards != mShardIndex) { 389 continue; 390 } 391 } 392 List<IMetricCollector> collectors = 393 CollectorHelper.cloneCollectors(mMetricCollectorList); 394 InstrumentationTest t = createInstrumentationTest(); 395 try { 396 // Copies all current argument values to the new runner that will be 397 // used to actually run the tests. 398 OptionCopier.copyOptions(InstalledInstrumentationsTest.this, t); 399 } catch (ConfigurationException e) { 400 // Bail out rather than run tests with unexpected options 401 throw new RuntimeException("failed to copy instrumentation options", e); 402 } 403 // Pass the collectors to each instrumentation, which will take care of init 404 t.setMetricCollectors(collectors); 405 String targetPackageName = target.packageName; 406 t.setPackageName(targetPackageName); 407 if (mRunTestsFailureMap != null) { 408 // Don't retry if there was no failure in a particular instrumentation. 409 if (mRunTestsFailureMap.containsKey(targetPackageName)) { 410 if (mRunTestsFailureMap.get(targetPackageName) == null) { 411 CLog.d("Skipping %s at retry, nothing to do.", targetPackageName); 412 continue; 413 } 414 // if possible reduce the scope of the retry to be more efficient. 415 if (t instanceof AndroidJUnitTest) { 416 AndroidJUnitTest filterable = (AndroidJUnitTest) t; 417 excludePassedTests( 418 filterable, mRunTestsFailureMap.get(targetPackageName)); 419 } 420 } 421 } 422 t.setRunnerName(target.runnerName); 423 t.setCoverageTarget(target.targetName); 424 if (mTotalShards > 0 && target.isShardable()) { 425 t.addInstrumentationArg("shardIndex", Integer.toString(mShardIndex)); 426 t.addInstrumentationArg("numShards", Integer.toString(mTotalShards)); 427 } 428 mTests.add(t); 429 } 430 } 431 } 432 } 433 excludePassedTests(ITestFilterReceiver test, Set<TestDescription> passedTests)434 private void excludePassedTests(ITestFilterReceiver test, Set<TestDescription> passedTests) { 435 // Exclude all passed tests for the retry. 436 for (TestDescription testCase : passedTests) { 437 String filter = String.format("%s#%s", testCase.getClassName(), testCase.getTestName()); 438 if (!(test instanceof ITestFileFilterReceiver)) { 439 test.addExcludeFilter(filter); 440 continue; 441 } 442 443 File excludeFilterFile = ((ITestFileFilterReceiver) test).getExcludeTestFile(); 444 if (excludeFilterFile == null) { 445 try { 446 excludeFilterFile = FileUtil.createTempFile("exclude-filter", ".txt"); 447 } catch (IOException e) { 448 throw new HarnessRuntimeException( 449 e.getMessage(), e, InfraErrorIdentifier.FAIL_TO_CREATE_FILE); 450 } 451 ((ITestFileFilterReceiver) test).setExcludeTestFile(excludeFilterFile); 452 } 453 try { 454 FileUtil.writeToFile(filter + "\n", excludeFilterFile, true); 455 } catch (IOException e) { 456 CLog.e(e); 457 continue; 458 } 459 } 460 } 461 462 /** 463 * Run the previously built tests. 464 * 465 * @param testInfo the {@link TestInformation} of the invocation. 466 * @param listener the {@link ITestInvocationListener} 467 * @throws DeviceNotAvailableException 468 */ doRun(TestInformation testInfo, ITestInvocationListener listener)469 private void doRun(TestInformation testInfo, ITestInvocationListener listener) 470 throws DeviceNotAvailableException { 471 while (!mTests.isEmpty()) { 472 InstrumentationTest test = mTests.get(0); 473 if (test instanceof ITestFilterReceiver) { 474 mConfiguration.getGlobalFilters().applyFiltersToTest((ITestFilterReceiver) test); 475 } 476 477 CLog.d("Running test %s on %s", test.getPackageName(), getDevice().getSerialNumber()); 478 479 test.setDevice(getDevice()); 480 test.setConfiguration(mConfiguration); 481 if (mTestClass != null) { 482 test.setClassName(mTestClass); 483 if (mTestPackageName != null) { 484 CLog.e( 485 "Ignoring --package option with value '%s' since it's incompatible " 486 + "with using --class at the same time.", 487 mTestPackageName); 488 } 489 } else if (mTestPackageName != null) { 490 test.setTestPackageName(mTestPackageName); 491 } 492 test.run(testInfo, listener); 493 // test completed, remove from list 494 mTests.remove(0); 495 } 496 mTests = null; 497 } 498 getShellTimeout()499 long getShellTimeout() { 500 return mShellTimeout; 501 } 502 getTestTimeout()503 long getTestTimeout() { 504 return mTestTimeout; 505 } 506 getTestSize()507 String getTestSize() { 508 return mTestSize; 509 } 510 511 /** 512 * Creates the {@link InstrumentationTest} to use. Exposed for unit testing. 513 */ createInstrumentationTest()514 InstrumentationTest createInstrumentationTest() { 515 // We do not know what kind of instrumentation we will find, so we don't enforce the ddmlib 516 // format for AndroidJUnitRunner. 517 InstrumentationTest test = null; 518 if (mDowngradeInstrumentation) { 519 test = new InstrumentationTest(); 520 } else { 521 test = new AndroidJUnitTest(); 522 } 523 test.setEnforceFormat(false); 524 return test; 525 } 526 527 @Override setMetricCollectors(List<IMetricCollector> collectors)528 public void setMetricCollectors(List<IMetricCollector> collectors) { 529 mMetricCollectorList = collectors; 530 } 531 532 /** {@inheritDoc} */ 533 @Override split(int shardCountHint)534 public Collection<IRemoteTest> split(int shardCountHint) { 535 if (shardCountHint > 1) { 536 Collection<IRemoteTest> shards = new ArrayList<>(shardCountHint); 537 for (int index = 0; index < shardCountHint; index++) { 538 shards.add(getTestShard(shardCountHint, index)); 539 } 540 return shards; 541 } 542 return null; 543 } 544 getTestShard(int shardCount, int shardIndex)545 private IRemoteTest getTestShard(int shardCount, int shardIndex) { 546 InstalledInstrumentationsTest shard = new InstalledInstrumentationsTest(); 547 try { 548 OptionCopier.copyOptions(this, shard); 549 } catch (ConfigurationException e) { 550 CLog.e("failed to copy instrumentation options: %s", e.getMessage()); 551 } 552 shard.mShardIndex = shardIndex; 553 shard.mTotalShards = shardCount; 554 return shard; 555 } 556 } 557