1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite.retry; 17 18 import static org.junit.Assert.assertNull; 19 20 import com.android.annotations.VisibleForTesting; 21 import com.android.tradefed.config.ConfigurationException; 22 import com.android.tradefed.config.ConfigurationFactory; 23 import com.android.tradefed.config.IConfiguration; 24 import com.android.tradefed.config.IConfigurationFactory; 25 import com.android.tradefed.config.IConfigurationReceiver; 26 import com.android.tradefed.config.Option; 27 import com.android.tradefed.config.Option.Importance; 28 import com.android.tradefed.device.DeviceNotAvailableException; 29 import com.android.tradefed.device.IDeviceSelection; 30 import com.android.tradefed.invoker.TestInformation; 31 import com.android.tradefed.log.FileLogger; 32 import com.android.tradefed.log.ILeveledLogOutput; 33 import com.android.tradefed.result.CollectingTestListener; 34 import com.android.tradefed.result.ITestInvocationListener; 35 import com.android.tradefed.result.TestDescription; 36 import com.android.tradefed.result.TestResult; 37 import com.android.tradefed.result.TestRunResult; 38 import com.android.tradefed.result.TextResultReporter; 39 import com.android.tradefed.testtype.IRemoteTest; 40 import com.android.tradefed.testtype.suite.BaseTestSuite; 41 import com.android.tradefed.testtype.suite.SuiteTestFilter; 42 import com.android.tradefed.util.AbiUtils; 43 import com.android.tradefed.util.QuotationAwareTokenizer; 44 45 import java.util.ArrayList; 46 import java.util.HashSet; 47 import java.util.LinkedHashMap; 48 import java.util.LinkedHashSet; 49 import java.util.List; 50 import java.util.Map; 51 import java.util.Map.Entry; 52 import java.util.Set; 53 54 /** 55 * A special runner that allows to reschedule a previous run tests that failed or where not 56 * executed. 57 */ 58 public final class RetryRescheduler implements IRemoteTest, IConfigurationReceiver { 59 60 /** The types of the tests that can be retried. */ 61 public enum RetryType { 62 FAILED, 63 NOT_EXECUTED, 64 } 65 66 @Option( 67 name = "retry-type", 68 description = 69 "used to retry tests of a certain status. Possible values include \"failed\" " 70 + "and \"not_executed\".") 71 private RetryType mRetryType = null; 72 73 @Option( 74 name = "new-parameterized-handling", 75 description = 76 "Feature flag to test out the newer parameterized method handling for retry.") 77 private boolean mParameterizedHandling = true; 78 79 @Option( 80 name = BaseTestSuite.MODULE_OPTION, 81 shortName = BaseTestSuite.MODULE_OPTION_SHORT_NAME, 82 description = "the test module to run. Only works for configuration in the tests dir." 83 ) 84 private String mModuleName = null; 85 86 /** 87 * It's possible to add extra exclusion from the rerun. But these tests will not change their 88 * state. 89 */ 90 @Option( 91 name = BaseTestSuite.EXCLUDE_FILTER_OPTION, 92 description = "the exclude module filters to apply.", 93 importance = Importance.ALWAYS 94 ) 95 private Set<String> mExcludeFilters = new HashSet<>(); 96 97 public static final String PREVIOUS_LOADER_NAME = "previous_loader"; 98 99 private IConfiguration mConfiguration; 100 101 private IConfigurationFactory mFactory; 102 103 private IConfiguration mRescheduledConfiguration; 104 105 @Override run( TestInformation testInfo , ITestInvocationListener listener )106 public void run( 107 TestInformation testInfo /* do not use - should be null */, 108 ITestInvocationListener listener /* do not use - should be null */) 109 throws DeviceNotAvailableException { 110 assertNull(testInfo); 111 assertNull(listener); 112 113 // Get the re-loader for previous results 114 Object loader = mConfiguration.getConfigurationObject(PREVIOUS_LOADER_NAME); 115 if (loader == null) { 116 throw new RuntimeException( 117 String.format( 118 "An <object> of type %s was expected in the retry.", 119 PREVIOUS_LOADER_NAME)); 120 } 121 if (!(loader instanceof ITestSuiteResultLoader)) { 122 throw new RuntimeException( 123 String.format( 124 "%s should be implementing %s", 125 loader.getClass().getCanonicalName(), 126 ITestSuiteResultLoader.class.getCanonicalName())); 127 } 128 129 ITestSuiteResultLoader previousLoader = (ITestSuiteResultLoader) loader; 130 // First init the reloader. 131 previousLoader.init(); 132 // Then get the command line of the previous run 133 String commandLine = previousLoader.getCommandLine(); 134 IConfiguration originalConfig; 135 try { 136 originalConfig = 137 getFactory() 138 .createConfigurationFromArgs( 139 QuotationAwareTokenizer.tokenizeLine(commandLine)); 140 // Transfer the sharding options from the original command. 141 originalConfig 142 .getCommandOptions() 143 .setShardCount(mConfiguration.getCommandOptions().getShardCount()); 144 originalConfig 145 .getCommandOptions() 146 .setShardIndex(mConfiguration.getCommandOptions().getShardIndex()); 147 IDeviceSelection requirements = mConfiguration.getDeviceRequirements(); 148 // It should be safe to use the current requirements against the old config because 149 // There will be more checks like fingerprint if it was supposed to run. 150 originalConfig.setDeviceRequirements(requirements); 151 152 // Transfer log level from retry to subconfig 153 ILeveledLogOutput originalLogger = originalConfig.getLogOutput(); 154 ILeveledLogOutput retryLogger = mConfiguration.getLogOutput(); 155 originalLogger.setLogLevel(retryLogger.getLogLevel()); 156 if (originalLogger instanceof FileLogger && retryLogger instanceof FileLogger) { 157 ((FileLogger) originalLogger) 158 .setLogLevelDisplay(((FileLogger) retryLogger).getLogLevelDisplay()); 159 } 160 161 handleExtraResultReporter(originalConfig, mConfiguration); 162 } catch (ConfigurationException e) { 163 throw new RuntimeException(e); 164 } 165 // Get previous results 166 CollectingTestListener collectedTests = previousLoader.loadPreviousResults(); 167 previousLoader.cleanUp(); 168 169 // Appropriately update the configuration 170 IRemoteTest test = originalConfig.getTests().get(0); 171 if (!(test instanceof BaseTestSuite)) { 172 throw new RuntimeException( 173 "RetryScheduler only works for BaseTestSuite implementations"); 174 } 175 BaseTestSuite suite = (BaseTestSuite) test; 176 ResultsPlayer replayer = new ResultsPlayer(); 177 updateRunner(suite, collectedTests, replayer); 178 collectedTests = null; 179 updateConfiguration(originalConfig, replayer); 180 // Do the customization of the configuration for specialized use cases. 181 customizeConfig(previousLoader, originalConfig); 182 183 mRescheduledConfiguration = originalConfig; 184 } 185 186 @Override setConfiguration(IConfiguration configuration)187 public void setConfiguration(IConfiguration configuration) { 188 mConfiguration = configuration; 189 } 190 getFactory()191 private IConfigurationFactory getFactory() { 192 if (mFactory != null) { 193 return mFactory; 194 } 195 return ConfigurationFactory.getInstance(); 196 } 197 198 @VisibleForTesting setConfigurationFactory(IConfigurationFactory factory)199 void setConfigurationFactory(IConfigurationFactory factory) { 200 mFactory = factory; 201 } 202 203 /** Returns the {@link IConfiguration} that should be retried. */ getRetryConfiguration()204 public final IConfiguration getRetryConfiguration() { 205 return mRescheduledConfiguration; 206 } 207 208 /** 209 * Update the configuration to be ready for re-run. 210 * 211 * @param suite The {@link BaseTestSuite} that will be re-run. 212 * @param results The results of the previous run. 213 * @param replayer The {@link ResultsPlayer} that will replay the non-retried use cases. 214 */ updateRunner( BaseTestSuite suite, CollectingTestListener results, ResultsPlayer replayer)215 private void updateRunner( 216 BaseTestSuite suite, CollectingTestListener results, ResultsPlayer replayer) { 217 List<RetryType> types = new ArrayList<>(); 218 if (mRetryType == null) { 219 types.add(RetryType.FAILED); 220 types.add(RetryType.NOT_EXECUTED); 221 } else { 222 types.add(mRetryType); 223 } 224 225 // Expand the --module option in case no abi is specified. 226 Set<String> expandedModuleOption = new HashSet<>(); 227 if (mModuleName != null) { 228 SuiteTestFilter moduleFilter = SuiteTestFilter.createFrom(mModuleName); 229 expandedModuleOption.add(mModuleName); 230 if (moduleFilter.getAbi() == null) { 231 Set<String> abis = AbiUtils.getAbisSupportedByCompatibility(); 232 for (String abi : abis) { 233 SuiteTestFilter namingFilter = 234 new SuiteTestFilter( 235 abi, moduleFilter.getName(), moduleFilter.getTest()); 236 expandedModuleOption.add(namingFilter.toString()); 237 } 238 } 239 } 240 241 // Expand the exclude-filter in case no abi is specified. 242 Set<String> extendedExcludeRetryFilters = new HashSet<>(); 243 for (String excludeFilter : mExcludeFilters) { 244 SuiteTestFilter suiteFilter = SuiteTestFilter.createFrom(excludeFilter); 245 // Keep the current exclude-filter 246 extendedExcludeRetryFilters.add(excludeFilter); 247 if (suiteFilter.getAbi() == null) { 248 // If no abi is specified, exclude them all. 249 Set<String> abis = AbiUtils.getAbisSupportedByCompatibility(); 250 for (String abi : abis) { 251 SuiteTestFilter namingFilter = 252 new SuiteTestFilter(abi, suiteFilter.getName(), suiteFilter.getTest()); 253 extendedExcludeRetryFilters.add(namingFilter.toString()); 254 } 255 } 256 } 257 258 // Prepare exclusion filters 259 for (TestRunResult moduleResult : results.getMergedTestRunResults()) { 260 // If the module is explicitly excluded from retries, preserve the original results. 261 if (!extendedExcludeRetryFilters.contains(moduleResult.getName()) 262 && (expandedModuleOption.isEmpty() 263 || expandedModuleOption.contains(moduleResult.getName())) 264 && RetryResultHelper.shouldRunModule(moduleResult, types)) { 265 if (types.contains(RetryType.NOT_EXECUTED)) { 266 // Clear the run failure since we are attempting to rerun all non-executed 267 moduleResult.resetRunFailure(); 268 } 269 270 Map<TestDescription, TestResult> parameterizedMethods = new LinkedHashMap<>(); 271 272 for (Entry<TestDescription, TestResult> result : 273 moduleResult.getTestResults().entrySet()) { 274 if (!mParameterizedHandling) { 275 // Put aside all parameterized methods 276 if (isParameterized(result.getKey())) { 277 parameterizedMethods.put(result.getKey(), result.getValue()); 278 continue; 279 } 280 } 281 if (!RetryResultHelper.shouldRunTest(result.getValue(), types)) { 282 addExcludeToConfig(suite, moduleResult, result.getKey().toString()); 283 replayer.addToReplay( 284 results.getModuleContextForRunResult(moduleResult.getName()), 285 moduleResult, 286 result); 287 } 288 } 289 290 if (!mParameterizedHandling) { 291 // Handle parameterized methods 292 for (Entry<String, Map<TestDescription, TestResult>> subMap : 293 sortMethodToClass(parameterizedMethods).entrySet()) { 294 boolean shouldNotrerunAnything = 295 subMap.getValue().entrySet().stream() 296 .noneMatch( 297 (v) -> 298 RetryResultHelper.shouldRunTest( 299 v.getValue(), types) 300 == true); 301 // If None of the base method need to be rerun exclude it 302 if (shouldNotrerunAnything) { 303 // Exclude the base method 304 addExcludeToConfig(suite, moduleResult, subMap.getKey()); 305 // Replay all test cases 306 for (Entry<TestDescription, TestResult> result : 307 subMap.getValue().entrySet()) { 308 replayer.addToReplay( 309 results.getModuleContextForRunResult( 310 moduleResult.getName()), 311 moduleResult, 312 result); 313 } 314 } 315 } 316 } 317 } else { 318 // Exclude the module completely - it will keep its current status 319 addExcludeToConfig(suite, moduleResult, null); 320 replayer.addToReplay( 321 results.getModuleContextForRunResult(moduleResult.getName()), 322 moduleResult, 323 null); 324 } 325 } 326 } 327 328 /** Update the configuration to put the replayer before all the actual real tests. */ updateConfiguration(IConfiguration config, ResultsPlayer replayer)329 private void updateConfiguration(IConfiguration config, ResultsPlayer replayer) { 330 List<IRemoteTest> tests = config.getTests(); 331 List<IRemoteTest> newList = new ArrayList<>(); 332 // Add the replayer first to replay all the tests cases first. 333 newList.add(replayer); 334 newList.addAll(tests); 335 config.setTests(newList); 336 } 337 338 /** Allow the specialized loader to customize the config before re-running it. */ customizeConfig(ITestSuiteResultLoader loader, IConfiguration originalConfig)339 private void customizeConfig(ITestSuiteResultLoader loader, IConfiguration originalConfig) { 340 loader.customizeConfiguration(originalConfig); 341 } 342 343 /** Add the filter to the suite. */ addExcludeToConfig( BaseTestSuite suite, TestRunResult moduleResult, String testDescription)344 private void addExcludeToConfig( 345 BaseTestSuite suite, TestRunResult moduleResult, String testDescription) { 346 String filter = moduleResult.getName(); 347 if (testDescription != null) { 348 filter = String.format("%s %s", filter, testDescription); 349 } 350 SuiteTestFilter testFilter = SuiteTestFilter.createFrom(filter); 351 Set<String> excludeFilter = new LinkedHashSet<>(); 352 excludeFilter.add(testFilter.toString()); 353 suite.setExcludeFilter(excludeFilter); 354 } 355 356 /** Returns True if a test case is a parameterized one. */ isParameterized(TestDescription description)357 private boolean isParameterized(TestDescription description) { 358 return !description.getTestName().equals(description.getTestNameWithoutParams()); 359 } 360 sortMethodToClass( Map<TestDescription, TestResult> paramMethods)361 private Map<String, Map<TestDescription, TestResult>> sortMethodToClass( 362 Map<TestDescription, TestResult> paramMethods) { 363 Map<String, Map<TestDescription, TestResult>> returnMap = new LinkedHashMap<>(); 364 for (Entry<TestDescription, TestResult> entry : paramMethods.entrySet()) { 365 String noParamName = 366 String.format( 367 "%s#%s", 368 entry.getKey().getClassName(), 369 entry.getKey().getTestNameWithoutParams()); 370 Map<TestDescription, TestResult> forClass = returnMap.get(noParamName); 371 if (forClass == null) { 372 forClass = new LinkedHashMap<>(); 373 returnMap.put(noParamName, forClass); 374 } 375 forClass.put(entry.getKey(), entry.getValue()); 376 } 377 return returnMap; 378 } 379 380 /** 381 * Fetch additional result_reporter from the retry configuration and add them to the original 382 * command. This is the only allowed modification of the original command: add more result 383 * end-points. 384 */ handleExtraResultReporter( IConfiguration originalConfig, IConfiguration retryConfig)385 private void handleExtraResultReporter( 386 IConfiguration originalConfig, IConfiguration retryConfig) { 387 // Since we always have 1 default reporter, avoid carrying it for no reason. Only carry 388 // reporters if some actual ones were specified. 389 if (retryConfig.getTestInvocationListeners().size() == 1 390 && (mConfiguration.getTestInvocationListeners().get(0) 391 instanceof TextResultReporter)) { 392 return; 393 } 394 List<ITestInvocationListener> listeners = originalConfig.getTestInvocationListeners(); 395 listeners.addAll(retryConfig.getTestInvocationListeners()); 396 originalConfig.setTestInvocationListeners(listeners); 397 } 398 } 399