1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.command; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider; 20 import com.android.compatibility.common.tradefed.result.SubPlanHelper; 21 import com.android.compatibility.common.tradefed.result.suite.CertificationResultXml; 22 import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite; 23 import com.android.compatibility.common.util.ResultHandler; 24 import com.android.tradefed.build.BuildRetrievalError; 25 import com.android.tradefed.build.IBuildInfo; 26 import com.android.tradefed.command.Console; 27 import com.android.tradefed.config.ArgsOptionParser; 28 import com.android.tradefed.config.ConfigurationException; 29 import com.android.tradefed.config.ConfigurationFactory; 30 import com.android.tradefed.config.IConfiguration; 31 import com.android.tradefed.config.IConfigurationFactory; 32 import com.android.tradefed.log.LogUtil.CLog; 33 import com.android.tradefed.result.suite.SuiteResultHolder; 34 import com.android.tradefed.testtype.Abi; 35 import com.android.tradefed.testtype.IAbi; 36 import com.android.tradefed.testtype.IRemoteTest; 37 import com.android.tradefed.testtype.IRuntimeHintProvider; 38 import com.android.tradefed.testtype.suite.ITestSuite; 39 import com.android.tradefed.testtype.suite.SuiteModuleLoader; 40 import com.android.tradefed.testtype.suite.TestSuiteInfo; 41 import com.android.tradefed.testtype.suite.params.ModuleParameters; 42 import com.android.tradefed.util.AbiUtils; 43 import com.android.tradefed.util.FileUtil; 44 import com.android.tradefed.util.MultiMap; 45 import com.android.tradefed.util.Pair; 46 import com.android.tradefed.util.RegexTrie; 47 import com.android.tradefed.util.TableFormatter; 48 import com.android.tradefed.util.TimeUtil; 49 import com.android.tradefed.util.VersionParser; 50 51 import com.google.common.base.Joiner; 52 53 import java.io.File; 54 import java.io.FileNotFoundException; 55 import java.io.IOException; 56 import java.io.PrintWriter; 57 import java.util.ArrayList; 58 import java.util.Arrays; 59 import java.util.Collections; 60 import java.util.Comparator; 61 import java.util.HashSet; 62 import java.util.Iterator; 63 import java.util.LinkedHashMap; 64 import java.util.LinkedHashSet; 65 import java.util.List; 66 import java.util.Map; 67 import java.util.Set; 68 69 /** 70 * An extension of Tradefed's console which adds features specific to compatibility testing. 71 */ 72 public class CompatibilityConsole extends Console { 73 74 /** 75 * Hard coded list of modules to be excluded from manual module sharding 76 * @see #splitModules(int) 77 */ 78 private final static Set<String> MODULE_SPLIT_EXCLUSIONS = new HashSet<>(); 79 static { 80 MODULE_SPLIT_EXCLUSIONS.add("CtsDeqpTestCases"); 81 } 82 private final static String ADD_PATTERN = "a(?:dd)?"; 83 private static final String LATEST_RESULT_DIR = "latest"; 84 private CompatibilityBuildHelper mBuildHelper; 85 private IBuildInfo mBuildInfo; 86 87 /** 88 * {@inheritDoc} 89 */ 90 @Override run()91 public void run() { 92 String buildNumber = TestSuiteInfo.getInstance().getBuildNumber(); 93 String versionFile = VersionParser.fetchVersion(); 94 if (versionFile != null) { 95 buildNumber = versionFile; 96 } 97 printLine( 98 String.format( 99 "Android %s %s (%s)", 100 TestSuiteInfo.getInstance().getFullName(), 101 TestSuiteInfo.getInstance().getVersion(), 102 buildNumber)); 103 printLine("Use \"help\" or \"help all\" to get more information on running commands."); 104 super.run(); 105 } 106 107 /** 108 * Adds the 'list plans', 'list modules' and 'list results' commands 109 */ 110 @Override setCustomCommands(RegexTrie<Runnable> trie, List<String> genericHelp, Map<String, String> commandHelp)111 protected void setCustomCommands(RegexTrie<Runnable> trie, List<String> genericHelp, 112 Map<String, String> commandHelp) { 113 114 genericHelp.add("Enter 'help add' for help with 'add subplan' commands"); 115 genericHelp.add("\t----- " + TestSuiteInfo.getInstance().getFullName() + " usage ----- "); 116 genericHelp.add("Usage: run <plan> [--module <module name>] [ options ]"); 117 genericHelp.add("Example: run cts --module CtsGestureTestCases --bugreport-on-failure"); 118 genericHelp.add(""); 119 120 trie.put(new Runnable() { 121 @Override 122 public void run() { 123 listPlans(); 124 } 125 }, LIST_PATTERN, "p(?:lans)?"); 126 trie.put( 127 new Runnable() { 128 @Override 129 public void run() { 130 listModules(null); 131 } 132 }, 133 LIST_PATTERN, 134 "m(?:odules)?"); 135 trie.put( 136 new ArgRunnable<CaptureList>() { 137 @Override 138 public void run(CaptureList args) { 139 String parameter = args.get(2).get(0); 140 listModules(parameter); 141 } 142 }, 143 LIST_PATTERN, 144 "m(?:odules)?", 145 "(.*)"); 146 trie.put(new Runnable() { 147 @Override 148 public void run() { 149 listResults(); 150 } 151 }, LIST_PATTERN, "r(?:esults)?"); 152 trie.put(new Runnable() { 153 @Override 154 public void run() { 155 listSubPlans(); 156 } 157 }, LIST_PATTERN, "s(?:ubplans)?"); 158 trie.put(new ArgRunnable<CaptureList>() { 159 @Override 160 public void run(CaptureList args) { 161 // Skip 2 tokens to get past split and modules pattern 162 String arg = args.get(2).get(0); 163 int shards = Integer.parseInt(arg); 164 if (shards <= 1) { 165 printLine("number of shards should be more than 1"); 166 return; 167 } 168 splitModules(shards); 169 } 170 }, "split", "m(?:odules)?", "(\\d+)"); 171 trie.put(new ArgRunnable<CaptureList>() { 172 @Override 173 public void run(CaptureList args) { 174 // Skip 2 tokens to get past "add" and "subplan" 175 String[] flatArgs = new String[args.size() - 2]; 176 for (int i = 2; i < args.size(); i++) { 177 flatArgs[i - 2] = args.get(i).get(0); 178 } 179 addSubPlan(flatArgs); 180 } 181 }, ADD_PATTERN, "s(?:ubplan)?", null); 182 trie.put(new ArgRunnable<CaptureList>() { 183 @Override 184 public void run(CaptureList args) { 185 printLine("'add subplan' requires parameters, use 'help add' to get more details"); 186 } 187 }, ADD_PATTERN, "s(?:ubplan)?"); 188 trie.put(new Runnable() { 189 @Override 190 public void run() { 191 printLine(String.format("Android %s %s (%s)", 192 TestSuiteInfo.getInstance().getFullName(), 193 TestSuiteInfo.getInstance().getVersion(), 194 TestSuiteInfo.getInstance().getBuildNumber())); 195 } 196 }, "version"); // override tradefed 'version' command to print test suite name and version 197 198 // find existing help for 'LIST_PATTERN' commands, and append these commands help 199 String listHelp = commandHelp.get(LIST_PATTERN); 200 if (listHelp == null) { 201 // no help? Unexpected, but soldier on 202 listHelp = new String(); 203 } 204 String combinedHelp = 205 listHelp 206 + LINE_SEPARATOR 207 + "\t----- " 208 + TestSuiteInfo.getInstance().getFullName() 209 + " specific options ----- " 210 + LINE_SEPARATOR 211 + "\tp[lans] List all plans available" 212 + LINE_SEPARATOR 213 + "\tm[odules] List all modules available" 214 + LINE_SEPARATOR 215 + String.format( 216 "\tm[odules] [module parameter] List all modules matching the " 217 + "parameter. (available params: %s)", 218 Arrays.asList(ModuleParameters.values())) 219 + LINE_SEPARATOR 220 + "\tr[esults] List all results" 221 + LINE_SEPARATOR; 222 commandHelp.put(LIST_PATTERN, combinedHelp); 223 224 // Update existing RUN_PATTERN with CTS specific extra run possibilities. 225 String runHelp = commandHelp.get(RUN_PATTERN); 226 if (runHelp == null) { 227 runHelp = new String(); 228 } 229 String combinedRunHelp = runHelp + 230 LINE_SEPARATOR + 231 "\t----- " + TestSuiteInfo.getInstance().getFullName() 232 + " specific options ----- " + LINE_SEPARATOR + 233 "\t<plan> --module/-m <module> Run a test module" + LINE_SEPARATOR + 234 "\t<plan> --module/-m <module> --test/-t <test_name> Run a specific test from" + 235 " the module. Test name can be <package>.<class>, <package>.<class>#<method> or " 236 + "<native_binary_name>" + LINE_SEPARATOR + 237 "\t\tAvailable Options:" + LINE_SEPARATOR + 238 "\t\t\t--serial/-s <device_id>: The device to run the test on" + LINE_SEPARATOR + 239 "\t\t\t--abi/-a <abi> : The ABI to run the test against" + LINE_SEPARATOR + 240 "\t\t\t--logcat-on-failure : Capture logcat when a test fails" 241 + LINE_SEPARATOR + 242 "\t\t\t--bugreport-on-failure : Capture a bugreport when a test fails" 243 + LINE_SEPARATOR + 244 "\t\t\t--screenshot-on-failure: Capture a screenshot when a test fails" 245 + LINE_SEPARATOR + 246 "\t\t\t--shard-count <shards>: Shards a run into the given number of independent " + 247 "chunks, to run on multiple devices in parallel." + LINE_SEPARATOR + 248 "\t ----- In order to retry a previous run -----" + LINE_SEPARATOR + 249 "\tretry --retry <session id to retry> [--retry-type <FAILED | NOT_EXECUTED>]" 250 + LINE_SEPARATOR + 251 "\t\tWithout --retry-type, retry will run both FAIL and NOT_EXECUTED tests" 252 + LINE_SEPARATOR; 253 commandHelp.put(RUN_PATTERN, combinedRunHelp); 254 255 commandHelp.put(ADD_PATTERN, String.format( 256 "%s help:" + LINE_SEPARATOR + 257 "\tadd s[ubplan]: create a subplan from a previous session" + LINE_SEPARATOR + 258 "\t\tAvailable Options:" + LINE_SEPARATOR + 259 "\t\t\t--session <session_id>: The session used to create a subplan" 260 + LINE_SEPARATOR + 261 "\t\t\t--name/-n <subplan_name>: The name of the new subplan" + LINE_SEPARATOR + 262 "\t\t\t--result-type <status>: Which results to include in the subplan. " 263 + "One of passed, failed, not_executed. Repeatable" + LINE_SEPARATOR, 264 ADD_PATTERN)); 265 } 266 267 /** 268 * {@inheritDoc} 269 */ 270 @Override getConsolePrompt()271 protected String getConsolePrompt() { 272 return String.format("%s-tf > ", TestSuiteInfo.getInstance().getName().toLowerCase()); 273 } 274 275 /** 276 * List all the modules available in the suite, if a specific parameter is requested, only 277 * display that one. 278 * 279 * @param moduleParameter The parameter requested to be displayed. Null if all should be shown. 280 */ listModules(String moduleParameter)281 private void listModules(String moduleParameter) { 282 CompatibilityTestSuite test = new CompatibilityTestSuite(); 283 Set<String> abiStrings = ITestSuite.getAbisForBuildTargetArchFromSuite(); 284 Set<IAbi> abis = new LinkedHashSet<>(); 285 for (String abi : abiStrings) { 286 if (AbiUtils.isAbiSupportedByCompatibility(abi)) { 287 abis.add(new Abi(abi, AbiUtils.getBitness(abi))); 288 } 289 } 290 test.setAbis(abis); 291 if (getBuild() != null) { 292 test.setEnableParameterizedModules(true); 293 test.setEnableOptionalParameterizedModules(true); 294 if (moduleParameter != null) { 295 test.setModuleParameter(ModuleParameters.valueOf(moduleParameter.toUpperCase())); 296 } 297 test.setBuild(getBuild()); 298 LinkedHashMap<String, IConfiguration> configs = test.loadTests(); 299 printLine(String.format("%s", Joiner.on("\n").join(configs.keySet()))); 300 } else { 301 printLine("Error fetching information about modules."); 302 } 303 } 304 listPlans()305 private void listPlans() { 306 printLine("Available plans include:"); 307 ConfigurationFactory.getInstance().printHelp(System.out); 308 } 309 splitModules(int shards)310 private void splitModules(int shards) { 311 File[] files = null; 312 try { 313 files = getBuildHelper().getTestsDir().listFiles(new SuiteModuleLoader.ConfigFilter()); 314 } catch (FileNotFoundException e) { 315 printLine(e.getMessage()); 316 e.printStackTrace(); 317 } 318 // parse through all config files to get runtime hints 319 if (files != null && files.length > 0) { 320 IConfigurationFactory configFactory = ConfigurationFactory.getInstance(); 321 List<Pair<String, Long>> moduleRuntime = new ArrayList<>(); 322 // parse through all config files to calculate module execution time 323 for (File file : files) { 324 IConfiguration config = null; 325 String moduleName = file.getName().split("\\.")[0]; 326 if (MODULE_SPLIT_EXCLUSIONS.contains(moduleName)) { 327 continue; 328 } 329 try { 330 config = configFactory.createConfigurationFromArgs(new String[]{ 331 file.getAbsolutePath(), 332 }); 333 } catch (ConfigurationException ce) { 334 printLine("Error loading config file: " + file.getAbsolutePath()); 335 CLog.e(ce); 336 continue; 337 } 338 long runtime = 0; 339 for (IRemoteTest test : config.getTests()) { 340 if (test instanceof IRuntimeHintProvider) { 341 runtime += ((IRuntimeHintProvider) test).getRuntimeHint(); 342 } else { 343 CLog.w("Using default 1m runtime estimation for test type %s", 344 test.getClass().getSimpleName()); 345 runtime += 60 * 1000; 346 } 347 } 348 moduleRuntime.add(new Pair<String, Long>(moduleName, runtime)); 349 } 350 // sort list modules in descending order of runtime hint 351 Collections.sort(moduleRuntime, new Comparator<Pair<String, Long>>() { 352 @Override 353 public int compare(Pair<String, Long> o1, Pair<String, Long> o2) { 354 return o2.second.compareTo(o1.second); 355 } 356 }); 357 // partition list of modules based on the runtime hint 358 List<List<Pair<String, Long>>> splittedModules = new ArrayList<>(); 359 for (int i = 0; i < shards; i++) { 360 splittedModules.add(new ArrayList<>()); 361 } 362 int shardIndex = 0; 363 int increment = 1; 364 long[] shardTimes = new long[shards]; 365 // go through the sorted list, distribute modules into shards in zig-zag pattern to get 366 // an even execution time among shards 367 for (Pair<String, Long> module : moduleRuntime) { 368 splittedModules.get(shardIndex).add(module); 369 // also collect total runtime per shard 370 shardTimes[shardIndex] += module.second; 371 shardIndex += increment; 372 // zig-zagging: first distribute modules from shard 0 to N, then N down to 0, repeat 373 if (shardIndex == shards) { 374 increment = -1; 375 shardIndex = shards - 1; 376 } 377 if (shardIndex == -1) { 378 increment = 1; 379 shardIndex = 0; 380 } 381 } 382 shardIndex = 0; 383 // print the final shared lists 384 for (List<Pair<String, Long>> shardedModules : splittedModules) { 385 StringBuilder lineBuffer = new StringBuilder(); 386 lineBuffer.append(String.format("shard #%d (%s):", 387 shardIndex, TimeUtil.formatElapsedTime(shardTimes[shardIndex]))); 388 Iterator<Pair<String, Long>> itr = shardedModules.iterator(); 389 lineBuffer.append(itr.next().first); 390 while (itr.hasNext()) { 391 lineBuffer.append(','); 392 lineBuffer.append(itr.next().first); 393 } 394 shardIndex++; 395 printLine(lineBuffer.toString()); 396 } 397 } else { 398 printLine("No modules found"); 399 } 400 } 401 listResults()402 private void listResults() { 403 TableFormatter tableFormatter = new TableFormatter(); 404 List<List<String>> table = new ArrayList<>(); 405 406 List<File> resultDirs = null; 407 Map<SuiteResultHolder, File> holders = new LinkedHashMap<>(); 408 try { 409 resultDirs = getResults(getBuildHelper().getResultsDir()); 410 } catch (FileNotFoundException e) { 411 throw new RuntimeException("Error while parsing results directory", e); 412 } 413 CertificationResultXml xmlParser = new CertificationResultXml(); 414 for (File resultDir : resultDirs) { 415 if (LATEST_RESULT_DIR.equals(resultDir.getName())) { 416 continue; 417 } 418 try { 419 holders.put(xmlParser.parseResults(resultDir, true), resultDir); 420 } catch (IOException e) { 421 e.printStackTrace(); 422 } 423 } 424 425 if (holders.isEmpty()) { 426 printLine(String.format("No results found")); 427 return; 428 } 429 int i = 0; 430 for (SuiteResultHolder holder : holders.keySet()) { 431 String moduleProgress = String.format("%d of %d", 432 holder.completeModules, holder.totalModules); 433 434 table.add( 435 Arrays.asList( 436 Integer.toString(i), 437 Long.toString(holder.passedTests), 438 Long.toString(holder.failedTests), 439 moduleProgress, 440 holders.get(holder).getName(), 441 holder.context 442 .getAttributes() 443 .get(CertificationResultXml.SUITE_PLAN_ATTR) 444 .get(0), 445 Joiner.on(", ").join(holder.context.getShardsSerials().values()), 446 printAttributes(holder.context.getAttributes(), "build_id"), 447 printAttributes(holder.context.getAttributes(), "build_product"))); 448 i++; 449 } 450 451 // add the table header to the beginning of the list 452 table.add(0, Arrays.asList("Session", "Pass", "Fail", "Modules Complete", 453 "Result Directory", "Test Plan", "Device serial(s)", "Build ID", "Product")); 454 tableFormatter.displayTable(table, new PrintWriter(System.out, true)); 455 } 456 printAttributes(MultiMap<String, String> map, String key)457 private String printAttributes(MultiMap<String, String> map, String key) { 458 if (map.get(key) == null) { 459 return "unknown"; 460 } 461 return map.get(key).get(0); 462 } 463 464 /** 465 * Returns the list of all results directories. 466 */ getResults(File resultsDir)467 private List<File> getResults(File resultsDir) { 468 return ResultHandler.getResultDirectories(resultsDir); 469 } 470 listSubPlans()471 private void listSubPlans() { 472 File[] files = null; 473 try { 474 files = getBuildHelper().getSubPlansDir().listFiles(); 475 } catch (FileNotFoundException e) { 476 printLine(e.getMessage()); 477 e.printStackTrace(); 478 } 479 if (files != null && files.length > 0) { 480 List<String> subPlans = new ArrayList<>(); 481 for (File subPlanFile : files) { 482 subPlans.add(FileUtil.getBaseName(subPlanFile.getName())); 483 } 484 Collections.sort(subPlans); 485 for (String subPlan : subPlans) { 486 printLine(subPlan); 487 } 488 } else { 489 printLine("No subplans found"); 490 } 491 } 492 addSubPlan(String[] flatArgs)493 private void addSubPlan(String[] flatArgs) { 494 SubPlanHelper creator = new SubPlanHelper(); 495 try { 496 ArgsOptionParser optionParser = new ArgsOptionParser(creator); 497 optionParser.parse(Arrays.asList(flatArgs)); 498 creator.createAndSerializeSubPlan(getBuildHelper()); 499 } catch (ConfigurationException e) { 500 printLine("Error: " + e.getMessage()); 501 printLine(ArgsOptionParser.getOptionHelp(false, creator)); 502 } 503 504 } 505 getBuildHelper()506 private CompatibilityBuildHelper getBuildHelper() { 507 if (mBuildHelper == null) { 508 IBuildInfo build = getBuild(); 509 if (build == null) { 510 return null; 511 } 512 mBuildHelper = new CompatibilityBuildHelper(build); 513 } 514 return mBuildHelper; 515 } 516 getBuild()517 private IBuildInfo getBuild() { 518 if (mBuildInfo == null) { 519 try { 520 CompatibilityBuildProvider buildProvider = new CompatibilityBuildProvider(); 521 mBuildInfo = buildProvider.getBuild(); 522 } catch (BuildRetrievalError e) { 523 e.printStackTrace(); 524 } 525 } 526 return mBuildInfo; 527 } 528 main(String[] args)529 public static void main(String[] args) throws InterruptedException, ConfigurationException { 530 Console console = new CompatibilityConsole(); 531 Console.startConsole(console, args); 532 } 533 } 534