1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.cluster; 18 19 import com.android.annotations.VisibleForTesting; 20 import com.android.helper.aoa.UsbDevice; 21 import com.android.helper.aoa.UsbHelper; 22 import com.android.tradefed.config.GlobalConfiguration; 23 import com.android.tradefed.config.IConfiguration; 24 import com.android.tradefed.config.IConfigurationReceiver; 25 import com.android.tradefed.config.Option; 26 import com.android.tradefed.config.OptionClass; 27 import com.android.tradefed.device.DeviceNotAvailableException; 28 import com.android.tradefed.device.ITestDevice; 29 import com.android.tradefed.device.LocalAndroidVirtualDevice; 30 import com.android.tradefed.invoker.IInvocationContext; 31 import com.android.tradefed.invoker.TestInformation; 32 import com.android.tradefed.log.LogUtil.CLog; 33 import com.android.tradefed.result.ITestInvocationListener; 34 import com.android.tradefed.testtype.IInvocationContextReceiver; 35 import com.android.tradefed.testtype.IRemoteTest; 36 import com.android.tradefed.util.ArrayUtil; 37 import com.android.tradefed.util.CommandResult; 38 import com.android.tradefed.util.CommandStatus; 39 import com.android.tradefed.util.FileIdleMonitor; 40 import com.android.tradefed.util.FileUtil; 41 import com.android.tradefed.util.IRunUtil; 42 import com.android.tradefed.util.QuotationAwareTokenizer; 43 import com.android.tradefed.util.RunUtil; 44 import com.android.tradefed.util.StreamUtil; 45 import com.android.tradefed.util.StringEscapeUtils; 46 import com.android.tradefed.util.StringUtil; 47 import com.android.tradefed.util.SubprocessEventHelper.InvocationFailedEventInfo; 48 import com.android.tradefed.util.SubprocessTestResultsParser; 49 50 import java.io.File; 51 import java.io.IOException; 52 import java.io.UncheckedIOException; 53 import java.nio.file.Files; 54 import java.nio.file.Path; 55 import java.time.Duration; 56 import java.util.ArrayList; 57 import java.util.Arrays; 58 import java.util.Iterator; 59 import java.util.LinkedHashMap; 60 import java.util.LinkedHashSet; 61 import java.util.List; 62 import java.util.Map; 63 import java.util.Map.Entry; 64 import java.util.Set; 65 import java.util.regex.Pattern; 66 import java.util.stream.Collectors; 67 import java.util.stream.Stream; 68 69 /** 70 * A {@link IRemoteTest} class to launch a command from TFC via a subprocess TF. FIXME: this needs 71 * to be extended to support multi-device tests. 72 */ 73 @OptionClass(alias = "cluster", global_namespace = false) 74 public class ClusterCommandLauncher 75 implements IRemoteTest, IInvocationContextReceiver, IConfigurationReceiver { 76 77 public static final String TF_JAR_DIR = "TF_JAR_DIR"; 78 public static final String TF_PATH = "TF_PATH"; 79 public static final String TEST_WORK_DIR = "TEST_WORK_DIR"; 80 public static final String ANDROID_SERIALS = "ANDROID_SERIALS"; 81 public static final String TF_DEVICE_COUNT = "TF_DEVICE_COUNT"; 82 83 private static final Duration MAX_EVENT_RECEIVER_WAIT_TIME = Duration.ofMinutes(30); 84 85 @Option(name = "root-dir", description = "A root directory", mandatory = true) 86 private File mRootDir; 87 88 @Option(name = "env-var", description = "Environment variables") 89 private Map<String, String> mEnvVars = new LinkedHashMap<>(); 90 91 @Option(name = "setup-script", description = "Setup scripts") 92 private List<String> mSetupScripts = new ArrayList<>(); 93 94 @Option(name = "script-timeout", description = "Script execution timeout", isTimeVal = true) 95 private long mScriptTimeout = 30 * 60 * 1000; 96 97 @Option(name = "jvm-option", description = "JVM options") 98 private List<String> mJvmOptions = new ArrayList<>(); 99 100 @Option(name = "java-property", description = "Java properties") 101 private Map<String, String> mJavaProperties = new LinkedHashMap<>(); 102 103 @Option(name = "command-line", description = "A command line to launch.", mandatory = true) 104 private String mCommandLine = null; 105 106 @Option( 107 name = "original-command-line", 108 description = 109 "Original command line. It may differ from command-line in retry invocations.") 110 private String mOriginalCommandLine = null; 111 112 @Option(name = "use-subprocess-reporting", description = "Use subprocess reporting.") 113 private boolean mUseSubprocessReporting = false; 114 115 @Option( 116 name = "output-idle-timeout", 117 description = "Maximum time to wait for an idle subprocess", 118 isTimeVal = true) 119 private long mOutputIdleTimeout = 0L; 120 121 @Option( 122 name = "exclude-file-in-java-classpath", 123 description = "The file not to include in the java classpath.") 124 private List<String> mExcludedFilesInClasspath = 125 new ArrayList<>(Arrays.asList("art-run-test.*", "art-gtest-jars.*")); 126 127 private IInvocationContext mInvocationContext; 128 private IConfiguration mConfiguration; 129 private IRunUtil mRunUtil; 130 131 @Override setInvocationContext(IInvocationContext invocationContext)132 public void setInvocationContext(IInvocationContext invocationContext) { 133 mInvocationContext = invocationContext; 134 } 135 136 @Override setConfiguration(IConfiguration configuration)137 public void setConfiguration(IConfiguration configuration) { 138 mConfiguration = configuration; 139 } 140 getEnvVar(String key)141 private String getEnvVar(String key) { 142 return getEnvVar(key, null); 143 } 144 getEnvVar(String key, String defaultValue)145 private String getEnvVar(String key, String defaultValue) { 146 String value = mEnvVars.getOrDefault(key, defaultValue); 147 if (value != null) { 148 value = StringUtil.expand(value, mEnvVars); 149 } 150 return value; 151 } 152 153 @Override run(TestInformation testInfo, ITestInvocationListener listener)154 public void run(TestInformation testInfo, ITestInvocationListener listener) 155 throws DeviceNotAvailableException { 156 // Prepare a IRunUtil instance for running subprocesses. 157 final IRunUtil runUtil = getRunUtil(); 158 runUtil.setWorkingDir(mRootDir); 159 // clear the TF_GLOBAL_CONFIG env, so another tradefed will not reuse the global config file 160 runUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_VARIABLE); 161 // Add device count to env var. 162 mEnvVars.put(TF_DEVICE_COUNT, String.valueOf(mInvocationContext.getDevices().size())); 163 for (final String key : mEnvVars.keySet()) { 164 runUtil.setEnvVariable(key, getEnvVar(key)); 165 } 166 // Add device serials to env var. 167 runUtil.setEnvVariable(ANDROID_SERIALS, String.join(",", mInvocationContext.getSerials())); 168 169 final File testWorkDir = new File(getEnvVar(TEST_WORK_DIR, mRootDir.getAbsolutePath())); 170 final File logDir = new File(mRootDir, "logs"); 171 logDir.mkdirs(); 172 File stdoutFile = new File(logDir, "stdout.txt"); 173 File stderrFile = new File(logDir, "stderr.txt"); 174 175 // Run setup scripts. 176 runSetupScripts(runUtil, stdoutFile, stderrFile); 177 178 FileIdleMonitor monitor = createFileMonitor(stdoutFile, stderrFile); 179 SubprocessTestResultsParser subprocessEventParser = null; 180 try { 181 String classpath = buildJavaClasspath(); 182 183 // TODO(b/129111645): use proto reporting if a test suite supports it. 184 if (mUseSubprocessReporting) { 185 subprocessEventParser = 186 createSubprocessTestResultsParser(listener, true, mInvocationContext); 187 final String port = Integer.toString(subprocessEventParser.getSocketServerPort()); 188 // Create injection jar for subprocess result reporter, which is used 189 // for pre-R xTS. The created jar is put in front position of the class path to 190 // override class with the same name. 191 final SubprocessReportingHelper mHelper = 192 new SubprocessReportingHelper(mCommandLine, classpath, testWorkDir, port); 193 final File subprocessReporterJar = mHelper.buildSubprocessReporterJar(); 194 classpath = 195 String.format("%s:%s", subprocessReporterJar.getAbsolutePath(), classpath); 196 } 197 198 List<String> javaCommandArgs = buildJavaCommandArgs(classpath, mCommandLine); 199 CLog.i("Running a command line: %s", mCommandLine); 200 CLog.i("args = %s", javaCommandArgs); 201 CLog.i("test working directory = %s", testWorkDir); 202 203 monitor.start(); 204 runUtil.setWorkingDir(testWorkDir); 205 CommandResult result = 206 runUtil.runTimedCmdWithInput( 207 mConfiguration.getCommandOptions().getInvocationTimeout(), 208 null, 209 stdoutFile, 210 stderrFile, 211 javaCommandArgs.toArray(new String[javaCommandArgs.size()])); 212 if (!result.getStatus().equals(CommandStatus.SUCCESS)) { 213 String error = null; 214 Throwable cause = null; 215 if (result.getStatus().equals(CommandStatus.TIMED_OUT)) { 216 error = 217 String.format( 218 "Command timed out after %sms", 219 mConfiguration.getCommandOptions().getInvocationTimeout()); 220 } else { 221 error = 222 String.format( 223 "Command finished unsuccessfully: status=%s, exit_code=%s", 224 result.getStatus(), result.getExitCode()); 225 InvocationFailedEventInfo errorInfo = 226 subprocessEventParser.getReportedInvocationFailedEventInfo(); 227 if (errorInfo != null) { 228 cause = errorInfo.mCause; 229 } else { 230 cause = new Throwable(FileUtil.readStringFromFile(stderrFile)); 231 } 232 } 233 throw new SubprocessCommandException(error, cause); 234 } 235 CLog.i("Successfully ran a command"); 236 237 } catch (IOException e) { 238 throw new RuntimeException(e); 239 } finally { 240 monitor.stop(); 241 if (subprocessEventParser != null) { 242 subprocessEventParser.joinReceiver( 243 MAX_EVENT_RECEIVER_WAIT_TIME.toMillis(), /* wait for connection */ false); 244 StreamUtil.close(subprocessEventParser); 245 } 246 } 247 } 248 runSetupScripts( final IRunUtil runUtil, final File stdoutFile, final File stderrFile)249 private void runSetupScripts( 250 final IRunUtil runUtil, final File stdoutFile, final File stderrFile) { 251 try { 252 long timeout = mScriptTimeout; 253 long startTime = System.currentTimeMillis(); 254 for (String script : mSetupScripts) { 255 script = StringUtil.expand(script, mEnvVars); 256 CLog.i("Running a setup script: %s", script); 257 File scriptFile = new File(QuotationAwareTokenizer.tokenizeLine(script)[0]); 258 if (scriptFile.isFile()) { 259 scriptFile.setExecutable(true); 260 } 261 // FIXME: Refactor command execution into a helper function. 262 CommandResult result = 263 runUtil.runTimedCmdWithInput( 264 timeout, 265 null, 266 stdoutFile, 267 stderrFile, 268 QuotationAwareTokenizer.tokenizeLine(script)); 269 if (!result.getStatus().equals(CommandStatus.SUCCESS)) { 270 String error = null; 271 if (result.getStatus().equals(CommandStatus.TIMED_OUT)) { 272 error = "timeout"; 273 } else { 274 error = FileUtil.readStringFromFile(stderrFile); 275 } 276 throw new RuntimeException(String.format("Script failed to run: %s", error)); 277 } 278 timeout -= (System.currentTimeMillis() - startTime); 279 if (timeout < 0) { 280 throw new RuntimeException( 281 String.format("Setup scripts failed to run in %sms", mScriptTimeout)); 282 } 283 } 284 } catch (IOException e) { 285 throw new UncheckedIOException("Error running setup scripts", e); 286 } 287 } 288 buildJavaClasspath()289 private String buildJavaClasspath() { 290 // Get an expanded TF_PATH value. 291 final String tfPath = getEnvVar(TF_PATH, System.getProperty(TF_JAR_DIR)); 292 if (tfPath == null) { 293 throw new RuntimeException("cannot find TF path!"); 294 } 295 296 // Construct a Java class path based on TF_PATH and exclude-file-in-java-classpath option. 297 // This expects TF_PATH to be a colon(:) separated list of paths where each path 298 // points to a specific jar file or folder. 299 // (example: path/to/tradefed.jar:path/to/tradefed/folder:...) 300 List<Pattern> excludedPatterns = new ArrayList<>(); 301 for (final String regex : mExcludedFilesInClasspath) { 302 excludedPatterns.add(Pattern.compile(StringUtil.expand(regex, mEnvVars))); 303 } 304 final Set<String> jars = new LinkedHashSet<>(); 305 for (final String path : tfPath.split(":")) { 306 final File jarFile = new File(path); 307 if (!jarFile.exists()) { 308 CLog.w("TF_PATH %s doesn't exist; ignoring", path); 309 continue; 310 } 311 if (jarFile.isFile() && !matchPatterns(excludedPatterns, jarFile.getAbsolutePath())) { 312 jars.add(jarFile.getAbsolutePath()); 313 } else { 314 try (Stream<Path> walk = Files.walk(jarFile.toPath())) { 315 List<String> result = 316 walk.map(Path::toString) 317 .filter( 318 f -> 319 f.toLowerCase().endsWith(".jar") 320 && !matchPatterns(excludedPatterns, f)) 321 .collect(Collectors.toList()); 322 jars.addAll(result); 323 } catch (IOException e) { 324 throw new RuntimeException( 325 String.format("failed to find jars from %s", jarFile), e); 326 } 327 } 328 } 329 if (jars.isEmpty()) { 330 throw new RuntimeException(String.format("cannot find any TF jars from %s!", tfPath)); 331 } 332 // TODO: remove after tradefed-no-fwk.jar is deprecated 333 List<String> finalJars = new ArrayList<>(); 334 Iterator<String> iterator = jars.iterator(); 335 while (iterator.hasNext()) { 336 final String jar = iterator.next(); 337 if (new File(jar).getName().equalsIgnoreCase("tradefed.jar")) { 338 finalJars.add(jar); 339 iterator.remove(); 340 } 341 } 342 if (!jars.isEmpty()) { 343 finalJars.add(String.join(":", jars)); 344 } 345 return String.join(":", finalJars); 346 } 347 348 /** Build a shell command line to invoke a TF process. */ buildJavaCommandArgs(String classpath, String tfCommandLine)349 private List<String> buildJavaCommandArgs(String classpath, String tfCommandLine) { 350 // Build a command line to invoke a TF process. 351 final List<String> cmdArgs = new ArrayList<>(); 352 final String javaHome = getEnvVar("JAVA_HOME", System.getProperty("java.home")); 353 final String javaPath = String.format("%s/bin/java", javaHome); 354 cmdArgs.add(new File(javaPath).getAbsolutePath()); 355 cmdArgs.add("-cp"); 356 cmdArgs.add(classpath); 357 cmdArgs.addAll(mJvmOptions); 358 359 // Use separate tmp directory for the subprocess to prevent clashing with other tmp files. 360 File tmpDir = new File(mRootDir, "tmp"); 361 tmpDir.mkdirs(); 362 cmdArgs.add("-Djava.io.tmpdir=" + tmpDir.getAbsolutePath()); 363 364 // Pass Java properties as -D options. 365 for (final Entry<String, String> entry : mJavaProperties.entrySet()) { 366 cmdArgs.add( 367 String.format( 368 "-D%s=%s", 369 entry.getKey(), StringUtil.expand(entry.getValue(), mEnvVars))); 370 } 371 cmdArgs.add("com.android.tradefed.command.CommandRunner"); 372 tfCommandLine = StringUtil.expand(tfCommandLine, mEnvVars); 373 cmdArgs.addAll(StringEscapeUtils.paramsToArgs(ArrayUtil.list(tfCommandLine))); 374 375 final Integer shardCount = mConfiguration.getCommandOptions().getShardCount(); 376 final Integer shardIndex = mConfiguration.getCommandOptions().getShardIndex(); 377 378 if (shardCount != null && shardCount > 1) { 379 cmdArgs.add("--shard-count"); 380 cmdArgs.add(Integer.toString(shardCount)); 381 if (shardIndex != null) { 382 cmdArgs.add("--shard-index"); 383 cmdArgs.add(Integer.toString(shardIndex)); 384 } 385 } 386 387 for (final ITestDevice device : mInvocationContext.getDevices()) { 388 // FIXME: Find a better way to support non-physical devices as well. 389 cmdArgs.add("--serial"); 390 cmdArgs.add(device.getSerialNumber()); 391 } 392 393 return cmdArgs; 394 } 395 396 /** Creates a file monitor which will perform a USB port reset if the subprocess is idle. */ createFileMonitor(File... files)397 private FileIdleMonitor createFileMonitor(File... files) { 398 // treat zero or negative timeout as infinite 399 long timeout = mOutputIdleTimeout > 0 ? mOutputIdleTimeout : Long.MAX_VALUE; 400 // reset USB ports if files are idle for too long 401 // TODO(peykov): consider making the callback customizable 402 return new FileIdleMonitor(Duration.ofMillis(timeout), this::resetDevices, files); 403 } 404 405 /** Performs reset on all devices. */ resetDevices()406 private void resetDevices() { 407 CLog.i("Subprocess output idle for %d ms, attempting device reset.", mOutputIdleTimeout); 408 List<ITestDevice> devices = mInvocationContext.getDevices(); 409 UsbHelper usb = null; 410 try { 411 for (ITestDevice device : devices) { 412 if (device instanceof LocalAndroidVirtualDevice) { 413 CLog.d("Shutting down local virtual device '%s'", device.getSerialNumber()); 414 ((LocalAndroidVirtualDevice) device).shutdown(); 415 } else { 416 if (usb == null) { 417 usb = new UsbHelper(); 418 } 419 resetUsbPort(usb, device.getSerialNumber()); 420 } 421 } 422 } catch (RuntimeException e) { 423 CLog.e(e); 424 } finally { 425 if (usb != null) { 426 usb.close(); 427 } 428 } 429 } 430 431 /** Performs a USB port reset on a device. */ resetUsbPort(UsbHelper usb, String serial)432 private void resetUsbPort(UsbHelper usb, String serial) { 433 try (UsbDevice device = usb.getDevice(serial)) { 434 if (device == null) { 435 CLog.w("Device '%s' not found during USB reset.", serial); 436 return; 437 } 438 CLog.d("Resetting USB port for device '%s'", serial); 439 device.reset(); 440 } 441 } 442 443 /** Returns true if the given string matches any of the patterns. */ matchPatterns(List<Pattern> patterns, String str)444 private static boolean matchPatterns(List<Pattern> patterns, String str) { 445 for (final Pattern pattern : patterns) { 446 if (pattern.matcher(str).find()) { 447 return true; 448 } 449 } 450 return false; 451 } 452 453 @VisibleForTesting getRunUtil()454 IRunUtil getRunUtil() { 455 if (mRunUtil == null) { 456 mRunUtil = new RunUtil(); 457 } 458 return mRunUtil; 459 } 460 461 @VisibleForTesting createSubprocessTestResultsParser( ITestInvocationListener listener, boolean streaming, IInvocationContext context)462 SubprocessTestResultsParser createSubprocessTestResultsParser( 463 ITestInvocationListener listener, boolean streaming, IInvocationContext context) 464 throws IOException { 465 return new SubprocessTestResultsParser(listener, streaming, context); 466 } 467 468 @VisibleForTesting getEnvVars()469 Map<String, String> getEnvVars() { 470 return mEnvVars; 471 } 472 473 @VisibleForTesting getSetupScripts()474 List<String> getSetupScripts() { 475 return mSetupScripts; 476 } 477 478 @VisibleForTesting getJvmOptions()479 List<String> getJvmOptions() { 480 return mJvmOptions; 481 } 482 483 @VisibleForTesting getJavaProperties()484 Map<String, String> getJavaProperties() { 485 return mJavaProperties; 486 } 487 488 @VisibleForTesting getCommandLine()489 String getCommandLine() { 490 return mCommandLine; 491 } 492 493 @VisibleForTesting useSubprocessReporting()494 boolean useSubprocessReporting() { 495 return mUseSubprocessReporting; 496 } 497 } 498