1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.python; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.config.GlobalConfiguration; 20 import com.android.tradefed.config.Option; 21 import com.android.tradefed.config.OptionClass; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.StubDevice; 24 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 25 import com.android.tradefed.invoker.TestInformation; 26 import com.android.tradefed.log.LogUtil.CLog; 27 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 28 import com.android.tradefed.result.ByteArrayInputStreamSource; 29 import com.android.tradefed.result.FailureDescription; 30 import com.android.tradefed.result.FileInputStreamSource; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.result.InputStreamSource; 33 import com.android.tradefed.result.LogDataType; 34 import com.android.tradefed.result.ResultForwarder; 35 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus; 36 import com.android.tradefed.testtype.IRemoteTest; 37 import com.android.tradefed.testtype.ITestFilterReceiver; 38 import com.android.tradefed.testtype.PythonUnitTestResultParser; 39 import com.android.tradefed.testtype.TestTimeoutEnforcer; 40 import com.android.tradefed.util.AdbUtils; 41 import com.android.tradefed.util.CommandResult; 42 import com.android.tradefed.util.FileUtil; 43 import com.android.tradefed.util.IRunUtil; 44 import com.android.tradefed.util.IRunUtil.EnvPriority; 45 import com.android.tradefed.util.RunUtil; 46 47 import com.google.common.base.Joiner; 48 import com.google.common.base.Strings; 49 50 import java.io.File; 51 import java.io.FileNotFoundException; 52 import java.io.FileOutputStream; 53 import java.io.IOException; 54 import java.time.Duration; 55 import java.util.ArrayList; 56 import java.util.Arrays; 57 import java.util.HashMap; 58 import java.util.HashSet; 59 import java.util.LinkedHashSet; 60 import java.util.List; 61 import java.util.Set; 62 import java.util.concurrent.TimeUnit; 63 import java.util.stream.Collectors; 64 65 /** 66 * Host test meant to run a python binary file from the Android Build system (Soong) 67 * 68 * <p>The test runner supports include-filter and exclude-filter. Note that exclude-filter works by 69 * ignoring the test result, instead of skipping the actual test. The tests specified in the 70 * exclude-filter will still be executed. 71 */ 72 @OptionClass(alias = "python-host") 73 public class PythonBinaryHostTest implements IRemoteTest, ITestFilterReceiver { 74 75 protected static final String ANDROID_SERIAL_VAR = "ANDROID_SERIAL"; 76 protected static final String LD_LIBRARY_PATH = "LD_LIBRARY_PATH"; 77 78 @VisibleForTesting static final String USE_TEST_OUTPUT_FILE_OPTION = "use-test-output-file"; 79 static final String TEST_OUTPUT_FILE_FLAG = "test-output-file"; 80 81 private static final String PYTHON_LOG_STDOUT_FORMAT = "%s-stdout"; 82 private static final String PYTHON_LOG_STDERR_FORMAT = "%s-stderr"; 83 private static final String PYTHON_LOG_TEST_OUTPUT_FORMAT = "%s-test-output"; 84 85 private Set<String> mIncludeFilters = new LinkedHashSet<>(); 86 private Set<String> mExcludeFilters = new LinkedHashSet<>(); 87 private String mLdLibraryPath = null; 88 89 @Option(name = "par-file-name", description = "The binary names inside the build info to run.") 90 private Set<String> mBinaryNames = new HashSet<>(); 91 92 @Option( 93 name = "python-binaries", 94 description = "The full path to a runnable python binary. Can be repeated." 95 ) 96 private Set<File> mBinaries = new HashSet<>(); 97 98 @Option( 99 name = "test-timeout", 100 description = "Timeout for a single par file to terminate.", 101 isTimeVal = true 102 ) 103 private long mTestTimeout = 20 * 1000L; 104 105 @Option( 106 name = "inject-serial-option", 107 description = "Whether or not to pass a -s <serialnumber> option to the binary") 108 private boolean mInjectSerial = false; 109 110 @Option( 111 name = "inject-android-serial", 112 description = "Whether or not to pass a ANDROID_SERIAL variable to the process.") 113 private boolean mInjectAndroidSerialVar = true; 114 115 @Option( 116 name = "python-options", 117 description = "Option string to be passed to the binary when running" 118 ) 119 private List<String> mTestOptions = new ArrayList<>(); 120 121 @Option( 122 name = "inject-build-key", 123 description = 124 "Link a file from the build by its key to the python subprocess via" 125 + " environment. This breaks test dependencies so shouldn't be used in" 126 + " standard suites.") 127 private Set<String> mBuildKeyToLink = new LinkedHashSet<String>(); 128 129 @Option( 130 name = USE_TEST_OUTPUT_FILE_OPTION, 131 description = 132 "Whether the test should write results to the file specified via the --" 133 + TEST_OUTPUT_FILE_FLAG 134 + " flag instead of stderr which could contain spurious messages that " 135 + "break result parsing. Using this option requires that the Python " 136 + "test have the necessary logic to accept the flag and write results " 137 + "in the expected format.") 138 private boolean mUseTestOutputFile = false; 139 140 @Option( 141 name = TestTimeoutEnforcer.TEST_CASE_TIMEOUT_OPTION, 142 description = TestTimeoutEnforcer.TEST_CASE_TIMEOUT_DESCRIPTION) 143 private Duration mTestCaseTimeout = Duration.ofSeconds(0L); 144 145 private TestInformation mTestInfo; 146 private IRunUtil mRunUtil; 147 148 /** {@inheritDoc} */ 149 @Override addIncludeFilter(String filter)150 public void addIncludeFilter(String filter) { 151 mIncludeFilters.add(filter); 152 } 153 154 /** {@inheritDoc} */ 155 @Override addExcludeFilter(String filter)156 public void addExcludeFilter(String filter) { 157 mExcludeFilters.add(filter); 158 } 159 160 /** {@inheritDoc} */ 161 @Override addAllIncludeFilters(Set<String> filters)162 public void addAllIncludeFilters(Set<String> filters) { 163 mIncludeFilters.addAll(filters); 164 } 165 166 /** {@inheritDoc} */ 167 @Override addAllExcludeFilters(Set<String> filters)168 public void addAllExcludeFilters(Set<String> filters) { 169 mExcludeFilters.addAll(filters); 170 } 171 172 /** {@inheritDoc} */ 173 @Override clearIncludeFilters()174 public void clearIncludeFilters() { 175 mIncludeFilters.clear(); 176 } 177 178 /** {@inheritDoc} */ 179 @Override clearExcludeFilters()180 public void clearExcludeFilters() { 181 mExcludeFilters.clear(); 182 } 183 184 /** {@inheritDoc} */ 185 @Override getIncludeFilters()186 public Set<String> getIncludeFilters() { 187 return mIncludeFilters; 188 } 189 190 /** {@inheritDoc} */ 191 @Override getExcludeFilters()192 public Set<String> getExcludeFilters() { 193 return mExcludeFilters; 194 } 195 196 @Override run(TestInformation testInfo, ITestInvocationListener listener)197 public final void run(TestInformation testInfo, ITestInvocationListener listener) 198 throws DeviceNotAvailableException { 199 mTestInfo = testInfo; 200 File testDir = mTestInfo.executionFiles().get(FilesKey.HOST_TESTS_DIRECTORY); 201 if (testDir == null || !testDir.exists()) { 202 testDir = mTestInfo.executionFiles().get(FilesKey.TESTS_DIRECTORY); 203 } 204 List<String> ldLibraryPath = new ArrayList<>(); 205 if (testDir != null && testDir.exists()) { 206 List<String> libPaths = 207 Arrays.asList("lib", "lib64", "host/testcases/lib", "host/testcases/lib64"); 208 for (String path : libPaths) { 209 File libDir = new File(testDir, path); 210 if (libDir.exists()) { 211 ldLibraryPath.add(libDir.getAbsolutePath()); 212 } 213 } 214 if (!ldLibraryPath.isEmpty()) { 215 mLdLibraryPath = Joiner.on(":").join(ldLibraryPath); 216 } 217 } 218 List<File> pythonFilesList = findParFiles(); 219 for (File pyFile : pythonFilesList) { 220 if (!pyFile.exists()) { 221 CLog.d( 222 "ignoring %s which doesn't look like a test file.", 223 pyFile.getAbsolutePath()); 224 continue; 225 } 226 // Complete the LD_LIBRARY_PATH with possible libs 227 String path = mLdLibraryPath; 228 List<String> paths = findAllSubdir(pyFile.getParentFile(), ldLibraryPath); 229 if (mLdLibraryPath != null) { 230 paths.add(0, mLdLibraryPath); 231 } 232 mLdLibraryPath = Joiner.on(":").join(paths); 233 pyFile.setExecutable(true); 234 runSinglePythonFile(listener, testInfo, pyFile); 235 mLdLibraryPath = path; 236 } 237 } 238 findParFiles()239 private List<File> findParFiles() { 240 List<File> files = new ArrayList<>(); 241 for (String parFileName : mBinaryNames) { 242 File res = null; 243 // search tests dir 244 try { 245 res = mTestInfo.getDependencyFile(parFileName, /* targetFirst */ false); 246 files.add(res); 247 } catch (FileNotFoundException e) { 248 throw new RuntimeException( 249 String.format("Couldn't find a par file %s", parFileName)); 250 } 251 } 252 files.addAll(mBinaries); 253 return files; 254 } 255 runSinglePythonFile( ITestInvocationListener listener, TestInformation testInfo, File pyFile)256 private void runSinglePythonFile( 257 ITestInvocationListener listener, TestInformation testInfo, File pyFile) { 258 List<String> commandLine = new ArrayList<>(); 259 commandLine.add(pyFile.getAbsolutePath()); 260 // If we have a physical device, pass it to the python test by serial 261 if (!(mTestInfo.getDevice().getIDevice() instanceof StubDevice) && mInjectSerial) { 262 // TODO: support multi-device python tests? 263 commandLine.add("-s"); 264 commandLine.add(mTestInfo.getDevice().getSerialNumber()); 265 } 266 // Set the process working dir as the directory of the main binary 267 getRunUtil().setWorkingDir(pyFile.getParentFile()); 268 // Set the parent dir on the PATH 269 String separator = System.getProperty("path.separator"); 270 List<String> paths = new ArrayList<>(); 271 // Bundle binaries / dependencies have priorities over existing PATH 272 paths.addAll(findAllSubdir(pyFile.getParentFile(), new ArrayList<>())); 273 paths.add(System.getenv("PATH")); 274 String path = paths.stream().distinct().collect(Collectors.joining(separator)); 275 CLog.d("Using updated $PATH: %s", path); 276 getRunUtil().setEnvVariablePriority(EnvPriority.SET); 277 getRunUtil().setEnvVariable("PATH", path); 278 279 if (mLdLibraryPath != null) { 280 getRunUtil().setEnvVariable(LD_LIBRARY_PATH, mLdLibraryPath); 281 } 282 if (mInjectAndroidSerialVar) { 283 getRunUtil() 284 .setEnvVariable(ANDROID_SERIAL_VAR, mTestInfo.getDevice().getSerialNumber()); 285 } 286 // This is not standard, but sometimes non-module data artifacts might be needed 287 for (String key : mBuildKeyToLink) { 288 if (mTestInfo.getBuildInfo().getFile(key) != null) { 289 getRunUtil() 290 .setEnvVariable( 291 key, mTestInfo.getBuildInfo().getFile(key).getAbsolutePath()); 292 } 293 } 294 295 File tempTestOutputFile = null; 296 if (mUseTestOutputFile) { 297 try { 298 tempTestOutputFile = FileUtil.createTempFile("python-test-output", ".txt"); 299 } catch (IOException e) { 300 throw new RuntimeException(e); 301 } 302 303 commandLine.add("--" + TEST_OUTPUT_FILE_FLAG); 304 commandLine.add(tempTestOutputFile.getAbsolutePath()); 305 } 306 307 AdbUtils.updateAdb(testInfo, getRunUtil(), getAdbPath()); 308 // Add all the other options 309 commandLine.addAll(mTestOptions); 310 311 // Prepare the parser if needed 312 String runName = pyFile.getName(); 313 PythonForwarder forwarder = new PythonForwarder(listener, runName); 314 ITestInvocationListener receiver = forwarder; 315 if (mTestCaseTimeout.toMillis() > 0L) { 316 receiver = 317 new TestTimeoutEnforcer( 318 mTestCaseTimeout.toMillis(), TimeUnit.MILLISECONDS, receiver); 319 } 320 PythonUnitTestResultParser pythonParser = 321 new PythonUnitTestResultParser( 322 Arrays.asList(receiver), "python-run", mIncludeFilters, mExcludeFilters); 323 324 CommandResult result = null; 325 File stderrFile = null; 326 try { 327 stderrFile = FileUtil.createTempFile("python-res", ".txt"); 328 if (mUseTestOutputFile) { 329 result = getRunUtil().runTimedCmd(mTestTimeout, commandLine.toArray(new String[0])); 330 } else { 331 try (FileOutputStream fileOutputParser = new FileOutputStream(stderrFile)) { 332 result = 333 getRunUtil() 334 .runTimedCmd( 335 mTestTimeout, 336 null, 337 fileOutputParser, 338 commandLine.toArray(new String[0])); 339 fileOutputParser.flush(); 340 } 341 } 342 343 if (!Strings.isNullOrEmpty(result.getStdout())) { 344 CLog.i("\nstdout:\n%s", result.getStdout()); 345 try (InputStreamSource data = 346 new ByteArrayInputStreamSource(result.getStdout().getBytes())) { 347 listener.testLog( 348 String.format(PYTHON_LOG_STDOUT_FORMAT, runName), 349 LogDataType.TEXT, 350 data); 351 } 352 } 353 if (!Strings.isNullOrEmpty(result.getStderr())) { 354 CLog.i("\nstderr:\n%s", result.getStderr()); 355 } 356 357 File testOutputFile = stderrFile; 358 if (mUseTestOutputFile) { 359 testOutputFile = tempTestOutputFile; 360 testLogFile( 361 listener, 362 String.format(PYTHON_LOG_TEST_OUTPUT_FORMAT, runName), 363 testOutputFile); 364 } 365 String testOutput = FileUtil.readStringFromFile(testOutputFile); 366 pythonParser.processNewLines(testOutput.split("\n")); 367 } catch (RuntimeException e) { 368 StringBuilder message = new StringBuilder(); 369 String stderr = ""; 370 try { 371 stderr = FileUtil.readStringFromFile(stderrFile); 372 } catch (IOException ioe) { 373 CLog.e(ioe); 374 } 375 message.append( 376 String.format( 377 "Failed to parse the python logs: %s. Please ensure that verbosity of " 378 + "output is high enough to be parsed." 379 + " Stderr: %s", 380 e.getMessage(), stderr)); 381 382 if (mUseTestOutputFile) { 383 message.append( 384 String.format( 385 " Make sure that your test writes its output to the file specified " 386 + "by the --%s flag and that its contents (%s) are in the " 387 + "format expected by the test runner.", 388 TEST_OUTPUT_FILE_FLAG, 389 String.format(PYTHON_LOG_TEST_OUTPUT_FORMAT, runName))); 390 } 391 392 reportFailure(listener, runName, message.toString()); 393 CLog.e(e); 394 } catch (IOException e) { 395 throw new RuntimeException(e); 396 } finally { 397 if (stderrFile != null) { 398 // Note that we still log stderr when parsing results from a test-written output 399 // file since it most likely contains useful debugging information. 400 try { 401 if (mUseTestOutputFile) { 402 FileUtil.writeToFile(result.getStderr(), stderrFile); 403 } 404 testLogFile( 405 listener, String.format(PYTHON_LOG_STDERR_FORMAT, runName), stderrFile); 406 } catch (IOException e) { 407 CLog.e(e); 408 } 409 } 410 FileUtil.deleteFile(stderrFile); 411 FileUtil.deleteFile(tempTestOutputFile); 412 } 413 } 414 415 @VisibleForTesting getRunUtil()416 IRunUtil getRunUtil() { 417 if (mRunUtil == null) { 418 mRunUtil = new RunUtil(); 419 } 420 return mRunUtil; 421 } 422 423 @VisibleForTesting getAdbPath()424 String getAdbPath() { 425 return GlobalConfiguration.getDeviceManagerInstance().getAdbPath(); 426 } 427 findAllSubdir(File parentDir, List<String> knownPaths)428 private List<String> findAllSubdir(File parentDir, List<String> knownPaths) { 429 List<String> subDir = new ArrayList<>(); 430 subDir.add(parentDir.getAbsolutePath()); 431 if (parentDir.listFiles() == null) { 432 return subDir; 433 } 434 for (File child : parentDir.listFiles()) { 435 if (child != null 436 && child.isDirectory() 437 && !knownPaths.contains(child.getAbsolutePath())) { 438 subDir.addAll(findAllSubdir(child, knownPaths)); 439 } 440 } 441 return subDir; 442 } 443 reportFailure( ITestInvocationListener listener, String runName, String errorMessage)444 private void reportFailure( 445 ITestInvocationListener listener, String runName, String errorMessage) { 446 listener.testRunStarted(runName, 0); 447 FailureDescription description = 448 FailureDescription.create(errorMessage, FailureStatus.TEST_FAILURE); 449 listener.testRunFailed(description); 450 listener.testRunEnded(0L, new HashMap<String, Metric>()); 451 } 452 testLogFile(ITestInvocationListener listener, String dataName, File f)453 private static void testLogFile(ITestInvocationListener listener, String dataName, File f) { 454 try (FileInputStreamSource data = new FileInputStreamSource(f)) { 455 listener.testLog(dataName, LogDataType.TEXT, data); 456 } 457 } 458 459 /** Result forwarder to replace the run name by the binary name. */ 460 public static class PythonForwarder extends ResultForwarder { 461 462 private String mRunName; 463 464 /** Ctor with the run name using the binary name. */ PythonForwarder(ITestInvocationListener listener, String name)465 public PythonForwarder(ITestInvocationListener listener, String name) { 466 super(listener); 467 mRunName = name; 468 } 469 470 @Override testRunStarted(String runName, int testCount)471 public void testRunStarted(String runName, int testCount) { 472 // Replace run name 473 testRunStarted(runName, testCount, 0); 474 } 475 476 @Override testRunStarted(String runName, int testCount, int attempt)477 public void testRunStarted(String runName, int testCount, int attempt) { 478 // Replace run name 479 testRunStarted(runName, testCount, attempt, System.currentTimeMillis()); 480 } 481 482 @Override testRunStarted(String runName, int testCount, int attempt, long startTime)483 public void testRunStarted(String runName, int testCount, int attempt, long startTime) { 484 // Replace run name 485 super.testRunStarted(mRunName, testCount, attempt, startTime); 486 } 487 } 488 } 489