1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.targetprep; 18 19 import com.android.ddmlib.IDevice; 20 import com.android.tradefed.build.BuildInfoKey.BuildInfoFileKey; 21 import com.android.tradefed.build.IBuildInfo; 22 import com.android.tradefed.build.IDeviceBuildInfo; 23 import com.android.tradefed.command.remote.DeviceDescriptor; 24 import com.android.tradefed.config.Option; 25 import com.android.tradefed.config.OptionClass; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.invoker.IInvocationContext; 29 import com.android.tradefed.invoker.TestInformation; 30 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 31 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 32 import com.android.tradefed.log.LogUtil.CLog; 33 import com.android.tradefed.observatory.IDiscoverDependencies; 34 import com.android.tradefed.result.error.DeviceErrorIdentifier; 35 import com.android.tradefed.result.error.ErrorIdentifier; 36 import com.android.tradefed.result.error.InfraErrorIdentifier; 37 import com.android.tradefed.testtype.IAbi; 38 import com.android.tradefed.testtype.IAbiReceiver; 39 import com.android.tradefed.testtype.IInvocationContextReceiver; 40 import com.android.tradefed.testtype.suite.ModuleDefinition; 41 import com.android.tradefed.util.AbiUtils; 42 import com.android.tradefed.util.FileUtil; 43 import com.android.tradefed.util.MultiMap; 44 45 import java.io.File; 46 import java.io.IOException; 47 import java.util.ArrayList; 48 import java.util.Arrays; 49 import java.util.Collection; 50 import java.util.HashSet; 51 import java.util.LinkedHashMap; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Set; 55 56 /** 57 * A {@link ITargetPreparer} that attempts to push any number of files from any host path to any 58 * device path. 59 * 60 * <p>Should be performed *after* a new build is flashed, and *after* DeviceSetup is run (if 61 * enabled) 62 */ 63 @OptionClass(alias = "push-file") 64 public class PushFilePreparer extends BaseTargetPreparer 65 implements IAbiReceiver, IInvocationContextReceiver, IDiscoverDependencies { 66 private static final String MEDIA_SCAN_INTENT = 67 "am broadcast -a android.intent.action.MEDIA_MOUNTED -d file://%s " 68 + "--receiver-include-background"; 69 70 private IAbi mAbi; 71 72 @Deprecated 73 @Option( 74 name = "push", 75 description = 76 "Deprecated. Please use push-file instead. A push-spec, formatted as " 77 + "'/localpath/to/srcfile.txt->/devicepath/to/destfile.txt' " 78 + "or '/localpath/to/srcfile.txt->/devicepath/to/destdir/'. " 79 + "May be repeated. The local path may be relative to the test cases " 80 + "build out directories " 81 + "($ANDROID_HOST_OUT_TESTCASES / $ANDROID_TARGET_OUT_TESTCASES)." 82 ) 83 private Collection<String> mPushSpecs = new ArrayList<>(); 84 85 @Option( 86 name = "push-file", 87 description = 88 "A push-spec, specifying the local file to the path where it should be pushed" 89 + " on device. May be repeated. If multiple files are configured to be" 90 + " pushed to the same remote path, the latest one will be pushed.") 91 private MultiMap<File, String> mPushFileSpecs = new MultiMap<>(); 92 93 @Option( 94 name = "skip-abi-filtering", 95 description = 96 "A bool to indicate we should or shouldn't skip files that match the " 97 + "architecture string name, e.g. x86, x86_64, arm64-v8. This " 98 + "is necessary when file or folder names match an architecture " 99 + "version but still need to be pushed to the device.") 100 private boolean mSkipAbiFiltering = false; 101 102 @Option( 103 name = "backup-file", 104 description = 105 "A key/value pair, the with key specifying a device file path to be backed up, " 106 + "and the value a device file path indicating where to save the file. " 107 + "During tear-down, the values will be executed in reverse, " 108 + "restoring the backup file location to the initial location. " 109 + "May be repeated.") 110 private Map<String, String> mBackupFileSpecs = new LinkedHashMap<>(); 111 112 @Option(name="post-push", description= 113 "A command to run on the device (with `adb shell (yourcommand)`) after all pushes " + 114 "have been attempted. Will not be run if a push fails with abort-on-push-failure " + 115 "enabled. May be repeated.") 116 private Collection<String> mPostPushCommands = new ArrayList<>(); 117 118 @Option(name="abort-on-push-failure", description= 119 "If false, continue if pushes fail. If true, abort the Invocation on any failure.") 120 private boolean mAbortOnFailure = true; 121 122 @Option(name="trigger-media-scan", description= 123 "After pushing files, trigger a media scan of external storage on device.") 124 private boolean mTriggerMediaScan = false; 125 126 @Option( 127 name = "cleanup", 128 description = 129 "Whether files pushed onto device should be cleaned up after test. Note that" 130 + " the preparer does not verify that files/directories have been deleted.") 131 private boolean mCleanup = true; 132 133 @Option( 134 name = "remount-system", 135 description = 136 "Remounts system partition to be writable " 137 + "so that files could be pushed there too") 138 private boolean mRemountSystem = false; 139 140 @Option( 141 name = "remount-vendor", 142 description = 143 "Remounts vendor partition to be writable " 144 + "so that files could be pushed there too") 145 private boolean mRemountVendor = false; 146 147 private Set<String> mFilesPushed = null; 148 /** If the preparer is part of a module, we can use the test module name as a search criteria */ 149 private String mModuleName = null; 150 151 /** 152 * Helper method to only throw if mAbortOnFailure is enabled. Callers should behave as if this 153 * method may return. 154 */ fail(String message, DeviceDescriptor descriptor, ErrorIdentifier identifier)155 private void fail(String message, DeviceDescriptor descriptor, ErrorIdentifier identifier) 156 throws TargetSetupError { 157 if (shouldAbortOnFailure()) { 158 throw new TargetSetupError(message, descriptor, identifier); 159 } else { 160 // Log the error and return 161 CLog.w(message); 162 } 163 } 164 165 /** Create the list of files to be pushed. */ getPushSpecs(ITestDevice device)166 public final Map<String, File> getPushSpecs(ITestDevice device) throws TargetSetupError { 167 Map<String, File> remoteToLocalMapping = new LinkedHashMap<>(); 168 for (String pushspec : mPushSpecs) { 169 String[] pair = pushspec.split("->"); 170 if (pair.length != 2) { 171 fail( 172 String.format("Invalid pushspec: '%s'", Arrays.asList(pair)), 173 device.getDeviceDescriptor(), 174 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 175 continue; 176 } 177 remoteToLocalMapping.put(pair[1], new File(pair[0])); 178 } 179 // Push the file structure 180 for (File local : mPushFileSpecs.keySet()) { 181 for (String remoteLocation : mPushFileSpecs.get(local)) { 182 remoteToLocalMapping.put(remoteLocation, local); 183 } 184 } 185 return remoteToLocalMapping; 186 } 187 188 /** Whether or not to abort on push failure. */ shouldAbortOnFailure()189 public boolean shouldAbortOnFailure() { 190 return mAbortOnFailure; 191 } 192 193 /** {@inheritDoc} */ 194 @Override setAbi(IAbi abi)195 public void setAbi(IAbi abi) { 196 mAbi = abi; 197 } 198 199 /** {@inheritDoc} */ 200 @Override getAbi()201 public IAbi getAbi() { 202 return mAbi; 203 } 204 205 /** {@inheritDoc} */ 206 @Override setInvocationContext(IInvocationContext invocationContext)207 public void setInvocationContext(IInvocationContext invocationContext) { 208 if (invocationContext.getAttributes().get(ModuleDefinition.MODULE_NAME) != null) { 209 // Only keep the module name 210 mModuleName = 211 invocationContext.getAttributes().get(ModuleDefinition.MODULE_NAME).get(0); 212 } 213 } 214 215 /** 216 * Resolve relative file path via {@link IBuildInfo} and test cases directories. 217 * 218 * @param buildInfo the build artifact information 219 * @param fileName relative file path to be resolved 220 * @return the file from the build info or test cases directories 221 */ resolveRelativeFilePath(IBuildInfo buildInfo, String fileName)222 public File resolveRelativeFilePath(IBuildInfo buildInfo, String fileName) { 223 File src = null; 224 if (buildInfo != null) { 225 src = buildInfo.getFile(fileName); 226 if (src != null && src.exists()) { 227 return src; 228 } 229 } 230 if (buildInfo instanceof IDeviceBuildInfo) { 231 IDeviceBuildInfo deviceBuild = (IDeviceBuildInfo) buildInfo; 232 File testDir = deviceBuild.getTestsDir(); 233 List<File> scanDirs = new ArrayList<>(); 234 // If it exists, always look first in the ANDROID_TARGET_OUT_TESTCASES 235 File targetTestCases = deviceBuild.getFile(BuildInfoFileKey.TARGET_LINKED_DIR); 236 if (targetTestCases != null) { 237 scanDirs.add(targetTestCases); 238 } 239 if (testDir != null) { 240 scanDirs.add(testDir); 241 } 242 243 if (mModuleName != null) { 244 // Use module name as a discriminant to find some files 245 if (testDir != null) { 246 try { 247 File moduleDir = 248 FileUtil.findDirectory( 249 mModuleName, scanDirs.toArray(new File[] {})); 250 if (moduleDir != null) { 251 // If the spec is pushing the module itself 252 if (mModuleName.equals(fileName)) { 253 // If that's the main binary generated by the target, we push the 254 // full directory 255 return moduleDir; 256 } 257 // Search the module directory if it exists use it in priority 258 src = FileUtil.findFile(fileName, null, moduleDir); 259 if (src != null) { 260 // Search again with filtering on ABI 261 File srcWithAbi = FileUtil.findFile(fileName, mAbi, moduleDir); 262 if (srcWithAbi != null 263 && !srcWithAbi 264 .getAbsolutePath() 265 .startsWith(src.getAbsolutePath())) { 266 // When multiple matches are found, return the one with matching 267 // ABI unless src is its parent directory. 268 return srcWithAbi; 269 } 270 return src; 271 } 272 } else { 273 CLog.d("Did not find any module directory for '%s'", mModuleName); 274 } 275 276 } catch (IOException e) { 277 CLog.w( 278 "Something went wrong while searching for the module '%s' " 279 + "directory.", 280 mModuleName); 281 } 282 } 283 } 284 // Search top-level matches 285 for (File searchDir : scanDirs) { 286 try { 287 Set<File> allMatch = FileUtil.findFilesObject(searchDir, fileName); 288 if (allMatch.size() > 1) { 289 CLog.d( 290 "Several match for filename '%s', searching for top-level match.", 291 fileName); 292 for (File f : allMatch) { 293 // Bias toward direct child / top level nodes 294 if (f.getParent().equals(searchDir.getAbsolutePath())) { 295 return f; 296 } 297 } 298 } else if (allMatch.size() == 1) { 299 return allMatch.iterator().next(); 300 } 301 } catch (IOException e) { 302 CLog.w("Failed to find test files from directory."); 303 } 304 } 305 // Fall-back to searching everything 306 try { 307 // Search the full tests dir if no target dir is available. 308 src = FileUtil.findFile(fileName, null, scanDirs.toArray(new File[] {})); 309 if (src != null) { 310 // Search again with filtering on ABI 311 File srcWithAbi = 312 FileUtil.findFile(fileName, mAbi, scanDirs.toArray(new File[] {})); 313 if (srcWithAbi != null 314 && !srcWithAbi.getAbsolutePath().startsWith(src.getAbsolutePath())) { 315 // When multiple matches are found, return the one with matching 316 // ABI unless src is its parent directory. 317 return srcWithAbi; 318 } 319 return src; 320 } 321 } catch (IOException e) { 322 CLog.w("Failed to find test files from directory."); 323 src = null; 324 } 325 326 if (src == null && testDir != null) { 327 // TODO(b/138416078): Once build dependency can be fixed and test required 328 // APKs are all under the test module directory, we can remove this fallback 329 // approach to do individual download from remote artifact. 330 // Try to stage the files from remote zip files. 331 src = buildInfo.stageRemoteFile(fileName, testDir); 332 if (src != null) { 333 InvocationMetricLogger.addInvocationMetrics( 334 InvocationMetricKey.STAGE_UNDEFINED_DEPENDENCY, fileName); 335 try { 336 // Search again with filtering on ABI 337 File srcWithAbi = FileUtil.findFile(fileName, mAbi, testDir); 338 if (srcWithAbi != null 339 && !srcWithAbi 340 .getAbsolutePath() 341 .startsWith(src.getAbsolutePath())) { 342 // When multiple matches are found, return the one with matching 343 // ABI unless src is its parent directory. 344 return srcWithAbi; 345 } 346 } catch (IOException e) { 347 CLog.w("Failed to find test files with matching ABI from directory."); 348 } 349 } 350 } 351 } 352 return src; 353 } 354 355 /** {@inheritDoc} */ 356 @Override setUp(TestInformation testInfo)357 public void setUp(TestInformation testInfo) 358 throws TargetSetupError, BuildError, DeviceNotAvailableException { 359 mFilesPushed = new HashSet<>(); 360 ITestDevice device = testInfo.getDevice(); 361 if (mRemountSystem) { 362 device.remountSystemWritable(); 363 } 364 if (mRemountVendor) { 365 device.remountVendorWritable(); 366 } 367 368 // Backup files 369 for (Map.Entry<String, String> entry : mBackupFileSpecs.entrySet()) { 370 device.executeShellCommand( 371 "mv \"" + entry.getKey() + "\" \"" + entry.getValue() + "\""); 372 } 373 374 Map<String, File> remoteToLocalMapping = getPushSpecs(device); 375 for (String remotePath : remoteToLocalMapping.keySet()) { 376 File local = remoteToLocalMapping.get(remotePath); 377 CLog.d("Trying to push local '%s' to remote '%s'", local.getPath(), remotePath); 378 evaluatePushingPair(device, testInfo.getBuildInfo(), local, remotePath); 379 } 380 381 for (String command : mPostPushCommands) { 382 device.executeShellCommand(command); 383 } 384 385 if (mTriggerMediaScan) { 386 String mountPoint = device.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); 387 device.executeShellCommand(String.format(MEDIA_SCAN_INTENT, mountPoint)); 388 } 389 } 390 391 /** {@inheritDoc} */ 392 @Override tearDown(TestInformation testInfo, Throwable e)393 public void tearDown(TestInformation testInfo, Throwable e) throws DeviceNotAvailableException { 394 ITestDevice device = testInfo.getDevice(); 395 if (!(e instanceof DeviceNotAvailableException) && mCleanup && mFilesPushed != null) { 396 if (mRemountSystem) { 397 device.remountSystemReadOnly(); 398 } 399 if (mRemountVendor) { 400 device.remountVendorReadOnly(); 401 } 402 for (String devicePath : mFilesPushed) { 403 device.deleteFile(devicePath); 404 } 405 // Restore files 406 for (Map.Entry<String, String> entry : mBackupFileSpecs.entrySet()) { 407 device.executeShellCommand( 408 "mv \"" + entry.getValue() + "\" \"" + entry.getKey() + "\""); 409 } 410 } 411 } 412 evaluatePushingPair( ITestDevice device, IBuildInfo buildInfo, File src, String remotePath)413 private void evaluatePushingPair( 414 ITestDevice device, IBuildInfo buildInfo, File src, String remotePath) 415 throws TargetSetupError, DeviceNotAvailableException { 416 String localPath = src.getPath(); 417 if (!src.isAbsolute()) { 418 src = resolveRelativeFilePath(buildInfo, localPath); 419 } 420 if (src == null || !src.exists()) { 421 fail( 422 String.format("Local source file '%s' does not exist", localPath), 423 device.getDeviceDescriptor(), 424 InfraErrorIdentifier.CONFIGURED_ARTIFACT_NOT_FOUND); 425 return; 426 } 427 if (src.isDirectory()) { 428 boolean deleteContentOnly = true; 429 if (!device.doesFileExist(remotePath)) { 430 device.executeShellCommand(String.format("mkdir -p \"%s\"", remotePath)); 431 deleteContentOnly = false; 432 } else if (!device.isDirectory(remotePath)) { 433 // File exists and is not a directory 434 throw new TargetSetupError( 435 String.format( 436 "Attempting to push dir '%s' to an existing device file '%s'", 437 src.getAbsolutePath(), remotePath), 438 device.getDeviceDescriptor(), 439 DeviceErrorIdentifier.FAIL_PUSH_FILE); 440 } 441 Set<String> filter = new HashSet<>(); 442 if (mAbi != null && !mSkipAbiFiltering) { 443 String currentArch = AbiUtils.getArchForAbi(mAbi.getName()); 444 filter.addAll(AbiUtils.getArchSupported()); 445 filter.remove(currentArch); 446 } 447 // TODO: Look into using syncFiles but that requires improving sync to work for unroot 448 if (!device.pushDir(src, remotePath, filter)) { 449 fail( 450 String.format( 451 "Failed to push local '%s' to remote '%s'", localPath, remotePath), 452 device.getDeviceDescriptor(), 453 DeviceErrorIdentifier.FAIL_PUSH_FILE); 454 return; 455 } else { 456 if (deleteContentOnly) { 457 remotePath += "/*"; 458 } 459 mFilesPushed.add(remotePath); 460 } 461 } else { 462 if (!device.pushFile(src, remotePath)) { 463 fail( 464 String.format( 465 "Failed to push local '%s' to remote '%s'", localPath, remotePath), 466 device.getDeviceDescriptor(), 467 DeviceErrorIdentifier.FAIL_PUSH_FILE); 468 return; 469 } else { 470 mFilesPushed.add(remotePath); 471 } 472 } 473 } 474 475 @Override reportDependencies()476 public Set<String> reportDependencies() { 477 Set<String> deps = new HashSet<>(); 478 try { 479 for (File f : getPushSpecs(null).values()) { 480 // Match the resolving logic when actually pushing 481 if (!f.isAbsolute()) { 482 deps.add(f.getName()); 483 } else { 484 CLog.d( 485 "%s detected as existing. Not reported as dependency.", 486 f.getAbsolutePath()); 487 } 488 } 489 } catch (TargetSetupError e) { 490 CLog.e(e); 491 } 492 return deps; 493 } 494 shouldRemountSystem()495 public boolean shouldRemountSystem() { 496 return mRemountSystem; 497 } 498 shouldRemountVendor()499 public boolean shouldRemountVendor() { 500 return mRemountVendor; 501 } 502 isCleanUpEnabled()503 public boolean isCleanUpEnabled() { 504 return mCleanup; 505 } 506 } 507