1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.csuite.core; 18 19 import com.android.csuite.core.DeviceUtils.DeviceTimestamp; 20 import com.android.csuite.core.DeviceUtils.DropboxEntry; 21 import com.android.tradefed.device.DeviceNotAvailableException; 22 import com.android.tradefed.invoker.TestInformation; 23 import com.android.tradefed.log.LogUtil.CLog; 24 import com.android.tradefed.result.ByteArrayInputStreamSource; 25 import com.android.tradefed.result.FileInputStreamSource; 26 import com.android.tradefed.result.InputStreamSource; 27 import com.android.tradefed.result.LogDataType; 28 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestLogData; 29 import com.android.tradefed.util.ZipUtil; 30 31 import com.google.common.annotations.VisibleForTesting; 32 33 import java.io.File; 34 import java.io.IOException; 35 import java.nio.file.Files; 36 import java.nio.file.Path; 37 import java.util.Arrays; 38 import java.util.Collections; 39 import java.util.List; 40 import java.util.concurrent.TimeUnit; 41 import java.util.function.BiFunction; 42 import java.util.function.Consumer; 43 import java.util.stream.Collectors; 44 import java.util.stream.Stream; 45 46 /** A utility class that contains common methods used by tests. */ 47 public class TestUtils { 48 private static final String GMS_PACKAGE_NAME = "com.google.android.gms"; 49 private final TestInformation mTestInformation; 50 private final TestArtifactReceiver mTestArtifactReceiver; 51 private final DeviceUtils mDeviceUtils; 52 private static final int MAX_CRASH_SNIPPET_LINES = 60; 53 54 public enum TakeEffectWhen { 55 NEVER, 56 ON_FAIL, 57 ON_PASS, 58 ALWAYS, 59 } 60 61 public enum RoboscriptSignal { 62 SUCCESS, 63 UNKNOWN, 64 FAIL 65 } 66 getInstance(TestInformation testInformation, TestLogData testLogData)67 public static TestUtils getInstance(TestInformation testInformation, TestLogData testLogData) { 68 return new TestUtils( 69 testInformation, 70 new TestLogDataTestArtifactReceiver(testLogData), 71 DeviceUtils.getInstance(testInformation.getDevice())); 72 } 73 getInstance( TestInformation testInformation, TestArtifactReceiver testArtifactReceiver)74 public static TestUtils getInstance( 75 TestInformation testInformation, TestArtifactReceiver testArtifactReceiver) { 76 return new TestUtils( 77 testInformation, 78 testArtifactReceiver, 79 DeviceUtils.getInstance(testInformation.getDevice())); 80 } 81 82 @VisibleForTesting TestUtils( TestInformation testInformation, TestArtifactReceiver testArtifactReceiver, DeviceUtils deviceUtils)83 TestUtils( 84 TestInformation testInformation, 85 TestArtifactReceiver testArtifactReceiver, 86 DeviceUtils deviceUtils) { 87 mTestInformation = testInformation; 88 mTestArtifactReceiver = testArtifactReceiver; 89 mDeviceUtils = deviceUtils; 90 } 91 92 /** 93 * Take a screenshot on the device and save it to the test result artifacts. 94 * 95 * @param prefix The file name prefix. 96 */ collectScreenshot(String prefix)97 public void collectScreenshot(String prefix) throws DeviceNotAvailableException { 98 try (InputStreamSource screenSource = mTestInformation.getDevice().getScreenshot()) { 99 mTestArtifactReceiver.addTestArtifact( 100 prefix + "_screenshot_" + mTestInformation.getDevice().getSerialNumber(), 101 LogDataType.PNG, 102 screenSource); 103 } 104 } 105 106 /** 107 * Record the device screen while running a task and save the video file to the test result 108 * artifacts. 109 * 110 * @param job A job to run while recording the screen. 111 * @param prefix The file name prefix. 112 * @throws DeviceNotAvailableException when device is lost 113 */ collectScreenRecord( DeviceUtils.RunnableThrowingDeviceNotAvailable job, String prefix)114 public void collectScreenRecord( 115 DeviceUtils.RunnableThrowingDeviceNotAvailable job, String prefix) 116 throws DeviceNotAvailableException { 117 collectScreenRecord(job, prefix, null); 118 } 119 120 /** 121 * Record the device screen while running a task and save the video file to the test result 122 * artifacts. 123 * 124 * @param job A job to run while recording the screen. 125 * @param prefix The file name prefix 126 * @param videoStartTimeConsumer A consumer function that accepts the video start time on the 127 * device. Can be null. 128 * @throws DeviceNotAvailableException when the device is lost 129 */ collectScreenRecord( DeviceUtils.RunnableThrowingDeviceNotAvailable job, String prefix, Consumer<DeviceUtils.DeviceTimestamp> videoStartTimeConsumer)130 public void collectScreenRecord( 131 DeviceUtils.RunnableThrowingDeviceNotAvailable job, 132 String prefix, 133 Consumer<DeviceUtils.DeviceTimestamp> videoStartTimeConsumer) 134 throws DeviceNotAvailableException { 135 mDeviceUtils.runWithScreenRecording( 136 job, 137 (video, videoStartTimeOnDevice) -> { 138 if (video != null && videoStartTimeOnDevice != null) { 139 if (videoStartTimeConsumer != null) { 140 videoStartTimeConsumer.accept(videoStartTimeOnDevice); 141 } 142 mTestArtifactReceiver.addTestArtifact( 143 prefix 144 + "_screenrecord_" 145 + mTestInformation.getDevice().getSerialNumber() 146 + "_(" 147 + videoStartTimeOnDevice.getFormatted( 148 "yyyy-MM-dd_HH.mm.ss_SSS") 149 + ")", 150 LogDataType.MP4, 151 video); 152 } else { 153 CLog.e("Failed to collect screen recording artifacts."); 154 } 155 }); 156 } 157 158 /** 159 * Saves test APK files when conditions on the test result is met. 160 * 161 * @param when Conditions to save the apks based on the test result. 162 * @param testPassed The test result. 163 * @param prefix Output file name prefix 164 * @param apks A list of files that can be files, directories, or a mix of both. 165 * @return true if apk files are saved as artifacts. False otherwise. 166 */ saveApks( TakeEffectWhen when, boolean testPassed, String prefix, List<File> apks)167 public boolean saveApks( 168 TakeEffectWhen when, boolean testPassed, String prefix, List<File> apks) { 169 if (apks.isEmpty() || when == TakeEffectWhen.NEVER) { 170 return false; 171 } 172 173 if ((when == TakeEffectWhen.ON_FAIL && testPassed) 174 || (when == TakeEffectWhen.ON_PASS && !testPassed)) { 175 return false; 176 } 177 178 try { 179 File outputZip = ZipUtil.createZip(apks); 180 getTestArtifactReceiver().addTestArtifact(prefix + "-apks", LogDataType.ZIP, outputZip); 181 return true; 182 } catch (IOException e) { 183 CLog.e("Failed to zip the apks: " + e); 184 } 185 186 return false; 187 } 188 189 /** 190 * Collect the GMS version name and version code, and save them as test result artifacts. 191 * 192 * @param prefix The file name prefix. 193 */ collectGmsVersion(String prefix)194 public void collectGmsVersion(String prefix) throws DeviceNotAvailableException { 195 String gmsVersionCode = mDeviceUtils.getPackageVersionCode(GMS_PACKAGE_NAME); 196 String gmsVersionName = mDeviceUtils.getPackageVersionName(GMS_PACKAGE_NAME); 197 CLog.i("GMS core versionCode=%s, versionName=%s", gmsVersionCode, gmsVersionName); 198 199 // Note: If the file name format needs to be modified, do it with cautions as some users may 200 // be parsing the output file name to get the version information. 201 mTestArtifactReceiver.addTestArtifact( 202 String.format("%s_[GMS_versionCode=%s]", prefix, gmsVersionCode), 203 LogDataType.HOST_LOG, 204 gmsVersionCode.getBytes()); 205 mTestArtifactReceiver.addTestArtifact( 206 String.format("%s_[GMS_versionName=%s]", prefix, gmsVersionName), 207 LogDataType.HOST_LOG, 208 gmsVersionName.getBytes()); 209 } 210 211 /** 212 * Collect the given package's version name and version code, and save them as test result 213 * artifacts. 214 * 215 * @param packageName The package name. 216 */ collectAppVersion(String packageName)217 public void collectAppVersion(String packageName) throws DeviceNotAvailableException { 218 String versionCode = mDeviceUtils.getPackageVersionCode(packageName); 219 String versionName = mDeviceUtils.getPackageVersionName(packageName); 220 CLog.i("Package %s versionCode=%s, versionName=%s", packageName, versionCode, versionName); 221 222 // Note: If the file name format needs to be modified, do it with cautions as some users may 223 // be parsing the output file name to get the version information. 224 mTestArtifactReceiver.addTestArtifact( 225 String.format("%s_[versionCode=%s]", packageName, versionCode), 226 LogDataType.HOST_LOG, 227 versionCode.getBytes()); 228 mTestArtifactReceiver.addTestArtifact( 229 String.format("%s_[versionName=%s]", packageName, versionName), 230 LogDataType.HOST_LOG, 231 versionName.getBytes()); 232 } 233 234 /** 235 * Looks for crash log of a package in the device's dropbox entries. 236 * 237 * @param packageName The package name of an app. 238 * @param startTimeOnDevice The device timestamp after which the check starts. Dropbox items 239 * before this device timestamp will be ignored. 240 * @param saveToFile whether to save the package's full dropbox crash logs to a test output 241 * file. 242 * @return A string of crash log if crash was found; null otherwise. 243 * @throws IOException unexpected IOException 244 */ getDropboxPackageCrashLog( String packageName, DeviceTimestamp startTimeOnDevice, boolean saveToFile)245 public String getDropboxPackageCrashLog( 246 String packageName, DeviceTimestamp startTimeOnDevice, boolean saveToFile) 247 throws IOException { 248 List<DropboxEntry> crashEntries = 249 mDeviceUtils.getDropboxEntries( 250 DeviceUtils.DROPBOX_APP_CRASH_TAGS, packageName, startTimeOnDevice, null); 251 return compileTestFailureMessage(packageName, crashEntries, saveToFile, null); 252 } 253 254 /** 255 * Compiles an error message from device's dropbox crash entries. 256 * 257 * @param packageName The package name of an app. 258 * @param entries Dropbox entries that indicate the package crashed. 259 * @param saveToFile whether to save the package's full dropbox crash logs to a test output 260 * file. 261 * @param screenRecordStartTime The screen record start time on the device, which is used to 262 * calculate the video time where the crashes happened. Can be null. 263 * @return A string of crash log if crash was found; null otherwise. 264 * @throws IOException unexpected IOException 265 */ compileTestFailureMessage( String packageName, List<DropboxEntry> entries, boolean saveToFile, DeviceTimestamp screenRecordStartTime)266 public String compileTestFailureMessage( 267 String packageName, 268 List<DropboxEntry> entries, 269 boolean saveToFile, 270 DeviceTimestamp screenRecordStartTime) 271 throws IOException { 272 if (entries.size() == 0) { 273 return null; 274 } 275 276 BiFunction<String, Integer, String> truncateFunction = 277 (text, maxLines) -> { 278 String[] lines = text.split("\\r?\\n"); 279 StringBuilder sb = new StringBuilder(); 280 for (int i = 0; i < maxLines && i < lines.length; i++) { 281 sb.append(lines[i]).append('\n'); 282 } 283 if (lines.length > maxLines) { 284 sb.append("... ") 285 .append(lines.length - maxLines) 286 .append(" more lines truncated ...\n"); 287 } 288 return sb.toString(); 289 }; 290 291 StringBuilder fullText = new StringBuilder(); 292 StringBuilder truncatedText = new StringBuilder(); 293 294 for (int i = 0; i < entries.size(); i++) { 295 DropboxEntry entry = entries.get(i); 296 String entryHeader = 297 String.format( 298 "\n============ Dropbox Entry (%s of %s) ============\n", 299 i + 1, entries.size()); 300 String videoTimeInfo = 301 screenRecordStartTime == null 302 ? "" 303 : "Time in screen record video: " 304 + convertToMinuteSecond( 305 entry.getTime() - screenRecordStartTime.get()) 306 + "\n"; 307 308 String fullEntry = entryHeader + videoTimeInfo + entry; 309 String truncatedEntry = truncateFunction.apply(fullEntry, MAX_CRASH_SNIPPET_LINES); 310 311 fullText.append(fullEntry); 312 truncatedText.append(truncatedEntry); 313 } 314 315 String videoCrashTimeMessage = 316 screenRecordStartTime == null 317 ? "" 318 : String.format( 319 "A screen recording video is available. The crashes happened" 320 + " at around the following video time: [%s]\n", 321 entries.stream() 322 .map(entry -> entry.getTime() - screenRecordStartTime.get()) 323 .sorted() 324 .map(videoTime -> convertToMinuteSecond(videoTime)) 325 .collect(Collectors.joining(", "))); 326 String summary = 327 String.format( 328 "Found a total of %s dropbox entries indicating the package %s may have run" 329 + " into an issue. Types of entries include: [%s].\n%sEntries:\n", 330 entries.size(), 331 packageName, 332 entries.stream() 333 .map(DropboxEntry::getTag) 334 .distinct() 335 .collect(Collectors.joining(", ")), 336 videoCrashTimeMessage); 337 338 if (saveToFile) { 339 mTestArtifactReceiver.addTestArtifact( 340 String.format("%s_dropbox_entries", packageName), 341 LogDataType.HOST_LOG, 342 (summary + fullText).getBytes()); 343 } 344 345 return summary + truncatedText; 346 } 347 convertToMinuteSecond(long milliseconds)348 private String convertToMinuteSecond(long milliseconds) { 349 long minutes = TimeUnit.MILLISECONDS.toMinutes(milliseconds); 350 long seconds = 351 TimeUnit.MILLISECONDS.toSeconds(milliseconds) - TimeUnit.MINUTES.toSeconds(minutes); 352 353 String formattedMinutes = String.format("%02d", minutes); 354 String formattedSeconds = String.format("%02d", seconds); 355 356 return formattedMinutes + ":" + formattedSeconds; 357 } 358 359 /** 360 * Generates a list of APK paths where the base.apk of split apk files are always on the first 361 * index if exists. 362 * 363 * <p>If the input path points to a single apk file, then the same path is returned. If the 364 * input path is a directory containing only one non-split apk file, the apk file path is 365 * returned. If the apk path is a directory containing split apk files for one package, then the 366 * list of apks are returned and the base.apk sits on the first index. If the path contains obb 367 * files, then they will be included at the end of the returned path list. If the apk path does 368 * not contain any apk files, or multiple apk files without base.apk, then an IOException is 369 * thrown. 370 * 371 * @return A list of APK paths with OBB files if available. 372 * @throws TestUtilsException If failed to read the apk path or unexpected number of apk files 373 * are found under the path. 374 */ listApks(Path root)375 public static List<Path> listApks(Path root) throws TestUtilsException { 376 // The apk path points to a non-split apk file. 377 if (Files.isRegularFile(root)) { 378 if (!root.toString().endsWith(".apk")) { 379 throw new TestUtilsException( 380 "The file on the given apk path is not an apk file: " + root); 381 } 382 return List.of(root); 383 } 384 385 List<Path> apksAndObbs; 386 CLog.d("APK path = " + root); 387 try (Stream<Path> fileTree = Files.walk(root)) { 388 apksAndObbs = 389 fileTree.filter(Files::isRegularFile) 390 .filter( 391 path -> 392 path.getFileName() 393 .toString() 394 .toLowerCase() 395 .endsWith(".apk") 396 || path.getFileName() 397 .toString() 398 .toLowerCase() 399 .endsWith(".obb")) 400 .collect(Collectors.toList()); 401 } catch (IOException e) { 402 throw new TestUtilsException("Failed to list apk files.", e); 403 } 404 405 List<Path> apkFiles = 406 apksAndObbs.stream() 407 .filter(path -> path.getFileName().toString().endsWith(".apk")) 408 .collect(Collectors.toList()); 409 410 if (apkFiles.isEmpty()) { 411 throw new TestUtilsException( 412 "Empty APK directory. Cannot find any APK files under " + root); 413 } 414 415 if (apkFiles.stream().map(path -> path.getParent().toString()).distinct().count() != 1) { 416 throw new TestUtilsException( 417 "Apk files are not all in the same folder: " 418 + Arrays.deepToString( 419 apksAndObbs.toArray(new Path[apksAndObbs.size()]))); 420 } 421 422 if (apkFiles.size() > 1 423 && apkFiles.stream() 424 .filter(path -> path.getFileName().toString().equals("base.apk")) 425 .count() 426 == 0) { 427 throw new TestUtilsException( 428 "Base apk is not found: " 429 + Arrays.deepToString( 430 apksAndObbs.toArray(new Path[apksAndObbs.size()]))); 431 } 432 433 if (apksAndObbs.stream() 434 .filter( 435 path -> 436 path.getFileName().toString().endsWith(".obb") 437 && path.getFileName().toString().startsWith("main")) 438 .count() 439 > 1) { 440 throw new TestUtilsException( 441 "Multiple main obb files are found: " 442 + Arrays.deepToString( 443 apksAndObbs.toArray(new Path[apksAndObbs.size()]))); 444 } 445 446 Collections.sort( 447 apksAndObbs, 448 (first, second) -> { 449 if (first.getFileName().toString().equals("base.apk")) { 450 return -1; 451 } else if (first.getFileName().toString().toLowerCase().endsWith(".obb")) { 452 return 1; 453 } else { 454 return first.getFileName().compareTo(second.getFileName()); 455 } 456 }); 457 458 return apksAndObbs; 459 } 460 461 /** Returns the test information. */ getTestInformation()462 public TestInformation getTestInformation() { 463 return mTestInformation; 464 } 465 466 /** Returns the test artifact receiver. */ getTestArtifactReceiver()467 public TestArtifactReceiver getTestArtifactReceiver() { 468 return mTestArtifactReceiver; 469 } 470 471 /** Returns the device utils. */ getDeviceUtils()472 public DeviceUtils getDeviceUtils() { 473 return mDeviceUtils; 474 } 475 476 /** An exception class representing exceptions thrown from the test utils. */ 477 public static final class TestUtilsException extends Exception { 478 /** 479 * Constructs a new {@link TestUtilsException} with a meaningful error message. 480 * 481 * @param message A error message describing the cause of the error. 482 */ TestUtilsException(String message)483 private TestUtilsException(String message) { 484 super(message); 485 } 486 487 /** 488 * Constructs a new {@link TestUtilsException} with a meaningful error message, and a cause. 489 * 490 * @param message A detailed error message. 491 * @param cause A {@link Throwable} capturing the original cause of the TestUtilsException. 492 */ TestUtilsException(String message, Throwable cause)493 private TestUtilsException(String message, Throwable cause) { 494 super(message, cause); 495 } 496 497 /** 498 * Constructs a new {@link TestUtilsException} with a cause. 499 * 500 * @param cause A {@link Throwable} capturing the original cause of the TestUtilsException. 501 */ TestUtilsException(Throwable cause)502 private TestUtilsException(Throwable cause) { 503 super(cause); 504 } 505 } 506 507 public static class TestLogDataTestArtifactReceiver implements TestArtifactReceiver { 508 @SuppressWarnings("hiding") 509 private final TestLogData mTestLogData; 510 TestLogDataTestArtifactReceiver(TestLogData testLogData)511 public TestLogDataTestArtifactReceiver(TestLogData testLogData) { 512 mTestLogData = testLogData; 513 } 514 515 @Override addTestArtifact(String name, LogDataType type, byte[] bytes)516 public void addTestArtifact(String name, LogDataType type, byte[] bytes) { 517 mTestLogData.addTestLog(name, type, new ByteArrayInputStreamSource(bytes)); 518 } 519 520 @Override addTestArtifact(String name, LogDataType type, File file)521 public void addTestArtifact(String name, LogDataType type, File file) { 522 mTestLogData.addTestLog(name, type, new FileInputStreamSource(file)); 523 } 524 525 @Override addTestArtifact(String name, LogDataType type, InputStreamSource source)526 public void addTestArtifact(String name, LogDataType type, InputStreamSource source) { 527 mTestLogData.addTestLog(name, type, source); 528 } 529 } 530 531 public interface TestArtifactReceiver { 532 533 /** 534 * Add a test artifact. 535 * 536 * @param name File name. 537 * @param type Output data type. 538 * @param bytes The output data. 539 */ addTestArtifact(String name, LogDataType type, byte[] bytes)540 void addTestArtifact(String name, LogDataType type, byte[] bytes); 541 542 /** 543 * Add a test artifact. 544 * 545 * @param name File name. 546 * @param type Output data type. 547 * @param inputStreamSource The inputStreamSource. 548 */ addTestArtifact(String name, LogDataType type, InputStreamSource inputStreamSource)549 void addTestArtifact(String name, LogDataType type, InputStreamSource inputStreamSource); 550 551 /** 552 * Add a test artifact. 553 * 554 * @param name File name. 555 * @param type Output data type. 556 * @param file The output file. 557 */ addTestArtifact(String name, LogDataType type, File file)558 void addTestArtifact(String name, LogDataType type, File file); 559 } 560 } 561