1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.presubmit; 17 18 import com.android.tradefed.build.IBuildInfo; 19 import com.android.tradefed.build.IDeviceBuildInfo; 20 import com.android.tradefed.config.ConfigurationDescriptor; 21 import com.android.tradefed.config.ConfigurationException; 22 import com.android.tradefed.config.ConfigurationFactory; 23 import com.android.tradefed.config.ConfigurationUtil; 24 import com.android.tradefed.config.IConfiguration; 25 import com.android.tradefed.config.IConfigurationFactory; 26 import com.android.tradefed.config.IDeviceConfiguration; 27 import com.android.tradefed.config.Option; 28 import com.android.tradefed.log.LogUtil; 29 import com.android.tradefed.targetprep.ITargetPreparer; 30 import com.android.tradefed.targetprep.PushFilePreparer; 31 import com.android.tradefed.targetprep.TestAppInstallSetup; 32 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; 33 import com.android.tradefed.testtype.IBuildReceiver; 34 import com.android.tradefed.testtype.IRemoteTest; 35 import com.android.tradefed.testtype.IsolatedHostTest; 36 import com.android.tradefed.testtype.suite.ITestSuite; 37 import com.android.tradefed.testtype.suite.ValidateSuiteConfigHelper; 38 import com.android.tradefed.testtype.suite.params.ModuleParameters; 39 import com.android.tradefed.util.FileUtil; 40 import com.android.tradefed.util.ModuleTestTypeUtil; 41 42 import com.google.common.base.Joiner; 43 44 import org.junit.Assume; 45 import org.junit.Test; 46 import org.junit.runner.RunWith; 47 48 import java.io.File; 49 import java.net.URI; 50 import java.util.ArrayList; 51 import java.util.Arrays; 52 import java.util.HashSet; 53 import java.util.List; 54 import java.util.Set; 55 56 /** 57 * Validation tests to run against the configuration in general-tests.zip to ensure they can all 58 * parse. 59 * 60 * <p>Do not add to UnitTests.java. This is meant to run standalone. 61 */ 62 @RunWith(DeviceJUnit4ClassRunner.class) 63 public class GeneralTestsConfigValidation implements IBuildReceiver { 64 65 @Option( 66 name = "config-extension", 67 description = "The expected extension from configuration to check.") 68 private String mConfigExtension = "config"; 69 70 @Option( 71 name = "disallowed-test-type", 72 description = "The disallowed test type for configs in general-tests.zip") 73 private List<String> mDisallowedTestTypes = new ArrayList<>(); 74 75 private IBuildInfo mBuild; 76 77 /** 78 * List of the officially supported runners in general-tests. Any new addition should go through 79 * a review to ensure all runners have a high quality bar. 80 */ 81 private static final Set<String> SUPPORTED_TEST_RUNNERS = 82 new HashSet<>( 83 Arrays.asList( 84 // Cts runners 85 "com.android.compatibility.common.tradefed.testtype.JarHostTest", 86 "com.android.compatibility.testtype.DalvikTest", 87 "com.android.compatibility.testtype.LibcoreTest", 88 "com.drawelements.deqp.runner.DeqpTestRunner", 89 // Tradefed runners 90 "com.android.tradefed.testtype.UiAutomatorTest", 91 "com.android.tradefed.testtype.InstrumentationTest", 92 "com.android.tradefed.testtype.AndroidJUnitTest", 93 "com.android.tradefed.testtype.HostTest", 94 "com.android.tradefed.testtype.GTest", 95 "com.android.tradefed.testtype.HostGTest", 96 "com.android.tradefed.testtype.GoogleBenchmarkTest", 97 "com.android.tradefed.testtype.IsolatedHostTest", 98 "com.android.tradefed.testtype.python.PythonBinaryHostTest", 99 "com.android.tradefed.testtype.binary.ExecutableHostTest", 100 "com.android.tradefed.testtype.binary.ExecutableTargetTest", 101 "com.android.tradefed.testtype.rust.RustBinaryHostTest", 102 "com.android.tradefed.testtype.rust.RustBinaryTest", 103 "com.android.tradefed.testtype.StubTest", 104 "com.android.tradefed.testtype.ArtRunTest", 105 "com.android.tradefed.testtype.ArtGTest", 106 "com.android.tradefed.testtype.mobly.MoblyBinaryHostTest", 107 "com.android.tradefed.testtype.pandora.PtsBotTest", 108 // VTS runners 109 "com.android.tradefed.testtype.binary.KernelTargetTest", 110 // Others 111 "com.google.android.deviceconfig.RebootTest", 112 "com.android.scenario.AppSetup", 113 "com.android.power.PowerRunner", 114 "com.android.boot.BootTimeTest")); 115 116 /** 117 * List of configs that will be exempted until they are converted to use MediaPreparers. 118 * (b/274674920) 119 */ 120 private static final Set<String> MEDIAPREPARER_EXEMPTED_CONFIGS = 121 new HashSet<>( 122 Arrays.asList( 123 "OpusHeaderTest.config", 124 "AmrnbEncoderTest.config", 125 "AmrnbDecoderTest.config", 126 "AmrwbEncoderTest.config", 127 "AmrwbDecoderTest.config", 128 "HEVCUtilsUnitTest.config", 129 "ExtractorUnitTest.config", 130 "MediaTranscoderBenchmark.config", 131 "TimedTextUnitTest.config", 132 "VorbisDecoderTest.config", 133 "MediaTrackTranscoderBenchmark.config", 134 "ID3Test.config", 135 "ExtractorFactoryTest.config", 136 "MediaSampleReaderBenchmark.config", 137 "Mpeg4H263EncoderTest.config", 138 "Mp3DecoderTest.config", 139 "Mpeg2tsUnitTest.config", 140 "Mpeg4H263DecoderTest.config")); 141 142 /** List of configs that will be exempted until b/274930471 is fixed. */ 143 private static final Set<String> EXEMPTED_PYTHON_TEST_MODULES = 144 new HashSet<>( 145 Arrays.asList( 146 "aidl_integration_test.config", 147 "hidl_test.config", 148 "hidl_test_java.config", 149 "fmq_test.config")); 150 /** List of configs to exclude until b/277261121 is fixed. */ 151 private static final Set<String> EXEMPTED_KERNEL_MODULES = 152 new HashSet<>( 153 Arrays.asList( 154 "vts_ltp_test_arm_64.config", 155 "vts_ltp_test_arm_64_lowmem.config", 156 "vts_ltp_test_arm_64_hwasan.config", 157 "vts_ltp_test_arm_64_lowmem_hwasan.config", 158 "vts_ltp_test_arm.config", 159 "vts_ltp_test_arm_lowmem.config", 160 "vts_ltp_test_x86_64.config", 161 "vts_ltp_test_x86.config", 162 "vts_linux_kselftest_arm_64.config", 163 "vts_linux_kselftest_arm_32.config", 164 "vts_linux_kselftest_x86_64.config", 165 "vts_linux_kselftest_x86_32.config", 166 "vts_linux_kselftest_riscv_64.config")); 167 168 /** 169 * Temporarily exempt the current configs so that the test can be submitted to block new 170 * configs. 171 */ 172 private static final Set<String> TEMP_EXEMPTED_MODULES = 173 new HashSet<>( 174 Arrays.asList( 175 "PtsStorageFuncTestCases.config", 176 "PtsPowerTestCases.config", 177 "PtsPerformanceLongTestCases.config", 178 "FirmwareDtboVerification.config", 179 "net_unittests_tester.config", 180 "PerfStressTests.config", 181 "binderHostDeviceTest.config", 182 "PerfUiGfxTests.config", 183 "PtsStorageUITestCases.config", 184 "PtsStoragePerfTestCases.config", 185 "PtsNgaTestCases.config", 186 "PerfUiMiscTests.config", 187 "GtsStatsdHostTestCases.config", 188 "PtsBackupHostSideTestCases.config", 189 "PtsStorageQualTestCases.config", 190 "PtsStoragePowerTestCases.config", 191 "PtsUipbUnitTests.config", 192 "PtsSensorHostTestCases.config", 193 "PerfCheckTests.config", 194 "cronet_unittests_tester.config", 195 "PerfUiPreconditionTest.config", 196 "PtsStorageLongTestCases.config", 197 "CtsAdServicesCUJTestCases.config", 198 "hwuimacro.config", 199 "libinputserialtracker_tests.config", 200 "MediaProviderTests.config", 201 "libsurfaceflinger_arc_test.config", 202 "PtsCoolingMapTests.config", 203 "hwuimicro.config", 204 "rustBinderTestService.config", 205 "hwui_unit_tests.config", 206 "libinputreader_arc_tests.config", 207 "PtsTpuPwrStateTests.config", 208 "CtsAdExtServicesCUJTestCases.config", 209 "InteractiveNeneTest.config", 210 "SdkSandboxPerfScenarioTests.config", 211 "libinputreporter_arc_tests.config", 212 "libwayland_service_tests.config", 213 "messagingtests.config", 214 "GtsPermissionTestCases.config", 215 "GtsReadLogStringTest.config", 216 "rustBinderTest.config", 217 "libsurfaceflinger_arc_backend_test.config", 218 "MicrodroidBenchmarkApp.config", 219 "OverlayHostTests.config", 220 "ComponentAliasTests.config", 221 "WMShellFlickerTests.config", 222 "AppEnumerationInternalTests.config", 223 "ComponentAliasTests2.config", 224 "ComponentAliasTests1.config", 225 "NeuralNetworksApiCrashTest.config", 226 "FrameworksServicesTests.config", 227 "MediaSampleQueueTests.config", 228 "HdrTranscodeTests.config", 229 "MediaSampleReaderNDKTests.config", 230 "MediaTrackTranscoderTests.config", 231 "PassthroughTrackTranscoderTests.config", 232 "MediaTranscoderTests.config", 233 "VideoTrackTranscoderTests.config", 234 "MediaSampleWriterTests.config", 235 "art-run-test-656-checker-simd-opt.config", 236 "PtsChreTestCases.config", 237 "chre_nanoapps_loaded.config", 238 "BiometricsMicrobenchmark.config", 239 "GoogleSearchPrebuiltDebug.config", 240 "SystemUIMicrobenchmark.config", 241 "PlatformScenarioTests.config", 242 "UiBenchMicrobenchmark_Internal.config", 243 "CellBroadcastReceiverGoogleUnitTests.config", 244 "fixed-appstartup-login-base.config", 245 "open-fixed-calculator.config", 246 "fixed-appstartup-base.config", 247 "open-prebuilt-maps.config", 248 "transition-coldlaunch-phone.config", 249 "transition-hot-applaunch-from-qs-base.config", 250 "open-fixed-messages-warm.config", 251 "transition-hotlaunch-gmail.config", 252 "open-fixed-chrome-hot.config", 253 "open-fixed-maps.config", 254 "open-prebuilt-photos.config", 255 "open-fixed-phone.config", 256 "prebuilt-appstartup-login-base.config", 257 "open-fixed-calculator-flicker.config", 258 "open-fixed-contacts.config", 259 "transition-hotlaunch-messages.config", 260 "open-fixed-gmail-hot.config", 261 "transition-hotlaunch-calculator.config", 262 "transition-hotlaunch-maps.config", 263 "open-fixed-gmail-warm.config", 264 "open-fixed-calculator-hot.config", 265 "open-prebuilt-gmail.config", 266 "transition-coldlaunch-chrome.config", 267 "open-fixed-chrome.config", 268 "transition-coldlaunch-maps.config", 269 "transition-coldlaunch-messages.config", 270 "transition-hotlaunch-from-qs-calculator.config", 271 "open-fixed-youtube.config", 272 "open-prebuilt-clock.config", 273 "open-prebuilt-youtube.config", 274 "transition-hotlaunch-phone.config", 275 "transition-hot-applaunch-from-qs-login-base.config", 276 "prebuilt-appstartup-base.config", 277 "open-prebuilt-contacts.config", 278 "transition-coldlaunch-gmail.config", 279 "open-fixed-calculator-warm.config", 280 "appstartup-base.config", 281 "transition-hotlaunch-from-qs-phone.config", 282 "AppMicrobenchmark.config", 283 "open-fixed-clock.config", 284 "open-prebuilt-phone.config", 285 "transition-hotlaunch-from-qs-gmail.config", 286 "open-prebuilt-calendar.config", 287 "open-fixed-chrome-warm.config", 288 "open-fixed-calendar.config", 289 "transition-hot-applaunch-login-base.config", 290 "transition-hotlaunch-chrome.config", 291 "open-prebuilt-calculator.config", 292 "open-fixed-phone-hot.config", 293 "transition-hot-applaunch-base.config", 294 "transition-coldlaunch-calculator.config", 295 "open-fixed-photos.config", 296 "transition-hotlaunch-from-qs-chrome.config", 297 "transition-hotlaunch-from-qs-maps.config", 298 "open-fixed-messages.config", 299 "open-prebuilt-camera.config", 300 "open-fixed-phone-warm.config", 301 "transition-hotlaunch-from-qs-messages.config", 302 "open-prebuilt-messages.config", 303 "open-fixed-messages-hot.config", 304 "open-fixed-camera.config", 305 "open-fixed-gmail.config", 306 "GoogleSearchPrebuiltDebugService.config", 307 "UiBenchJankTests_Internal.config", 308 "HubUIScenarioTests.config", 309 "LauncherMicrobenchmark.config", 310 "MultitaskingTests.config", 311 "art-run-test-156-register-dex-file-multi-loader.config", 312 "PtsKmsVBlankTestCases.config", 313 "PtsGemBltTestCases.config", 314 "PtsSyncobjBasicTestCases.config", 315 "PtsKmsAddfbBasicTestCases.config", 316 "PtsKmsAtomicTransitionTestCases.config", 317 "PtsKmsThroughputTestCases.config", 318 "CollectorsHelperTest.config", 319 "PtsKmsAtomicInterruptibleTestCases.config", 320 "PtsKmsAtomicTestCases.config", 321 "PtsKmsPropBlobTestCases.config", 322 "PtsSyncobjWaitTestCases.config", 323 "PtsKmsPropertiesTestCases.config", 324 "PtsKmsPlaneScalingTestCases.config", 325 "PtsCoreAuthTestCases.config", 326 "PtsCoreGetclientTestCases.config", 327 "PtsKmsGetfbTestCases.config", 328 "PtsKmsFlipTestCases.config", 329 "s2-geometry-library-java-tests.config")); 330 331 @Override setBuild(IBuildInfo buildInfo)332 public void setBuild(IBuildInfo buildInfo) { 333 mBuild = buildInfo; 334 } 335 336 /** Get all the configuration copied to the build tests dir and check if they load. */ 337 @Test testConfigsLoad()338 public void testConfigsLoad() throws Exception { 339 List<String> errors = new ArrayList<>(); 340 Assume.assumeTrue(mBuild instanceof IDeviceBuildInfo); 341 342 IConfigurationFactory configFactory = ConfigurationFactory.getInstance(); 343 List<File> configs = new ArrayList<>(); 344 IDeviceBuildInfo deviceBuildInfo = (IDeviceBuildInfo) mBuild; 345 File testsDir = deviceBuildInfo.getTestsDir(); 346 List<File> extraTestCasesDirs = Arrays.asList(testsDir); 347 String configPattern = ".*\\." + mConfigExtension + "$"; 348 // include config files with same name, but with different contents (for example: host and 349 // device variants of the same config). 350 configs.addAll( 351 ConfigurationUtil.getConfigNamesFileFromDirs( 352 null, extraTestCasesDirs, Arrays.asList(configPattern), true)); 353 for (File config : configs) { 354 try { 355 IConfiguration c = 356 configFactory.createConfigurationFromArgs( 357 new String[] {config.getAbsolutePath()}); 358 // All configurations in general-tests.zip should be module since they are generated 359 // from AndroidTest.xml 360 ValidateSuiteConfigHelper.validateConfig(c); 361 362 for (IDeviceConfiguration dConfig : c.getDeviceConfig()) { 363 validatePreparers(c, config, dConfig.getTargetPreparers()); 364 } 365 // Check that all the tests runners are well supported. 366 checkRunners(c.getTests(), "general-tests"); 367 368 ConfigurationDescriptor cd = c.getConfigurationDescription(); 369 checkModuleParameters(c.getName(), cd.getMetaData(ITestSuite.PARAMETER_KEY)); 370 371 // Check for disallowed test types 372 checkDisallowedTestType(c, mDisallowedTestTypes); 373 374 // Add more checks if necessary 375 } catch (ConfigurationException e) { 376 errors.add(String.format("\t%s: %s", config.getName(), e.getMessage())); 377 } 378 } 379 380 // If any errors report them in a final exception. 381 if (!errors.isEmpty()) { 382 throw new ConfigurationException( 383 String.format("Fail configuration check:\n%s", Joiner.on("\n").join(errors))); 384 } 385 } 386 validatePreparers( IConfiguration c, File config, List<ITargetPreparer> preparers)387 public static void validatePreparers( 388 IConfiguration c, File config, List<ITargetPreparer> preparers) throws Exception { 389 if (EXEMPTED_PYTHON_TEST_MODULES.contains(config.getName())) { 390 LogUtil.CLog.w( 391 "Module %s is a python_test_host module. Ignoring until b/274930471 is fixed.s", 392 config.getName()); 393 return; 394 } 395 if (EXEMPTED_KERNEL_MODULES.contains(config.getName())) { 396 LogUtil.CLog.w("Ignoring module %s until b/277261121 is fixed.s", config.getName()); 397 return; 398 } 399 if (MEDIAPREPARER_EXEMPTED_CONFIGS.contains(config.getName())) { 400 LogUtil.CLog.w( 401 "Module %s is exempted until b/274674920 is fixed. Please Fix the config.", 402 config.getName()); 403 return; 404 } 405 if (TEMP_EXEMPTED_MODULES.contains(config.getName())) { 406 LogUtil.CLog.w("Ignoring module %s temporarily.", config.getName()); 407 return; 408 } 409 boolean isPerfModule = ModuleTestTypeUtil.isPerformanceModule(c); 410 for (ITargetPreparer preparer : preparers) { 411 if (preparer instanceof TestAppInstallSetup) { 412 List<File> apkNames = new ArrayList<>(); 413 TestAppInstallSetup installer = (TestAppInstallSetup) preparer; 414 // Ensure clean up is enabled 415 if (!installer.isCleanUpEnabled()) { 416 throw new ConfigurationException( 417 String.format("Config: %s should set cleanup-apks=true.", config)); 418 } 419 apkNames.addAll(((TestAppInstallSetup) preparer).getTestsFileName()); 420 421 // Ensure all apk dependencies are specified 422 for (File apk : apkNames) { 423 String apkName = apk.getName(); 424 File apkFile = FileUtil.findFile(config.getParentFile(), apkName); 425 if (apkFile == null || !apkFile.exists()) { 426 // allow performance modules to specify dynamic links 427 if (isPerfModule) { 428 URI uri = new URI(apk.getPath()); 429 if (uri.getScheme() != null 430 && (uri.getScheme().contains("gs") 431 || uri.getScheme().contains("http"))) { 432 continue; 433 } 434 } 435 throw new ConfigurationException( 436 String.format( 437 "Module %s is trying to install %s which does not " 438 + "exists in testcases/. Make sure that it's added " 439 + "in the Android.bp file of the module under " 440 + "'data' field.", 441 config.getName(), apkName)); 442 } 443 } 444 } 445 if (preparer instanceof PushFilePreparer) { 446 PushFilePreparer pusher = (PushFilePreparer) preparer; 447 if (!pusher.isCleanUpEnabled()) { 448 throw new ConfigurationException( 449 String.format( 450 "Config: %s should set cleanup=true for file pusher.", config)); 451 } 452 for (File f : pusher.getPushSpecs(null).values()) { 453 String path = f.getPath(); 454 // Use findFiles to also match top-level dir, which is a valid push spec 455 Set<String> toBePushed = FileUtil.findFiles(config.getParentFile(), path); 456 if (toBePushed.isEmpty()) { 457 // allow performance modules to specify dynamic links 458 if (isPerfModule) { 459 URI uri = new URI(path); 460 if (uri.getScheme() != null 461 && (uri.getScheme().contains("gs") 462 || uri.getScheme().contains("http"))) { 463 continue; 464 } 465 } 466 // See if binary files exists 467 File file32 = FileUtil.findFile(config.getParentFile(), path + "32"); 468 File file64 = FileUtil.findFile(config.getParentFile(), path + "64"); 469 if (file32 == null || file64 == null) { 470 throw new ConfigurationException( 471 String.format( 472 "File %s wasn't found in module dependencies while it's" 473 + " expected to be pushed as part of %s. Make" 474 + " sure that it's added in the Android.bp file" 475 + " of the module under 'data_device_bins_both'" 476 + " field if it's a binary file or under 'data'" 477 + " field for all other files.", 478 path, config.getName())); 479 } 480 } 481 } 482 } 483 } 484 } 485 checkRunners(List<IRemoteTest> tests, String name)486 public static void checkRunners(List<IRemoteTest> tests, String name) 487 throws ConfigurationException { 488 for (IRemoteTest test : tests) { 489 // Check that all the tests runners are well supported. 490 if (!SUPPORTED_TEST_RUNNERS.contains(test.getClass().getCanonicalName())) { 491 throw new ConfigurationException( 492 String.format( 493 "testtype %s is not officially supported in %s. " 494 + "The supported ones are: %s", 495 test.getClass().getCanonicalName(), name, SUPPORTED_TEST_RUNNERS)); 496 } 497 if (test instanceof IsolatedHostTest 498 && ((IsolatedHostTest) test).useRobolectricResources()) { 499 throw new ConfigurationException( 500 String.format( 501 "Robolectric tests aren't supported in general-tests yet. They" 502 + " have their own setup.")); 503 } 504 } 505 } 506 checkModuleParameters(String configName, List<String> parameters)507 public static void checkModuleParameters(String configName, List<String> parameters) 508 throws ConfigurationException { 509 if (parameters == null) { 510 return; 511 } 512 for (String param : parameters) { 513 try { 514 ModuleParameters.valueOf(param.toUpperCase()); 515 } catch (IllegalArgumentException e) { 516 throw new ConfigurationException( 517 String.format( 518 "Config: %s includes an unknown parameter '%s'.", 519 configName, param)); 520 } 521 } 522 } 523 524 /** 525 * Check the {@link config} to ensure it's not declared as one of the {#link 526 * disallowedTestTypes}. 527 * 528 * @param config The config to check. 529 * @param ConfigurationException The disallowed test types to check against. 530 * @throws ConfigurationException if the config is of disallowed test types. 531 */ checkDisallowedTestType( IConfiguration config, List<String> disallowedTestTypes)532 public static void checkDisallowedTestType( 533 IConfiguration config, List<String> disallowedTestTypes) throws ConfigurationException { 534 if (disallowedTestTypes == null || disallowedTestTypes.isEmpty()) { 535 return; 536 } 537 538 List<String> matched = 539 ModuleTestTypeUtil.getMatchedConfigTestTypes(config, disallowedTestTypes); 540 if (!matched.isEmpty()) { 541 throw new ConfigurationException( 542 String.format( 543 "Config %s of test type '%s' is not allowed.", 544 config.getName(), Joiner.on(", ").join(matched))); 545 } 546 } 547 } 548