1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.presubmit; 17 18 import static org.junit.Assert.assertTrue; 19 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.build.IDeviceBuildInfo; 22 import com.android.tradefed.config.ConfigurationException; 23 import com.android.tradefed.config.ConfigurationFactory; 24 import com.android.tradefed.config.ConfigurationUtil; 25 import com.android.tradefed.config.IConfiguration; 26 import com.android.tradefed.config.IConfigurationFactory; 27 import com.android.tradefed.config.Option; 28 import com.android.tradefed.targetprep.ITargetPreparer; 29 import com.android.tradefed.targetprep.PushFilePreparer; 30 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; 31 import com.android.tradefed.testtype.IBuildReceiver; 32 import com.android.tradefed.testtype.IRemoteTest; 33 import com.android.tradefed.testtype.suite.ValidateSuiteConfigHelper; 34 import com.android.tradefed.util.FileUtil; 35 import com.android.tradefed.util.testmapping.TestInfo; 36 import com.android.tradefed.util.testmapping.TestMapping; 37 38 import com.google.common.base.Joiner; 39 import com.google.common.collect.ImmutableSet; 40 41 import org.junit.Assume; 42 import org.junit.Test; 43 import org.junit.runner.RunWith; 44 45 import java.io.File; 46 import java.util.ArrayList; 47 import java.util.Arrays; 48 import java.util.Collections; 49 import java.util.HashMap; 50 import java.util.HashSet; 51 import java.util.List; 52 import java.util.Map; 53 import java.util.Set; 54 55 /** 56 * Validation tests to run against the configuration in host-unit-tests.zip to ensure they can all 57 * parse. 58 * 59 * <p>Do not add to UnitTests.java. This is meant to run standalone. 60 */ 61 @RunWith(DeviceJUnit4ClassRunner.class) 62 public class HostUnitTestsConfigValidation implements IBuildReceiver { 63 64 @Option( 65 name = "disallowed-test-type", 66 description = "The disallowed test type for configs in host-unit-tests.zip") 67 private List<String> mDisallowedTestTypes = new ArrayList<>(); 68 69 private IBuildInfo mBuild; 70 71 /** 72 * List of the officially supported runners in general-tests. Any new addition should go through 73 * a review to ensure all runners have a high quality bar. 74 */ 75 private static final Set<String> SUPPORTED_TEST_RUNNERS = 76 new HashSet<>( 77 Arrays.asList( 78 // Only accept runners that can be pure host-tests. 79 "com.android.tradefed.testtype.HostGTest", 80 "com.android.tradefed.testtype.IsolatedHostTest", 81 "com.android.tradefed.testtype.python.PythonBinaryHostTest", 82 "com.android.tradefed.testtype.binary.ExecutableHostTest", 83 "com.android.tradefed.testtype.rust.RustBinaryHostTest")); 84 85 @Override setBuild(IBuildInfo buildInfo)86 public void setBuild(IBuildInfo buildInfo) { 87 mBuild = buildInfo; 88 } 89 90 /** Get all the configuration copied to the build tests dir and check if they load. */ 91 @Test testConfigsLoad()92 public void testConfigsLoad() throws Exception { 93 List<String> errors = new ArrayList<>(); 94 Assume.assumeTrue(mBuild instanceof IDeviceBuildInfo); 95 96 IConfigurationFactory configFactory = ConfigurationFactory.getInstance(); 97 List<String> configs = new ArrayList<>(); 98 IDeviceBuildInfo deviceBuildInfo = (IDeviceBuildInfo) mBuild; 99 File testsDir = deviceBuildInfo.getTestsDir(); 100 List<File> extraTestCasesDirs = Arrays.asList(testsDir); 101 configs.addAll(ConfigurationUtil.getConfigNamesFromDirs(null, extraTestCasesDirs)); 102 for (String configName : configs) { 103 try { 104 IConfiguration c = 105 configFactory.createConfigurationFromArgs(new String[] {configName}); 106 // All configurations in host-unit-tests.zip should be module since they are 107 // generated from AndroidTest.xml 108 ValidateSuiteConfigHelper.validateConfig(c); 109 110 checkPreparers(c.getTargetPreparers(), "host-unit-tests"); 111 // Check that all the tests runners are well supported. 112 checkRunners(c.getTests(), "host-unit-tests"); 113 114 // Check for disallowed test types 115 GeneralTestsConfigValidation.checkDisallowedTestType(c, mDisallowedTestTypes); 116 117 // Add more checks if necessary 118 } catch (ConfigurationException e) { 119 errors.add(String.format("\t%s: %s", configName, e.getMessage())); 120 } 121 } 122 123 // If any errors report them in a final exception. 124 if (!errors.isEmpty()) { 125 throw new ConfigurationException( 126 String.format("Fail configuration check:\n%s", Joiner.on("\n").join(errors))); 127 } 128 } 129 checkPreparers(List<ITargetPreparer> preparers, String name)130 private static void checkPreparers(List<ITargetPreparer> preparers, String name) 131 throws ConfigurationException { 132 for (ITargetPreparer preparer : preparers) { 133 // Check that all preparers are supported. 134 if (preparer instanceof PushFilePreparer) { 135 throw new ConfigurationException( 136 String.format( 137 "preparer %s is not supported in %s.", 138 preparer.getClass().getCanonicalName(), name)); 139 } 140 } 141 } 142 checkRunners(List<IRemoteTest> tests, String name)143 private static void checkRunners(List<IRemoteTest> tests, String name) 144 throws ConfigurationException { 145 for (IRemoteTest test : tests) { 146 // Check that all the tests runners are well supported. 147 if (!SUPPORTED_TEST_RUNNERS.contains(test.getClass().getCanonicalName())) { 148 throw new ConfigurationException( 149 String.format( 150 "testtype %s is not officially supported in %s. " 151 + "The supported ones are: %s", 152 test.getClass().getCanonicalName(), name, SUPPORTED_TEST_RUNNERS)); 153 } 154 } 155 } 156 157 // This list contains exemption to the duplication of host-unit-tests & TEST_MAPPING. 158 // This will be used when migrating default and clean up as we clear the TEST_MAPPING files. 159 private static final Set<String> EXEMPTION_LIST = Collections.emptySet(); 160 161 // These are the final modules allowed to be in host test mapping to prevent new addition. 162 private static final Set<String> FINAL_MODULE_LIST = 163 ImmutableSet.of("hello_world_test", "tvts-tradefed-tests"); 164 165 /** 166 * This test ensures that unit tests are not also running as part of test mapping to avoid 167 * double running them. 168 */ 169 @Test testNotInTestMappingPresubmit()170 public void testNotInTestMappingPresubmit() { 171 List<String> errors = getErrors("presubmit"); 172 if (!errors.isEmpty()) { 173 String message = 174 String.format("Fail configuration check:\n%s", Joiner.on("\n").join(errors)); 175 assertTrue(message, errors.isEmpty()); 176 } 177 } 178 179 /** 180 * This test ensures that unit tests are not also running as part of test mapping to avoid 181 * double running them. 182 */ 183 @Test testNotInTestMappingPostsubmit()184 public void testNotInTestMappingPostsubmit() { 185 List<String> errors = getErrors("postsubmit"); 186 if (!errors.isEmpty()) { 187 String message = 188 String.format("Fail configuration check:\n%s", Joiner.on("\n").join(errors)); 189 assertTrue(message, errors.isEmpty()); 190 } 191 } 192 getErrors(String group)193 private List<String> getErrors(String group) { 194 // We need the test mapping files for this test. 195 Assume.assumeNotNull(mBuild.getFile("test_mappings.zip")); 196 197 TestMapping testMapping = new TestMapping(); 198 Set<TestInfo> testInfosToRun = 199 testMapping.getTests( 200 mBuild, 201 group, /* host */ 202 true, /* keywords */ 203 new HashSet<>(), /* ignoreKeywords */ 204 new HashSet<>()); 205 206 List<String> errors = new ArrayList<>(); 207 List<String> configs = new ArrayList<>(); 208 IDeviceBuildInfo deviceBuildInfo = (IDeviceBuildInfo) mBuild; 209 File testsDir = deviceBuildInfo.getTestsDir(); 210 List<File> extraTestCasesDirs = Arrays.asList(testsDir); 211 configs.addAll(ConfigurationUtil.getConfigNamesFromDirs(null, extraTestCasesDirs)); 212 213 Map<String, Set<String>> infos = new HashMap<>(); 214 testInfosToRun.stream().forEach(e -> infos.put(e.getName(), e.getSources())); 215 for (String configName : configs) { 216 String moduleName = FileUtil.getBaseName(new File(configName).getName()); 217 if (infos.containsKey(moduleName) && !EXEMPTION_LIST.contains(moduleName)) { 218 errors.add( 219 String.format( 220 "Target '%s' is already running in host-unit-tests, it doesn't " 221 + "need the test mapping config: %s", 222 moduleName, infos.get(moduleName))); 223 } else if (infos.containsKey(moduleName) && !FINAL_MODULE_LIST.contains(moduleName)) { 224 errors.add( 225 String.format( 226 "Target '%s' is attempted to be added to host test mapping. We do" 227 + " not currently allow new addition, consider using unit_tests" 228 + " setup instead.", 229 moduleName)); 230 } 231 } 232 return errors; 233 } 234 } 235