1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite;
17 
18 import static org.junit.Assert.assertEquals;
19 import static org.junit.Assert.assertTrue;
20 import static org.mockito.Mockito.mock;
21 import static org.mockito.Mockito.when;
22 
23 import com.android.tradefed.build.IDeviceBuildInfo;
24 import com.android.tradefed.config.IConfiguration;
25 import com.android.tradefed.config.OptionSetter;
26 import com.android.tradefed.device.DeviceNotAvailableException;
27 import com.android.tradefed.device.ITestDevice;
28 import com.android.tradefed.result.ITestInvocationListener;
29 import com.android.tradefed.targetprep.ITargetPreparer;
30 import com.android.tradefed.testtype.Abi;
31 import com.android.tradefed.testtype.IAbi;
32 import com.android.tradefed.testtype.IRemoteTest;
33 import com.android.tradefed.testtype.InstrumentationTest;
34 import com.android.tradefed.testtype.UiAutomatorTest;
35 import com.android.tradefed.util.AbiUtils;
36 import com.android.tradefed.util.CommandResult;
37 import com.android.tradefed.util.CommandStatus;
38 import com.android.tradefed.util.FileUtil;
39 
40 import org.junit.Before;
41 import org.junit.Rule;
42 import org.junit.Test;
43 import org.junit.rules.TemporaryFolder;
44 import org.junit.runner.RunWith;
45 import org.junit.runners.JUnit4;
46 
47 import java.io.File;
48 import java.io.IOException;
49 import java.util.ArrayList;
50 import java.util.HashSet;
51 import java.util.LinkedHashMap;
52 import java.util.LinkedHashSet;
53 import java.util.List;
54 import java.util.Set;
55 
56 /** Unit tests for {@link AtestRunner}. */
57 @RunWith(JUnit4.class)
58 public class AtestRunnerTest {
59 
60     @Rule public TemporaryFolder mTempFolder = new TemporaryFolder();
61 
62     private static final String TEST_CONFIG =
63         "<configuration description=\"Runs a stub tests part of some suite\">\n"
64             + "    <test class=\"com.android.tradefed.testtype.suite.SuiteModuleLoaderTest"
65             + "$TestInject\" />\n"
66             + "</configuration>";
67 
68     private static final String ABI = "armeabi-v7a";
69     private static final String TEST_NAME_FMT = ABI + " %s";
70     private static final String INSTRUMENTATION_TEST_NAME =
71             String.format(TEST_NAME_FMT, "tf/instrumentation");
72 
73     private AbiAtestRunner mRunner;
74     private OptionSetter setter;
75     private IDeviceBuildInfo mBuildInfo;
76     private ITestDevice mMockDevice;
77     private String classA = "fully.qualified.classA";
78     private String classB = "fully.qualified.classB";
79     private String method1 = "method1";
80 
81     /**
82      * Test AtestRunner that hardcodes the abis to avoid failures related to running the tests
83      * against a particular abi build of tradefed.
84      */
85     public static class AbiAtestRunner extends AtestRunner {
86         @Override
getAbis(ITestDevice device)87         public Set<IAbi> getAbis(ITestDevice device) throws DeviceNotAvailableException {
88             Set<IAbi> abis = new LinkedHashSet<>();
89             abis.add(new Abi(ABI, AbiUtils.getBitness(ABI)));
90             return abis;
91         }
92     }
93 
94     @Before
setUp()95     public void setUp() throws Exception {
96         mRunner = new AbiAtestRunner();
97         mBuildInfo = mock(IDeviceBuildInfo.class);
98         mMockDevice = mock(ITestDevice.class);
99         mRunner.setBuild(mBuildInfo);
100         mRunner.setDevice(mMockDevice);
101 
102         when(mBuildInfo.getTestsDir()).thenReturn(mTempFolder.newFolder());
103 
104         CommandResult result = new CommandResult(CommandStatus.SUCCESS);
105         result.setStdout("Supported states: [\n" +
106                 " DeviceState{identifier=0, name='DEFAULT'},\n" +
107                 "]\n");
108         when(mMockDevice.executeShellV2Command("cmd device_state print-states"))
109                 .thenReturn(result);
110     }
111 
112     @Test
testLoadTests_one()113     public void testLoadTests_one() throws Exception {
114         setter = new OptionSetter(mRunner);
115         setter.setOptionValue("suite-config-prefix", "tf");
116         setter.setOptionValue("include-filter", "tf/fake");
117         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
118         assertEquals(1, configMap.size());
119         assertTrue(configMap.containsKey(String.format(TEST_NAME_FMT, "tf/fake")));
120     }
121 
122     @Test
testLoadTests_two()123     public void testLoadTests_two() throws Exception {
124         setter = new OptionSetter(mRunner);
125         setter.setOptionValue("suite-config-prefix", "tf");
126         setter.setOptionValue("include-filter", "tf/fake");
127         setter.setOptionValue("include-filter", "tf/func");
128         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
129         assertEquals(2, configMap.size());
130         assertTrue(configMap.containsKey(String.format(TEST_NAME_FMT, "tf/fake")));
131         assertTrue(configMap.containsKey(String.format(TEST_NAME_FMT, "tf/func")));
132     }
133 
134     @Test
testLoadTests_filter()135     public void testLoadTests_filter() throws Exception {
136         setter = new OptionSetter(mRunner);
137         setter.setOptionValue("suite-config-prefix", "tf");
138         setter.setOptionValue("include-filter", "tf/uiautomator");
139         setter.setOptionValue("atest-include-filter", "tf/uiautomator:" + classA);
140         setter.setOptionValue("atest-include-filter", "tf/uiautomator:" + classB + "#" + method1);
141         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
142         assertEquals(1, configMap.size());
143         String testName = String.format(TEST_NAME_FMT, "tf/uiautomator");
144         assertTrue(configMap.containsKey(testName));
145         IConfiguration config = configMap.get(testName);
146         List<IRemoteTest> tests = config.getTests();
147         assertEquals(1, tests.size());
148         UiAutomatorTest test = (UiAutomatorTest) tests.get(0);
149         List<String> classFilters = new ArrayList<>();
150         classFilters.add(classA);
151         classFilters.add(classB + "#" + method1);
152         assertEquals(classFilters, test.getClassNames());
153     }
154 
155     @Test
testLoadTests_WithTFConfigSpecified()156     public void testLoadTests_WithTFConfigSpecified() throws Exception {
157         setter = new OptionSetter(mRunner);
158         setter.setOptionValue("suite-config-prefix", "suite");
159         setter.setOptionValue("tf-config-path", "suite/base-suite1");
160         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
161         assertEquals(1, configMap.size());
162         String testName = String.format(TEST_NAME_FMT, "suite/base-suite1");
163         assertTrue(configMap.containsKey(testName));
164     }
165 
166     @Test
testLoadTests_WithModuleAndTFConfigSpecified()167     public void testLoadTests_WithModuleAndTFConfigSpecified() throws Exception {
168         File tmpDir = FileUtil.createTempDir("some-dir");
169         String filePath = createModuleConfig(tmpDir, "TestModule");
170         try {
171             mRunner.setupFilters(tmpDir);
172             setter = new OptionSetter(mRunner);
173             setter.setOptionValue("suite-config-prefix", "suite");
174             setter.setOptionValue("tf-config-path", "suite/base-suite1");
175             setter.setOptionValue("module-config-path", filePath);
176             LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
177             assertEquals(2, configMap.size());
178             String testName = String.format(TEST_NAME_FMT, "TestModule");
179             assertTrue(configMap.containsKey(testName));
180             testName = String.format(TEST_NAME_FMT, "suite/base-suite1");
181             assertTrue(configMap.containsKey(testName));
182         } finally {
183             FileUtil.recursiveDelete(tmpDir);
184         }
185     }
186 
187     @Test
testLoadTests_WithModuleConfigSpecified()188     public void testLoadTests_WithModuleConfigSpecified() throws Exception {
189         File tmpDir = FileUtil.createTempDir("some-dir");
190         String filePath = createModuleConfig(tmpDir, "TestModule");
191         try {
192             mRunner.setupFilters(tmpDir);
193             setter = new OptionSetter(mRunner);
194             setter.setOptionValue("module-config-path", filePath);
195             LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
196             assertEquals(1, configMap.size());
197             String testName = String.format(TEST_NAME_FMT, "TestModule");
198             assertTrue(configMap.containsKey(testName));
199         } finally {
200             FileUtil.recursiveDelete(tmpDir);
201         }
202     }
203 
204     @Test
testLoadTests_ignoreFilter()205     public void testLoadTests_ignoreFilter() throws Exception {
206         setter = new OptionSetter(mRunner);
207         setter.setOptionValue("suite-config-prefix", "suite");
208         setter.setOptionValue("include-filter", "suite/base-suite1");
209         setter.setOptionValue("atest-include-filter", "suite/base-suite1:" + classA);
210         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
211         assertEquals(1, configMap.size());
212         String testName = String.format(TEST_NAME_FMT, "suite/base-suite1");
213         assertTrue(configMap.containsKey(testName));
214         IConfiguration config = configMap.get(testName);
215         List<IRemoteTest> tests = config.getTests();
216         assertEquals(1, tests.size());
217         BaseTestSuite test = (BaseTestSuite) tests.get(0);
218         assertEquals(new HashSet<String>(), test.getIncludeFilter());
219     }
220 
221     @Test
testWaitForDebugger()222     public void testWaitForDebugger() throws Exception {
223         OptionSetter setter = new OptionSetter(mRunner);
224         setter.setOptionValue("suite-config-prefix", "tf");
225         setter.setOptionValue("wait-for-debugger", "true");
226         setter.setOptionValue("include-filter", "tf/instrumentation");
227         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
228         assertEquals(1, configMap.size());
229         IConfiguration config = configMap.get(INSTRUMENTATION_TEST_NAME);
230         IRemoteTest test = config.getTests().get(0);
231         assertTrue(((InstrumentationTest) test).getDebug());
232     }
233 
234     @Test
testdisableTargetPreparers()235     public void testdisableTargetPreparers() throws Exception {
236         OptionSetter setter = new OptionSetter(mRunner);
237         setter.setOptionValue("suite-config-prefix", "tf");
238         setter.setOptionValue("disable-target-preparers", "true");
239         setter.setOptionValue("include-filter", "tf/instrumentation");
240         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
241         assertEquals(1, configMap.size());
242         IConfiguration config = configMap.get(INSTRUMENTATION_TEST_NAME);
243         for (ITargetPreparer targetPreparer : config.getTargetPreparers()) {
244             assertTrue(targetPreparer.isDisabled());
245         }
246     }
247 
248     @Test
testdisableTargetPreparersUnset()249     public void testdisableTargetPreparersUnset() throws Exception {
250         OptionSetter setter = new OptionSetter(mRunner);
251         setter.setOptionValue("suite-config-prefix", "tf");
252         setter.setOptionValue("include-filter", "tf/instrumentation");
253         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
254         assertEquals(1, configMap.size());
255         IConfiguration config = configMap.get(INSTRUMENTATION_TEST_NAME);
256         for (ITargetPreparer targetPreparer : config.getTargetPreparers()) {
257             assertTrue(!targetPreparer.isDisabled());
258         }
259     }
260 
261     @Test
testDisableTearDown()262     public void testDisableTearDown() throws Exception {
263         OptionSetter setter = new OptionSetter(mRunner);
264         setter.setOptionValue("suite-config-prefix", "tf");
265         setter.setOptionValue("disable-teardown", "true");
266         setter.setOptionValue("include-filter", "tf/instrumentation");
267         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
268         IConfiguration config = configMap.get(INSTRUMENTATION_TEST_NAME);
269         for (ITargetPreparer targetPreparer : config.getTargetPreparers()) {
270             assertTrue(targetPreparer.isTearDownDisabled());
271         }
272     }
273 
274     @Test
testDisableTearDownUnset()275     public void testDisableTearDownUnset() throws Exception {
276         OptionSetter setter = new OptionSetter(mRunner);
277         setter.setOptionValue("suite-config-prefix", "tf");
278         setter.setOptionValue("include-filter", "tf/instrumentation");
279         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
280         assertEquals(1, configMap.size());
281         IConfiguration config = configMap.get(INSTRUMENTATION_TEST_NAME);
282         for (ITargetPreparer targetPreparer : config.getTargetPreparers()) {
283             assertTrue(!targetPreparer.isTearDownDisabled());
284         }
285     }
286 
287     @Test
testCreateModuleListener()288     public void testCreateModuleListener() throws Exception {
289         OptionSetter setter = new OptionSetter(mRunner);
290         setter.setOptionValue("subprocess-report-port", "55555");
291         List<ITestInvocationListener> listeners = mRunner.createModuleListeners();
292         assertEquals(1, listeners.size());
293     }
294 
createModuleConfig(File dir, String moduleName)295     private String createModuleConfig(File dir, String moduleName) throws IOException {
296         File moduleConfig = new File(dir, moduleName + SuiteModuleLoader.CONFIG_EXT);
297         FileUtil.writeToFile(TEST_CONFIG, moduleConfig);
298         return moduleConfig.getAbsolutePath();
299     }
300 }
301