1 /* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.test; 18 19 import android.app.Instrumentation; 20 import android.content.Context; 21 import android.os.Bundle; 22 import android.test.mock.MockContext; 23 import android.test.suitebuilder.ListTestCaseNames; 24 import android.test.suitebuilder.ListTestCaseNames.TestDescriptor; 25 26 import androidx.test.filters.SmallTest; 27 28 import junit.framework.Test; 29 import junit.framework.TestCase; 30 import junit.framework.TestSuite; 31 32 import java.util.List; 33 34 /** 35 * Tests for {@link InstrumentationTestRunner} 36 */ 37 @SmallTest 38 public class InstrumentationTestRunnerTest extends TestCase { 39 private StubInstrumentationTestRunner mInstrumentationTestRunner; 40 private StubAndroidTestRunner mStubAndroidTestRunner; 41 private String mTargetContextPackageName; 42 setUp()43 protected void setUp() throws Exception { 44 super.setUp(); 45 mStubAndroidTestRunner = new StubAndroidTestRunner(); 46 mTargetContextPackageName = "android.test.suitebuilder.examples"; 47 mInstrumentationTestRunner = new StubInstrumentationTestRunner( 48 new StubContext("com.google.foo.tests"), 49 new StubContext(mTargetContextPackageName), mStubAndroidTestRunner); 50 } 51 testOverrideTestToRunWithClassArgument()52 public void testOverrideTestToRunWithClassArgument() throws Exception { 53 String expectedTestClassName = PlaceHolderTest.class.getName(); 54 mInstrumentationTestRunner.onCreate(createBundle( 55 InstrumentationTestRunner.ARGUMENT_TEST_CLASS, expectedTestClassName)); 56 57 assertTestRunnerCalledWithExpectedParameters(expectedTestClassName, "testPlaceHolder"); 58 } 59 testOverrideTestToRunWithClassAndMethodArgument()60 public void testOverrideTestToRunWithClassAndMethodArgument() throws Exception { 61 String expectedTestClassName = PlaceHolderTest.class.getName(); 62 String expectedTestMethodName = "testPlaceHolder"; 63 String classAndMethod = expectedTestClassName + "#" + expectedTestMethodName; 64 mInstrumentationTestRunner.onCreate(createBundle( 65 InstrumentationTestRunner.ARGUMENT_TEST_CLASS, classAndMethod)); 66 67 assertTestRunnerCalledWithExpectedParameters(expectedTestClassName, 68 expectedTestMethodName); 69 } 70 testUseSelfAsTestSuiteProviderWhenNoMetaDataOrClassArgument()71 public void testUseSelfAsTestSuiteProviderWhenNoMetaDataOrClassArgument() throws Exception { 72 TestSuite testSuite = new TestSuite(); 73 testSuite.addTestSuite(PlaceHolderTest.class); 74 mInstrumentationTestRunner.setAllTestsSuite(testSuite); 75 mInstrumentationTestRunner.onCreate(null); 76 assertTestRunnerCalledWithExpectedParameters( 77 PlaceHolderTest.class.getName(), "testPlaceHolder"); 78 } 79 testMultipleTestClass()80 public void testMultipleTestClass() throws Exception { 81 String classArg = PlaceHolderTest.class.getName() + "," + 82 PlaceHolderTest2.class.getName(); 83 mInstrumentationTestRunner.onCreate(createBundle( 84 InstrumentationTestRunner.ARGUMENT_TEST_CLASS, classArg)); 85 86 Test test = mStubAndroidTestRunner.getTest(); 87 88 assertContentsInOrder(ListTestCaseNames.getTestNames((TestSuite) test), 89 new TestDescriptor(PlaceHolderTest.class.getName(), "testPlaceHolder"), 90 new TestDescriptor(PlaceHolderTest2.class.getName(), "testPlaceHolder2")); 91 92 } 93 94 /** 95 * Test that runtime exceptions during runTest are handled gracefully 96 */ testUnhandledException()97 public void testUnhandledException() throws Exception { 98 StubAndroidTestRunner stubAndroidTestRunner = new StubAndroidTestRunner() { 99 @Override 100 public void runTest() { 101 throw new RuntimeException(); 102 } 103 }; 104 StubInstrumentationTestRunner instrumentationTestRunner = new StubInstrumentationTestRunner( 105 new StubContext("com.google.foo.tests"), 106 new StubContext(mTargetContextPackageName), stubAndroidTestRunner); 107 instrumentationTestRunner.onCreate(new Bundle()); 108 instrumentationTestRunner.onStart(); 109 assertTrue("Instrumentation did not finish", instrumentationTestRunner.isFinished()); 110 // ensure a meaningful error message placed in results 111 String resultsData = instrumentationTestRunner.mResults.getString( 112 Instrumentation.REPORT_KEY_STREAMRESULT); 113 assertTrue("Instrumentation results is missing RuntimeException", 114 resultsData.contains("RuntimeException")); 115 } 116 117 /** 118 * Test that specifying a method which does not exist is handled gracefully 119 */ testBadMethodArgument()120 public void testBadMethodArgument() throws Exception { 121 String testClassName = PlaceHolderTest.class.getName(); 122 String invalidMethodName = "testNoExist"; 123 String classAndMethod = testClassName + "#" + invalidMethodName; 124 mInstrumentationTestRunner.onCreate(createBundle( 125 InstrumentationTestRunner.ARGUMENT_TEST_CLASS, classAndMethod)); 126 assertTestRunnerCalledWithExpectedParameters(testClassName, 127 invalidMethodName); 128 } 129 testDelayParameter()130 public void testDelayParameter() throws Exception { 131 int delayMsec = 1000; 132 Bundle args = new Bundle(); 133 args.putInt(InstrumentationTestRunner.ARGUMENT_DELAY_MSEC, delayMsec); 134 args.putString(InstrumentationTestRunner.ARGUMENT_TEST_CLASS, 135 PlaceHolderTest.class.getName() + "," + 136 PlaceHolderTest2.class.getName()); 137 mInstrumentationTestRunner.onCreate(args); 138 Thread t = new Thread() { public void run() { mInstrumentationTestRunner.onStart(); } }; 139 140 // Should delay three times: before, between, and after the two tests. 141 long beforeTest = System.currentTimeMillis(); 142 t.start(); 143 t.join(); 144 assertTrue(System.currentTimeMillis() > beforeTest + delayMsec * 3); 145 assertTrue(mInstrumentationTestRunner.isStarted()); 146 assertTrue(mInstrumentationTestRunner.isFinished()); 147 assertTrue(mStubAndroidTestRunner.isRun()); 148 } 149 150 /** 151 * Test that the -e {@link InstrumentationTestRunner.ARGUMENT_ANNOTATION} parameter properly 152 * selects tests. 153 */ testAnnotationParameter()154 public void testAnnotationParameter() throws Exception { 155 String expectedTestClassName = AnnotationTest.class.getName(); 156 Bundle args = new Bundle(); 157 args.putString(InstrumentationTestRunner.ARGUMENT_TEST_CLASS, expectedTestClassName); 158 args.putString(InstrumentationTestRunner.ARGUMENT_ANNOTATION, FlakyTest.class.getName()); 159 mInstrumentationTestRunner.onCreate(args); 160 assertTestRunnerCalledWithExpectedParameters(expectedTestClassName, "testAnnotated"); 161 } 162 163 /** 164 * Test that the -e {@link InstrumentationTestRunner.ARGUMENT_NOT_ANNOTATION} parameter 165 * properly excludes tests. 166 */ testNotAnnotationParameter()167 public void testNotAnnotationParameter() throws Exception { 168 String expectedTestClassName = AnnotationTest.class.getName(); 169 Bundle args = new Bundle(); 170 args.putString(InstrumentationTestRunner.ARGUMENT_TEST_CLASS, expectedTestClassName); 171 args.putString(InstrumentationTestRunner.ARGUMENT_NOT_ANNOTATION, 172 FlakyTest.class.getName()); 173 mInstrumentationTestRunner.onCreate(args); 174 assertTestRunnerCalledWithExpectedParameters(expectedTestClassName, "testNotAnnotated"); 175 } 176 assertContentsInOrder(List<TestDescriptor> actual, TestDescriptor... source)177 private void assertContentsInOrder(List<TestDescriptor> actual, TestDescriptor... source) { 178 TestDescriptor[] clonedSource = source.clone(); 179 assertEquals("Unexpected number of items.", clonedSource.length, actual.size()); 180 for (int i = 0; i < actual.size(); i++) { 181 TestDescriptor actualItem = actual.get(i); 182 TestDescriptor sourceItem = clonedSource[i]; 183 assertEquals("Unexpected item. Index: " + i, sourceItem, actualItem); 184 } 185 } 186 assertTestRunnerCalledWithExpectedParameters( String expectedTestClassName, String expectedTestMethodName)187 private void assertTestRunnerCalledWithExpectedParameters( 188 String expectedTestClassName, String expectedTestMethodName) { 189 Test test = mStubAndroidTestRunner.getTest(); 190 assertContentsInOrder(ListTestCaseNames.getTestNames((TestSuite) test), 191 new TestDescriptor(expectedTestClassName, expectedTestMethodName)); 192 assertTrue(mInstrumentationTestRunner.isStarted()); 193 assertFalse(mInstrumentationTestRunner.isFinished()); 194 } 195 createBundle(String key, String value)196 private Bundle createBundle(String key, String value) { 197 Bundle bundle = new Bundle(); 198 bundle.putString(key, value); 199 return bundle; 200 } 201 202 private static class StubInstrumentationTestRunner extends InstrumentationTestRunner { 203 private Context mContext; 204 private Context mTargetContext; 205 private boolean mStarted; 206 private boolean mFinished; 207 private AndroidTestRunner mAndroidTestRunner; 208 private TestSuite mTestSuite; 209 private TestSuite mDefaultTestSuite; 210 private String mPackageNameForDefaultTests; 211 private Bundle mResults; 212 StubInstrumentationTestRunner(Context context, Context targetContext, AndroidTestRunner androidTestRunner)213 public StubInstrumentationTestRunner(Context context, Context targetContext, 214 AndroidTestRunner androidTestRunner) { 215 this.mContext = context; 216 this.mTargetContext = targetContext; 217 this.mAndroidTestRunner = androidTestRunner; 218 } 219 getContext()220 public Context getContext() { 221 return mContext; 222 } 223 getAllTests()224 public TestSuite getAllTests() { 225 return mTestSuite; 226 } 227 getTargetContext()228 public Context getTargetContext() { 229 return mTargetContext; 230 } 231 getAndroidTestRunner()232 protected AndroidTestRunner getAndroidTestRunner() { 233 return mAndroidTestRunner; 234 } 235 start()236 public void start() { 237 mStarted = true; 238 } 239 finish(int resultCode, Bundle results)240 public void finish(int resultCode, Bundle results) { 241 mFinished = true; 242 mResults = results; 243 } 244 isStarted()245 public boolean isStarted() { 246 return mStarted; 247 } 248 isFinished()249 public boolean isFinished() { 250 return mFinished; 251 } 252 setAllTestsSuite(TestSuite testSuite)253 public void setAllTestsSuite(TestSuite testSuite) { 254 mTestSuite = testSuite; 255 } 256 setDefaultTestsSuite(TestSuite testSuite)257 public void setDefaultTestsSuite(TestSuite testSuite) { 258 mDefaultTestSuite = testSuite; 259 } 260 getPackageNameForDefaultTests()261 public String getPackageNameForDefaultTests() { 262 return mPackageNameForDefaultTests; 263 } 264 265 @Override prepareLooper()266 void prepareLooper() { 267 // ignore 268 } 269 } 270 271 private static class StubContext extends MockContext { 272 private String mPackageName; 273 StubContext(String packageName)274 public StubContext(String packageName) { 275 this.mPackageName = packageName; 276 } 277 278 @Override getPackageCodePath()279 public String getPackageCodePath() { 280 return mPackageName; 281 } 282 283 @Override getPackageName()284 public String getPackageName() { 285 return mPackageName; 286 } 287 288 @Override getClassLoader()289 public ClassLoader getClassLoader() { 290 return getClass().getClassLoader(); 291 } 292 } 293 294 private static class StubAndroidTestRunner extends AndroidTestRunner { 295 private Test mTest; 296 private boolean mRun; 297 isRun()298 public boolean isRun() { 299 return mRun; 300 } 301 setTest(Test test)302 public void setTest(Test test) { 303 super.setTest(test); 304 mTest = test; 305 } 306 getTest()307 public Test getTest() { 308 return mTest; 309 } 310 runTest()311 public void runTest() { 312 super.runTest(); 313 mRun = true; 314 } 315 } 316 317 /** 318 * Empty test used for validation 319 */ 320 public static class PlaceHolderTest extends TestCase { 321 PlaceHolderTest()322 public PlaceHolderTest() { 323 super("testPlaceHolder"); 324 } 325 testPlaceHolder()326 public void testPlaceHolder() throws Exception { 327 328 } 329 } 330 331 /** 332 * Empty test used for validation 333 */ 334 public static class PlaceHolderTest2 extends TestCase { 335 PlaceHolderTest2()336 public PlaceHolderTest2() { 337 super("testPlaceHolder2"); 338 } 339 testPlaceHolder2()340 public void testPlaceHolder2() throws Exception { 341 342 } 343 } 344 345 /** 346 * Annotated test used for validation. 347 */ 348 public static class AnnotationTest extends TestCase { 349 testNotAnnotated()350 public void testNotAnnotated() throws Exception { 351 } 352 353 @FlakyTest testAnnotated()354 public void testAnnotated() throws Exception { 355 } 356 } 357 } 358