1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import static org.junit.Assert.assertEquals; 20 import static org.junit.Assert.assertNotNull; 21 import static org.junit.Assert.assertTrue; 22 import static org.mockito.Mockito.verify; 23 24 import com.android.tradefed.TestAppConstants; 25 import com.android.tradefed.config.OptionSetter; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.DeviceUnresponsiveException; 28 import com.android.tradefed.device.ITestDevice; 29 import com.android.tradefed.device.ITestDevice.RecoveryMode; 30 import com.android.tradefed.invoker.TestInformation; 31 import com.android.tradefed.log.LogUtil.CLog; 32 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 33 import com.android.tradefed.result.CollectingTestListener; 34 import com.android.tradefed.result.ITestInvocationListener; 35 import com.android.tradefed.result.TestDescription; 36 import com.android.tradefed.result.TestStatus; 37 import com.android.tradefed.util.RunUtil; 38 39 import org.junit.Before; 40 import org.junit.Ignore; 41 import org.junit.Test; 42 import org.junit.runner.RunWith; 43 import org.mockito.InOrder; 44 import org.mockito.Mock; 45 import org.mockito.Mockito; 46 import org.mockito.MockitoAnnotations; 47 48 import java.io.IOException; 49 import java.util.HashMap; 50 51 /** Functional tests for {@link InstrumentationTest}. */ 52 @RunWith(DeviceJUnit4ClassRunner.class) 53 public class InstrumentationTestFuncTest implements IDeviceTest, ITestInformationReceiver { 54 55 private static final long SHELL_TIMEOUT = 2500; 56 private static final int TEST_TIMEOUT = 2000; 57 private static final long WAIT_FOR_DEVICE_AVAILABLE = 5 * 60 * 1000; 58 59 private ITestDevice mDevice; 60 private TestInformation mTestInfo; 61 62 /** The {@link InstrumentationTest} under test */ 63 private InstrumentationTest mInstrumentationTest; 64 65 @Mock ITestInvocationListener mMockListener; 66 67 @Override setTestInformation(TestInformation testInformation)68 public void setTestInformation(TestInformation testInformation) { 69 mTestInfo = testInformation; 70 } 71 72 @Override getTestInformation()73 public TestInformation getTestInformation() { 74 return mTestInfo; 75 } 76 77 @Override setDevice(ITestDevice device)78 public void setDevice(ITestDevice device) { 79 mDevice = device; 80 } 81 82 @Override getDevice()83 public ITestDevice getDevice() { 84 return mDevice; 85 } 86 87 @Before setUp()88 public void setUp() throws Exception { 89 MockitoAnnotations.initMocks(this); 90 91 mInstrumentationTest = new InstrumentationTest(); 92 mInstrumentationTest.setPackageName(TestAppConstants.TESTAPP_PACKAGE); 93 mInstrumentationTest.setDevice(getDevice()); 94 // use no timeout by default 95 mInstrumentationTest.setShellTimeout(-1); 96 // set to no rerun by default 97 mInstrumentationTest.setRerunMode(false); 98 99 getDevice().disableKeyguard(); 100 } 101 102 /** Test normal run scenario with a single passed test result. */ 103 @Ignore 104 @Test testRun()105 public void testRun() throws DeviceNotAvailableException { 106 CLog.i("testRun"); 107 TestDescription expectedTest = 108 new TestDescription( 109 TestAppConstants.TESTAPP_CLASS, TestAppConstants.PASSED_TEST_METHOD); 110 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 111 mInstrumentationTest.setMethodName(TestAppConstants.PASSED_TEST_METHOD); 112 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 113 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 114 115 mInstrumentationTest.run(getTestInformation(), mMockListener); 116 InOrder inOrder = Mockito.inOrder(mMockListener); 117 inOrder.verify(mMockListener).testRunStarted(TestAppConstants.TESTAPP_PACKAGE, 1); 118 inOrder.verify(mMockListener).testStarted(Mockito.eq(expectedTest)); 119 inOrder.verify(mMockListener) 120 .testEnded(Mockito.eq(expectedTest), Mockito.<HashMap<String, Metric>>any()); 121 inOrder.verify(mMockListener) 122 .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any()); 123 124 verify(mMockListener).testRunStarted(TestAppConstants.TESTAPP_PACKAGE, 1); 125 verify(mMockListener).testStarted(Mockito.eq(expectedTest)); 126 verify(mMockListener) 127 .testEnded(Mockito.eq(expectedTest), Mockito.<HashMap<String, Metric>>any()); 128 verify(mMockListener) 129 .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any()); 130 } 131 132 /** Test normal run scenario with a single failed test result. */ 133 @Ignore 134 @Test testRun_testFailed()135 public void testRun_testFailed() throws DeviceNotAvailableException { 136 CLog.i("testRun_testFailed"); 137 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 138 mInstrumentationTest.setMethodName(TestAppConstants.FAILED_TEST_METHOD); 139 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 140 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 141 String[] error = new String[1]; 142 error[0] = null; 143 mInstrumentationTest.run( 144 getTestInformation(), 145 new ITestInvocationListener() { 146 @Override 147 public void testFailed(TestDescription test, String trace) { 148 error[0] = trace; 149 } 150 }); 151 assertNotNull("testFailed was not called", error[0]); 152 assertTrue(error[0].contains("junit.framework.AssertionFailedError: test failed")); 153 } 154 155 /** Test run scenario where test process crashes. */ 156 @Ignore 157 @Test testRun_testCrash()158 public void testRun_testCrash() throws DeviceNotAvailableException { 159 CLog.i("testRun_testCrash"); 160 TestDescription expectedTest = 161 new TestDescription( 162 TestAppConstants.TESTAPP_CLASS, TestAppConstants.CRASH_TEST_METHOD); 163 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 164 mInstrumentationTest.setMethodName(TestAppConstants.CRASH_TEST_METHOD); 165 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 166 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 167 168 try { 169 mInstrumentationTest.run(getTestInformation(), mMockListener); 170 InOrder inOrder = Mockito.inOrder(mMockListener); 171 inOrder.verify(mMockListener).testRunStarted(TestAppConstants.TESTAPP_PACKAGE, 1); 172 inOrder.verify(mMockListener).testStarted(Mockito.eq(expectedTest)); 173 if (getDevice().getApiLevel() <= 23) { 174 // Before N handling of instrumentation crash is slightly different. 175 inOrder.verify(mMockListener) 176 .testFailed(Mockito.eq(expectedTest), Mockito.contains("RuntimeException")); 177 inOrder.verify(mMockListener) 178 .testEnded(Mockito.eq(expectedTest), Mockito.<HashMap<String, Metric>>any()); 179 inOrder.verify(mMockListener) 180 .testRunFailed( 181 Mockito.eq( 182 "Instrumentation run failed due to" 183 + " 'java.lang.RuntimeException'")); 184 } else { 185 inOrder.verify(mMockListener) 186 .testFailed(Mockito.eq(expectedTest), Mockito.contains("Process crashed.")); 187 inOrder.verify(mMockListener) 188 .testEnded(Mockito.eq(expectedTest), Mockito.<HashMap<String, Metric>>any()); 189 inOrder.verify(mMockListener) 190 .testRunFailed( 191 Mockito.eq("Instrumentation run failed due to 'Process crashed.'")); 192 } 193 inOrder.verify(mMockListener) 194 .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any()); 195 196 verify(mMockListener).testRunStarted(TestAppConstants.TESTAPP_PACKAGE, 1); 197 verify(mMockListener).testStarted(Mockito.eq(expectedTest)); 198 verify(mMockListener) 199 .testFailed(Mockito.eq(expectedTest), Mockito.contains("RuntimeException")); 200 verify(mMockListener) 201 .testEnded(Mockito.eq(expectedTest), Mockito.<HashMap<String, Metric>>any()); 202 verify(mMockListener) 203 .testRunFailed( 204 Mockito.eq( 205 "Instrumentation run failed due to" 206 + " 'java.lang.RuntimeException'")); 207 verify(mMockListener) 208 .testFailed(Mockito.eq(expectedTest), Mockito.contains("Process crashed.")); 209 verify(mMockListener) 210 .testEnded(Mockito.eq(expectedTest), Mockito.<HashMap<String, Metric>>any()); 211 verify(mMockListener) 212 .testRunFailed( 213 Mockito.eq("Instrumentation run failed due to 'Process crashed.'")); 214 verify(mMockListener) 215 .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any()); 216 } finally { 217 getDevice().waitForDeviceAvailable(); 218 } 219 } 220 221 /** Test run scenario where test run hangs indefinitely, and times out. */ 222 @Ignore 223 @Test testRun_testTimeout()224 public void testRun_testTimeout() throws DeviceNotAvailableException { 225 CLog.i("testRun_testTimeout"); 226 RecoveryMode initMode = getDevice().getRecoveryMode(); 227 getDevice().setRecoveryMode(RecoveryMode.NONE); 228 try { 229 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 230 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 231 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 232 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 233 234 String[] error = new String[1]; 235 error[0] = null; 236 mInstrumentationTest.run( 237 getTestInformation(), 238 new ITestInvocationListener() { 239 @Override 240 public void testFailed(TestDescription test, String trace) { 241 error[0] = trace; 242 } 243 }); 244 assertEquals( 245 "Test failed to run to completion. Reason: 'Failed to receive adb shell test" 246 + " output within 2500 ms. Test may have timed out, or adb connection to" 247 + " device became unresponsive'. Check device logcat for details", 248 error[0]); 249 } finally { 250 getDevice().setRecoveryMode(initMode); 251 RunUtil.getDefault().sleep(500); 252 } 253 } 254 255 /** Test run scenario where device reboots during test run. */ 256 @Ignore 257 @Test testRun_deviceReboot()258 public void testRun_deviceReboot() throws Exception { 259 CLog.i("testRun_deviceReboot"); 260 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 261 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 262 mInstrumentationTest.setShellTimeout(0); 263 mInstrumentationTest.setTestTimeout(0); 264 // Set a max timeout to avoid hanging forever for safety 265 // OptionSetter setter = new OptionSetter(mInstrumentationTest); 266 // setter.setOptionValue("max-timeout", "600000"); 267 268 // fork off a thread to do the reboot 269 Thread rebootThread = 270 new Thread() { 271 @Override 272 public void run() { 273 // wait for test run to begin 274 try { 275 // Give time to the instrumentation to start 276 Thread.sleep(2000); 277 getDevice().reboot(); 278 } catch (InterruptedException e) { 279 CLog.w("interrupted"); 280 } catch (DeviceNotAvailableException dnae) { 281 CLog.w("Device did not come back online after reboot"); 282 } 283 } 284 }; 285 rebootThread.setName("InstrumentationTestFuncTest#testRun_deviceReboot"); 286 rebootThread.start(); 287 try { 288 String[] error = new String[1]; 289 error[0] = null; 290 mInstrumentationTest.run( 291 getTestInformation(), 292 new ITestInvocationListener() { 293 @Override 294 public void testRunFailed(String errorMessage) { 295 error[0] = errorMessage; 296 } 297 }); 298 assertEquals("Test run failed to complete. Expected 1 tests, received 0", error[0]); 299 } catch (DeviceUnresponsiveException expected) { 300 // expected 301 } finally { 302 rebootThread.join(WAIT_FOR_DEVICE_AVAILABLE); 303 getDevice().waitForDeviceAvailable(); 304 } 305 } 306 307 /** Test that when a max-timeout is set the instrumentation is stopped. */ 308 @Ignore 309 @Test testRun_maxTimeout()310 public void testRun_maxTimeout() throws Exception { 311 CLog.i("testRun_maxTimeout"); 312 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 313 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 314 mInstrumentationTest.setShellTimeout(0); 315 mInstrumentationTest.setTestTimeout(0); 316 OptionSetter setter = new OptionSetter(mInstrumentationTest); 317 setter.setOptionValue("max-timeout", "5000"); 318 final String[] called = new String[1]; 319 called[0] = null; 320 mInstrumentationTest.run( 321 getTestInformation(), 322 new ITestInvocationListener() { 323 @Override 324 public void testRunFailed(String errorMessage) { 325 called[0] = errorMessage; 326 } 327 }); 328 assertEquals( 329 "com.android.ddmlib.TimeoutException: executeRemoteCommand timed out after 5000ms", 330 called[0]); 331 } 332 333 /** Test run scenario where device runtime resets during test run. */ 334 @Test 335 @Ignore testRun_deviceRuntimeReset()336 public void testRun_deviceRuntimeReset() throws Exception { 337 CLog.i("testRun_deviceRuntimeReset"); 338 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 339 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 340 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 341 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 342 343 // fork off a thread to do the runtime reset 344 Thread resetThread = 345 new Thread() { 346 @Override 347 public void run() { 348 // wait for test run to begin 349 try { 350 Thread.sleep(1000); 351 Runtime.getRuntime() 352 .exec( 353 String.format( 354 "adb -s %s shell stop", 355 getDevice().getIDevice().getSerialNumber())); 356 Thread.sleep(500); 357 Runtime.getRuntime() 358 .exec( 359 String.format( 360 "adb -s %s shell start", 361 getDevice().getIDevice().getSerialNumber())); 362 } catch (InterruptedException e) { 363 CLog.w("interrupted"); 364 } catch (IOException e) { 365 CLog.w("IOException when rebooting"); 366 } 367 } 368 }; 369 resetThread.setName("InstrumentationTestFuncTest#testRun_deviceRuntimeReset"); 370 resetThread.start(); 371 try { 372 String[] error = new String[1]; 373 error[0] = null; 374 mInstrumentationTest.run( 375 getTestInformation(), 376 new ITestInvocationListener() { 377 @Override 378 public void testRunFailed(String errorMessage) { 379 error[0] = errorMessage; 380 } 381 }); 382 assertEquals( 383 "Failed to receive adb shell test output within 120000 ms. Test may have " 384 + "timed out, or adb connection to device became unresponsive", 385 error[0]); 386 } finally { 387 resetThread.join(WAIT_FOR_DEVICE_AVAILABLE); 388 RunUtil.getDefault().sleep(5000); 389 getDevice().waitForDeviceAvailable(); 390 } 391 } 392 393 /** 394 * Test running all the tests with rerun on. At least one method will cause run to stop 395 * (currently TIMEOUT_TEST_METHOD and CRASH_TEST_METHOD). Verify that results are recorded for 396 * all tests in the suite. 397 */ 398 @Ignore 399 @Test testRun_rerun()400 public void testRun_rerun() throws Exception { 401 CLog.i("testRun_rerun"); 402 // run all tests in class 403 RecoveryMode initMode = getDevice().getRecoveryMode(); 404 getDevice().setRecoveryMode(RecoveryMode.NONE); 405 try { 406 OptionSetter setter = new OptionSetter(mInstrumentationTest); 407 setter.setOptionValue("collect-tests-timeout", Long.toString(SHELL_TIMEOUT)); 408 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 409 mInstrumentationTest.setRerunMode(true); 410 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 411 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 412 CollectingTestListener listener = new CollectingTestListener(); 413 mInstrumentationTest.run(getTestInformation(), listener); 414 assertEquals(TestAppConstants.TOTAL_TEST_CLASS_TESTS, listener.getNumTotalTests()); 415 assertEquals( 416 TestAppConstants.TOTAL_TEST_CLASS_PASSED_TESTS, 417 listener.getNumTestsInState(TestStatus.PASSED)); 418 } finally { 419 getDevice().setRecoveryMode(initMode); 420 } 421 } 422 423 /** 424 * Test a run that crashes when collecting tests. 425 * 426 * <p>Expect run to proceed, but be reported as a run failure 427 */ 428 @Ignore 429 @Test testRun_rerunCrash()430 public void testRun_rerunCrash() throws Exception { 431 CLog.i("testRun_rerunCrash"); 432 mInstrumentationTest.setClassName(TestAppConstants.CRASH_ON_INIT_TEST_CLASS); 433 mInstrumentationTest.setMethodName(TestAppConstants.CRASH_ON_INIT_TEST_METHOD); 434 mInstrumentationTest.setRerunMode(false); 435 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 436 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 437 CollectingTestListener listener = new CollectingTestListener(); 438 mInstrumentationTest.run(getTestInformation(), listener); 439 assertEquals(0, listener.getNumTotalTests()); 440 assertNotNull(listener.getCurrentRunResults()); 441 assertEquals(TestAppConstants.TESTAPP_PACKAGE, listener.getCurrentRunResults().getName()); 442 assertTrue(listener.getCurrentRunResults().isRunFailure()); 443 assertTrue(listener.getCurrentRunResults().isRunComplete()); 444 } 445 446 /** 447 * Test a run that hangs when collecting tests. 448 * 449 * <p>Expect a run failure to be reported 450 */ 451 @Ignore 452 @Test testRun_rerunHang()453 public void testRun_rerunHang() throws Exception { 454 CLog.i("testRun_rerunHang"); 455 RecoveryMode initMode = getDevice().getRecoveryMode(); 456 getDevice().setRecoveryMode(RecoveryMode.NONE); 457 try { 458 OptionSetter setter = new OptionSetter(mInstrumentationTest); 459 setter.setOptionValue("collect-tests-timeout", Long.toString(SHELL_TIMEOUT)); 460 mInstrumentationTest.setClassName(TestAppConstants.HANG_ON_INIT_TEST_CLASS); 461 mInstrumentationTest.setRerunMode(false); 462 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 463 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 464 CollectingTestListener listener = new CollectingTestListener(); 465 mInstrumentationTest.run(getTestInformation(), listener); 466 assertEquals(0, listener.getNumTotalTests()); 467 assertEquals( 468 TestAppConstants.TESTAPP_PACKAGE, listener.getCurrentRunResults().getName()); 469 assertTrue(listener.getCurrentRunResults().isRunFailure()); 470 assertTrue(listener.getCurrentRunResults().isRunComplete()); 471 } finally { 472 getDevice().setRecoveryMode(initMode); 473 } 474 } 475 } 476