1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.view.inputmethod.cts; 18 19 import static android.provider.Settings.Secure.STYLUS_HANDWRITING_DEFAULT_VALUE; 20 import static android.provider.Settings.Secure.STYLUS_HANDWRITING_ENABLED; 21 import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED; 22 import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED; 23 import static android.view.inputmethod.Flags.FLAG_CONNECTIONLESS_HANDWRITING; 24 import static android.view.inputmethod.Flags.FLAG_HOME_SCREEN_HANDWRITING_DELEGATOR; 25 import static android.view.inputmethod.Flags.initiationWithoutInputConnection; 26 import static android.view.inputmethod.InputMethodInfo.ACTION_STYLUS_HANDWRITING_SETTINGS; 27 28 import static com.android.cts.mockime.ImeEventStreamTestUtils.editorMatcher; 29 import static com.android.cts.mockime.ImeEventStreamTestUtils.eventMatcher; 30 import static com.android.cts.mockime.ImeEventStreamTestUtils.expectBindInput; 31 import static com.android.cts.mockime.ImeEventStreamTestUtils.expectCommand; 32 import static com.android.cts.mockime.ImeEventStreamTestUtils.expectEvent; 33 import static com.android.cts.mockime.ImeEventStreamTestUtils.notExpectEvent; 34 import static com.android.cts.mockime.ImeEventStreamTestUtils.withDescription; 35 import static com.android.text.flags.Flags.FLAG_HANDWRITING_END_OF_LINE_TAP; 36 import static com.android.text.flags.Flags.FLAG_HANDWRITING_UNSUPPORTED_MESSAGE; 37 38 import static com.google.common.truth.Truth.assertThat; 39 40 import static org.junit.Assert.assertEquals; 41 import static org.junit.Assert.assertFalse; 42 import static org.junit.Assert.assertNotNull; 43 import static org.junit.Assert.assertTrue; 44 import static org.junit.Assume.assumeFalse; 45 import static org.junit.Assume.assumeTrue; 46 import static org.mockito.Mockito.mock; 47 48 import android.Manifest; 49 import android.app.Activity; 50 import android.app.Instrumentation; 51 import android.content.ComponentName; 52 import android.content.Context; 53 import android.content.Intent; 54 import android.content.pm.PackageManager; 55 import android.graphics.Color; 56 import android.hardware.input.InputManager; 57 import android.inputmethodservice.InputMethodService; 58 import android.os.Process; 59 import android.platform.test.annotations.AppModeFull; 60 import android.platform.test.annotations.AppModeSdkSandbox; 61 import android.platform.test.annotations.RequiresFlagsEnabled; 62 import android.platform.test.flag.junit.CheckFlagsRule; 63 import android.platform.test.flag.junit.DeviceFlagsValueProvider; 64 import android.provider.Settings; 65 import android.text.InputType; 66 import android.text.TextUtils; 67 import android.util.DisplayMetrics; 68 import android.util.Pair; 69 import android.view.Display; 70 import android.view.InputDevice; 71 import android.view.KeyEvent; 72 import android.view.MotionEvent; 73 import android.view.View; 74 import android.view.ViewConfiguration; 75 import android.view.ViewGroup; 76 import android.view.inputmethod.ConnectionlessHandwritingCallback; 77 import android.view.inputmethod.CursorAnchorInfo; 78 import android.view.inputmethod.EditorInfo; 79 import android.view.inputmethod.Flags; 80 import android.view.inputmethod.InputConnection; 81 import android.view.inputmethod.InputMethodInfo; 82 import android.view.inputmethod.InputMethodManager; 83 import android.view.inputmethod.cts.util.EndToEndImeTestBase; 84 import android.view.inputmethod.cts.util.MockTestActivityUtil; 85 import android.view.inputmethod.cts.util.NoOpInputConnection; 86 import android.view.inputmethod.cts.util.TestActivity; 87 import android.view.inputmethod.cts.util.TestActivity2; 88 import android.view.inputmethod.cts.util.TestUtils; 89 import android.widget.EditText; 90 import android.widget.LinearLayout; 91 92 import androidx.annotation.ColorInt; 93 import androidx.annotation.NonNull; 94 import androidx.test.filters.FlakyTest; 95 import androidx.test.platform.app.InstrumentationRegistry; 96 97 import com.android.compatibility.common.util.ApiTest; 98 import com.android.compatibility.common.util.CommonTestUtils; 99 import com.android.compatibility.common.util.GestureNavSwitchHelper; 100 import com.android.compatibility.common.util.SystemUtil; 101 import com.android.cts.input.UinputStylus; 102 import com.android.cts.input.UinputTouchDevice; 103 import com.android.cts.input.UinputTouchScreen; 104 import com.android.cts.mockime.ImeEvent; 105 import com.android.cts.mockime.ImeEventStream; 106 import com.android.cts.mockime.ImeEventStreamTestUtils.DescribedPredicate; 107 import com.android.cts.mockime.ImeSettings; 108 import com.android.cts.mockime.MockImeSession; 109 110 import org.junit.After; 111 import org.junit.Before; 112 import org.junit.Rule; 113 import org.junit.Test; 114 115 import java.util.ArrayList; 116 import java.util.Iterator; 117 import java.util.List; 118 import java.util.concurrent.CountDownLatch; 119 import java.util.concurrent.TimeUnit; 120 import java.util.concurrent.TimeoutException; 121 import java.util.concurrent.atomic.AtomicReference; 122 import java.util.function.Predicate; 123 124 /** 125 * IMF and end-to-end Stylus handwriting tests. 126 */ 127 @AppModeSdkSandbox(reason = "Allow test in the SDK sandbox (does not prevent other modes).") 128 public class StylusHandwritingTest extends EndToEndImeTestBase { 129 private static final long TIMEOUT_IN_SECONDS = 5; 130 private static final long TIMEOUT = TimeUnit.SECONDS.toMillis(TIMEOUT_IN_SECONDS); 131 private static final long TIMEOUT_6_S = TimeUnit.SECONDS.toMillis(6); 132 private static final long TIMEOUT_1_S = TimeUnit.SECONDS.toMillis(1); 133 private static final long NOT_EXPECT_TIMEOUT_IN_SECONDS = 3; 134 private static final long NOT_EXPECT_TIMEOUT = 135 TimeUnit.SECONDS.toMillis(NOT_EXPECT_TIMEOUT_IN_SECONDS); 136 private static final int SETTING_VALUE_ON = 1; 137 private static final int SETTING_VALUE_OFF = 0; 138 private static final int HANDWRITING_BOUNDS_OFFSET_PX = 20; 139 // A timeout greater than HandwritingModeController#HANDWRITING_DELEGATION_IDLE_TIMEOUT_MS. 140 private static final long DELEGATION_AFTER_IDLE_TIMEOUT_MS = 3100; 141 private static final int NUMBER_OF_INJECTED_EVENTS = 5; 142 private static final String TEST_LAUNCHER_COMPONENT = 143 "android.view.inputmethod.ctstestlauncher/" 144 + "android.view.inputmethod.ctstestlauncher.LauncherActivity"; 145 146 private Context mContext; 147 private int mHwInitialState; 148 private boolean mShouldRestoreInitialHwState; 149 private String mDefaultLauncherToRestore; 150 151 private static final GestureNavSwitchHelper sGestureNavRule = new GestureNavSwitchHelper(); 152 153 @Rule 154 public final CheckFlagsRule mCheckFlagsRule = DeviceFlagsValueProvider.createCheckFlagsRule(); 155 156 @Before setup()157 public void setup() { 158 mContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); 159 assumeFalse(mContext.getPackageManager().hasSystemFeature( 160 PackageManager.FEATURE_LEANBACK_ONLY)); 161 assumeFalse(mContext.getPackageManager().hasSystemFeature( 162 PackageManager.FEATURE_AUTOMOTIVE)); 163 164 mHwInitialState = Settings.Secure.getInt(mContext.getContentResolver(), 165 STYLUS_HANDWRITING_ENABLED, STYLUS_HANDWRITING_DEFAULT_VALUE); 166 if (mHwInitialState != SETTING_VALUE_ON) { 167 SystemUtil.runWithShellPermissionIdentity(() -> { 168 Settings.Secure.putInt(mContext.getContentResolver(), 169 STYLUS_HANDWRITING_ENABLED, SETTING_VALUE_ON); 170 }, Manifest.permission.WRITE_SECURE_SETTINGS); 171 mShouldRestoreInitialHwState = true; 172 } 173 } 174 175 @After tearDown()176 public void tearDown() { 177 MockTestActivityUtil.forceStopPackage(); 178 if (mShouldRestoreInitialHwState) { 179 mShouldRestoreInitialHwState = false; 180 SystemUtil.runWithShellPermissionIdentity(() -> { 181 Settings.Secure.putInt(mContext.getContentResolver(), 182 STYLUS_HANDWRITING_ENABLED, mHwInitialState); 183 }, Manifest.permission.WRITE_SECURE_SETTINGS); 184 } 185 if (mDefaultLauncherToRestore != null) { 186 setDefaultLauncher(mDefaultLauncherToRestore); 187 mDefaultLauncherToRestore = null; 188 } 189 } 190 191 /** 192 * Verify current IME has {@link InputMethodInfo} for stylus handwriting, settings. 193 */ 194 @Test 195 @ApiTest(apis = {"android.view.inputmethod.InputMethodInfo#supportsStylusHandwriting", 196 "android.view.inputmethod.InputMethodInfo#ACTION_STYLUS_HANDWRITING_SETTINGS", 197 "android.view.inputmethod.InputMethodInfo#createStylusHandwritingSettingsActivityIntent" 198 }) testHandwritingInfo()199 public void testHandwritingInfo() throws Exception { 200 try (MockImeSession imeSession = MockImeSession.create( 201 InstrumentationRegistry.getInstrumentation().getContext(), 202 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 203 new ImeSettings.Builder())) { 204 InputMethodInfo info = imeSession.getInputMethodInfo(); 205 assertTrue(info.supportsStylusHandwriting()); 206 // TODO(b/217957587): migrate CtsMockInputMethodLib to android_library and use 207 // string resource. 208 Intent stylusSettingsIntent = info.createStylusHandwritingSettingsActivityIntent(); 209 assertEquals(ACTION_STYLUS_HANDWRITING_SETTINGS, stylusSettingsIntent.getAction()); 210 assertEquals("handwriting_settings", 211 stylusSettingsIntent.getComponent().getClassName()); 212 } 213 } 214 215 @Test testIsStylusHandwritingAvailable_prefDisabled()216 public void testIsStylusHandwritingAvailable_prefDisabled() throws Exception { 217 try (MockImeSession imeSession = MockImeSession.create( 218 InstrumentationRegistry.getInstrumentation().getContext(), 219 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 220 new ImeSettings.Builder())) { 221 imeSession.openEventStream(); 222 223 // Disable pref 224 SystemUtil.runWithShellPermissionIdentity(() -> { 225 Settings.Secure.putInt(mContext.getContentResolver(), 226 STYLUS_HANDWRITING_ENABLED, SETTING_VALUE_OFF); 227 }, Manifest.permission.WRITE_SECURE_SETTINGS); 228 mShouldRestoreInitialHwState = true; 229 230 launchTestActivity(getTestMarker()); 231 assertFalse( 232 "should return false for isStylusHandwritingAvailable() when pref is disabled", 233 mContext.getSystemService( 234 InputMethodManager.class).isStylusHandwritingAvailable()); 235 } 236 } 237 238 @Test testIsStylusHandwritingAvailable()239 public void testIsStylusHandwritingAvailable() throws Exception { 240 try (MockImeSession imeSession = MockImeSession.create( 241 InstrumentationRegistry.getInstrumentation().getContext(), 242 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 243 new ImeSettings.Builder())) { 244 imeSession.openEventStream(); 245 246 launchTestActivity(getTestMarker()); 247 assertTrue("Mock IME should return true for isStylusHandwritingAvailable() ", 248 mContext.getSystemService( 249 InputMethodManager.class).isStylusHandwritingAvailable()); 250 } 251 } 252 253 @Test 254 @RequiresFlagsEnabled(FLAG_CONNECTIONLESS_HANDWRITING) testIsConnectionlessStylusHandwritingAvailable_prefDisabled()255 public void testIsConnectionlessStylusHandwritingAvailable_prefDisabled() throws Exception { 256 try (MockImeSession imeSession = MockImeSession.create( 257 InstrumentationRegistry.getInstrumentation().getContext(), 258 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 259 new ImeSettings.Builder())) { 260 imeSession.openEventStream(); 261 262 // Disable pref 263 SystemUtil.runWithShellPermissionIdentity(() -> { 264 Settings.Secure.putInt(mContext.getContentResolver(), 265 STYLUS_HANDWRITING_ENABLED, SETTING_VALUE_OFF); 266 }, Manifest.permission.WRITE_SECURE_SETTINGS); 267 mShouldRestoreInitialHwState = true; 268 269 launchTestActivity(getTestMarker()); 270 assertFalse( 271 "Mock IME should return false for isConnectionlessStylusHandwritingAvailable() " 272 + "when pref is disabled", 273 mContext.getSystemService( 274 InputMethodManager.class).isConnectionlessStylusHandwritingAvailable()); 275 } 276 } 277 278 @Test 279 @RequiresFlagsEnabled(FLAG_CONNECTIONLESS_HANDWRITING) testIsConnectionlessStylusHandwritingAvailable()280 public void testIsConnectionlessStylusHandwritingAvailable() throws Exception { 281 try (MockImeSession imeSession = MockImeSession.create( 282 InstrumentationRegistry.getInstrumentation().getContext(), 283 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 284 new ImeSettings.Builder())) { 285 imeSession.openEventStream(); 286 287 launchTestActivity(getTestMarker()); 288 assertTrue( 289 "Mock IME should return true for isConnectionlessStylusHandwritingAvailable()", 290 mContext.getSystemService( 291 InputMethodManager.class).isConnectionlessStylusHandwritingAvailable()); 292 } 293 } 294 295 /** 296 * Test to verify that we dont init handwriting on devices that dont have any supported stylus. 297 */ 298 @Test testHandwritingNoInitOnDeviceWithNoStylus()299 public void testHandwritingNoInitOnDeviceWithNoStylus() { 300 assumeTrue("Skipping test on devices that do not have stylus support", 301 hasSupportedStylus()); 302 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 303 try (MockImeSession imeSession = MockImeSession.create( 304 InstrumentationRegistry.getInstrumentation().getContext(), 305 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 306 new ImeSettings.Builder())) { 307 final ImeEventStream stream = imeSession.openEventStream(); 308 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 309 final EditText editText = launchTestActivity(marker); 310 imm.startStylusHandwriting(editText); 311 // Handwriting should not start since there are no stylus devices registered. 312 notExpectEvent( 313 stream, 314 editorMatcher("onStartStylusHandwriting", marker), 315 NOT_EXPECT_TIMEOUT); 316 } catch (Exception e) { 317 } 318 } 319 320 @Test testHandwritingDoesNotStartWhenNoStylusDown()321 public void testHandwritingDoesNotStartWhenNoStylusDown() throws Exception { 322 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 323 try (MockImeSession imeSession = MockImeSession.create( 324 InstrumentationRegistry.getInstrumentation().getContext(), 325 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 326 new ImeSettings.Builder())) { 327 final ImeEventStream stream = imeSession.openEventStream(); 328 329 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 330 final EditText editText = launchTestActivity(marker); 331 332 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 333 notExpectEvent( 334 stream, 335 editorMatcher("onStartInputView", marker), 336 NOT_EXPECT_TIMEOUT); 337 338 addVirtualStylusIdForTestSession(); 339 imm.startStylusHandwriting(editText); 340 341 // Handwriting should not start 342 notExpectEvent( 343 stream, 344 editorMatcher("onStartStylusHandwriting", marker), 345 NOT_EXPECT_TIMEOUT); 346 347 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 348 } 349 } 350 351 @Test testHandwritingStartAndFinish()352 public void testHandwritingStartAndFinish() throws Exception { 353 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 354 try (MockImeSession imeSession = MockImeSession.create( 355 InstrumentationRegistry.getInstrumentation().getContext(), 356 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 357 new ImeSettings.Builder())) { 358 final ImeEventStream stream = imeSession.openEventStream(); 359 360 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 361 final EditText editText = launchTestActivity(marker); 362 363 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 364 notExpectEvent( 365 stream, 366 editorMatcher("onStartInputView", marker), 367 NOT_EXPECT_TIMEOUT); 368 369 addVirtualStylusIdForTestSession(); 370 // Touch down with a stylus 371 final int startX = editText.getWidth() / 2; 372 final int startY = editText.getHeight() / 2; 373 TestUtils.injectStylusDownEvent(editText, startX, startY); 374 375 try { 376 imm.startStylusHandwriting(editText); 377 // keyboard shouldn't show up. 378 notExpectEvent( 379 stream, 380 editorMatcher("onStartInputView", marker), 381 NOT_EXPECT_TIMEOUT); 382 383 // Handwriting should start 384 expectEvent( 385 stream, 386 editorMatcher("onPrepareStylusHandwriting", marker), 387 TIMEOUT); 388 expectEvent( 389 stream, 390 editorMatcher("onStartStylusHandwriting", marker), 391 TIMEOUT); 392 393 verifyStylusHandwritingWindowIsShown(stream, imeSession); 394 } finally { 395 // Release the stylus pointer 396 TestUtils.injectStylusUpEvent(editText, startX, startY); 397 } 398 399 // Verify calling finishStylusHandwriting() calls onFinishStylusHandwriting(). 400 imeSession.callFinishStylusHandwriting(); 401 expectEvent( 402 stream, 403 editorMatcher("onFinishStylusHandwriting", marker), 404 TIMEOUT); 405 } 406 } 407 408 /** 409 * Verifies that stylus hover events initializes the InkWindow. 410 * @throws Exception 411 */ 412 @Test testStylusHoverInitInkWindow()413 public void testStylusHoverInitInkWindow() throws Exception { 414 try (MockImeSession imeSession = MockImeSession.create( 415 InstrumentationRegistry.getInstrumentation().getContext(), 416 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 417 new ImeSettings.Builder())) { 418 final ImeEventStream stream = imeSession.openEventStream(); 419 420 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 421 final EditText editText = launchTestActivity(marker); 422 423 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 424 notExpectEvent( 425 stream, 426 editorMatcher("onStartInputView", marker), 427 NOT_EXPECT_TIMEOUT); 428 429 addVirtualStylusIdForTestSession(); 430 // Verify there is no handwriting window before stylus is added. 431 assertFalse(expectCommand( 432 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT_1_S) 433 .getReturnBooleanValue()); 434 // Stylus hover 435 final int startX = editText.getWidth() / 2; 436 final int startY = editText.getHeight() / 2; 437 TestUtils.injectStylusHoverEvents(editText, startX, startY); 438 // keyboard shouldn't show up. 439 notExpectEvent( 440 stream, 441 editorMatcher("onStartInputView", marker), 442 NOT_EXPECT_TIMEOUT); 443 444 // Handwriting prep should start for stylus onHover 445 expectEvent( 446 stream, 447 editorMatcher("onPrepareStylusHandwriting", marker), 448 TIMEOUT); 449 notExpectEvent( 450 stream, 451 editorMatcher("onStartStylusHandwriting", marker), 452 NOT_EXPECT_TIMEOUT); 453 454 // Verify handwriting window exists but not shown. 455 assertTrue(expectCommand( 456 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT_1_S) 457 .getReturnBooleanValue()); 458 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 459 } 460 } 461 462 /** 463 * Call {@link InputMethodManager#startStylusHandwriting(View)} and inject Stylus touch events 464 * on screen. Make sure {@link InputMethodService#onStylusHandwritingMotionEvent(MotionEvent)} 465 * receives those events via Spy window surface. 466 * @throws Exception 467 */ 468 @Test testHandwritingStylusEvents_onStylusHandwritingMotionEvent()469 public void testHandwritingStylusEvents_onStylusHandwritingMotionEvent() throws Exception { 470 testHandwritingStylusEvents(false /* verifyOnInkView */); 471 } 472 473 /** 474 * Call {@link InputMethodManager#startStylusHandwriting(View)} and inject Stylus touch events 475 * on screen. Make sure Inking view receives those events via Spy window surface. 476 * @throws Exception 477 */ 478 @Test testHandwritingStylusEvents_dispatchToInkView()479 public void testHandwritingStylusEvents_dispatchToInkView() throws Exception { 480 testHandwritingStylusEvents(false /* verifyOnInkView */); 481 } 482 verifyStylusHandwritingWindowIsShown(ImeEventStream stream, MockImeSession imeSession)483 private void verifyStylusHandwritingWindowIsShown(ImeEventStream stream, 484 MockImeSession imeSession) throws InterruptedException, TimeoutException { 485 CommonTestUtils.waitUntil("Stylus handwriting window should be shown", TIMEOUT_IN_SECONDS, 486 () -> expectCommand( 487 stream, imeSession.callGetStylusHandwritingWindowVisibility(), TIMEOUT) 488 .getReturnBooleanValue()); 489 } 490 verifyStylusHandwritingWindowIsNotShown(ImeEventStream stream, MockImeSession imeSession)491 private void verifyStylusHandwritingWindowIsNotShown(ImeEventStream stream, 492 MockImeSession imeSession) throws InterruptedException, TimeoutException { 493 CommonTestUtils.waitUntil("Stylus handwriting window should not be shown", 494 NOT_EXPECT_TIMEOUT_IN_SECONDS, 495 () -> !expectCommand( 496 stream, imeSession.callGetStylusHandwritingWindowVisibility(), TIMEOUT) 497 .getReturnBooleanValue()); 498 } 499 testHandwritingStylusEvents(boolean verifyOnInkView)500 private void testHandwritingStylusEvents(boolean verifyOnInkView) throws Exception { 501 final InputMethodManager imm = InstrumentationRegistry.getInstrumentation() 502 .getTargetContext().getSystemService(InputMethodManager.class); 503 try (MockImeSession imeSession = MockImeSession.create( 504 InstrumentationRegistry.getInstrumentation().getContext(), 505 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 506 new ImeSettings.Builder())) { 507 final ImeEventStream stream = imeSession.openEventStream(); 508 509 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 510 final EditText editText = launchTestActivity(marker); 511 512 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 513 notExpectEvent( 514 stream, 515 editorMatcher("onStartInputView", marker), 516 NOT_EXPECT_TIMEOUT); 517 518 addVirtualStylusIdForTestSession(); 519 final List<MotionEvent> injectedEvents = new ArrayList<>(); 520 // Touch down with a stylus 521 final int touchSlop = getTouchSlop(); 522 final int startX = editText.getWidth() / 2; 523 final int startY = editText.getHeight() / 2; 524 final int endX = startX + 2 * touchSlop; 525 final int endY = startY; 526 final int number = 5; 527 injectedEvents.add(TestUtils.injectStylusDownEvent(editText, startX, startY)); 528 529 try { 530 imm.startStylusHandwriting(editText); 531 532 // Handwriting should start 533 expectEvent( 534 stream, 535 editorMatcher("onStartStylusHandwriting", marker), 536 TIMEOUT); 537 538 verifyStylusHandwritingWindowIsShown(stream, imeSession); 539 540 if (verifyOnInkView) { 541 // Set IME stylus Ink view 542 assertTrue(expectCommand( 543 stream, 544 imeSession.callSetStylusHandwritingInkView(), 545 TIMEOUT).getReturnBooleanValue()); 546 } 547 548 injectedEvents.addAll( 549 TestUtils.injectStylusMoveEvents(editText, startX, startY, endX, endY, 550 number)); 551 } finally { 552 injectedEvents.add(TestUtils.injectStylusUpEvent(editText, endX, endY)); 553 } 554 555 expectEvent(stream, eventMatcher("onStylusMotionEvent"), TIMEOUT); 556 557 // get Stylus events from Ink view, splitting any batched events. 558 final ArrayList<MotionEvent> capturedBatchedEvents = expectCommand( 559 stream, imeSession.callGetStylusHandwritingEvents(), TIMEOUT) 560 .getReturnParcelableArrayListValue(); 561 assertNotNull(capturedBatchedEvents); 562 final ArrayList<MotionEvent> capturedEvents = new ArrayList<>(); 563 capturedBatchedEvents.forEach( 564 e -> capturedEvents.addAll(TestUtils.splitBatchedMotionEvent(e))); 565 566 // captured events should be same as injected. 567 assertEquals(injectedEvents.size(), capturedEvents.size()); 568 569 // Verify MotionEvents as well. 570 // Note: we cannot just use equals() since some MotionEvent fields can change after 571 // dispatch. 572 Iterator<MotionEvent> capturedIt = capturedEvents.iterator(); 573 Iterator<MotionEvent> injectedIt = injectedEvents.iterator(); 574 while (injectedIt.hasNext() && capturedIt.hasNext()) { 575 MotionEvent injected = injectedIt.next(); 576 MotionEvent captured = capturedIt.next(); 577 assertEquals("X should be same for MotionEvent", injected.getX(), captured.getX(), 578 5.0f); 579 assertEquals("Y should be same for MotionEvent", injected.getY(), captured.getY(), 580 5.0f); 581 assertEquals("Action should be same for MotionEvent", 582 injected.getAction(), captured.getAction()); 583 } 584 } 585 } 586 587 @FlakyTest(bugId = 210039666) 588 @Test 589 /** 590 * Inject Stylus events on top of focused editor and verify Handwriting is started and InkWindow 591 * is displayed. 592 */ testHandwritingEndToEnd()593 public void testHandwritingEndToEnd() throws Exception { 594 try (MockImeSession imeSession = MockImeSession.create( 595 InstrumentationRegistry.getInstrumentation().getContext(), 596 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 597 new ImeSettings.Builder())) { 598 final ImeEventStream stream = imeSession.openEventStream(); 599 600 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 601 final EditText editText = launchTestActivity(marker); 602 603 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 604 notExpectEvent( 605 stream, 606 editorMatcher("onStartInputView", marker), 607 NOT_EXPECT_TIMEOUT); 608 609 addVirtualStylusIdForTestSession(); 610 611 injectStylusEventToEditorAndVerify(editText, stream, imeSession, marker, 612 true /* verifyHandwritingStart */, true /* verifyHandwritingWindowShown */, 613 false /* verifyHandwritingWindowNotShown */); 614 } 615 } 616 617 /** 618 * Inject stylus tap on a focused non-empty EditText and verify that handwriting is started. 619 */ 620 @Test 621 @RequiresFlagsEnabled(FLAG_HANDWRITING_END_OF_LINE_TAP) testHandwriting_endOfLineTap()622 public void testHandwriting_endOfLineTap() throws Exception { 623 try (MockImeSession imeSession = MockImeSession.create( 624 InstrumentationRegistry.getInstrumentation().getContext(), 625 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 626 new ImeSettings.Builder())) { 627 final ImeEventStream stream = imeSession.openEventStream(); 628 629 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 630 final EditText editText = launchTestActivity(marker); 631 editText.setText("a"); 632 editText.setSelection(1); 633 634 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 635 notExpectEvent(stream, editorMatcher("onStartInputView", marker), NOT_EXPECT_TIMEOUT); 636 637 addVirtualStylusIdForTestSession(); 638 639 // Stylus tap must be after the end of the line. 640 final int x = editText.getWidth() / 2; 641 final int y = editText.getHeight() / 2; 642 TestUtils.injectStylusDownEvent(editText, x, y); 643 TestUtils.injectStylusUpEvent(editText, x, y); 644 645 notExpectEvent(stream, editorMatcher("onStartInputView", marker), NOT_EXPECT_TIMEOUT); 646 expectEvent(stream, editorMatcher("onStartStylusHandwriting", marker), TIMEOUT); 647 verifyStylusHandwritingWindowIsShown(stream, imeSession); 648 } 649 } 650 651 @FlakyTest(bugId = 222840964) 652 @Test 653 /** 654 * Inject Stylus events on top of focused editor and verify Handwriting can be initiated 655 * multiple times. 656 */ testHandwritingInitMultipleTimes()657 public void testHandwritingInitMultipleTimes() throws Exception { 658 try (MockImeSession imeSession = MockImeSession.create( 659 InstrumentationRegistry.getInstrumentation().getContext(), 660 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 661 new ImeSettings.Builder())) { 662 final ImeEventStream stream = imeSession.openEventStream(); 663 664 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 665 final EditText editText = launchTestActivity(marker); 666 667 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 668 notExpectEvent( 669 stream, 670 editorMatcher("onStartInputView", marker), 671 NOT_EXPECT_TIMEOUT); 672 673 final int touchSlop = getTouchSlop(); 674 final int startX = editText.getWidth() / 2; 675 final int startY = editText.getHeight() / 2; 676 final int endX = startX + 2 * touchSlop; 677 final int endY = startY; 678 final int number = 5; 679 680 // Try to init handwriting for multiple times. 681 for (int i = 0; i < 3; ++i) { 682 addVirtualStylusIdForTestSession(); 683 684 injectStylusEventToEditorAndVerify(editText, stream, imeSession, marker, 685 true /* verifyHandwritingStart */, true /* verifyHandwritingWindowShown */, 686 false /* verifyHandwritingWindowNotShown */); 687 688 imeSession.callFinishStylusHandwriting(); 689 expectEvent( 690 stream, 691 editorMatcher("onFinishStylusHandwriting", marker), 692 TIMEOUT); 693 } 694 } 695 } 696 697 @Test 698 /** 699 * Inject Stylus events on top of focused editor's handwriting bounds and verify 700 * Handwriting is started and InkWindow is displayed. 701 */ testHandwritingInOffsetHandwritingBounds()702 public void testHandwritingInOffsetHandwritingBounds() throws Exception { 703 try (MockImeSession imeSession = MockImeSession.create( 704 InstrumentationRegistry.getInstrumentation().getContext(), 705 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 706 new ImeSettings.Builder())) { 707 final ImeEventStream stream = imeSession.openEventStream(); 708 709 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 710 final EditText editText = launchTestActivity(marker); 711 712 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 713 notExpectEvent( 714 stream, 715 editorMatcher("onStartInputView", marker), 716 NOT_EXPECT_TIMEOUT); 717 718 addVirtualStylusIdForTestSession(); 719 injectStylusEventToEditorAndVerify(editText, stream, imeSession, marker, 720 true /* verifyHandwritingStart */, true /* verifyHandwritingWindowShown */, 721 false /* verifyHandwritingWindowNotShown */); 722 } 723 } 724 725 /** 726 * Inject Stylus events on top of focused editor and verify Handwriting is started and then 727 * inject events on navbar to swipe to home and make sure motionEvents are consumed by 728 * Handwriting window. 729 */ 730 @Test testStylusSession_stylusWouldNotTriggerNavbarGestures()731 public void testStylusSession_stylusWouldNotTriggerNavbarGestures() throws Exception { 732 assumeTrue(sGestureNavRule.isGestureMode()); 733 734 try (MockImeSession imeSession = MockImeSession.create( 735 InstrumentationRegistry.getInstrumentation().getContext(), 736 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 737 new ImeSettings.Builder())) { 738 final ImeEventStream stream = imeSession.openEventStream(); 739 740 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 741 final EditText editText = launchTestActivity(marker); 742 743 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 744 notExpectEvent( 745 stream, 746 editorMatcher("onStartInputView", marker), 747 NOT_EXPECT_TIMEOUT); 748 749 addVirtualStylusIdForTestSession(); 750 751 injectStylusEventToEditorAndVerify(editText, stream, imeSession, marker, 752 true /* verifyHandwritingStart */, 753 false /* verifyHandwritingWindowShown */, 754 false /* verifyHandwritingWindowNotShown */); 755 756 // Inject stylus swipe up on navbar. 757 TestUtils.injectNavBarToHomeGestureEvents( 758 ((Activity) editText.getContext()), MotionEvent.TOOL_TYPE_STYLUS); 759 760 // Handwriting is finished if navigation gesture is executed. 761 // Make sure handwriting isn't finished. 762 notExpectEvent( 763 stream, 764 editorMatcher("onFinishStylusHandwriting", marker), 765 TIMEOUT_1_S); 766 } 767 } 768 769 /** 770 * Inject Stylus events on top of focused editor and verify Handwriting is started and then 771 * inject finger touch events on navbar to swipe to home and make sure user can swipe to home. 772 */ 773 @Test testStylusSession_fingerTriggersNavbarGestures()774 public void testStylusSession_fingerTriggersNavbarGestures() throws Exception { 775 assumeTrue(sGestureNavRule.isGestureMode()); 776 777 try (MockImeSession imeSession = MockImeSession.create( 778 InstrumentationRegistry.getInstrumentation().getContext(), 779 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 780 new ImeSettings.Builder())) { 781 final ImeEventStream stream = imeSession.openEventStream(); 782 783 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 784 final EditText editText = launchTestActivity(marker); 785 786 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 787 notExpectEvent( 788 stream, 789 editorMatcher("onStartInputView", marker), 790 NOT_EXPECT_TIMEOUT); 791 792 addVirtualStylusIdForTestSession(); 793 injectStylusEventToEditorAndVerify(editText, stream, imeSession, marker, 794 true /* verifyHandwritingStart */, 795 false /* verifyHandwritingWindowShown */, 796 false /* verifyHandwritingWindowNotShown */); 797 798 // Inject finger swipe up on navbar. 799 TestUtils.injectNavBarToHomeGestureEvents( 800 ((Activity) editText.getContext()), MotionEvent.TOOL_TYPE_FINGER); 801 802 // Handwriting is finished if navigation gesture is executed. 803 // Make sure handwriting is finished to ensure swipe to home works. 804 expectEvent( 805 stream, 806 editorMatcher("onFinishStylusHandwriting", marker), 807 // BlastSyncEngine has a 5s timeout when launcher fails to sync its 808 // transaction, exceeding it avoids flakes when that happens. 809 TIMEOUT_6_S); 810 } 811 } 812 813 @Test 814 /** 815 * Inject stylus events to a focused EditText that disables autoHandwriting. 816 * {@link InputMethodManager#startStylusHandwriting(View)} should not be called. 817 */ testAutoHandwritingDisabled()818 public void testAutoHandwritingDisabled() throws Exception { 819 try (MockImeSession imeSession = MockImeSession.create( 820 InstrumentationRegistry.getInstrumentation().getContext(), 821 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 822 new ImeSettings.Builder())) { 823 final ImeEventStream stream = imeSession.openEventStream(); 824 825 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 826 final EditText editText = launchTestActivity(marker); 827 editText.setAutoHandwritingEnabled(false); 828 829 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 830 notExpectEvent( 831 stream, 832 editorMatcher("onStartInputView", marker), 833 NOT_EXPECT_TIMEOUT); 834 835 addVirtualStylusIdForTestSession(); 836 TestUtils.injectStylusEvents(editText); 837 838 // TODO(215439842): check that keyboard is not shown. 839 // Handwriting should not start 840 notExpectEvent( 841 stream, 842 editorMatcher("onStartStylusHandwriting", marker), 843 NOT_EXPECT_TIMEOUT); 844 845 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 846 } 847 } 848 849 @Test 850 /** 851 * Inject stylus events out of a focused editor's view bound. 852 * {@link InputMethodManager#startStylusHandwriting(View)} should not be called for this editor. 853 */ testAutoHandwritingOutOfBound()854 public void testAutoHandwritingOutOfBound() throws Exception { 855 try (MockImeSession imeSession = MockImeSession.create( 856 InstrumentationRegistry.getInstrumentation().getContext(), 857 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 858 new ImeSettings.Builder())) { 859 final ImeEventStream stream = imeSession.openEventStream(); 860 861 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 862 final EditText editText = launchTestActivity(marker); 863 editText.setAutoHandwritingEnabled(false); 864 865 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 866 notExpectEvent( 867 stream, 868 editorMatcher("onStartInputView", marker), 869 NOT_EXPECT_TIMEOUT); 870 871 addVirtualStylusIdForTestSession(); 872 // Inject stylus events out of the editor boundary. 873 TestUtils.injectStylusEvents(editText, editText.getWidth() / 2, 874 -HANDWRITING_BOUNDS_OFFSET_PX - 50); 875 // keyboard shouldn't show up. 876 notExpectEvent( 877 stream, 878 editorMatcher("onStartInputView", marker), 879 NOT_EXPECT_TIMEOUT); 880 // Handwriting should not start 881 notExpectEvent( 882 stream, 883 editorMatcher("onStartStylusHandwriting", marker), 884 NOT_EXPECT_TIMEOUT); 885 886 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 887 } 888 } 889 890 @Test 891 /** 892 * Inject Stylus events on top of an unfocused editor and verify Handwriting is started and 893 * InkWindow is displayed. 894 */ testHandwriting_unfocusedEditText()895 public void testHandwriting_unfocusedEditText() throws Exception { 896 try (MockImeSession imeSession = MockImeSession.create( 897 InstrumentationRegistry.getInstrumentation().getContext(), 898 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 899 new ImeSettings.Builder())) { 900 final ImeEventStream stream = imeSession.openEventStream(); 901 902 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 903 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 904 final Pair<EditText, EditText> editTextPair = 905 launchTestActivity(focusedMarker, unfocusedMarker); 906 final EditText unfocusedEditText = editTextPair.second; 907 908 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 909 notExpectEvent( 910 stream, 911 editorMatcher("onStartInputView", focusedMarker), 912 NOT_EXPECT_TIMEOUT); 913 914 addVirtualStylusIdForTestSession(); 915 final int touchSlop = getTouchSlop(); 916 final int startX = unfocusedEditText.getWidth() / 2; 917 final int startY = 2 * touchSlop; 918 // (endX, endY) is out of bound to avoid that unfocusedEditText is focused due to the 919 // stylus touch. 920 final int endX = startX; 921 final int endY = unfocusedEditText.getHeight() + 2 * touchSlop; 922 final int number = 5; 923 924 TestUtils.injectStylusDownEvent(unfocusedEditText, startX, startY); 925 TestUtils.injectStylusMoveEvents(unfocusedEditText, startX, startY, 926 endX, endY, number); 927 try { 928 // Handwriting should already be initiated before ACTION_UP. 929 // unfocusedEditor is focused and triggers onStartInput. 930 expectEvent(stream, editorMatcher("onStartInput", unfocusedMarker), TIMEOUT); 931 // keyboard shouldn't show up. 932 notExpectEvent( 933 stream, 934 editorMatcher("onStartInputView", unfocusedMarker), 935 NOT_EXPECT_TIMEOUT); 936 // Handwriting should start on the unfocused EditText. 937 expectEvent( 938 stream, 939 editorMatcher("onStartStylusHandwriting", unfocusedMarker), 940 TIMEOUT); 941 verifyStylusHandwritingWindowIsShown(stream, imeSession); 942 } finally { 943 TestUtils.injectStylusUpEvent(unfocusedEditText, endX, endY); 944 } 945 } 946 } 947 948 /** 949 * Inject stylus events on top of an unfocused password EditText and verify keyboard is shown 950 * and handwriting is not started 951 */ 952 @Test 953 @RequiresFlagsEnabled(FLAG_HANDWRITING_UNSUPPORTED_MESSAGE) testHandwriting_unfocusedEditText_password()954 public void testHandwriting_unfocusedEditText_password() throws Exception { 955 try (MockImeSession imeSession = MockImeSession.create( 956 InstrumentationRegistry.getInstrumentation().getContext(), 957 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 958 new ImeSettings.Builder())) { 959 final ImeEventStream stream = imeSession.openEventStream(); 960 961 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 962 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 963 final Pair<EditText, EditText> editTextPair = 964 launchTestActivity(focusedMarker, unfocusedMarker); 965 final EditText unfocusedEditText = editTextPair.second; 966 unfocusedEditText.post(() -> unfocusedEditText.setInputType( 967 InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_VARIATION_PASSWORD)); 968 969 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 970 notExpectEvent( 971 stream, 972 editorMatcher("onStartInputView", focusedMarker), 973 NOT_EXPECT_TIMEOUT); 974 975 addVirtualStylusIdForTestSession(); 976 final int touchSlop = getTouchSlop(); 977 final int startX = unfocusedEditText.getWidth() / 2; 978 final int startY = 2 * touchSlop; 979 // (endX, endY) is out of bound to avoid that unfocusedEditText is focused due to the 980 // stylus touch. 981 final int endX = startX; 982 final int endY = unfocusedEditText.getHeight() + 2 * touchSlop; 983 final int number = 5; 984 985 TestUtils.injectStylusDownEvent(unfocusedEditText, startX, startY); 986 TestUtils.injectStylusMoveEvents(unfocusedEditText, startX, startY, 987 endX, endY, number); 988 989 // Handwriting is not started since it is not supported for password fields, but it is 990 // focused and the soft keyboard is shown. 991 expectEvent(stream, editorMatcher("onStartInput", unfocusedMarker), TIMEOUT); 992 expectEvent(stream, editorMatcher("onStartInputView", unfocusedMarker), TIMEOUT); 993 notExpectEvent( 994 stream, 995 editorMatcher("onStartStylusHandwriting", unfocusedMarker), 996 NOT_EXPECT_TIMEOUT); 997 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 998 999 TestUtils.injectStylusUpEvent(unfocusedEditText, endX, endY); 1000 } 1001 } 1002 1003 /** 1004 * With handwriting setting disabled, inject stylus events on top of an unfocused EditText and 1005 * verify handwriting is not started and keyboard is not shown. 1006 */ 1007 @Test 1008 @RequiresFlagsEnabled(FLAG_HANDWRITING_UNSUPPORTED_MESSAGE) testHandwriting_unfocusedEditText_prefDisabled()1009 public void testHandwriting_unfocusedEditText_prefDisabled() throws Exception { 1010 try (MockImeSession imeSession = MockImeSession.create( 1011 InstrumentationRegistry.getInstrumentation().getContext(), 1012 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1013 new ImeSettings.Builder())) { 1014 final ImeEventStream stream = imeSession.openEventStream(); 1015 1016 // Disable preference 1017 SystemUtil.runWithShellPermissionIdentity(() -> { 1018 Settings.Secure.putInt(mContext.getContentResolver(), 1019 STYLUS_HANDWRITING_ENABLED, SETTING_VALUE_OFF); 1020 }, Manifest.permission.WRITE_SECURE_SETTINGS); 1021 mShouldRestoreInitialHwState = true; 1022 1023 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 1024 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 1025 final Pair<EditText, EditText> editTextPair = 1026 launchTestActivity(focusedMarker, unfocusedMarker); 1027 final EditText unfocusedEditText = editTextPair.second; 1028 1029 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 1030 notExpectEvent( 1031 stream, 1032 editorMatcher("onStartInputView", focusedMarker), 1033 NOT_EXPECT_TIMEOUT); 1034 1035 addVirtualStylusIdForTestSession(); 1036 final int touchSlop = getTouchSlop(); 1037 final int x = unfocusedEditText.getWidth() / 2; 1038 final int startY = 2 * touchSlop; 1039 // (endX, endY) is out of bound to avoid that unfocusedEditText is focused due to the 1040 // stylus touch. 1041 final int endY = unfocusedEditText.getHeight() + 2 * touchSlop; 1042 final int number = 5; 1043 1044 TestUtils.injectStylusDownEvent(unfocusedEditText, x, startY); 1045 TestUtils.injectStylusMoveEvents(unfocusedEditText, x, startY, x, endY, number); 1046 1047 // Handwriting is not started, 1048 notExpectEvent(stream, editorMatcher("onStartInput", unfocusedMarker), 1049 NOT_EXPECT_TIMEOUT); 1050 notExpectEvent(stream, editorMatcher("onStartInputView", unfocusedMarker), 1051 NOT_EXPECT_TIMEOUT); 1052 notExpectEvent( 1053 stream, 1054 editorMatcher("onStartStylusHandwriting", unfocusedMarker), 1055 NOT_EXPECT_TIMEOUT); 1056 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 1057 1058 TestUtils.injectStylusUpEvent(unfocusedEditText, x, endY); 1059 } 1060 } 1061 1062 /** 1063 * Inject stylus events on top of an unfocused editor which disabled the autoHandwriting and 1064 * verify keyboard is shown and handwriting is not started. 1065 */ 1066 @Test 1067 @RequiresFlagsEnabled(FLAG_HANDWRITING_UNSUPPORTED_MESSAGE) testHandwriting_unfocusedEditText_autoHandwritingDisabled()1068 public void testHandwriting_unfocusedEditText_autoHandwritingDisabled() throws Exception { 1069 try (MockImeSession imeSession = MockImeSession.create( 1070 InstrumentationRegistry.getInstrumentation().getContext(), 1071 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1072 new ImeSettings.Builder())) { 1073 final ImeEventStream stream = imeSession.openEventStream(); 1074 1075 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 1076 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 1077 final Pair<EditText, EditText> editTextPair = 1078 launchTestActivity(focusedMarker, unfocusedMarker); 1079 final EditText unfocusedEditText = editTextPair.second; 1080 unfocusedEditText.setAutoHandwritingEnabled(false); 1081 1082 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 1083 notExpectEvent( 1084 stream, 1085 editorMatcher("onStartInputView", focusedMarker), 1086 NOT_EXPECT_TIMEOUT); 1087 1088 addVirtualStylusIdForTestSession(); 1089 final int touchSlop = getTouchSlop(); 1090 final int startX = unfocusedEditText.getWidth() / 2; 1091 final int startY = 2 * touchSlop; 1092 // (endX, endY) is out of bound to avoid that unfocusedEditText is focused due to the 1093 // stylus touch. 1094 final int endX = startX; 1095 final int endY = -2 * touchSlop; 1096 final int number = 5; 1097 TestUtils.injectStylusDownEvent(unfocusedEditText, startX, startY); 1098 TestUtils.injectStylusMoveEvents(unfocusedEditText, startX, startY, 1099 endX, endY, number); 1100 TestUtils.injectStylusUpEvent(unfocusedEditText, endX, endY); 1101 1102 // Handwriting is not started since it is disabled for the EditText, but it is focused 1103 // and the soft keyboard is shown. 1104 expectEvent(stream, editorMatcher("onStartInput", unfocusedMarker), TIMEOUT); 1105 expectEvent(stream, editorMatcher("onStartInputView", unfocusedMarker), TIMEOUT); 1106 notExpectEvent( 1107 stream, 1108 editorMatcher("onStartStylusHandwriting", unfocusedMarker), 1109 NOT_EXPECT_TIMEOUT); 1110 1111 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 1112 } 1113 } 1114 1115 /** 1116 * Inject finger taps during ongoing stylus handwriting and make sure those taps are ignored 1117 * until stylus ACTION_UP. 1118 */ 1119 @Test testHandwriting_fingerTouchIsIgnored()1120 public void testHandwriting_fingerTouchIsIgnored() throws Exception { 1121 int displayId = 0; 1122 String initialUserRotation = null; 1123 Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); 1124 try (MockImeSession imeSession = MockImeSession.create( 1125 instrumentation.getContext(), 1126 instrumentation.getUiAutomation(), 1127 new ImeSettings.Builder())) { 1128 final ImeEventStream stream = imeSession.openEventStream(); 1129 1130 final String focusedMarker = getTestMarker(); 1131 final String unfocusedMarker = getTestMarker(); 1132 final Pair<EditText, EditText> editTextPair = 1133 launchTestActivity(focusedMarker, unfocusedMarker); 1134 final EditText focusedEditText = editTextPair.first; 1135 final EditText unfocusedEditText = editTextPair.second; 1136 Context context = focusedEditText.getContext(); 1137 displayId = context.getDisplayId(); 1138 1139 1140 final Display display = context.getDisplay(); 1141 try (UinputTouchDevice touch = new UinputTouchScreen(instrumentation, display); 1142 UinputTouchDevice stylus = new UinputStylus(instrumentation, display)) { 1143 initialUserRotation = 1144 getInitialRotationAndAwaitExpectedRotation(displayId, context); 1145 1146 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 1147 notExpectEvent( 1148 stream, 1149 editorMatcher("onStartInputView", focusedMarker), 1150 NOT_EXPECT_TIMEOUT); 1151 1152 addVirtualStylusIdForTestSession(); 1153 final int touchSlop = getTouchSlop(); 1154 int startX = focusedEditText.getWidth() / 2; 1155 final int startY = 2 * touchSlop; 1156 int endX = startX; 1157 int endY = focusedEditText.getHeight() + 2 * touchSlop; 1158 final int number = 5; 1159 1160 // set a longer idle-timeout for handwriting session. 1161 assertTrue(expectCommand( 1162 stream, imeSession.callSetStylusHandwritingTimeout(TIMEOUT * 2), 1163 TIMEOUT).getReturnBooleanValue()); 1164 TestUtils.injectStylusDownEvent(stylus, focusedEditText, startX, startY); 1165 TestUtils.injectStylusMoveEvents(stylus, focusedEditText, startX, startY, 1166 endX, endY, number); 1167 1168 // Handwriting should start on the focused EditText. 1169 expectEvent( 1170 stream, 1171 editorMatcher("onStartStylusHandwriting", focusedMarker), 1172 TIMEOUT); 1173 1174 // Set IME stylus Ink view to listen for ACTION_UP MotionEvent 1175 assertTrue(expectCommand( 1176 stream, 1177 imeSession.callSetStylusHandwritingInkView(), 1178 TIMEOUT).getReturnBooleanValue()); 1179 1180 TestUtils.injectStylusUpEvent(stylus); 1181 waitForStylusAction(MotionEvent.ACTION_UP, stream, imeSession, endX, endY); 1182 1183 // Finger tap on unfocused editor. 1184 TestUtils.injectFingerEventOnViewCenter( 1185 touch, unfocusedEditText, MotionEvent.ACTION_DOWN); 1186 TestUtils.injectFingerEventOnViewCenter( 1187 touch, unfocusedEditText, MotionEvent.ACTION_UP); 1188 1189 // Finger tap should passthrough and unfocused editor should steal focus. 1190 TestUtils.waitOnMainUntil(unfocusedEditText::hasFocus, 1191 TIMEOUT, "unfocusedEditText should gain focus on finger tap"); 1192 1193 // reset focus back to focusedEditText. 1194 focusedEditText.post(focusedEditText::requestFocus); 1195 1196 // Inject a lot of stylus events async. 1197 TestUtils.injectStylusDownEvent(stylus, focusedEditText, startX, startY); 1198 TestUtils.injectStylusMoveEvents(stylus, focusedEditText, startX, startY, 1199 endX, endY, number); 1200 // Set IME stylus Ink view to listen for ACTION_MOVE MotionEvents. 1201 // (This can only be set on Handwriting window, which exists for the duration of 1202 // session). 1203 assertTrue(expectCommand( 1204 stream, 1205 imeSession.callSetStylusHandwritingInkView(), 1206 TIMEOUT).getReturnBooleanValue()); 1207 // After handwriting has started, inject another ACTION_MOVE so we receive that on 1208 // InkView. 1209 TestUtils.injectStylusMoveEvents(stylus, focusedEditText, endX, endY, 1210 endX, endY, number); 1211 waitForStylusAction(MotionEvent.ACTION_MOVE, stream, imeSession, endX, endY); 1212 1213 // Finger tap on unfocused editor while stylus is still injecting events. 1214 TestUtils.injectFingerEventOnViewCenter( 1215 touch, unfocusedEditText, MotionEvent.ACTION_DOWN); 1216 TestUtils.injectFingerEventOnViewCenter( 1217 touch, unfocusedEditText, MotionEvent.ACTION_UP); 1218 // Finger tap should be ignored and unfocused editor shouldn't steal focus. 1219 TestUtils.waitOnMainUntil(() -> !unfocusedEditText.hasFocus(), 1220 TIMEOUT_1_S, "Finger tap on unfocusedEditText should be ignored"); 1221 TestUtils.injectStylusUpEvent(stylus); 1222 1223 notExpectEvent( 1224 stream, 1225 editorMatcher("finishStylusHandwriting", unfocusedMarker), 1226 NOT_EXPECT_TIMEOUT); 1227 } finally { 1228 imeSession.callFinishStylusHandwriting(); 1229 } 1230 } finally { 1231 TestUtils.setRotation(displayId, initialUserRotation); 1232 } 1233 } 1234 1235 // wait for stylus action to be delivered to IME. waitForStylusAction( int action, ImeEventStream stream, MockImeSession imeSession, int x, int y)1236 private void waitForStylusAction( 1237 int action, ImeEventStream stream, MockImeSession imeSession, int x, int y) 1238 throws TimeoutException { 1239 long elapsedMs = 0; 1240 int sleepDurationMs = 50; 1241 while (elapsedMs < TIMEOUT_1_S) { 1242 final ArrayList<MotionEvent> capturedBatchedEvents = 1243 expectCommand(stream, imeSession.callGetStylusHandwritingEvents(), TIMEOUT) 1244 .getReturnParcelableArrayListValue(); 1245 assertNotNull(capturedBatchedEvents); 1246 assertFalse("captured events shouldn't be empty", capturedBatchedEvents.isEmpty()); 1247 1248 MotionEvent lastEvent = capturedBatchedEvents.get(capturedBatchedEvents.size() - 1); 1249 for (MotionEvent event : capturedBatchedEvents) { 1250 if (lastEvent.getAction() == action && event.getX() == x && event.getY() == y) { 1251 break; 1252 } 1253 } 1254 1255 elapsedMs += sleepDurationMs; 1256 try { 1257 Thread.sleep(sleepDurationMs); 1258 } catch (InterruptedException e) { 1259 throw new RuntimeException(e); 1260 } 1261 } 1262 } 1263 1264 getInitialRotationAndAwaitExpectedRotation(int displayId, Context context)1265 private String getInitialRotationAndAwaitExpectedRotation(int displayId, Context context) { 1266 String expectedRotation = "0"; 1267 String initialUserRotation = TestUtils.getRotation(displayId); 1268 if (!expectedRotation.equals(initialUserRotation)) { 1269 // Set device-default rotation for UinputTouchDevice to work as expected. 1270 TestUtils.setLockedRotation(displayId, expectedRotation); 1271 waitUntilActivityReadyForInput((Activity) context); 1272 } 1273 return initialUserRotation; 1274 } 1275 waitUntilActivityReadyForInput(Activity activity)1276 private void waitUntilActivityReadyForInput(Activity activity) { 1277 // If we requested an orientation change, just waiting for the window to be visible is not 1278 // sufficient. We should first wait for the transitions to stop, and the for app's UI thread 1279 // to process them before making sure the window is visible. 1280 try { 1281 TestUtils.waitUntilActivityReadyForInputInjection( 1282 activity, StylusHandwritingTest.this.getClass().getName(), 1283 "test: " + StylusHandwritingTest.this.mTestName.getMethodName() 1284 + ", virtualDisplayId=" + activity.getDisplayId() 1285 ); 1286 } catch (InterruptedException e) { 1287 throw new RuntimeException(e); 1288 } 1289 } 1290 1291 /** 1292 * Inject stylus top on an editor and verify stylus source is detected with 1293 * {@link InputMethodService#onUpdateEditorToolType(int)} lifecycle method. 1294 */ 1295 @Test 1296 @FlakyTest testOnViewClicked_withStylusTap()1297 public void testOnViewClicked_withStylusTap() throws Exception { 1298 UinputTouchDevice stylus = null; 1299 int displayId = 0; 1300 String initialUserRotation = null; 1301 try (MockImeSession imeSession = MockImeSession.create( 1302 InstrumentationRegistry.getInstrumentation().getContext(), 1303 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1304 new ImeSettings.Builder())) { 1305 final ImeEventStream stream = imeSession.openEventStream(); 1306 1307 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 1308 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 1309 final Pair<EditText, EditText> pair = 1310 launchTestActivity(focusedMarker, unfocusedMarker); 1311 final EditText focusedEditText = pair.first; 1312 final EditText unfocusedEditText = pair.second; 1313 Context context = focusedEditText.getContext(); 1314 displayId = context.getDisplayId(); 1315 1316 int x = focusedEditText.getWidth() / 2; 1317 int y = focusedEditText.getHeight() / 2; 1318 1319 // Tap with stylus on focused editor 1320 stylus = new UinputStylus(InstrumentationRegistry.getInstrumentation(), 1321 focusedEditText.getDisplay()); 1322 initialUserRotation = getInitialRotationAndAwaitExpectedRotation(displayId, context); 1323 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 1324 notExpectEvent( 1325 stream, 1326 editorMatcher("onStartInputView", focusedMarker), 1327 NOT_EXPECT_TIMEOUT); 1328 addVirtualStylusIdForTestSession(); 1329 1330 TestUtils.injectStylusDownEvent(stylus, focusedEditText, x, y); 1331 TestUtils.injectStylusUpEvent(stylus); 1332 1333 int toolType = MotionEvent.TOOL_TYPE_STYLUS; 1334 expectEvent( 1335 stream, 1336 onUpdateEditorToolTypeMatcher(toolType), 1337 TIMEOUT); 1338 1339 // Tap with stylus on unfocused editor 1340 x = unfocusedEditText.getWidth() / 2; 1341 y = unfocusedEditText.getHeight() / 2; 1342 TestUtils.injectStylusDownEvent(stylus, unfocusedEditText, x, y); 1343 TestUtils.injectStylusUpEvent(stylus); 1344 if (Flags.useHandwritingListenerForTooltype()) { 1345 expectEvent(stream, startInputInitialEditorToolMatcher(toolType, unfocusedMarker), 1346 TIMEOUT); 1347 } else { 1348 expectEvent(stream, onStartInputMatcher(toolType, unfocusedMarker), TIMEOUT); 1349 } 1350 expectEvent( 1351 stream, 1352 onUpdateEditorToolTypeMatcher(toolType), 1353 TIMEOUT); 1354 } finally { 1355 if (stylus != null) { 1356 stylus.close(); 1357 } 1358 if (initialUserRotation != null) { 1359 TestUtils.setRotation(displayId, initialUserRotation); 1360 } 1361 } 1362 } 1363 1364 /** 1365 * Inject finger top on an editor and verify stylus source is detected with 1366 * {@link InputMethodService#onUpdateEditorToolType(int)} lifecycle method. 1367 */ 1368 @Test 1369 @FlakyTest testOnViewClicked_withFingerTap()1370 public void testOnViewClicked_withFingerTap() throws Exception { 1371 UinputTouchDevice touch = null; 1372 int displayId = 0; 1373 String initialUserRotation = null; 1374 try (MockImeSession imeSession = MockImeSession.create( 1375 InstrumentationRegistry.getInstrumentation().getContext(), 1376 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1377 new ImeSettings.Builder())) { 1378 final ImeEventStream stream = imeSession.openEventStream(); 1379 1380 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 1381 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 1382 final Pair<EditText, EditText> pair = 1383 launchTestActivity(focusedMarker, unfocusedMarker); 1384 final EditText focusedEditText = pair.first; 1385 final EditText unfocusedEditText = pair.second; 1386 Context context = focusedEditText.getContext(); 1387 displayId = context.getDisplayId(); 1388 1389 touch = new UinputTouchScreen( 1390 InstrumentationRegistry.getInstrumentation(), unfocusedEditText.getDisplay()); 1391 initialUserRotation = getInitialRotationAndAwaitExpectedRotation(displayId, context); 1392 int toolTypeFinger = MotionEvent.TOOL_TYPE_FINGER; 1393 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 1394 notExpectEvent( 1395 stream, 1396 editorMatcher("onStartInputView", focusedMarker), 1397 NOT_EXPECT_TIMEOUT); 1398 TestUtils.injectFingerEventOnViewCenter( 1399 touch, focusedEditText, MotionEvent.ACTION_DOWN); 1400 TestUtils.injectFingerEventOnViewCenter( 1401 touch, focusedEditText, MotionEvent.ACTION_UP); 1402 1403 expectEvent( 1404 stream, 1405 onUpdateEditorToolTypeMatcher(MotionEvent.TOOL_TYPE_FINGER), 1406 TIMEOUT); 1407 1408 // tap on unfocused editor 1409 TestUtils.injectFingerEventOnViewCenter( 1410 touch, unfocusedEditText, MotionEvent.ACTION_DOWN); 1411 TestUtils.injectFingerEventOnViewCenter( 1412 touch, unfocusedEditText, MotionEvent.ACTION_UP); 1413 expectEvent(stream, onStartInputMatcher(toolTypeFinger, unfocusedMarker), TIMEOUT); 1414 expectEvent( 1415 stream, 1416 onUpdateEditorToolTypeMatcher(MotionEvent.TOOL_TYPE_FINGER), 1417 TIMEOUT); 1418 } finally { 1419 if (touch != null) { 1420 touch.close(); 1421 } 1422 if (initialUserRotation != null) { 1423 TestUtils.setRotation(displayId, initialUserRotation); 1424 } 1425 } 1426 } 1427 1428 /** 1429 * Inject stylus handwriting event on an editor and verify stylus source is detected with 1430 * {@link InputMethodService#onUpdateEditorToolType(int)} on next startInput(). 1431 */ 1432 @Test 1433 @FlakyTest testOnViewClicked_withStylusHandwriting()1434 public void testOnViewClicked_withStylusHandwriting() throws Exception { 1435 try (MockImeSession imeSession = MockImeSession.create( 1436 InstrumentationRegistry.getInstrumentation().getContext(), 1437 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1438 new ImeSettings.Builder())) { 1439 final ImeEventStream stream = imeSession.openEventStream(); 1440 1441 addVirtualStylusIdForTestSession(); 1442 1443 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 1444 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 1445 final Pair<EditText, EditText> pair = 1446 launchTestActivity(focusedMarker, unfocusedMarker); 1447 final EditText focusedEditText = pair.first; 1448 final EditText unfocusedEditText = pair.second; 1449 1450 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 1451 notExpectEvent( 1452 stream, 1453 editorMatcher("onStartInputView", focusedMarker), 1454 NOT_EXPECT_TIMEOUT); 1455 1456 // Finger tap on editor and verify onUpdateEditorToolType 1457 TestUtils.injectFingerEventOnViewCenter(focusedEditText, MotionEvent.ACTION_DOWN); 1458 MotionEvent upEvent = 1459 TestUtils.injectFingerEventOnViewCenter(focusedEditText, MotionEvent.ACTION_UP); 1460 int toolTypeFinger = upEvent.getToolType(upEvent.getActionIndex()); 1461 assertEquals( 1462 "tool type finger must match", MotionEvent.TOOL_TYPE_FINGER, toolTypeFinger); 1463 expectEvent( 1464 stream, 1465 onUpdateEditorToolTypeMatcher(toolTypeFinger), 1466 TIMEOUT); 1467 1468 // Start handwriting on same focused editor 1469 final int touchSlop = getTouchSlop(); 1470 int startX = focusedEditText.getWidth() / 2; 1471 int startY = focusedEditText.getHeight() / 2; 1472 int endX = startX + 2 * touchSlop; 1473 int endY = startY + 2 * touchSlop; 1474 final int number = 5; 1475 TestUtils.injectStylusDownEvent(focusedEditText, startX, startY); 1476 TestUtils.injectStylusMoveEvents(focusedEditText, startX, startY, 1477 endX, endY, number); 1478 try { 1479 // Handwriting should start. 1480 expectEvent( 1481 stream, 1482 editorMatcher("onStartStylusHandwriting", focusedMarker), 1483 TIMEOUT); 1484 } finally { 1485 TestUtils.injectStylusUpEvent(focusedEditText, endX, endY); 1486 } 1487 imeSession.callFinishStylusHandwriting(); 1488 expectEvent( 1489 stream, 1490 editorMatcher("onFinishStylusHandwriting", focusedMarker), 1491 TIMEOUT_1_S); 1492 1493 addVirtualStylusIdForTestSession(); 1494 // Now start handwriting on unfocused editor and verify toolType is available in 1495 // EditorInfo 1496 startX = unfocusedEditText.getWidth() / 2; 1497 startY = unfocusedEditText.getHeight() / 2; 1498 endX = startX + 2 * touchSlop; 1499 endY = startY + 2 * touchSlop; 1500 1501 TestUtils.injectStylusDownEvent(unfocusedEditText, startX, startY); 1502 TestUtils.injectStylusMoveEvents(unfocusedEditText, startX, startY, 1503 endX, endY, number); 1504 try { 1505 expectEvent(stream, editorMatcher("onStartInput", unfocusedMarker), TIMEOUT); 1506 1507 // toolType should be updated on next stylus handwriting start 1508 expectEvent(stream, onStartStylusHandwritingMatcher( 1509 MotionEvent.TOOL_TYPE_STYLUS, unfocusedMarker), TIMEOUT); 1510 } finally { 1511 TestUtils.injectStylusUpEvent(unfocusedEditText, endX, endY); 1512 } 1513 } 1514 } 1515 1516 /** 1517 * Inject KeyEvent and Stylus tap verify toolType is detected with 1518 * {@link InputMethodService#onUpdateEditorToolType(int)} lifecycle method. 1519 */ 1520 @Test testOnViewClicked_withKeyEvent()1521 public void testOnViewClicked_withKeyEvent() throws Exception { 1522 assumeTrue("skipping test when flag useHandwritingListenerForTooltype is disabled", 1523 Flags.useHandwritingListenerForTooltype()); 1524 final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); 1525 try (MockImeSession imeSession = MockImeSession.create( 1526 instrumentation.getContext(), instrumentation.getUiAutomation(), 1527 new ImeSettings.Builder())) { 1528 final ImeEventStream stream = imeSession.openEventStream(); 1529 1530 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 1531 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 1532 final Pair<EditText, EditText> pair = 1533 launchTestActivityNoEditorFocus(focusedMarker, unfocusedMarker); 1534 final EditText firstEditText = pair.first; 1535 1536 // Send any KeyEvent when editor isn't focused. 1537 instrumentation.sendKeyDownUpSync(KeyEvent.KEYCODE_0); 1538 1539 // KeyEvents are identified as unknown tooltype. 1540 int toolType = MotionEvent.TOOL_TYPE_UNKNOWN; 1541 expectEvent( 1542 stream, 1543 onUpdateEditorToolTypeMatcher(toolType), 1544 TIMEOUT); 1545 } 1546 } 1547 onStartInputMatcher(int toolType, String marker)1548 private static DescribedPredicate<ImeEvent> onStartInputMatcher(int toolType, String marker) { 1549 Predicate<ImeEvent> matcher = event -> { 1550 if (!TextUtils.equals("onStartInput", event.getEventName())) { 1551 return false; 1552 } 1553 EditorInfo info = event.getArguments().getParcelable("editorInfo"); 1554 return info.getInitialToolType() == toolType 1555 && TextUtils.equals(marker, info.privateImeOptions); 1556 }; 1557 return withDescription( 1558 "onStartInput(initialToolType=" + toolType + ",marker=" + marker + ")", matcher); 1559 } 1560 1561 startInputInitialEditorToolMatcher( int expectedToolType, @NonNull String marker)1562 private static DescribedPredicate<ImeEvent> startInputInitialEditorToolMatcher( 1563 int expectedToolType, @NonNull String marker) { 1564 return withDescription("onStartInput()" + "(marker=" + marker + ")", event -> { 1565 if (!TextUtils.equals("onStartInput", event.getEventName())) { 1566 return false; 1567 } 1568 final EditorInfo editorInfo = event.getArguments().getParcelable("editorInfo"); 1569 return expectedToolType == editorInfo.getInitialToolType(); 1570 }); 1571 } 1572 1573 private static DescribedPredicate<ImeEvent> onStartStylusHandwritingMatcher( 1574 int toolType, String marker) { 1575 Predicate<ImeEvent> matcher = event -> { 1576 if (!TextUtils.equals("onStartStylusHandwriting", event.getEventName())) { 1577 return false; 1578 } 1579 EditorInfo info = event.getArguments().getParcelable("editorInfo"); 1580 return info.getInitialToolType() == toolType 1581 && TextUtils.equals(marker, info.privateImeOptions); 1582 }; 1583 return withDescription( 1584 "onStartStylusHandwriting(initialToolType=" + toolType 1585 + ", marker=" + marker + ")", matcher); 1586 } 1587 1588 private static DescribedPredicate<ImeEvent> onUpdateEditorToolTypeMatcher(int expectedToolType) { 1589 Predicate<ImeEvent> matcher = event -> { 1590 if (!TextUtils.equals("onUpdateEditorToolType", event.getEventName())) { 1591 return false; 1592 } 1593 final int actualToolType = event.getArguments().getInt("toolType"); 1594 return actualToolType == expectedToolType; 1595 }; 1596 return withDescription("onUpdateEditorToolType(toolType=" + expectedToolType + ")", 1597 matcher); 1598 } 1599 1600 /** 1601 * Inject stylus events on top of a focused custom editor and verify handwriting is started and 1602 * stylus handwriting window is displayed. 1603 */ 1604 @Test 1605 public void testHandwriting_focusedCustomEditor() throws Exception { 1606 try (MockImeSession imeSession = MockImeSession.create( 1607 InstrumentationRegistry.getInstrumentation().getContext(), 1608 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1609 new ImeSettings.Builder())) { 1610 final ImeEventStream stream = imeSession.openEventStream(); 1611 1612 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 1613 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 1614 final Pair<CustomEditorView, CustomEditorView> customEditorPair = 1615 launchTestActivityWithCustomEditors(focusedMarker, unfocusedMarker); 1616 final CustomEditorView focusedCustomEditor = customEditorPair.first; 1617 1618 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 1619 notExpectEvent( 1620 stream, 1621 editorMatcher("onStartInputView", focusedMarker), 1622 NOT_EXPECT_TIMEOUT); 1623 1624 addVirtualStylusIdForTestSession(); 1625 1626 injectStylusEventToEditorAndVerify(focusedCustomEditor, stream, imeSession, 1627 focusedMarker, true /* verifyHandwritingStart */, 1628 true /* verifyHandwritingWindowShown */, 1629 false /* verifyHandwritingWindowNotShown */); 1630 1631 // Verify that stylus move events are swallowed by the handwriting initiator once 1632 // handwriting has been initiated and not dispatched to the view tree. 1633 assertThat(focusedCustomEditor.mStylusMoveEventCount) 1634 .isLessThan(NUMBER_OF_INJECTED_EVENTS); 1635 } 1636 } 1637 1638 /** 1639 * Inject stylus events on top of a handwriting initiation delegate view and verify handwriting 1640 * is started on the delegator editor and stylus handwriting window is displayed. 1641 */ 1642 @ApiTest(apis = { 1643 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 1644 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegationAsync", 1645 "android.view.inputmethod.InputMethodManager#prepareStylusHandwritingDelegation", 1646 "android.view.View#setHandwritingDelegatorCallback", 1647 "android.view.View#setIsHandwritingDelegate"}) 1648 @Test 1649 public void testHandwriting_delegate() throws Exception { 1650 try (MockImeSession imeSession = MockImeSession.create( 1651 InstrumentationRegistry.getInstrumentation().getContext(), 1652 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1653 new ImeSettings.Builder())) { 1654 final ImeEventStream stream = imeSession.openEventStream(); 1655 1656 final String editTextMarker = getTestMarker(); 1657 final View delegateView = 1658 launchTestActivityWithDelegate( 1659 editTextMarker, null /* delegateLatch */, 0 /* delegateDelayMs */); 1660 expectBindInput(stream, Process.myPid(), TIMEOUT); 1661 addVirtualStylusIdForTestSession(); 1662 1663 // After injecting DOWN and MOVE events, the handwriting initiator should trigger the 1664 // delegate view's callback which creates the EditText and requests focus, which should 1665 // then initiate handwriting for the EditText. 1666 injectStylusEventToEditorAndVerify(delegateView, stream, imeSession, 1667 editTextMarker, true /* verifyHandwritingStart */, 1668 true /* verifyHandwritingWindowShown */, 1669 false /* verifyHandwritingWindowNotShown */); 1670 } 1671 } 1672 1673 /** 1674 * When the IME supports connectionless handwriting sessions, inject stylus events on top of a 1675 * handwriting initiation delegator view and verify a connectionless handwriting session is 1676 * started. When the session is finished, verify that the delegation transition os triggered 1677 * and the recognised text is committed. 1678 */ 1679 @Test 1680 @RequiresFlagsEnabled(FLAG_CONNECTIONLESS_HANDWRITING) 1681 @ApiTest(apis = { 1682 "android.view.inputmethod.InputMethodManager" 1683 + "#startConnectionlessStylusHandwritingForDelegation", 1684 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 1685 "android.view.inputmethod.InputMethodService#onStartConnectionlessStylusHandwriting", 1686 "android.view.inputmethod.InputMethodService#finishConnectionlessStylusHandwriting"}) 1687 @FlakyTest(bugId = 329267066) 1688 public void testHandwriting_delegate_connectionless() throws Exception { 1689 try (MockImeSession imeSession = MockImeSession.create( 1690 InstrumentationRegistry.getInstrumentation().getContext(), 1691 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1692 new ImeSettings.Builder().setConnectionlessHandwritingEnabled(true))) { 1693 final ImeEventStream stream = imeSession.openEventStream(); 1694 1695 final String delegateMarker = getTestMarker(); 1696 final View delegatorView = 1697 launchTestActivityWithDelegate( 1698 delegateMarker, null /* delegateLatch */, 0 /* delegateDelayMs */); 1699 expectBindInput(stream, Process.myPid(), TIMEOUT); 1700 addVirtualStylusIdForTestSession(); 1701 1702 int touchSlop = getTouchSlop(); 1703 int startX = delegatorView.getWidth() / 2; 1704 int startY = delegatorView.getHeight() / 2; 1705 int endX = startX + 2 * touchSlop; 1706 int endY = startY + 2 * touchSlop; 1707 TestUtils.injectStylusDownEvent(delegatorView, startX, startY); 1708 TestUtils.injectStylusMoveEvents(delegatorView, startX, startY, endX, endY, 5); 1709 1710 try { 1711 expectEvent( 1712 stream, 1713 eventMatcher("onPrepareStylusHandwriting"), 1714 TIMEOUT); 1715 expectEvent( 1716 stream, 1717 eventMatcher("onStartConnectionlessStylusHandwriting"), 1718 TIMEOUT); 1719 verifyStylusHandwritingWindowIsShown(stream, imeSession); 1720 // The transition to show the real edit text shouldn't occur yet. 1721 notExpectEvent( 1722 stream, editorMatcher("onStartInput", delegateMarker), NOT_EXPECT_TIMEOUT); 1723 } finally { 1724 TestUtils.injectStylusUpEvent(delegatorView, endX, endY); 1725 } 1726 imeSession.callFinishConnectionlessStylusHandwriting("abc"); 1727 1728 // Finishing the handwriting session triggers the transition to show the real edit text. 1729 expectEvent( 1730 stream, 1731 eventMatcher("onFinishStylusHandwriting"), 1732 TIMEOUT); 1733 expectEvent(stream, editorMatcher("onStartInput", delegateMarker), TIMEOUT); 1734 // When the real edit text start its input connection, the recognised text from the 1735 // connectionless handwriting session is committed. 1736 EditText delegate = 1737 ((View) delegatorView.getParent()).findViewById(R.id.handwriting_delegate); 1738 TestUtils.waitOnMainUntil(() -> delegate.getText().toString().equals("abc"), 1739 TIMEOUT_IN_SECONDS, "Delegate should receive text"); 1740 } 1741 } 1742 1743 /** 1744 * When the IME supports connectionless handwriting sessions, start a connectionless handwriting 1745 * session for delegation. When the session is finished and a delegate editor view is focused, 1746 * verify that the recognised text is committed to the delegate. 1747 */ 1748 @Test 1749 @ApiTest(apis = { 1750 "android.view.inputmethod.InputMethodManager" 1751 + "#startConnectionlessStylusHandwritingForDelegation", 1752 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 1753 "android.view.inputmethod.InputMethodService#onStartConnectionlessStylusHandwriting", 1754 "android.view.inputmethod.InputMethodService#finishConnectionlessStylusHandwriting"}) 1755 @FlakyTest(bugId = 328765068) 1756 public void testHandwriting_delegate_connectionless_direct() throws Exception { 1757 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 1758 try (MockImeSession imeSession = MockImeSession.create( 1759 InstrumentationRegistry.getInstrumentation().getContext(), 1760 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1761 new ImeSettings.Builder().setConnectionlessHandwritingEnabled(true))) { 1762 final ImeEventStream stream = imeSession.openEventStream(); 1763 1764 final String delegateMarker = getTestMarker(); 1765 final View view = 1766 launchTestActivityWithDelegate( 1767 delegateMarker, null /* delegateLatch */, 0 /* delegateDelayMs */); 1768 expectBindInput(stream, Process.myPid(), TIMEOUT); 1769 addVirtualStylusIdForTestSession(); 1770 1771 TestUtils.injectStylusDownEvent(view, 0, 0); 1772 CursorAnchorInfo cursorAnchorInfo = new CursorAnchorInfo.Builder().build(); 1773 TestCallback callback = new TestCallback(); 1774 imm.startConnectionlessStylusHandwritingForDelegation( 1775 view, cursorAnchorInfo, view::post, callback); 1776 1777 expectEvent( 1778 stream, 1779 eventMatcher("onPrepareStylusHandwriting"), 1780 TIMEOUT); 1781 expectEvent( 1782 stream, 1783 eventMatcher("onStartConnectionlessStylusHandwriting"), 1784 TIMEOUT); 1785 verifyStylusHandwritingWindowIsShown(stream, imeSession); 1786 1787 TestUtils.injectStylusUpEvent(view, 0, 0); 1788 imeSession.callFinishConnectionlessStylusHandwriting("abc"); 1789 1790 expectEvent( 1791 stream, 1792 eventMatcher("onFinishStylusHandwriting"), 1793 TIMEOUT); 1794 1795 view.post(() -> view.getHandwritingDelegatorCallback().run()); 1796 1797 expectEvent(stream, editorMatcher("onStartInput", delegateMarker), TIMEOUT); 1798 // When the real edit text start its input connection, the recognised text from the 1799 // connectionless handwriting session is committed. 1800 EditText delegate = 1801 ((View) view.getParent()).findViewById(R.id.handwriting_delegate); 1802 TestUtils.waitOnMainUntil(() -> delegate.getText().toString().equals("abc"), 1803 TIMEOUT_IN_SECONDS, "Delegate should receive text"); 1804 } 1805 } 1806 1807 /** 1808 * Inject stylus events on top of a handwriting initiation delegator view and verify handwriting 1809 * is started on the delegate editor, even though delegate took a little time to 1810 * acceptStylusHandwriting(). 1811 */ 1812 @ApiTest(apis = { 1813 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 1814 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegationAsync", 1815 "android.view.inputmethod.InputMethodManager#prepareStylusHandwritingDelegation", 1816 "android.view.View#setHandwritingDelegatorCallback", 1817 "android.view.View#setIsHandwritingDelegate"}) 1818 @Test 1819 public void testHandwriting_delegateDelayed() throws Exception { 1820 try (MockImeSession imeSession = MockImeSession.create( 1821 InstrumentationRegistry.getInstrumentation().getContext(), 1822 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1823 new ImeSettings.Builder())) { 1824 final ImeEventStream stream = imeSession.openEventStream(); 1825 1826 final String editTextMarker = getTestMarker(); 1827 final CountDownLatch latch = new CountDownLatch(1); 1828 // Use a delegate that executes after 1 second delay. 1829 final View delegatorView = 1830 launchTestActivityWithDelegate(editTextMarker, latch, TIMEOUT_1_S); 1831 expectBindInput(stream, Process.myPid(), TIMEOUT); 1832 addVirtualStylusIdForTestSession(); 1833 1834 final int touchSlop = getTouchSlop(); 1835 final int startX = delegatorView.getWidth() / 2; 1836 final int startY = delegatorView.getHeight() / 2; 1837 final int endX = startX + 2 * touchSlop; 1838 final int endY = startY + 2 * touchSlop; 1839 final int number = 5; 1840 TestUtils.injectStylusDownEvent(delegatorView, startX, startY); 1841 TestUtils.injectStylusMoveEvents(delegatorView, startX, startY, endX, endY, number); 1842 try { 1843 // Wait until delegate makes request. 1844 latch.await(DELEGATION_AFTER_IDLE_TIMEOUT_MS, TimeUnit.MILLISECONDS); 1845 // Keyboard shouldn't show up. 1846 notExpectEvent( 1847 stream, editorMatcher("onStartInputView", editTextMarker), 1848 NOT_EXPECT_TIMEOUT); 1849 // Handwriting should start since delegation was delayed (but still before timeout). 1850 expectEvent( 1851 stream, editorMatcher("onStartStylusHandwriting", editTextMarker), TIMEOUT); 1852 verifyStylusHandwritingWindowIsShown(stream, imeSession); 1853 } finally { 1854 TestUtils.injectStylusUpEvent(delegatorView, endX, endY); 1855 } 1856 } 1857 } 1858 1859 /** 1860 * Inject stylus events on top of a handwriting initiation delegator view and verify handwriting 1861 * is not started on the delegate editor after delegate idle-timeout. 1862 */ 1863 @ApiTest(apis = { 1864 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 1865 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegationAsync", 1866 "android.view.inputmethod.InputMethodManager#prepareStylusHandwritingDelegation", 1867 "android.view.View#setHandwritingDelegatorCallback", 1868 "android.view.View#setIsHandwritingDelegate"}) 1869 @Test 1870 public void testHandwriting_delegateAfterTimeout() throws Exception { 1871 try (MockImeSession imeSession = MockImeSession.create( 1872 InstrumentationRegistry.getInstrumentation().getContext(), 1873 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 1874 new ImeSettings.Builder())) { 1875 final ImeEventStream stream = imeSession.openEventStream(); 1876 1877 final String editTextMarker = getTestMarker(); 1878 final CountDownLatch latch = new CountDownLatch(1); 1879 // Use a delegate that executes after idle-timeout. 1880 final View delegatorView = 1881 launchTestActivityWithDelegate( 1882 editTextMarker, latch, DELEGATION_AFTER_IDLE_TIMEOUT_MS); 1883 expectBindInput(stream, Process.myPid(), TIMEOUT); 1884 addVirtualStylusIdForTestSession(); 1885 1886 final int touchSlop = getTouchSlop(); 1887 final int startX = delegatorView.getWidth() / 2; 1888 final int startY = delegatorView.getHeight() / 2; 1889 final int endX = startX + 2 * touchSlop; 1890 final int endY = startY + 2 * touchSlop; 1891 final int number = 5; 1892 TestUtils.injectStylusDownEvent(delegatorView, startX, startY); 1893 TestUtils.injectStylusMoveEvents(delegatorView, startX, startY, endX, endY, number); 1894 try { 1895 // Wait until delegate makes request. 1896 latch.await(DELEGATION_AFTER_IDLE_TIMEOUT_MS, TimeUnit.MILLISECONDS); 1897 // Keyboard shouldn't show up. 1898 notExpectEvent( 1899 stream, editorMatcher("onStartInputView", editTextMarker), 1900 NOT_EXPECT_TIMEOUT); 1901 // Handwriting should *not* start since delegation was idle timed-out. 1902 notExpectEvent( 1903 stream, editorMatcher("onStartStylusHandwriting", editTextMarker), TIMEOUT); 1904 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 1905 } finally { 1906 TestUtils.injectStylusUpEvent(delegatorView, endX, endY); 1907 } 1908 } 1909 } 1910 1911 /** 1912 * Tap on a view with stylus to launch a new activity with Editor. The editor's 1913 * editor ToolType should match stylus. 1914 */ 1915 @Test 1916 public void testHandwriting_editorToolTypeOnNewWindow() throws Exception { 1917 assumeTrue(Flags.useHandwritingListenerForTooltype()); 1918 Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); 1919 UinputTouchDevice stylus = null; 1920 try (MockImeSession imeSession = MockImeSession.create( 1921 instrumentation.getContext(), 1922 instrumentation.getUiAutomation(), 1923 new ImeSettings.Builder())) { 1924 final ImeEventStream stream = imeSession.openEventStream(); 1925 1926 final String editTextMarker = getTestMarker(); 1927 final CountDownLatch latch = new CountDownLatch(1); 1928 1929 // Use a clickable view that launches activity and focuses an editor. 1930 final AtomicReference<View> clickableViewRef = new AtomicReference<>(); 1931 final AtomicReference<View> editorViewRef = new AtomicReference<>(); 1932 TestActivity.startSync(activity -> { 1933 final LinearLayout layout = new LinearLayout(activity); 1934 final View clickableView = new View(activity); 1935 clickableViewRef.set(clickableView); 1936 clickableView.setBackgroundColor(Color.GREEN); 1937 clickableView.setOnClickListener(v -> { 1938 final EditText editText = new EditText(activity); 1939 editText.setPrivateImeOptions(editTextMarker); 1940 editText.setHint("editText"); 1941 layout.addView(editText); 1942 editorViewRef.set(editText); 1943 editText.requestFocus(); 1944 latch.countDown(); 1945 }); 1946 1947 LinearLayout.LayoutParams layoutParams = 1948 new LinearLayout.LayoutParams( 1949 LinearLayout.LayoutParams.WRAP_CONTENT, 1950 LinearLayout.LayoutParams.WRAP_CONTENT); 1951 layout.addView(clickableView, layoutParams); 1952 return layout; 1953 }); 1954 addVirtualStylusIdForTestSession(); 1955 View clickableView = clickableViewRef.get(); 1956 expectBindInput(stream, Process.myPid(), TIMEOUT); 1957 // click on view with stylus to launch new activity 1958 stylus = new UinputStylus(instrumentation, clickableView.getDisplay()); 1959 final int x = clickableView.getWidth() / 2; 1960 final int y = clickableView.getHeight() / 2; 1961 TestUtils.injectStylusDownEvent(stylus, clickableView, x, y); 1962 TestUtils.injectStylusUpEvent(stylus); 1963 // Wait until editor on next activity has focus. 1964 latch.await(TIMEOUT_1_S, TimeUnit.MILLISECONDS); 1965 1966 // call showSoftInput and make sure onUpdateToolType is stylus. 1967 final InputMethodManager imm = 1968 mContext.getSystemService(InputMethodManager.class); 1969 imm.showSoftInput(editorViewRef.get(), 0); 1970 // verify editor on new activity has editorToolType as stylus. 1971 expectEvent( 1972 stream, 1973 onUpdateEditorToolTypeMatcher(MotionEvent.TOOL_TYPE_STYLUS), 1974 TIMEOUT); 1975 } finally { 1976 if (stylus != null) { 1977 stylus.close(); 1978 } 1979 } 1980 } 1981 1982 /** 1983 * Inject stylus events on top of a handwriting initiation delegate view and verify handwriting 1984 * is started on the delegator editor [in different package] and stylus handwriting is 1985 * started. 1986 * TODO(b/210039666): support instant apps for this test. 1987 */ 1988 @AppModeFull(reason = "Launching external activity from this test is not yet supported.") 1989 @ApiTest(apis = { 1990 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 1991 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegationAsync", 1992 "android.view.inputmethod.InputMethodManager#prepareStylusHandwritingDelegation", 1993 "android.view.View#setAllowedHandwritingDelegatePackage", 1994 "android.view.View#setAllowedHandwritingDelegatorPackage", 1995 "android.view.View#setHandwritingDelegatorCallback", 1996 "android.view.View#setIsHandwritingDelegate"}) 1997 @Test 1998 public void testHandwriting_delegateToDifferentPackage() throws Exception { 1999 testHandwriting_delegateToDifferentPackage(true /* setAllowedDelegatorPackage */); 2000 } 2001 2002 /** 2003 * Inject stylus events on top of a handwriting initiation delegate view and verify handwriting 2004 * is not started on the delegator editor [in different package] because allowed package wasn't 2005 * set. 2006 * TODO(b/210039666): support instant apps for this test. 2007 */ 2008 @AppModeFull(reason = "Launching external activity from this test is not yet supported.") 2009 @ApiTest(apis = { 2010 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 2011 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegationAsync", 2012 "android.view.inputmethod.InputMethodManager#prepareStylusHandwritingDelegation", 2013 "android.view.View#setAllowedHandwritingDelegatePackage", 2014 "android.view.View#setAllowedHandwritingDelegatorPackage", 2015 "android.view.View#setHandwritingDelegatorCallback", 2016 "android.view.View#setIsHandwritingDelegate"}) 2017 @Test 2018 public void testHandwriting_delegateToDifferentPackage_fail() throws Exception { 2019 testHandwriting_delegateToDifferentPackage(false /* setAllowedDelegatorPackage */); 2020 } 2021 2022 private void testHandwriting_delegateToDifferentPackage(boolean setAllowedDelegatorPackage) 2023 throws Exception { 2024 try (MockImeSession imeSession = MockImeSession.create( 2025 InstrumentationRegistry.getInstrumentation().getContext(), 2026 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2027 new ImeSettings.Builder())) { 2028 final ImeEventStream stream = imeSession.openEventStream(); 2029 2030 final String editTextMarker = getTestMarker(); 2031 final View delegateView = 2032 launchTestActivityInExternalPackage(editTextMarker, setAllowedDelegatorPackage); 2033 expectBindInput(stream, Process.myPid(), TIMEOUT); 2034 addVirtualStylusIdForTestSession(); 2035 2036 final int touchSlop = getTouchSlop(); 2037 final int startX = delegateView.getWidth() / 2; 2038 final int startY = delegateView.getHeight() / 2; 2039 final int endX = startX + 2 * touchSlop; 2040 final int endY = startY + 2 * touchSlop; 2041 final int number = 5; 2042 2043 TestUtils.injectStylusDownEvent(delegateView, startX, startY); 2044 TestUtils.injectStylusMoveEvents(delegateView, startX, startY, endX, endY, number); 2045 2046 try { 2047 // Keyboard shouldn't show up. 2048 notExpectEvent( 2049 stream, editorMatcher("onStartInputView", editTextMarker), 2050 NOT_EXPECT_TIMEOUT); 2051 2052 if (setAllowedDelegatorPackage) { 2053 if (initiationWithoutInputConnection()) { 2054 // There will be no active InputConnection when handwriting starts 2055 expectEvent( 2056 stream, 2057 eventMatcher("onStartStylusHandwriting"), 2058 TIMEOUT); 2059 } else { 2060 expectEvent( 2061 stream, editorMatcher("onStartStylusHandwriting", editTextMarker), 2062 TIMEOUT); 2063 } 2064 verifyStylusHandwritingWindowIsShown(stream, imeSession); 2065 } else { 2066 if (initiationWithoutInputConnection()) { 2067 // There will be no active InputConnection if handwriting starts 2068 notExpectEvent( 2069 stream, 2070 eventMatcher("onStartStylusHandwriting"), 2071 NOT_EXPECT_TIMEOUT); 2072 } else { 2073 notExpectEvent( 2074 stream, editorMatcher("onStartStylusHandwriting", editTextMarker), 2075 NOT_EXPECT_TIMEOUT); 2076 } 2077 } 2078 } finally { 2079 TestUtils.injectStylusUpEvent(delegateView, endX, endY); 2080 } 2081 } 2082 } 2083 2084 /** 2085 * Inject stylus events on top of a handwriting initiation delegator view in the default 2086 * launcher activity, and verify stylus handwriting is started on the delegate editor (in a 2087 * different package]. 2088 * TODO(b/210039666): Support instant apps for this test. 2089 */ 2090 @Test 2091 @ApiTest(apis = { 2092 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 2093 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegationAsync", 2094 "android.view.inputmethod.InputMethodManager#prepareStylusHandwritingDelegation", 2095 "android.view.View#setAllowedHandwritingDelegatePackage", 2096 "android.view.View#setAllowedHandwritingDelegatorPackage", 2097 "android.view.View#setHandwritingDelegateFlags", 2098 "android.view.View#setHandwritingDelegatorCallback", 2099 "android.view.View#setIsHandwritingDelegate"}) 2100 @RequiresFlagsEnabled(FLAG_HOME_SCREEN_HANDWRITING_DELEGATOR) 2101 @AppModeFull(reason = "Launching external activity from this test is not yet supported.") 2102 public void testHandwriting_delegateFromHomePackage() throws Exception { 2103 testHandwriting_delegateFromHomePackage(/* setHomeDelegatorAllowed= */ true); 2104 } 2105 2106 /** 2107 * Inject stylus events on top of a handwriting initiation delegator view in the default 2108 * launcher activity, and verify stylus handwriting is not started on the delegate editor (in a 2109 * different package] because {@link View#setHomeScreenHandwritingDelegatorAllowed} wasn't set. 2110 * TODO(b/210039666): Support instant apps for this test. 2111 */ 2112 @Test 2113 @ApiTest(apis = { 2114 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegation", 2115 "android.view.inputmethod.InputMethodManager#acceptStylusHandwritingDelegationAsync", 2116 "android.view.inputmethod.InputMethodManager#prepareStylusHandwritingDelegation", 2117 "android.view.View#setAllowedHandwritingDelegatePackage", 2118 "android.view.View#setAllowedHandwritingDelegatorPackage", 2119 "android.view.View#setHandwritingDelegateFlags", 2120 "android.view.View#setHandwritingDelegatorCallback", 2121 "android.view.View#setIsHandwritingDelegate"}) 2122 @RequiresFlagsEnabled(FLAG_HOME_SCREEN_HANDWRITING_DELEGATOR) 2123 @AppModeFull(reason = "Launching external activity from this test is not yet supported.") 2124 public void testHandwriting_delegateFromHomePackage_fail() throws Exception { 2125 testHandwriting_delegateFromHomePackage(/* setHomeDelegatorAllowed= */ false); 2126 } 2127 2128 public void testHandwriting_delegateFromHomePackage(boolean setHomeDelegatorAllowed) 2129 throws Exception { 2130 mDefaultLauncherToRestore = getDefaultLauncher(); 2131 setDefaultLauncher(TEST_LAUNCHER_COMPONENT); 2132 2133 try (MockImeSession imeSession = MockImeSession.create( 2134 InstrumentationRegistry.getInstrumentation().getContext(), 2135 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2136 new ImeSettings.Builder())) { 2137 ImeEventStream stream = imeSession.openEventStream(); 2138 2139 String editTextMarker = getTestMarker(); 2140 2141 // Start launcher activity 2142 Intent intent = new Intent(Intent.ACTION_MAIN); 2143 intent.addCategory(Intent.CATEGORY_HOME); 2144 intent.addCategory(Intent.CATEGORY_DEFAULT); 2145 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); 2146 // LauncherActivity passes these three extras to the ctstestapp MainActivity 2147 intent.putExtra(MockTestActivityUtil.EXTRA_KEY_PRIVATE_IME_OPTIONS, editTextMarker); 2148 intent.putExtra(MockTestActivityUtil.EXTRA_HANDWRITING_DELEGATE, true); 2149 intent.putExtra( 2150 MockTestActivityUtil.EXTRA_HOME_HANDWRITING_DELEGATOR_ALLOWED, 2151 setHomeDelegatorAllowed); 2152 InstrumentationRegistry.getInstrumentation().getContext().startActivity(intent); 2153 2154 expectBindInput(stream, Process.myPid(), TIMEOUT); 2155 addVirtualStylusIdForTestSession(); 2156 2157 // Launcher activity displays a full screen handwriting delegator view. Stylus events 2158 // are injected in the center of the screen to trigger the delegator callback, which 2159 // launches the ctstestapp MainActivity with the delegate editor with editTextMarker. 2160 DisplayMetrics metrics = mContext.getResources().getDisplayMetrics(); 2161 int touchSlop = getTouchSlop(); 2162 int startX = metrics.widthPixels / 2; 2163 int startY = metrics.heightPixels / 2; 2164 int endX = startX + 2 * touchSlop; 2165 int endY = startY + 2 * touchSlop; 2166 View mockView = mock(View.class); 2167 TestUtils.injectStylusDownEvent(mockView, startX, startY); 2168 TestUtils.injectStylusMoveEvents(mockView, startX, startY, endX, endY, 5); 2169 2170 try { 2171 // Keyboard shouldn't show up. 2172 notExpectEvent( 2173 stream, editorMatcher("onStartInputView", editTextMarker), 2174 NOT_EXPECT_TIMEOUT); 2175 if (setHomeDelegatorAllowed) { 2176 if (initiationWithoutInputConnection()) { 2177 // There will be no active InputConnection when handwriting starts. 2178 expectEvent( 2179 stream, 2180 eventMatcher("onStartStylusHandwriting"), 2181 TIMEOUT); 2182 } else { 2183 expectEvent( 2184 stream, editorMatcher("onStartStylusHandwriting", editTextMarker), 2185 TIMEOUT); 2186 } 2187 verifyStylusHandwritingWindowIsShown(stream, imeSession); 2188 } else { 2189 if (initiationWithoutInputConnection()) { 2190 // There will be no active InputConnection if handwriting starts. 2191 notExpectEvent( 2192 stream, 2193 eventMatcher("onStartStylusHandwriting"), 2194 NOT_EXPECT_TIMEOUT); 2195 } else { 2196 notExpectEvent( 2197 stream, editorMatcher("onStartStylusHandwriting", editTextMarker), 2198 NOT_EXPECT_TIMEOUT); 2199 } 2200 } 2201 } finally { 2202 TestUtils.injectStylusUpEvent(mockView, endX, endY); 2203 } 2204 } 2205 } 2206 2207 @Test 2208 @RequiresFlagsEnabled(FLAG_CONNECTIONLESS_HANDWRITING) 2209 @ApiTest(apis = { 2210 "android.view.inputmethod.InputMethodManager#startConnectionlessStylusHandwriting", 2211 "android.view.inputmethod.InputMethodManager#onStartConnectionlessStylusHandwriting", 2212 "android.view.inputmethod.InputMethodManager#finishConnectionlessStylusHandwriting"}) 2213 public void testHandwriting_connectionless_standalone() throws Exception { 2214 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 2215 try (MockImeSession imeSession = MockImeSession.create( 2216 InstrumentationRegistry.getInstrumentation().getContext(), 2217 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2218 new ImeSettings.Builder().setConnectionlessHandwritingEnabled(true))) { 2219 final ImeEventStream stream = imeSession.openEventStream(); 2220 2221 final View view = 2222 launchTestActivityWithDelegate( 2223 getTestMarker(), null /* delegateLatch */, 0 /* delegateDelayMs */); 2224 expectBindInput(stream, Process.myPid(), TIMEOUT); 2225 addVirtualStylusIdForTestSession(); 2226 2227 TestUtils.injectStylusDownEvent(view, 0, 0); 2228 try { 2229 CursorAnchorInfo cursorAnchorInfo = new CursorAnchorInfo.Builder().build(); 2230 TestCallback callback = new TestCallback(); 2231 imm.startConnectionlessStylusHandwriting(view, cursorAnchorInfo, view::post, 2232 callback); 2233 2234 expectEvent( 2235 stream, 2236 eventMatcher("onPrepareStylusHandwriting"), 2237 TIMEOUT); 2238 expectEvent( 2239 stream, 2240 eventMatcher("onStartConnectionlessStylusHandwriting"), 2241 TIMEOUT); 2242 verifyStylusHandwritingWindowIsShown(stream, imeSession); 2243 2244 imeSession.callFinishConnectionlessStylusHandwriting("abc"); 2245 2246 expectEvent( 2247 stream, 2248 eventMatcher("onFinishStylusHandwriting"), 2249 TIMEOUT); 2250 assertThat(callback.mResultText).isEqualTo("abc"); 2251 assertThat(callback.mErrorCode).isEqualTo(-1); 2252 } finally { 2253 TestUtils.injectStylusUpEvent(view, 0, 0); 2254 } 2255 } 2256 } 2257 2258 @Test 2259 @RequiresFlagsEnabled(FLAG_CONNECTIONLESS_HANDWRITING) 2260 @ApiTest(apis = { 2261 "android.view.inputmethod.InputMethodManager#startConnectionlessStylusHandwriting", 2262 "android.view.inputmethod.InputMethodManager#onStartConnectionlessStylusHandwriting", 2263 "android.view.inputmethod.InputMethodManager#finishConnectionlessStylusHandwriting"}) 2264 public void testHandwriting_connectionless_standalone_error() throws Exception { 2265 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 2266 try (MockImeSession imeSession = MockImeSession.create( 2267 InstrumentationRegistry.getInstrumentation().getContext(), 2268 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2269 new ImeSettings.Builder().setConnectionlessHandwritingEnabled(true))) { 2270 final ImeEventStream stream = imeSession.openEventStream(); 2271 2272 final View view = 2273 launchTestActivityWithDelegate( 2274 getTestMarker(), null /* delegateLatch */, 0 /* delegateDelayMs */); 2275 expectBindInput(stream, Process.myPid(), TIMEOUT); 2276 addVirtualStylusIdForTestSession(); 2277 2278 TestUtils.injectStylusDownEvent(view, 0, 0); 2279 try { 2280 CursorAnchorInfo cursorAnchorInfo = new CursorAnchorInfo.Builder().build(); 2281 TestCallback callback = new TestCallback(); 2282 imm.startConnectionlessStylusHandwriting(view, cursorAnchorInfo, view::post, 2283 callback); 2284 2285 expectEvent( 2286 stream, 2287 eventMatcher("onPrepareStylusHandwriting"), 2288 TIMEOUT); 2289 expectEvent( 2290 stream, 2291 eventMatcher("onStartConnectionlessStylusHandwriting"), 2292 TIMEOUT); 2293 verifyStylusHandwritingWindowIsShown(stream, imeSession); 2294 2295 // Finish the session with no text recognized. 2296 imeSession.callFinishConnectionlessStylusHandwriting(""); 2297 2298 expectEvent( 2299 stream, 2300 eventMatcher("onFinishStylusHandwriting"), 2301 TIMEOUT); 2302 assertThat(callback.mResultText).isNull(); 2303 assertThat(callback.mErrorCode) 2304 .isEqualTo(CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED); 2305 } finally { 2306 TestUtils.injectStylusUpEvent(view, 0, 0); 2307 } 2308 } 2309 } 2310 2311 @Test 2312 @RequiresFlagsEnabled(FLAG_CONNECTIONLESS_HANDWRITING) 2313 @ApiTest(apis = { 2314 "android.view.inputmethod.InputMethodManager#startConnectionlessStylusHandwriting", 2315 "android.view.inputmethod.InputMethodService#onStartConnectionlessStylusHandwriting", 2316 "android.view.inputmethod.InputMethodManager#finishConnectionlessStylusHandwriting"}) 2317 public void testHandwriting_connectionless_standalone_unsupported() throws Exception { 2318 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 2319 try (MockImeSession imeSession = MockImeSession.create( 2320 InstrumentationRegistry.getInstrumentation().getContext(), 2321 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2322 new ImeSettings.Builder().setConnectionlessHandwritingEnabled(false))) { 2323 final ImeEventStream stream = imeSession.openEventStream(); 2324 2325 final View view = 2326 launchTestActivityWithDelegate( 2327 getTestMarker(), null /* delegateLatch */, 0 /* delegateDelayMs */); 2328 expectBindInput(stream, Process.myPid(), TIMEOUT); 2329 addVirtualStylusIdForTestSession(); 2330 2331 TestUtils.injectStylusDownEvent(view, 0, 0); 2332 try { 2333 CursorAnchorInfo cursorAnchorInfo = new CursorAnchorInfo.Builder().build(); 2334 TestCallback callback = new TestCallback(); 2335 imm.startConnectionlessStylusHandwriting(view, cursorAnchorInfo, view::post, 2336 callback); 2337 2338 // onPrepareStylusHandwriting and onStartConnectionlessStylusHandwriting are called, 2339 // but onStartConnectionlessStylusHandwriting returns false so handwriting 2340 // does not start. 2341 expectEvent( 2342 stream, 2343 eventMatcher("onPrepareStylusHandwriting"), 2344 TIMEOUT); 2345 expectEvent( 2346 stream, 2347 eventMatcher("onStartConnectionlessStylusHandwriting"), 2348 TIMEOUT); 2349 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 2350 assertThat(callback.mResultText).isNull(); 2351 assertThat(callback.mErrorCode) 2352 .isEqualTo(CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED); 2353 } finally { 2354 TestUtils.injectStylusUpEvent(view, 0, 0); 2355 } 2356 } 2357 } 2358 2359 /** 2360 * Verify that system times-out Handwriting session after given timeout. 2361 */ 2362 @Test 2363 public void testHandwritingSessionIdleTimeout() throws Exception { 2364 try (MockImeSession imeSession = MockImeSession.create( 2365 InstrumentationRegistry.getInstrumentation().getContext(), 2366 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2367 new ImeSettings.Builder())) { 2368 final ImeEventStream stream = imeSession.openEventStream(); 2369 2370 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 2371 final EditText editText = launchTestActivity(marker); 2372 2373 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 2374 notExpectEvent( 2375 stream, 2376 editorMatcher("onStartInputView", marker), 2377 NOT_EXPECT_TIMEOUT); 2378 2379 addVirtualStylusIdForTestSession(); 2380 // update handwriting session timeout 2381 assertTrue(expectCommand( 2382 stream, 2383 imeSession.callSetStylusHandwritingTimeout(100 /* timeoutMs */), 2384 TIMEOUT).getReturnBooleanValue()); 2385 2386 injectStylusEventToEditorAndVerify(editText, stream, imeSession, 2387 marker, true /* verifyHandwritingStart */, 2388 false /* verifyHandwritingWindowShown */, 2389 false /* verifyHandwritingWindowNotShown */); 2390 2391 // Handwriting should finish soon. 2392 expectEvent( 2393 stream, 2394 editorMatcher("onFinishStylusHandwriting", marker), 2395 TIMEOUT_1_S); 2396 2397 // test setting extremely large timeout and verify we limit it to 2398 // STYLUS_HANDWRITING_IDLE_TIMEOUT_MS 2399 assertTrue(expectCommand( 2400 stream, imeSession.callSetStylusHandwritingTimeout( 2401 InputMethodService.getStylusHandwritingIdleTimeoutMax().toMillis() 2402 * 10), 2403 TIMEOUT).getReturnBooleanValue()); 2404 assertEquals("Stylus handwriting timeout must be equal to max value.", 2405 InputMethodService.getStylusHandwritingIdleTimeoutMax().toMillis(), 2406 expectCommand( 2407 stream, imeSession.callGetStylusHandwritingTimeout(), TIMEOUT) 2408 .getReturnLongValue()); 2409 } 2410 } 2411 2412 @Test 2413 public void testHandwritingFinishesOnUnbind() throws Exception { 2414 try (MockImeSession imeSession = MockImeSession.create( 2415 InstrumentationRegistry.getInstrumentation().getContext(), 2416 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2417 new ImeSettings.Builder())) { 2418 final ImeEventStream stream = imeSession.openEventStream(); 2419 2420 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 2421 final EditText editText = launchTestActivity(marker); 2422 2423 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 2424 notExpectEvent( 2425 stream, 2426 editorMatcher("onStartInputView", marker), 2427 NOT_EXPECT_TIMEOUT); 2428 2429 addVirtualStylusIdForTestSession(); 2430 2431 final int touchSlop = getTouchSlop(); 2432 final int startX = editText.getWidth() / 2; 2433 final int startY = editText.getHeight() / 2; 2434 final int endX = startX + 2 * touchSlop; 2435 final int endY = startY; 2436 final int number = 5; 2437 TestUtils.injectStylusDownEvent(editText, startX, startY); 2438 TestUtils.injectStylusMoveEvents(editText, startX, startY, 2439 endX, endY, number); 2440 2441 try { 2442 expectEvent( 2443 stream, 2444 editorMatcher("onStartStylusHandwriting", marker), 2445 TIMEOUT); 2446 // Unbind IME and verify finish is called 2447 ((Activity) editText.getContext()).finish(); 2448 2449 // Handwriting should finish soon. 2450 expectEvent( 2451 stream, 2452 editorMatcher("onFinishStylusHandwriting", marker), 2453 TIMEOUT_1_S); 2454 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 2455 } finally { 2456 TestUtils.injectStylusUpEvent(editText, endX, endY); 2457 } 2458 } 2459 } 2460 2461 /** 2462 * Verify that system remove handwriting window immediately when timeout is small 2463 */ 2464 @Test 2465 public void testHandwritingWindowRemoval_immediate() throws Exception { 2466 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 2467 try (MockImeSession imeSession = MockImeSession.create( 2468 InstrumentationRegistry.getInstrumentation().getContext(), 2469 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2470 new ImeSettings.Builder())) { 2471 final ImeEventStream stream = imeSession.openEventStream(); 2472 2473 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 2474 final EditText editText = launchTestActivity(marker); 2475 2476 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 2477 notExpectEvent( 2478 stream, 2479 editorMatcher("onStartInputView", marker), 2480 NOT_EXPECT_TIMEOUT); 2481 2482 addVirtualStylusIdForTestSession(); 2483 // update handwriting window timeout to a small value so that it is removed immediately. 2484 SystemUtil.runWithShellPermissionIdentity(() -> 2485 imm.setStylusWindowIdleTimeoutForTest(100)); 2486 2487 final int touchSlop = getTouchSlop(); 2488 final int startX = editText.getWidth() / 2; 2489 final int startY = editText.getHeight() / 2; 2490 final int endX = startX + 2 * touchSlop; 2491 final int endY = startY; 2492 final int number = 5; 2493 TestUtils.injectStylusDownEvent(editText, startX, startY); 2494 TestUtils.injectStylusMoveEvents(editText, startX, startY, 2495 endX, endY, number); 2496 try { 2497 // Handwriting should already be initiated before ACTION_UP. 2498 // keyboard shouldn't show up. 2499 notExpectEvent( 2500 stream, 2501 editorMatcher("onStartInputView", marker), 2502 NOT_EXPECT_TIMEOUT); 2503 // Handwriting should start 2504 expectEvent( 2505 stream, 2506 editorMatcher("onStartStylusHandwriting", marker), 2507 TIMEOUT); 2508 } finally { 2509 TestUtils.injectStylusUpEvent(editText, endX, endY); 2510 } 2511 2512 // Handwriting should finish soon. 2513 expectEvent( 2514 stream, 2515 editorMatcher("onFinishStylusHandwriting", marker), 2516 TIMEOUT_1_S); 2517 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 2518 // Verify handwriting window is removed. 2519 assertFalse(expectCommand( 2520 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT_1_S) 2521 .getReturnBooleanValue()); 2522 } 2523 } 2524 2525 2526 /** 2527 * Verify that system remove handwriting window after timeout 2528 */ 2529 @Test 2530 public void testHandwritingWindowRemoval_afterDelay() throws Exception { 2531 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 2532 try (MockImeSession imeSession = MockImeSession.create( 2533 InstrumentationRegistry.getInstrumentation().getContext(), 2534 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2535 new ImeSettings.Builder())) { 2536 // skip this test if device doesn't have stylus. 2537 // stylus is required, otherwise stylus virtual deviceId is removed on finishInput and 2538 // we cannot test InkWindow living beyond finishHandwriting. 2539 assumeTrue("Skipping test on devices that don't have stylus connected.", 2540 hasSupportedStylus()); 2541 final ImeEventStream stream = imeSession.openEventStream(); 2542 2543 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 2544 final EditText editText = launchTestActivity(marker); 2545 2546 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 2547 notExpectEvent( 2548 stream, 2549 editorMatcher("onStartInputView", marker), 2550 NOT_EXPECT_TIMEOUT); 2551 2552 final int touchSlop = getTouchSlop(); 2553 final int startX = editText.getWidth() / 2; 2554 final int startY = editText.getHeight() / 2; 2555 final int endX = startX + 2 * touchSlop; 2556 final int endY = startY; 2557 final int number = 5; 2558 2559 // Set a larger timeout and verify handwriting window exists after unbind. 2560 SystemUtil.runWithShellPermissionIdentity(() -> 2561 imm.setStylusWindowIdleTimeoutForTest(TIMEOUT)); 2562 2563 TestUtils.injectStylusDownEvent(editText, startX, startY); 2564 TestUtils.injectStylusMoveEvents(editText, startX, startY, 2565 endX, endY, number); 2566 try { 2567 // Handwriting should already be initiated before ACTION_UP. 2568 // Handwriting should start 2569 expectEvent( 2570 stream, 2571 editorMatcher("onStartStylusHandwriting", marker), 2572 TIMEOUT); 2573 } finally { 2574 TestUtils.injectStylusUpEvent(editText, endX, endY); 2575 } 2576 2577 // Handwriting should finish soon. 2578 notExpectEvent( 2579 stream, 2580 editorMatcher("onFinishStylusHandwriting", marker), 2581 TIMEOUT_1_S); 2582 verifyStylusHandwritingWindowIsShown(stream, imeSession); 2583 // Verify handwriting window exists. 2584 assertTrue(expectCommand( 2585 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT_1_S) 2586 .getReturnBooleanValue()); 2587 2588 // Finish activity and IME window should be invisible. 2589 ((Activity) editText.getContext()).finish(); 2590 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 2591 // Verify handwriting window isn't removed immediately. 2592 assertTrue(expectCommand( 2593 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT_1_S) 2594 .getReturnBooleanValue()); 2595 // Verify handwriting window is eventually removed (within timeout). 2596 CommonTestUtils.waitUntil("Stylus handwriting window should be removed", 2597 TIMEOUT_IN_SECONDS, 2598 () -> !expectCommand( 2599 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT) 2600 .getReturnBooleanValue()); 2601 } 2602 } 2603 2604 /** 2605 * Verify that when system has no stylus, there is no handwriting window. 2606 */ 2607 @Test 2608 @ApiTest(apis = {"android.view.inputmethod.InputMethodManager#startStylusHandwriting", 2609 "android.inputmethodservice.InputMethodService#onStartStylusHandwriting", 2610 "android.inputmethodservice.InputMethodService#onFinishStylusHandwriting"}) 2611 public void testNoStylusNoHandwritingWindow() throws Exception { 2612 // skip this test if device already has stylus. 2613 assumeFalse("Skipping test on devices that have stylus connected.", 2614 hasSupportedStylus()); 2615 2616 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 2617 try (MockImeSession imeSession = MockImeSession.create( 2618 InstrumentationRegistry.getInstrumentation().getContext(), 2619 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2620 new ImeSettings.Builder())) { 2621 final ImeEventStream stream = imeSession.openEventStream(); 2622 2623 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 2624 final EditText editText = launchTestActivity(marker); 2625 2626 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 2627 notExpectEvent( 2628 stream, 2629 editorMatcher("onStartInputView", marker), 2630 NOT_EXPECT_TIMEOUT); 2631 2632 // Verify there is no handwriting window before stylus is added. 2633 assertFalse(expectCommand( 2634 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT_1_S) 2635 .getReturnBooleanValue()); 2636 2637 addVirtualStylusIdForTestSession(); 2638 2639 injectStylusEventToEditorAndVerify(editText, stream, imeSession, 2640 marker, true /* verifyHandwritingStart */, 2641 true /* verifyHandwritingWindowShown */, 2642 false /* verifyHandwritingWindowNotShown */); 2643 2644 // Finish handwriting to remove test stylus id. 2645 imeSession.callFinishStylusHandwriting(); 2646 expectEvent( 2647 stream, 2648 editorMatcher("onFinishStylusHandwriting", marker), 2649 TIMEOUT_1_S); 2650 2651 // Verify no handwriting window after stylus is removed from device. 2652 assertFalse(expectCommand( 2653 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT_1_S) 2654 .getReturnBooleanValue()); 2655 2656 } 2657 } 2658 2659 /** 2660 * Verifies that in split-screen multi-window mode, unfocused activity can start handwriting 2661 */ 2662 @Test 2663 @ApiTest(apis = {"android.view.inputmethod.InputMethodManager#startStylusHandwriting", 2664 "android.inputmethodservice.InputMethodService#onStartStylusHandwriting", 2665 "android.inputmethodservice.InputMethodService#onFinishStylusHandwriting"}) 2666 public void testMultiWindow_unfocusedWindowCanStartHandwriting() throws Exception { 2667 assumeTrue(TestUtils.supportsSplitScreenMultiWindow()); 2668 2669 try (MockImeSession imeSession = MockImeSession.create( 2670 InstrumentationRegistry.getInstrumentation().getContext(), 2671 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2672 new ImeSettings.Builder())) { 2673 final ImeEventStream stream = imeSession.openEventStream(); 2674 final String primaryMarker = getTestMarker(FIRST_EDIT_TEXT_TAG); 2675 final String secondaryMarker = getTestMarker(SECOND_EDIT_TEXT_TAG); 2676 2677 // Launch an editor activity to be on the split primary task. 2678 final TestActivity splitPrimaryActivity = TestActivity.startSync(activity -> { 2679 final LinearLayout layout = new LinearLayout(activity); 2680 layout.setOrientation(LinearLayout.VERTICAL); 2681 final EditText editText = new EditText(activity); 2682 layout.addView(editText); 2683 editText.setHint("focused editText"); 2684 editText.setPrivateImeOptions(primaryMarker); 2685 editText.requestFocus(); 2686 return layout; 2687 }); 2688 expectEvent(stream, editorMatcher("onStartInput", primaryMarker), TIMEOUT); 2689 notExpectEvent(stream, editorMatcher("onStartInputView", primaryMarker), 2690 NOT_EXPECT_TIMEOUT); 2691 2692 TestUtils.waitOnMainUntil(() -> splitPrimaryActivity.hasWindowFocus(), TIMEOUT); 2693 2694 // Launch another activity to be on the split secondary task, expect stylus gesture on 2695 // it can steal focus from primary and start handwriting. 2696 final AtomicReference<EditText> editTextRef = new AtomicReference<>(); 2697 final TestActivity splitSecondaryActivity = new TestActivity.Starter() 2698 .asMultipleTask() 2699 .withAdditionalFlags(Intent.FLAG_ACTIVITY_LAUNCH_ADJACENT) 2700 .startSync(splitPrimaryActivity, activity -> { 2701 final LinearLayout layout = new LinearLayout(activity); 2702 layout.setOrientation(LinearLayout.VERTICAL); 2703 final EditText editText = new EditText(activity); 2704 editTextRef.set(editText); 2705 layout.addView(editText); 2706 editText.setHint("unfocused editText"); 2707 editText.setPrivateImeOptions(secondaryMarker); 2708 return layout; 2709 }, TestActivity2.class); 2710 notExpectEvent(stream, eventMatcher("onStartInputView"), 2711 NOT_EXPECT_TIMEOUT); 2712 TestUtils.waitOnMainUntil(() -> splitSecondaryActivity.hasWindowFocus(), TIMEOUT); 2713 TestUtils.waitOnMainUntil(() -> !splitPrimaryActivity.hasWindowFocus(), TIMEOUT_1_S); 2714 2715 addVirtualStylusIdForTestSession(); 2716 2717 final EditText editText = editTextRef.get(); 2718 2719 injectStylusEventToEditorAndVerify(editText, stream, imeSession, 2720 secondaryMarker, true /* verifyHandwritingStart */, 2721 true /* verifyHandwritingWindowShown */, 2722 false /* verifyHandwritingWindowNotShown */); 2723 2724 // Finish handwriting to remove test stylus id. 2725 imeSession.callFinishStylusHandwriting(); 2726 expectEvent( 2727 stream, 2728 editorMatcher("onFinishStylusHandwriting", secondaryMarker), 2729 TIMEOUT_1_S); 2730 } 2731 } 2732 2733 /** 2734 * Verifies that in split-screen multi-window mode, an unfocused window can't steal ongoing 2735 * handwriting session. 2736 */ 2737 @Test 2738 @ApiTest(apis = {"android.view.inputmethod.InputMethodManager#startStylusHandwriting", 2739 "android.inputmethodservice.InputMethodService#onStartStylusHandwriting", 2740 "android.inputmethodservice.InputMethodService#onFinishStylusHandwriting"}) 2741 public void testMultiWindow_unfocusedWindowCannotStealOngoingHandwriting() throws Exception { 2742 assumeTrue(TestUtils.supportsSplitScreenMultiWindow()); 2743 2744 try (MockImeSession imeSession = MockImeSession.create( 2745 InstrumentationRegistry.getInstrumentation().getContext(), 2746 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2747 new ImeSettings.Builder())) { 2748 final ImeEventStream stream = imeSession.openEventStream(); 2749 final String primaryMarker = getTestMarker(FIRST_EDIT_TEXT_TAG); 2750 final String secondaryMarker = getTestMarker(SECOND_EDIT_TEXT_TAG); 2751 2752 // Launch an editor activity to be on the split primary task. 2753 final AtomicReference<EditText> editTextPrimaryRef = new AtomicReference<>(); 2754 final TestActivity splitPrimaryActivity = TestActivity.startSync(activity -> { 2755 final LinearLayout layout = new LinearLayout(activity); 2756 layout.setOrientation(LinearLayout.VERTICAL); 2757 final EditText editText = new EditText(activity); 2758 layout.addView(editText); 2759 editTextPrimaryRef.set(editText); 2760 editText.setHint("focused editText"); 2761 editText.setPrivateImeOptions(primaryMarker); 2762 return layout; 2763 }); 2764 notExpectEvent(stream, 2765 editorMatcher("onStartInput", primaryMarker), NOT_EXPECT_TIMEOUT); 2766 notExpectEvent(stream, 2767 editorMatcher("onStartInputView", primaryMarker), NOT_EXPECT_TIMEOUT); 2768 2769 TestUtils.waitOnMainUntil(() -> splitPrimaryActivity.hasWindowFocus(), TIMEOUT); 2770 2771 // Launch another activity to be on the split secondary task, expect stylus gesture on 2772 // it can steal focus from primary and start handwriting. 2773 final AtomicReference<EditText> editTextSecondaryRef = new AtomicReference<>(); 2774 new TestActivity.Starter() 2775 .asMultipleTask() 2776 .withAdditionalFlags(Intent.FLAG_ACTIVITY_LAUNCH_ADJACENT) 2777 .startSync(splitPrimaryActivity, activity -> { 2778 final LinearLayout layout = new LinearLayout(activity); 2779 layout.setOrientation(LinearLayout.VERTICAL); 2780 final EditText editText = new EditText(activity); 2781 editTextSecondaryRef.set(editText); 2782 layout.addView(editText); 2783 editText.setHint("unfocused editText"); 2784 editText.setPrivateImeOptions(secondaryMarker); 2785 return layout; 2786 }, TestActivity2.class); 2787 notExpectEvent(stream, eventMatcher("onStartInputView"), NOT_EXPECT_TIMEOUT); 2788 2789 addVirtualStylusIdForTestSession(); 2790 2791 // Inject events on primary to start handwriting. 2792 final EditText editTextPrimary = editTextPrimaryRef.get(); 2793 2794 injectStylusEventToEditorAndVerify(editTextPrimary, stream, imeSession, 2795 primaryMarker, true /* verifyHandwritingStart */, 2796 false /* verifyHandwritingWindowShown */, 2797 false /* verifyHandwritingWindowNotShown */); 2798 2799 TestUtils.waitOnMainUntil(() -> splitPrimaryActivity.hasWindowFocus(), TIMEOUT_1_S); 2800 2801 // Inject events on secondary shouldn't start handwriting on secondary 2802 // (since primary is already ongoing). 2803 final EditText editTextSecondary = editTextSecondaryRef.get(); 2804 2805 injectStylusEventToEditorAndVerify(editTextSecondary, stream, imeSession, 2806 secondaryMarker, false /* verifyHandwritingStart */, 2807 false /* verifyHandwritingWindowShown */, 2808 false /* verifyHandwritingWindowNotShown */); 2809 2810 TestUtils.waitOnMainUntil(() -> splitPrimaryActivity.hasWindowFocus(), TIMEOUT_1_S); 2811 2812 // Finish handwriting to remove test stylus id. 2813 imeSession.callFinishStylusHandwriting(); 2814 expectEvent( 2815 stream, 2816 editorMatcher("onFinishStylusHandwriting", primaryMarker), 2817 TIMEOUT_1_S); 2818 } 2819 } 2820 2821 /** 2822 * Verify that once stylus hasn't been used for more than idle-timeout, there is no handwriting 2823 * window. 2824 */ 2825 @Test 2826 @ApiTest(apis = {"android.view.inputmethod.InputMethodManager#startStylusHandwriting", 2827 "android.inputmethodservice.InputMethodService#onStartStylusHandwriting", 2828 "android.inputmethodservice.InputMethodService#onFinishStylusHandwriting"}) 2829 public void testNoHandwritingWindow_afterIdleTimeout() throws Exception { 2830 // skip this test if device doesn't have stylus. 2831 assumeTrue("Skipping test on devices that don't stylus connected.", 2832 hasSupportedStylus()); 2833 2834 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 2835 try (MockImeSession imeSession = MockImeSession.create( 2836 InstrumentationRegistry.getInstrumentation().getContext(), 2837 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2838 new ImeSettings.Builder())) { 2839 final ImeEventStream stream = imeSession.openEventStream(); 2840 2841 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 2842 final EditText editText = launchTestActivity(marker); 2843 2844 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 2845 notExpectEvent( 2846 stream, 2847 editorMatcher("onStartInputView", marker), 2848 NOT_EXPECT_TIMEOUT); 2849 2850 SystemUtil.runWithShellPermissionIdentity(() -> 2851 imm.setStylusWindowIdleTimeoutForTest(TIMEOUT)); 2852 2853 injectStylusEventToEditorAndVerify(editText, stream, imeSession, 2854 marker, true /* verifyHandwritingStart */, 2855 true /* verifyHandwritingWindowShown */, 2856 false /* verifyHandwritingWindowNotShown */); 2857 2858 // Finish handwriting to remove test stylus id. 2859 imeSession.callFinishStylusHandwriting(); 2860 expectEvent( 2861 stream, 2862 editorMatcher("onFinishStylusHandwriting", marker), 2863 TIMEOUT_1_S); 2864 2865 // Verify handwriting window is removed after stylus handwriting idle-timeout. 2866 TestUtils.waitOnMainUntil(() -> { 2867 try { 2868 // wait until callHasStylusHandwritingWindow returns false 2869 return !expectCommand(stream, imeSession.callHasStylusHandwritingWindow(), 2870 TIMEOUT).getReturnBooleanValue(); 2871 } catch (TimeoutException e) { 2872 e.printStackTrace(); 2873 } 2874 // handwriting window is still around. 2875 return true; 2876 }, TIMEOUT); 2877 2878 // reset idle-timeout 2879 SystemUtil.runWithShellPermissionIdentity(() -> 2880 imm.setStylusWindowIdleTimeoutForTest(0)); 2881 } 2882 } 2883 2884 /** 2885 * Verify that Ink window is around before timeout 2886 */ 2887 @Test 2888 @ApiTest(apis = {"android.view.inputmethod.InputMethodManager#startStylusHandwriting", 2889 "android.inputmethodservice.InputMethodService#onStartStylusHandwriting", 2890 "android.inputmethodservice.InputMethodService#onFinishStylusHandwriting"}) 2891 public void testHandwritingWindow_beforeTimeout() throws Exception { 2892 // skip this test if device doesn't have stylus. 2893 assumeTrue("Skipping test on devices that don't stylus connected.", 2894 hasSupportedStylus()); 2895 2896 final InputMethodManager imm = mContext.getSystemService(InputMethodManager.class); 2897 try (MockImeSession imeSession = MockImeSession.create( 2898 InstrumentationRegistry.getInstrumentation().getContext(), 2899 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2900 new ImeSettings.Builder())) { 2901 final ImeEventStream stream = imeSession.openEventStream(); 2902 2903 final String marker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 2904 final EditText editText = launchTestActivity(marker); 2905 2906 expectEvent(stream, editorMatcher("onStartInput", marker), TIMEOUT); 2907 notExpectEvent( 2908 stream, 2909 editorMatcher("onStartInputView", marker), 2910 NOT_EXPECT_TIMEOUT); 2911 2912 SystemUtil.runWithShellPermissionIdentity(() -> 2913 imm.setStylusWindowIdleTimeoutForTest(TIMEOUT)); 2914 2915 injectStylusEventToEditorAndVerify(editText, stream, imeSession, 2916 marker, true /* verifyHandwritingStart */, 2917 true /* verifyHandwritingWindowShown */, 2918 false /* verifyHandwritingWindowNotShown */); 2919 2920 // Finish handwriting to remove test stylus id. 2921 imeSession.callFinishStylusHandwriting(); 2922 expectEvent( 2923 stream, 2924 editorMatcher("onFinishStylusHandwriting", marker), 2925 TIMEOUT_1_S); 2926 2927 // Just any stylus events to delay idle-timeout 2928 TestUtils.injectStylusDownEvent(editText, 0, 0); 2929 TestUtils.injectStylusUpEvent(editText, 0, 0); 2930 2931 // Verify handwriting window is still around as stylus was used recently. 2932 assertTrue(expectCommand( 2933 stream, imeSession.callHasStylusHandwritingWindow(), TIMEOUT_1_S) 2934 .getReturnBooleanValue()); 2935 2936 // Reset idle-timeout 2937 SystemUtil.runWithShellPermissionIdentity(() -> 2938 imm.setStylusWindowIdleTimeoutForTest(0)); 2939 } 2940 } 2941 2942 /** 2943 * Inject stylus events on top of an unfocused custom editor and verify handwriting is started 2944 * and stylus handwriting window is displayed. 2945 */ 2946 @Test 2947 public void testHandwriting_unfocusedCustomEditor() throws Exception { 2948 try (MockImeSession imeSession = MockImeSession.create( 2949 InstrumentationRegistry.getInstrumentation().getContext(), 2950 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 2951 new ImeSettings.Builder())) { 2952 final ImeEventStream stream = imeSession.openEventStream(); 2953 2954 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 2955 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 2956 final Pair<CustomEditorView, CustomEditorView> customEditorPair = 2957 launchTestActivityWithCustomEditors(focusedMarker, unfocusedMarker); 2958 final CustomEditorView unfocusedCustomEditor = customEditorPair.second; 2959 2960 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 2961 notExpectEvent( 2962 stream, 2963 editorMatcher("onStartInputView", focusedMarker), 2964 NOT_EXPECT_TIMEOUT); 2965 2966 addVirtualStylusIdForTestSession(); 2967 final int touchSlop = getTouchSlop(); 2968 final int startX = unfocusedCustomEditor.getWidth() / 2; 2969 final int startY = unfocusedCustomEditor.getHeight() / 2; 2970 final int endX = startX + 2 * touchSlop; 2971 final int endY = startY + 2 * touchSlop; 2972 final int number = 5; 2973 TestUtils.injectStylusDownEvent(unfocusedCustomEditor, startX, startY); 2974 TestUtils.injectStylusMoveEvents(unfocusedCustomEditor, startX, startY, 2975 endX, endY, number); 2976 try { 2977 // Handwriting should already be initiated before ACTION_UP. 2978 // unfocusedCustomEditor is focused and triggers onStartInput. 2979 expectEvent(stream, editorMatcher("onStartInput", unfocusedMarker), TIMEOUT); 2980 // Keyboard shouldn't show up. 2981 notExpectEvent( 2982 stream, 2983 editorMatcher("onStartInputView", unfocusedMarker), 2984 NOT_EXPECT_TIMEOUT); 2985 // Handwriting should start. 2986 expectEvent( 2987 stream, 2988 editorMatcher("onStartStylusHandwriting", unfocusedMarker), 2989 TIMEOUT); 2990 2991 verifyStylusHandwritingWindowIsShown(stream, imeSession); 2992 2993 // Verify that stylus move events are swallowed by the handwriting initiator once 2994 // handwriting has been initiated and not dispatched to the view tree. 2995 assertThat(unfocusedCustomEditor.mStylusMoveEventCount).isLessThan(number); 2996 } finally { 2997 TestUtils.injectStylusUpEvent(unfocusedCustomEditor, endX, endY); 2998 } 2999 } 3000 } 3001 3002 /** 3003 * Inject stylus events on top of a focused custom editor that disables auto handwriting. 3004 * 3005 * @link InputMethodManager#startStylusHandwriting(View)} should not be called. 3006 */ 3007 @Test 3008 public void testAutoHandwritingDisabled_customEditor() throws Exception { 3009 try (MockImeSession imeSession = MockImeSession.create( 3010 InstrumentationRegistry.getInstrumentation().getContext(), 3011 InstrumentationRegistry.getInstrumentation().getUiAutomation(), 3012 new ImeSettings.Builder())) { 3013 final ImeEventStream stream = imeSession.openEventStream(); 3014 3015 final String focusedMarker = getTestMarker(FOCUSED_EDIT_TEXT_TAG); 3016 final String unfocusedMarker = getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG); 3017 final Pair<CustomEditorView, CustomEditorView> customEditorPair = 3018 launchTestActivityWithCustomEditors(focusedMarker, unfocusedMarker); 3019 final CustomEditorView focusedCustomEditor = customEditorPair.first; 3020 focusedCustomEditor.setAutoHandwritingEnabled(false); 3021 3022 expectEvent(stream, editorMatcher("onStartInput", focusedMarker), TIMEOUT); 3023 notExpectEvent( 3024 stream, 3025 editorMatcher("onStartInputView", focusedMarker), 3026 NOT_EXPECT_TIMEOUT); 3027 3028 addVirtualStylusIdForTestSession(); 3029 3030 injectStylusEventToEditorAndVerify( 3031 focusedCustomEditor, stream, imeSession, focusedMarker, 3032 false /* verifyHandwritingStart */, false, 3033 false /* verifyHandwritingWindowIsShown */); 3034 3035 // Verify that all stylus move events are dispatched to the view tree. 3036 assertThat(focusedCustomEditor.mStylusMoveEventCount) 3037 .isEqualTo(NUMBER_OF_INJECTED_EVENTS); 3038 } 3039 } 3040 3041 private void injectStylusEventToEditorAndVerify( 3042 View editor, ImeEventStream stream, MockImeSession imeSession, String marker, 3043 boolean verifyHandwritingStart, boolean verifyHandwritingWindowIsShown, 3044 boolean verifyHandwritingWindowNotShown) throws Exception { 3045 final int touchSlop = getTouchSlop(); 3046 final int startX = editor.getWidth() / 2; 3047 final int startY = editor.getHeight() / 2; 3048 final int endX = startX + 2 * touchSlop; 3049 final int endY = startY + 2 * touchSlop; 3050 TestUtils.injectStylusDownEvent(editor, startX, startY); 3051 TestUtils.injectStylusMoveEvents( 3052 editor, startX, startY, endX, endY, NUMBER_OF_INJECTED_EVENTS); 3053 try { 3054 // Handwriting should already be initiated before ACTION_UP. 3055 // keyboard shouldn't show up. 3056 notExpectEvent( 3057 stream, 3058 editorMatcher("onStartInputView", marker), 3059 NOT_EXPECT_TIMEOUT); 3060 if (verifyHandwritingStart) { 3061 // Handwriting should start 3062 expectEvent( 3063 stream, 3064 editorMatcher("onStartStylusHandwriting", marker), 3065 TIMEOUT); 3066 } else { 3067 // Handwriting should not start 3068 notExpectEvent( 3069 stream, 3070 editorMatcher("onStartStylusHandwriting", marker), 3071 NOT_EXPECT_TIMEOUT); 3072 } 3073 if (verifyHandwritingWindowIsShown) { 3074 verifyStylusHandwritingWindowIsShown(stream, imeSession); 3075 } else if (verifyHandwritingWindowNotShown) { 3076 verifyStylusHandwritingWindowIsNotShown(stream, imeSession); 3077 } 3078 } finally { 3079 TestUtils.injectStylusUpEvent(editor, endX, endY); 3080 } 3081 } 3082 3083 private EditText launchTestActivity(@NonNull String marker) { 3084 return launchTestActivity(marker, getTestMarker(NON_FOCUSED_EDIT_TEXT_TAG)).first; 3085 } 3086 3087 private static int getTouchSlop() { 3088 final Context context = InstrumentationRegistry.getInstrumentation().getContext(); 3089 // Some tests require stylus movements to exceed the touch slop so that they are not 3090 // interpreted as clicks. Other tests require the movements to exceed the handwriting slop 3091 // to trigger handwriting initiation. Using the larger value allows all tests to pass. 3092 return Math.max( 3093 ViewConfiguration.get(context).getScaledTouchSlop(), 3094 ViewConfiguration.get(context).getScaledHandwritingSlop()); 3095 } 3096 3097 private Pair<EditText, EditText> launchTestActivityNoEditorFocus(@NonNull String focusedMarker, 3098 @NonNull String nonFocusedMarker) { 3099 return launchTestActivity(focusedMarker, nonFocusedMarker, false /* isEditorFocused */); 3100 } 3101 3102 private Pair<EditText, EditText> launchTestActivity(@NonNull String focusedMarker, 3103 @NonNull String nonFocusedMarker) { 3104 return launchTestActivity(focusedMarker, nonFocusedMarker, true /* isEditorFocused */); 3105 } 3106 3107 private Pair<EditText, EditText> launchTestActivity(@NonNull String focusedMarker, 3108 @NonNull String nonFocusedMarker, boolean isEditorFocused) { 3109 final AtomicReference<EditText> focusedEditTextRef = new AtomicReference<>(); 3110 final AtomicReference<EditText> nonFocusedEditTextRef = new AtomicReference<>(); 3111 TestActivity.startSync(activity -> { 3112 final LinearLayout layout = new LinearLayout(activity); 3113 layout.setOrientation(LinearLayout.VERTICAL); 3114 // Adding some top padding tests that inject stylus event out of the view boundary. 3115 layout.setPadding(0, 100, 0, 0); 3116 3117 final EditText focusedEditText = new EditText(activity); 3118 focusedEditText.setHint("focused editText"); 3119 focusedEditText.setPrivateImeOptions(focusedMarker); 3120 if (isEditorFocused) { 3121 focusedEditText.requestFocus(); 3122 } 3123 focusedEditText.setAutoHandwritingEnabled(true); 3124 focusedEditText.setHandwritingBoundsOffsets( 3125 HANDWRITING_BOUNDS_OFFSET_PX, 3126 HANDWRITING_BOUNDS_OFFSET_PX, 3127 HANDWRITING_BOUNDS_OFFSET_PX, 3128 HANDWRITING_BOUNDS_OFFSET_PX); 3129 focusedEditTextRef.set(focusedEditText); 3130 layout.addView(focusedEditText); 3131 3132 final EditText nonFocusedEditText = new EditText(activity); 3133 nonFocusedEditText.setPrivateImeOptions(nonFocusedMarker); 3134 nonFocusedEditText.setHint("target editText"); 3135 nonFocusedEditText.setAutoHandwritingEnabled(true); 3136 nonFocusedEditTextRef.set(nonFocusedEditText); 3137 nonFocusedEditText.setHandwritingBoundsOffsets( 3138 HANDWRITING_BOUNDS_OFFSET_PX, 3139 HANDWRITING_BOUNDS_OFFSET_PX, 3140 HANDWRITING_BOUNDS_OFFSET_PX, 3141 HANDWRITING_BOUNDS_OFFSET_PX); 3142 // Leave margin between the EditTexts so that their extended handwriting bounds do not 3143 // overlap. 3144 LinearLayout.LayoutParams layoutParams = new LinearLayout.LayoutParams( 3145 ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT); 3146 layoutParams.topMargin = 3 * HANDWRITING_BOUNDS_OFFSET_PX; 3147 layout.addView(nonFocusedEditText, layoutParams); 3148 return layout; 3149 }); 3150 return new Pair<>(focusedEditTextRef.get(), nonFocusedEditTextRef.get()); 3151 } 3152 3153 private Pair<CustomEditorView, CustomEditorView> launchTestActivityWithCustomEditors( 3154 @NonNull String focusedMarker, @NonNull String unfocusedMarker) { 3155 final AtomicReference<CustomEditorView> focusedCustomEditorRef = new AtomicReference<>(); 3156 final AtomicReference<CustomEditorView> unfocusedCustomEditorRef = new AtomicReference<>(); 3157 TestActivity.startSync(activity -> { 3158 final LinearLayout layout = new LinearLayout(activity); 3159 layout.setOrientation(LinearLayout.VERTICAL); 3160 // Add some top padding for tests that inject stylus event out of the view boundary. 3161 layout.setPadding(0, 100, 0, 0); 3162 3163 final CustomEditorView focusedCustomEditor = 3164 new CustomEditorView(activity, focusedMarker, Color.RED); 3165 focusedCustomEditor.setAutoHandwritingEnabled(true); 3166 focusedCustomEditor.requestFocus(); 3167 focusedCustomEditorRef.set(focusedCustomEditor); 3168 layout.addView(focusedCustomEditor); 3169 3170 final CustomEditorView unfocusedCustomEditor = 3171 new CustomEditorView(activity, unfocusedMarker, Color.BLUE); 3172 unfocusedCustomEditor.setAutoHandwritingEnabled(true); 3173 unfocusedCustomEditorRef.set(unfocusedCustomEditor); 3174 layout.addView(unfocusedCustomEditor); 3175 3176 return layout; 3177 }); 3178 return new Pair<>(focusedCustomEditorRef.get(), unfocusedCustomEditorRef.get()); 3179 } 3180 3181 private View launchTestActivityWithDelegate( 3182 @NonNull String editTextMarker, CountDownLatch delegateLatch, long delegateDelayMs) { 3183 final AtomicReference<View> delegatorViewRef = new AtomicReference<>(); 3184 TestActivity.startSync(activity -> { 3185 final LinearLayout layout = new LinearLayout(activity); 3186 layout.setOrientation(LinearLayout.VERTICAL); 3187 3188 final View delegatorView = new View(activity); 3189 delegatorViewRef.set(delegatorView); 3190 delegatorView.setBackgroundColor(Color.GREEN); 3191 delegatorView.setHandwritingDelegatorCallback( 3192 () -> { 3193 final EditText editText = new EditText(activity); 3194 editText.setIsHandwritingDelegate(true); 3195 editText.setPrivateImeOptions(editTextMarker); 3196 editText.setHint("editText"); 3197 editText.setId(R.id.handwriting_delegate); 3198 layout.addView(editText); 3199 editText.postDelayed(() -> { 3200 editText.requestFocus(); 3201 if (delegateLatch != null) { 3202 delegateLatch.countDown(); 3203 } 3204 }, delegateDelayMs); 3205 }); 3206 3207 LinearLayout.LayoutParams layoutParams = 3208 new LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, 40); 3209 // Add space so that stylus motion on the delegate view is not within the edit text's 3210 // extended handwriting bounds. 3211 layoutParams.bottomMargin = 200; 3212 layout.addView(delegatorView, layoutParams); 3213 return layout; 3214 }); 3215 return delegatorViewRef.get(); 3216 } 3217 3218 private View launchTestActivityInExternalPackage( 3219 @NonNull final String editTextMarker, final boolean setAllowedDelegatorPackage) { 3220 final AtomicReference<View> delegateViewRef = new AtomicReference<>(); 3221 TestActivity.startSync(activity -> { 3222 final LinearLayout layout = new LinearLayout(activity); 3223 layout.setOrientation(LinearLayout.VERTICAL); 3224 3225 final View delegatorView = new View(activity); 3226 delegateViewRef.set(delegatorView); 3227 delegatorView.setBackgroundColor(Color.GREEN); 3228 3229 delegatorView.setHandwritingDelegatorCallback(()-> { 3230 // launch activity in a different package. 3231 Intent intent = new Intent(Intent.ACTION_MAIN); 3232 intent.setComponent(new ComponentName( 3233 "android.view.inputmethod.ctstestapp", 3234 "android.view.inputmethod.ctstestapp.MainActivity")); 3235 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); 3236 intent.putExtra(MockTestActivityUtil.EXTRA_KEY_PRIVATE_IME_OPTIONS, editTextMarker); 3237 intent.putExtra(MockTestActivityUtil.EXTRA_HANDWRITING_DELEGATE, true); 3238 activity.startActivity(intent); 3239 }); 3240 if (setAllowedDelegatorPackage) { 3241 delegatorView.setAllowedHandwritingDelegatePackage( 3242 "android.view.inputmethod.ctstestapp"); 3243 } 3244 layout.addView( 3245 delegatorView, 3246 new ViewGroup.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, 40)); 3247 return layout; 3248 }); 3249 return delegateViewRef.get(); 3250 } 3251 3252 private boolean hasSupportedStylus() { 3253 final InputManager im = mContext.getSystemService(InputManager.class); 3254 for (int id : im.getInputDeviceIds()) { 3255 InputDevice inputDevice = im.getInputDevice(id); 3256 if (inputDevice != null && isStylusDevice(inputDevice)) { 3257 return true; 3258 } 3259 } 3260 return false; 3261 } 3262 3263 private static boolean isStylusDevice(InputDevice inputDevice) { 3264 return inputDevice.supportsSource(InputDevice.SOURCE_STYLUS) 3265 || inputDevice.supportsSource(InputDevice.SOURCE_BLUETOOTH_STYLUS); 3266 } 3267 3268 private void addVirtualStylusIdForTestSession() { 3269 SystemUtil.runWithShellPermissionIdentity(() -> { 3270 mContext.getSystemService(InputMethodManager.class) 3271 .addVirtualStylusIdForTestSession(); 3272 }, Manifest.permission.TEST_INPUT_METHOD); 3273 } 3274 3275 private String getDefaultLauncher() throws Exception { 3276 final String prefix = "Launcher: ComponentInfo{"; 3277 final String postfix = "}"; 3278 for (String s : 3279 SystemUtil.runShellCommand("cmd shortcut get-default-launcher").split("\n")) { 3280 if (s.startsWith(prefix) && s.endsWith(postfix)) { 3281 return s.substring(prefix.length(), s.length() - postfix.length()); 3282 } 3283 } 3284 throw new Exception("Default launcher not found"); 3285 } 3286 3287 private void setDefaultLauncher(String component) { 3288 SystemUtil.runShellCommand("cmd package set-home-activity " + component); 3289 } 3290 3291 private static final class CustomEditorView extends View { 3292 private final String mMarker; 3293 private int mStylusMoveEventCount = 0; 3294 3295 private CustomEditorView(Context context, @NonNull String marker, 3296 @ColorInt int backgroundColor) { 3297 super(context); 3298 mMarker = marker; 3299 setFocusable(true); 3300 setFocusableInTouchMode(true); 3301 setBackgroundColor(backgroundColor); 3302 } 3303 3304 @Override 3305 public boolean onCheckIsTextEditor() { 3306 return true; 3307 } 3308 3309 @Override 3310 public InputConnection onCreateInputConnection(EditorInfo outAttrs) { 3311 outAttrs.inputType = EditorInfo.TYPE_CLASS_TEXT; 3312 outAttrs.privateImeOptions = mMarker; 3313 return new NoOpInputConnection(); 3314 } 3315 3316 @Override 3317 public void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { 3318 // This View needs a valid size to be focusable. 3319 setMeasuredDimension(300, 100); 3320 } 3321 3322 @Override 3323 public boolean onTouchEvent(MotionEvent event) { 3324 if (event.getToolType(event.getActionIndex()) == MotionEvent.TOOL_TYPE_STYLUS) { 3325 if (event.getActionMasked() == MotionEvent.ACTION_DOWN) { 3326 // Return true to receive ACTION_MOVE events. 3327 return true; 3328 } else if (event.getActionMasked() == MotionEvent.ACTION_MOVE) { 3329 mStylusMoveEventCount++; 3330 } 3331 } 3332 return super.onTouchEvent(event); 3333 } 3334 } 3335 3336 private static final class TestCallback implements ConnectionlessHandwritingCallback { 3337 private CharSequence mResultText; 3338 public int mErrorCode = -1; 3339 3340 @Override 3341 public void onResult(@NonNull CharSequence text) { 3342 assertNoCallbackMethodsPreviouslyCalled(); 3343 mResultText = text; 3344 } 3345 3346 @Override 3347 public void onError(int errorCode) { 3348 assertNoCallbackMethodsPreviouslyCalled(); 3349 mErrorCode = errorCode; 3350 } 3351 3352 // Used to verify that the callback only receives a single result. 3353 private void assertNoCallbackMethodsPreviouslyCalled() { 3354 assertThat(mResultText).isNull(); 3355 assertThat(mErrorCode).isEqualTo(-1); 3356 } 3357 } 3358 } 3359