1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.view; 18 19 import static com.android.text.flags.Flags.handwritingCursorPosition; 20 import static com.android.text.flags.Flags.handwritingUnsupportedMessage; 21 22 import android.annotation.FlaggedApi; 23 import android.annotation.NonNull; 24 import android.annotation.Nullable; 25 import android.content.Context; 26 import android.graphics.Matrix; 27 import android.graphics.Rect; 28 import android.graphics.RectF; 29 import android.graphics.Region; 30 import android.text.TextUtils; 31 import android.view.inputmethod.ConnectionlessHandwritingCallback; 32 import android.view.inputmethod.CursorAnchorInfo; 33 import android.view.inputmethod.Flags; 34 import android.view.inputmethod.InputMethodManager; 35 import android.widget.EditText; 36 import android.widget.Editor; 37 import android.widget.TextView; 38 import android.widget.Toast; 39 40 import com.android.internal.R; 41 import com.android.internal.annotations.VisibleForTesting; 42 43 import java.lang.ref.WeakReference; 44 import java.util.ArrayList; 45 import java.util.Iterator; 46 import java.util.List; 47 import java.util.function.Consumer; 48 49 /** 50 * Initiates handwriting mode once it detects stylus movement in handwritable areas. 51 * 52 * It is designed to be used by {@link ViewRootImpl}. For every stylus related MotionEvent that is 53 * dispatched to view tree, ViewRootImpl should call {@link #onTouchEvent} method of this class. 54 * And it will automatically request to enter the handwriting mode when the conditions meet. 55 * 56 * Notice that ViewRootImpl should still dispatch MotionEvents to view tree as usual. 57 * And if it successfully enters the handwriting mode, the ongoing MotionEvent stream will be 58 * routed to the input method. Input system will fabricate an ACTION_CANCEL and send to 59 * ViewRootImpl. 60 * 61 * This class does nothing if: 62 * a) MotionEvents are not from stylus. 63 * b) The user taps or long-clicks with a stylus etc. 64 * c) Stylus pointer down position is not within a handwritable area. 65 * 66 * Used by InputMethodManager. 67 * @hide 68 */ 69 public class HandwritingInitiator { 70 /** 71 * The maximum amount of distance a stylus touch can wander before it is considered 72 * handwriting. 73 */ 74 private final int mHandwritingSlop; 75 /** 76 * The timeout used to distinguish tap or long click from handwriting. If the stylus doesn't 77 * move before this timeout, it's not considered as handwriting. 78 */ 79 private final long mHandwritingTimeoutInMillis; 80 81 private State mState; 82 private final HandwritingAreaTracker mHandwritingAreasTracker = new HandwritingAreaTracker(); 83 84 /** The reference to the View that currently has the input connection. */ 85 @Nullable 86 @VisibleForTesting 87 public WeakReference<View> mConnectedView = null; 88 89 /** 90 * When InputConnection restarts for a View, View#onInputConnectionCreatedInternal 91 * might be called before View#onInputConnectionClosedInternal, so we need to count the input 92 * connections and only set mConnectedView to null when mConnectionCount is zero. 93 */ 94 private int mConnectionCount = 0; 95 96 /** 97 * The reference to the View that currently has focus. 98 * This replaces mConnecteView when {@code Flags#intitiationWithoutInputConnection()} is 99 * enabled. 100 */ 101 @Nullable 102 @VisibleForTesting 103 public WeakReference<View> mFocusedView = null; 104 105 private final InputMethodManager mImm; 106 107 private final int[] mTempLocation = new int[2]; 108 109 private final Rect mTempRect = new Rect(); 110 111 private final RectF mTempRectF = new RectF(); 112 113 private final Region mTempRegion = new Region(); 114 115 private final Matrix mTempMatrix = new Matrix(); 116 117 /** 118 * The handwrite-able View that is currently the target of a hovering stylus pointer. This is 119 * used to help determine whether the handwriting PointerIcon should be shown in 120 * {@link #onResolvePointerIcon(Context, MotionEvent)} so that we can reduce the number of calls 121 * to {@link #findBestCandidateView(float, float, boolean)}. 122 */ 123 @Nullable 124 private WeakReference<View> mCachedHoverTarget = null; 125 126 /** 127 * Whether to show the hover icon for the current connected view. 128 * Hover icon should be hidden for the current connected view after handwriting is initiated 129 * for it until one of the following events happens: 130 * a) user performs a click or long click. In other words, if it receives a series of motion 131 * events that don't trigger handwriting, show hover icon again. 132 * b) the stylus hovers on another editor that supports handwriting (or a handwriting delegate). 133 * c) the current connected editor lost focus. 134 * 135 * If the stylus is hovering on an unconnected editor that supports handwriting, we always show 136 * the hover icon. 137 * TODO(b/308827131): Rename to FocusedView after Flag is flipped. 138 */ 139 private boolean mShowHoverIconForConnectedView = true; 140 141 /** When flag is enabled, touched editors don't wait for InputConnection for initiation. 142 * However, delegation still waits for InputConnection. 143 */ 144 private final boolean mInitiateWithoutConnection = Flags.initiationWithoutInputConnection(); 145 146 @VisibleForTesting HandwritingInitiator(@onNull ViewConfiguration viewConfiguration, @NonNull InputMethodManager inputMethodManager)147 public HandwritingInitiator(@NonNull ViewConfiguration viewConfiguration, 148 @NonNull InputMethodManager inputMethodManager) { 149 mHandwritingSlop = viewConfiguration.getScaledHandwritingSlop(); 150 mHandwritingTimeoutInMillis = ViewConfiguration.getLongPressTimeout(); 151 mImm = inputMethodManager; 152 } 153 154 /** 155 * Notify the HandwritingInitiator that a new MotionEvent has arrived. 156 * 157 * <p>The return value indicates whether the event has been fully handled by the 158 * HandwritingInitiator and should not be dispatched to the view tree. This will be true for 159 * ACTION_MOVE events from a stylus gesture after handwriting mode has been initiated, in order 160 * to suppress other actions such as scrolling. 161 * 162 * <p>If HandwritingInitiator triggers the handwriting mode, a fabricated ACTION_CANCEL event 163 * will be sent to the ViewRootImpl. 164 * 165 * @param motionEvent the stylus {@link MotionEvent} 166 * @return true if the event has been fully handled by the {@link HandwritingInitiator} and 167 * should not be dispatched to the {@link View} tree, or false if the event should be dispatched 168 * to the {@link View} tree as usual 169 */ 170 @VisibleForTesting onTouchEvent(@onNull MotionEvent motionEvent)171 public boolean onTouchEvent(@NonNull MotionEvent motionEvent) { 172 final int maskedAction = motionEvent.getActionMasked(); 173 switch (maskedAction) { 174 case MotionEvent.ACTION_DOWN: 175 case MotionEvent.ACTION_POINTER_DOWN: 176 mState = null; 177 if (!motionEvent.isStylusPointer()) { 178 // The motion event is not from a stylus event, ignore it. 179 return false; 180 } 181 mState = new State(motionEvent); 182 break; 183 case MotionEvent.ACTION_POINTER_UP: 184 final int pointerId = motionEvent.getPointerId(motionEvent.getActionIndex()); 185 if (mState == null || pointerId != mState.mStylusPointerId) { 186 // ACTION_POINTER_UP is from another stylus pointer, ignore the event. 187 return false; 188 } 189 // Deliberately fall through. 190 case MotionEvent.ACTION_CANCEL: 191 case MotionEvent.ACTION_UP: 192 // If it's ACTION_CANCEL or ACTION_UP, all the pointers go up. There is no need to 193 // check whether the stylus we are tracking goes up. 194 if (mState != null) { 195 mState.mShouldInitHandwriting = false; 196 if (!mState.mHandled) { 197 // The user just did a click, long click or another stylus gesture, 198 // show hover icon again for the connected view. 199 mShowHoverIconForConnectedView = true; 200 } 201 } 202 return false; 203 case MotionEvent.ACTION_MOVE: 204 if (mState == null) { 205 return false; 206 } 207 208 // Either we've already tried to initiate handwriting, or the ongoing MotionEvent 209 // sequence is considered to be tap, long-click or other gestures. 210 if (!mState.mShouldInitHandwriting || mState.mExceedHandwritingSlop) { 211 return mState.mHandled; 212 } 213 214 final long timeElapsed = 215 motionEvent.getEventTime() - mState.mStylusDownTimeInMillis; 216 if (timeElapsed > mHandwritingTimeoutInMillis) { 217 mState.mShouldInitHandwriting = false; 218 return mState.mHandled; 219 } 220 221 final int pointerIndex = motionEvent.findPointerIndex(mState.mStylusPointerId); 222 final float x = motionEvent.getX(pointerIndex); 223 final float y = motionEvent.getY(pointerIndex); 224 if (largerThanTouchSlop(x, y, mState.mStylusDownX, mState.mStylusDownY)) { 225 mState.mExceedHandwritingSlop = true; 226 View candidateView = findBestCandidateView(mState.mStylusDownX, 227 mState.mStylusDownY, /* isHover */ false); 228 if (candidateView != null && candidateView.isEnabled()) { 229 boolean candidateHasFocus = candidateView.hasFocus(); 230 if (!candidateView.isStylusHandwritingAvailable()) { 231 mState.mShouldInitHandwriting = false; 232 return false; 233 } else if (shouldShowHandwritingUnavailableMessageForView(candidateView)) { 234 int messagesResId = (candidateView instanceof TextView tv 235 && tv.isAnyPasswordInputType()) 236 ? R.string.error_handwriting_unsupported_password 237 : R.string.error_handwriting_unsupported; 238 Toast.makeText(candidateView.getContext(), messagesResId, 239 Toast.LENGTH_SHORT).show(); 240 if (!candidateView.hasFocus()) { 241 requestFocusWithoutReveal(candidateView); 242 } 243 mImm.showSoftInput(candidateView, 0); 244 mState.mHandled = true; 245 mState.mShouldInitHandwriting = false; 246 motionEvent.setAction((motionEvent.getAction() 247 & MotionEvent.ACTION_POINTER_INDEX_MASK) 248 | MotionEvent.ACTION_CANCEL); 249 candidateView.getRootView().dispatchTouchEvent(motionEvent); 250 } else if (candidateView == getConnectedOrFocusedView()) { 251 if (!candidateHasFocus) { 252 requestFocusWithoutReveal(candidateView); 253 } 254 startHandwriting(candidateView); 255 } else if (candidateView.getHandwritingDelegatorCallback() != null) { 256 prepareDelegation(candidateView); 257 } else { 258 if (mInitiateWithoutConnection) { 259 if (!candidateHasFocus) { 260 // schedule for view focus. 261 mState.mPendingFocusedView = new WeakReference<>(candidateView); 262 requestFocusWithoutReveal(candidateView); 263 } 264 } else { 265 mState.mPendingConnectedView = new WeakReference<>(candidateView); 266 if (!candidateHasFocus) { 267 requestFocusWithoutReveal(candidateView); 268 } 269 } 270 } 271 } 272 } 273 return mState.mHandled; 274 } 275 return false; 276 } 277 278 @Nullable getConnectedView()279 private View getConnectedView() { 280 if (mConnectedView == null) return null; 281 return mConnectedView.get(); 282 } 283 clearConnectedView()284 private void clearConnectedView() { 285 mConnectedView = null; 286 mConnectionCount = 0; 287 } 288 289 /** 290 * Notify HandwritingInitiator that a delegate view (see {@link View#isHandwritingDelegate}) 291 * gained focus. 292 */ onDelegateViewFocused(@onNull View view)293 public void onDelegateViewFocused(@NonNull View view) { 294 if (mInitiateWithoutConnection) { 295 onEditorFocused(view); 296 } 297 if (view == getConnectedView()) { 298 tryAcceptStylusHandwritingDelegation(view); 299 } 300 } 301 302 /** 303 * Notify HandwritingInitiator that a new InputConnection is created. 304 * The caller of this method should guarantee that each onInputConnectionCreated call 305 * is paired with a onInputConnectionClosed call. 306 * @param view the view that created the current InputConnection. 307 * @see #onInputConnectionClosed(View) 308 */ onInputConnectionCreated(@onNull View view)309 public void onInputConnectionCreated(@NonNull View view) { 310 if (mInitiateWithoutConnection && !view.isHandwritingDelegate()) { 311 // When flag is enabled, only delegation continues to wait for InputConnection. 312 return; 313 } 314 if (!view.isAutoHandwritingEnabled()) { 315 clearConnectedView(); 316 return; 317 } 318 319 final View connectedView = getConnectedView(); 320 if (connectedView == view) { 321 ++mConnectionCount; 322 } else { 323 mConnectedView = new WeakReference<>(view); 324 mConnectionCount = 1; 325 // A new view just gain focus. By default, we should show hover icon for it. 326 mShowHoverIconForConnectedView = true; 327 if (view.isHandwritingDelegate() && tryAcceptStylusHandwritingDelegation(view)) { 328 // tryAcceptStylusHandwritingDelegation should set boolean below, however, we 329 // cannot mock IMM to return true for acceptStylusDelegation(). 330 // TODO(b/324670412): we should move any dependent tests to integration and remove 331 // the assignment below. 332 mShowHoverIconForConnectedView = false; 333 return; 334 } 335 if (!mInitiateWithoutConnection && mState != null 336 && mState.mPendingConnectedView != null 337 && mState.mPendingConnectedView.get() == view) { 338 startHandwriting(view); 339 } 340 } 341 } 342 343 /** 344 * Notify HandwritingInitiator that a new editor is focused. 345 * @param view the view that received focus. 346 */ 347 @VisibleForTesting onEditorFocused(@onNull View view)348 public void onEditorFocused(@NonNull View view) { 349 if (!mInitiateWithoutConnection) { 350 return; 351 } 352 353 if (!view.isAutoHandwritingEnabled()) { 354 clearFocusedView(view); 355 return; 356 } 357 358 final View focusedView = getFocusedView(); 359 if (focusedView == view) { 360 return; 361 } 362 updateFocusedView(view); 363 364 if (mState != null && mState.mPendingFocusedView != null 365 && mState.mPendingFocusedView.get() == view) { 366 startHandwriting(view); 367 } 368 } 369 370 /** 371 * Notify HandwritingInitiator that the InputConnection has closed for the given view. 372 * The caller of this method should guarantee that each onInputConnectionClosed call 373 * is paired with a onInputConnectionCreated call. 374 * @param view the view that closed the InputConnection. 375 */ onInputConnectionClosed(@onNull View view)376 public void onInputConnectionClosed(@NonNull View view) { 377 if (mInitiateWithoutConnection && !view.isHandwritingDelegate()) { 378 return; 379 } 380 final View connectedView = getConnectedView(); 381 if (connectedView == null) return; 382 if (connectedView == view) { 383 --mConnectionCount; 384 if (mConnectionCount == 0) { 385 clearConnectedView(); 386 } 387 } else { 388 // Unexpected branch, set mConnectedView to null to avoid further problem. 389 clearConnectedView(); 390 } 391 } 392 393 @Nullable getFocusedView()394 private View getFocusedView() { 395 if (mFocusedView == null) return null; 396 return mFocusedView.get(); 397 } 398 399 /** 400 * Clear the tracked focused view tracked for handwriting initiation. 401 * @param view the focused view. 402 */ clearFocusedView(View view)403 public void clearFocusedView(View view) { 404 if (view == null || mFocusedView == null) { 405 return; 406 } 407 if (mFocusedView.get() == view) { 408 mFocusedView = null; 409 } 410 } 411 412 /** 413 * Called when new {@link Editor} is focused. 414 * @return {@code true} if handwriting can initiate for given view. 415 */ 416 @VisibleForTesting updateFocusedView(@onNull View view)417 public boolean updateFocusedView(@NonNull View view) { 418 if (!view.shouldInitiateHandwriting()) { 419 mFocusedView = null; 420 return false; 421 } 422 423 final View focusedView = getFocusedView(); 424 if (focusedView != view) { 425 mFocusedView = new WeakReference<>(view); 426 // A new view just gain focus. By default, we should show hover icon for it. 427 mShowHoverIconForConnectedView = true; 428 } 429 430 return true; 431 } 432 433 /** Starts a stylus handwriting session for the view. */ 434 @VisibleForTesting startHandwriting(@onNull View view)435 public void startHandwriting(@NonNull View view) { 436 mImm.startStylusHandwriting(view); 437 mState.mHandled = true; 438 mState.mShouldInitHandwriting = false; 439 mShowHoverIconForConnectedView = false; 440 if (view instanceof TextView) { 441 ((TextView) view).hideHint(); 442 } 443 } 444 prepareDelegation(View view)445 private void prepareDelegation(View view) { 446 String delegatePackageName = view.getAllowedHandwritingDelegatePackageName(); 447 if (delegatePackageName == null) { 448 delegatePackageName = view.getContext().getOpPackageName(); 449 } 450 if (mImm.isConnectionlessStylusHandwritingAvailable()) { 451 // No other view should have focus during the connectionless handwriting session, as 452 // this could cause user confusion about the input target for the session. 453 view.getViewRootImpl().getView().clearFocus(); 454 mImm.startConnectionlessStylusHandwritingForDelegation( 455 view, getCursorAnchorInfoForConnectionless(view), delegatePackageName, 456 view::post, new DelegationCallback(view, delegatePackageName)); 457 mState.mShouldInitHandwriting = false; 458 } else { 459 mImm.prepareStylusHandwritingDelegation(view, delegatePackageName); 460 view.getHandwritingDelegatorCallback().run(); 461 } 462 mState.mHandled = true; 463 } 464 465 /** 466 * Starts a stylus handwriting session for the delegate view, if {@link 467 * InputMethodManager#prepareStylusHandwritingDelegation} was previously called. 468 */ 469 @VisibleForTesting tryAcceptStylusHandwritingDelegation(@onNull View view)470 public boolean tryAcceptStylusHandwritingDelegation(@NonNull View view) { 471 if (Flags.useZeroJankProxy()) { 472 tryAcceptStylusHandwritingDelegationAsync(view); 473 } else { 474 return tryAcceptStylusHandwritingDelegationInternal(view); 475 } 476 return false; 477 } 478 tryAcceptStylusHandwritingDelegationInternal(@onNull View view)479 private boolean tryAcceptStylusHandwritingDelegationInternal(@NonNull View view) { 480 String delegatorPackageName = 481 view.getAllowedHandwritingDelegatorPackageName(); 482 if (delegatorPackageName == null) { 483 delegatorPackageName = view.getContext().getOpPackageName(); 484 } 485 if (mImm.acceptStylusHandwritingDelegation(view, delegatorPackageName)) { 486 onDelegationAccepted(view); 487 return true; 488 } 489 return false; 490 } 491 492 @FlaggedApi(Flags.FLAG_USE_ZERO_JANK_PROXY) tryAcceptStylusHandwritingDelegationAsync(@onNull View view)493 private void tryAcceptStylusHandwritingDelegationAsync(@NonNull View view) { 494 String delegatorPackageName = 495 view.getAllowedHandwritingDelegatorPackageName(); 496 if (delegatorPackageName == null) { 497 delegatorPackageName = view.getContext().getOpPackageName(); 498 } 499 WeakReference<View> viewRef = new WeakReference<>(view); 500 Consumer<Boolean> consumer = delegationAccepted -> { 501 if (delegationAccepted) { 502 onDelegationAccepted(viewRef.get()); 503 } 504 }; 505 mImm.acceptStylusHandwritingDelegation(view, delegatorPackageName, view::post, consumer); 506 } 507 onDelegationAccepted(View view)508 private void onDelegationAccepted(View view) { 509 if (mState != null) { 510 mState.mHandled = true; 511 mState.mShouldInitHandwriting = false; 512 } 513 if (view == null) { 514 // can be null if view was detached and was GCed. 515 return; 516 } 517 if (view instanceof TextView) { 518 ((TextView) view).hideHint(); 519 } 520 // A handwriting delegate view is accepted and handwriting starts; hide the 521 // hover icon. 522 mShowHoverIconForConnectedView = false; 523 } 524 525 /** 526 * Notify that the handwriting area for the given view might be updated. 527 * @param view the view whose handwriting area might be updated. 528 */ updateHandwritingAreasForView(@onNull View view)529 public void updateHandwritingAreasForView(@NonNull View view) { 530 mHandwritingAreasTracker.updateHandwritingAreaForView(view); 531 } 532 shouldTriggerStylusHandwritingForView(@onNull View view)533 private static boolean shouldTriggerStylusHandwritingForView(@NonNull View view) { 534 if (!view.shouldInitiateHandwriting()) { 535 return false; 536 } 537 // The view may be a handwriting initiation delegator, in which case it is not the editor 538 // view for which handwriting would be started. However, in almost all cases, the return 539 // values of View#isStylusHandwritingAvailable will be the same for the delegator view and 540 // the delegate editor view. So the delegator view can be used to decide whether handwriting 541 // should be triggered. 542 return view.isStylusHandwritingAvailable(); 543 } 544 shouldShowHandwritingUnavailableMessageForView(@onNull View view)545 private static boolean shouldShowHandwritingUnavailableMessageForView(@NonNull View view) { 546 return (view instanceof TextView) && !shouldTriggerStylusHandwritingForView(view); 547 } 548 shouldTriggerHandwritingOrShowUnavailableMessageForView( @onNull View view)549 private static boolean shouldTriggerHandwritingOrShowUnavailableMessageForView( 550 @NonNull View view) { 551 return (view instanceof TextView) || shouldTriggerStylusHandwritingForView(view); 552 } 553 554 /** 555 * Returns the pointer icon for the motion event, or null if it doesn't specify the icon. 556 * This gives HandwritingInitiator a chance to show the stylus handwriting icon over a 557 * handwrite-able area. 558 */ onResolvePointerIcon(Context context, MotionEvent event)559 public PointerIcon onResolvePointerIcon(Context context, MotionEvent event) { 560 final View hoverView = findHoverView(event); 561 if (hoverView == null || !shouldTriggerStylusHandwritingForView(hoverView)) { 562 return null; 563 } 564 565 if (mShowHoverIconForConnectedView) { 566 return PointerIcon.getSystemIcon(context, PointerIcon.TYPE_HANDWRITING); 567 } 568 569 if (hoverView != getConnectedOrFocusedView()) { 570 // The stylus is hovering on another view that supports handwriting. We should show 571 // hover icon. Also reset the mShowHoverIconForFocusedView so that hover 572 // icon is displayed again next time when the stylus hovers on focused view. 573 mShowHoverIconForConnectedView = true; 574 return PointerIcon.getSystemIcon(context, PointerIcon.TYPE_HANDWRITING); 575 } 576 return null; 577 } 578 579 // TODO(b/308827131): Remove once Flag is flipped. getConnectedOrFocusedView()580 private View getConnectedOrFocusedView() { 581 if (mInitiateWithoutConnection) { 582 return mFocusedView == null ? null : mFocusedView.get(); 583 } else { 584 return mConnectedView == null ? null : mConnectedView.get(); 585 } 586 } 587 getCachedHoverTarget()588 private View getCachedHoverTarget() { 589 if (mCachedHoverTarget == null) { 590 return null; 591 } 592 return mCachedHoverTarget.get(); 593 } 594 findHoverView(MotionEvent event)595 private View findHoverView(MotionEvent event) { 596 if (!event.isStylusPointer() || !event.isHoverEvent()) { 597 return null; 598 } 599 600 if (event.getActionMasked() == MotionEvent.ACTION_HOVER_ENTER 601 || event.getActionMasked() == MotionEvent.ACTION_HOVER_MOVE) { 602 final float hoverX = event.getX(event.getActionIndex()); 603 final float hoverY = event.getY(event.getActionIndex()); 604 605 final View cachedHoverTarget = getCachedHoverTarget(); 606 if (cachedHoverTarget != null) { 607 final Rect handwritingArea = mTempRect; 608 if (getViewHandwritingArea(cachedHoverTarget, handwritingArea) 609 && isInHandwritingArea(handwritingArea, hoverX, hoverY, cachedHoverTarget, 610 /* isHover */ true) 611 && shouldTriggerStylusHandwritingForView(cachedHoverTarget)) { 612 return cachedHoverTarget; 613 } 614 } 615 616 final View candidateView = findBestCandidateView(hoverX, hoverY, /* isHover */ true); 617 618 if (candidateView != null) { 619 if (!handwritingUnsupportedMessage()) { 620 mCachedHoverTarget = new WeakReference<>(candidateView); 621 } 622 return candidateView; 623 } 624 } 625 626 mCachedHoverTarget = null; 627 return null; 628 } 629 requestFocusWithoutReveal(View view)630 private void requestFocusWithoutReveal(View view) { 631 if (!handwritingCursorPosition() && view instanceof EditText editText 632 && !mState.mStylusDownWithinEditorBounds) { 633 // If the stylus down point was inside the EditText's bounds, then the EditText will 634 // automatically set its cursor position nearest to the stylus down point when it 635 // gains focus. If the stylus down point was outside the EditText's bounds (within 636 // the extended handwriting bounds), then we must calculate and set the cursor 637 // position manually. 638 view.getLocationInWindow(mTempLocation); 639 int offset = editText.getOffsetForPosition( 640 mState.mStylusDownX - mTempLocation[0], 641 mState.mStylusDownY - mTempLocation[1]); 642 editText.setSelection(offset); 643 } 644 if (view.getRevealOnFocusHint()) { 645 view.setRevealOnFocusHint(false); 646 view.requestFocus(); 647 view.setRevealOnFocusHint(true); 648 } else { 649 view.requestFocus(); 650 } 651 if (handwritingCursorPosition() && view instanceof EditText editText) { 652 // Move the cursor to the end of the paragraph closest to the stylus down point. 653 view.getLocationInWindow(mTempLocation); 654 int line = editText.getLineAtCoordinate(mState.mStylusDownY - mTempLocation[1]); 655 int paragraphEnd = TextUtils.indexOf(editText.getText(), '\n', 656 editText.getLayout().getLineStart(line)); 657 if (paragraphEnd < 0) { 658 paragraphEnd = editText.getText().length(); 659 } 660 editText.setSelection(paragraphEnd); 661 } 662 } 663 664 /** 665 * Given the location of the stylus event, return the best candidate view to initialize 666 * handwriting mode or show the handwriting unavailable error message. 667 * 668 * @param x the x coordinates of the stylus event, in the coordinates of the window. 669 * @param y the y coordinates of the stylus event, in the coordinates of the window. 670 */ 671 @Nullable findBestCandidateView(float x, float y, boolean isHover)672 private View findBestCandidateView(float x, float y, boolean isHover) { 673 // TODO(b/308827131): Rename to FocusedView after Flag is flipped. 674 // If the connectedView is not null and do not set any handwriting area, it will check 675 // whether the connectedView's boundary contains the initial stylus position. If true, 676 // directly return the connectedView. 677 final View connectedOrFocusedView = getConnectedOrFocusedView(); 678 if (connectedOrFocusedView != null) { 679 Rect handwritingArea = mTempRect; 680 if (getViewHandwritingArea(connectedOrFocusedView, handwritingArea) 681 && isInHandwritingArea(handwritingArea, x, y, connectedOrFocusedView, isHover) 682 && shouldTriggerHandwritingOrShowUnavailableMessageForView( 683 connectedOrFocusedView)) { 684 if (!isHover && mState != null) { 685 mState.mStylusDownWithinEditorBounds = 686 contains(handwritingArea, x, y, 0f, 0f, 0f, 0f); 687 } 688 return connectedOrFocusedView; 689 } 690 } 691 692 float minDistance = Float.MAX_VALUE; 693 View bestCandidate = null; 694 // Check the registered handwriting areas. 695 final List<HandwritableViewInfo> handwritableViewInfos = 696 mHandwritingAreasTracker.computeViewInfos(); 697 for (HandwritableViewInfo viewInfo : handwritableViewInfos) { 698 final View view = viewInfo.getView(); 699 final Rect handwritingArea = viewInfo.getHandwritingArea(); 700 if (!isInHandwritingArea(handwritingArea, x, y, view, isHover) 701 || !shouldTriggerHandwritingOrShowUnavailableMessageForView(view)) { 702 continue; 703 } 704 705 final float distance = distance(handwritingArea, x, y); 706 if (distance == 0f) { 707 if (!isHover && mState != null) { 708 mState.mStylusDownWithinEditorBounds = true; 709 } 710 return view; 711 } 712 if (distance < minDistance) { 713 minDistance = distance; 714 bestCandidate = view; 715 } 716 } 717 return bestCandidate; 718 } 719 720 /** 721 * Return the square of the distance from point (x, y) to the given rect, which is mainly used 722 * for comparison. The distance is defined to be: the shortest distance between (x, y) to any 723 * point on rect. When (x, y) is contained by the rect, return 0f. 724 */ distance(@onNull Rect rect, float x, float y)725 private static float distance(@NonNull Rect rect, float x, float y) { 726 if (contains(rect, x, y, 0f, 0f, 0f, 0f)) { 727 return 0f; 728 } 729 730 /* The distance between point (x, y) and rect, there are 2 basic cases: 731 * a) The distance is the distance from (x, y) to the closest corner on rect. 732 * o | | 733 * ---+-----+--- 734 * | | 735 * ---+-----+--- 736 * | | 737 * b) The distance is the distance from (x, y) to the closest edge on rect. 738 * | o | 739 * ---+-----+--- 740 * | | 741 * ---+-----+--- 742 * | | 743 * We define xDistance as following(similar for yDistance): 744 * If x is in [left, right) 0, else min(abs(x - left), abs(x - y)) 745 * For case a, sqrt(xDistance^2 + yDistance^2) is the final distance. 746 * For case b, distance should be yDistance, which is also equal to 747 * sqrt(xDistance^2 + yDistance^2) because xDistance is 0. 748 */ 749 final float xDistance; 750 if (x >= rect.left && x < rect.right) { 751 xDistance = 0f; 752 } else if (x < rect.left) { 753 xDistance = rect.left - x; 754 } else { 755 xDistance = x - rect.right; 756 } 757 758 final float yDistance; 759 if (y >= rect.top && y < rect.bottom) { 760 yDistance = 0f; 761 } else if (y < rect.top) { 762 yDistance = rect.top - y; 763 } else { 764 yDistance = y - rect.bottom; 765 } 766 // We can omit sqrt here because we only need the distance for comparison. 767 return xDistance * xDistance + yDistance * yDistance; 768 } 769 770 /** 771 * Return the handwriting area of the given view, represented in the window's coordinate. 772 * If the view didn't set any handwriting area, it will return the view's boundary. 773 * 774 * <p> The handwriting area is clipped to its visible part. 775 * Notice that the returned rectangle is the view's original handwriting area without the 776 * view's handwriting area extends. </p> 777 * 778 * @param view the {@link View} whose handwriting area we want to compute. 779 * @param rect the {@link Rect} to receive the result. 780 * 781 * @return true if the view's handwriting area is still visible, or false if it's clipped and 782 * fully invisible. This method only consider the clip by given view's parents, but not the case 783 * where a view is covered by its sibling view. 784 */ getViewHandwritingArea(@onNull View view, @NonNull Rect rect)785 private static boolean getViewHandwritingArea(@NonNull View view, @NonNull Rect rect) { 786 final ViewParent viewParent = view.getParent(); 787 if (viewParent != null && view.isAttachedToWindow() && view.isAggregatedVisible()) { 788 final Rect localHandwritingArea = view.getHandwritingArea(); 789 if (localHandwritingArea != null) { 790 rect.set(localHandwritingArea); 791 } else { 792 rect.set(0, 0, view.getWidth(), view.getHeight()); 793 } 794 return viewParent.getChildVisibleRect(view, rect, null); 795 } 796 return false; 797 } 798 799 /** 800 * Return true if the (x, y) is inside by the given {@link Rect} with the View's 801 * handwriting bounds with offsets applied. 802 */ isInHandwritingArea(@ullable Rect handwritingArea, float x, float y, View view, boolean isHover)803 private boolean isInHandwritingArea(@Nullable Rect handwritingArea, 804 float x, float y, View view, boolean isHover) { 805 if (handwritingArea == null) return false; 806 807 if (!contains(handwritingArea, x, y, 808 view.getHandwritingBoundsOffsetLeft(), 809 view.getHandwritingBoundsOffsetTop(), 810 view.getHandwritingBoundsOffsetRight(), 811 view.getHandwritingBoundsOffsetBottom())) { 812 return false; 813 } 814 815 // The returned handwritingArea computed by ViewParent#getChildVisibleRect didn't consider 816 // the case where a view is stacking on top of the editor. (e.g. DrawerLayout, popup) 817 // We must check the hit region of the editor again, and avoid the case where another 818 // view on top of the editor is handling MotionEvents. 819 ViewParent parent = view.getParent(); 820 if (parent == null) { 821 return true; 822 } 823 824 Region region = mTempRegion; 825 mTempRegion.set(0, 0, view.getWidth(), view.getHeight()); 826 Matrix matrix = mTempMatrix; 827 matrix.reset(); 828 if (!parent.getChildLocalHitRegion(view, region, matrix, isHover)) { 829 return false; 830 } 831 832 // It's not easy to extend the region by the given handwritingBoundsOffset. Instead, we 833 // create a rectangle surrounding the motion event location and check if this rectangle 834 // overlaps with the hit region of the editor. 835 float left = x - view.getHandwritingBoundsOffsetRight(); 836 float top = y - view.getHandwritingBoundsOffsetBottom(); 837 float right = Math.max(x + view.getHandwritingBoundsOffsetLeft(), left + 1); 838 float bottom = Math.max(y + view.getHandwritingBoundsOffsetTop(), top + 1); 839 RectF rectF = mTempRectF; 840 rectF.set(left, top, right, bottom); 841 matrix.mapRect(rectF); 842 843 return region.op(Math.round(rectF.left), Math.round(rectF.top), 844 Math.round(rectF.right), Math.round(rectF.bottom), Region.Op.INTERSECT); 845 } 846 847 /** 848 * Return true if the (x, y) is inside by the given {@link Rect} offset by the given 849 * offsetLeft, offsetTop, offsetRight and offsetBottom. 850 */ contains(@onNull Rect rect, float x, float y, float offsetLeft, float offsetTop, float offsetRight, float offsetBottom)851 private static boolean contains(@NonNull Rect rect, float x, float y, 852 float offsetLeft, float offsetTop, float offsetRight, float offsetBottom) { 853 return x >= rect.left - offsetLeft && x < rect.right + offsetRight 854 && y >= rect.top - offsetTop && y < rect.bottom + offsetBottom; 855 } 856 largerThanTouchSlop(float x1, float y1, float x2, float y2)857 private boolean largerThanTouchSlop(float x1, float y1, float x2, float y2) { 858 float dx = x1 - x2; 859 float dy = y1 - y2; 860 return dx * dx + dy * dy > mHandwritingSlop * mHandwritingSlop; 861 } 862 863 /** Object that keeps the MotionEvent related states for HandwritingInitiator. */ 864 private static class State { 865 /** 866 * Whether it should initiate handwriting mode for the current MotionEvent sequence. 867 * (A series of MotionEvents from ACTION_DOWN to ACTION_UP) 868 * 869 * The purpose of this boolean value is: 870 * a) We should only request to start handwriting mode ONCE for each MotionEvent sequence. 871 * If we've already requested to enter handwriting mode for the ongoing MotionEvent 872 * sequence, this boolean is set to false. And it won't request to start handwriting again. 873 * 874 * b) If the MotionEvent sequence is considered to be tap, long-click or other gestures. 875 * This boolean will be set to false, and it won't request to start handwriting. 876 */ 877 private boolean mShouldInitHandwriting; 878 879 /** 880 * Whether the current MotionEvent sequence has been handled by the handwriting initiator, 881 * either by initiating handwriting mode, or by preparing handwriting delegation. 882 */ 883 private boolean mHandled; 884 885 /** 886 * Whether the current ongoing stylus MotionEvent sequence already exceeds the 887 * handwriting slop. 888 * It's used for the case where the stylus exceeds handwriting slop before the target View 889 * built InputConnection. 890 */ 891 private boolean mExceedHandwritingSlop; 892 893 /** 894 * Whether the stylus down point of the MotionEvent sequence was within the editor's bounds 895 * (not including the extended handwriting bounds). 896 */ 897 private boolean mStylusDownWithinEditorBounds; 898 899 /** 900 * A view which has requested focus and is pending input connection creation. When an input 901 * connection is created for the view, a handwriting session should be started for the view. 902 */ 903 private WeakReference<View> mPendingConnectedView = null; 904 905 /** 906 * A view which has requested focus and is yet to receive it. 907 * When view receives focus, a handwriting session should be started for the view. 908 */ 909 private WeakReference<View> mPendingFocusedView = null; 910 911 /** The pointer id of the stylus pointer that is being tracked. */ 912 private final int mStylusPointerId; 913 /** The time stamp when the stylus pointer goes down. */ 914 private final long mStylusDownTimeInMillis; 915 /** The initial location where the stylus pointer goes down. */ 916 private final float mStylusDownX; 917 private final float mStylusDownY; 918 State(MotionEvent motionEvent)919 private State(MotionEvent motionEvent) { 920 final int actionIndex = motionEvent.getActionIndex(); 921 mStylusPointerId = motionEvent.getPointerId(actionIndex); 922 mStylusDownTimeInMillis = motionEvent.getEventTime(); 923 mStylusDownX = motionEvent.getX(actionIndex); 924 mStylusDownY = motionEvent.getY(actionIndex); 925 926 mShouldInitHandwriting = true; 927 mHandled = false; 928 mExceedHandwritingSlop = false; 929 } 930 } 931 932 /** The helper method to check if the given view is still active for handwriting. */ isViewActive(@ullable View view)933 private static boolean isViewActive(@Nullable View view) { 934 return view != null && view.isAttachedToWindow() && view.isAggregatedVisible() 935 && view.shouldTrackHandwritingArea(); 936 } 937 getCursorAnchorInfoForConnectionless(View view)938 private CursorAnchorInfo getCursorAnchorInfoForConnectionless(View view) { 939 CursorAnchorInfo.Builder builder = new CursorAnchorInfo.Builder(); 940 // Fake editor views will usually display hint text. The hint text view can be used to 941 // populate the CursorAnchorInfo. 942 TextView textView = findFirstTextViewDescendent(view); 943 if (textView != null) { 944 textView.getCursorAnchorInfo(0, builder, mTempMatrix); 945 if (textView.getSelectionStart() < 0) { 946 // Insertion marker location is not populated if selection start is negative, so 947 // make a best guess. 948 float bottom = textView.getHeight() - textView.getExtendedPaddingBottom(); 949 builder.setInsertionMarkerLocation( 950 /* horizontalPosition= */ textView.getCompoundPaddingStart(), 951 /* lineTop= */ textView.getExtendedPaddingTop(), 952 /* lineBaseline= */ bottom, 953 /* lineBottom= */ bottom, 954 /* flags= */ 0); 955 } 956 } else { 957 // If there is no TextView descendent, just populate the insertion marker with the start 958 // edge of the view. 959 mTempMatrix.reset(); 960 view.transformMatrixToGlobal(mTempMatrix); 961 builder.setMatrix(mTempMatrix); 962 builder.setInsertionMarkerLocation( 963 /* horizontalPosition= */ view.isLayoutRtl() ? view.getWidth() : 0, 964 /* lineTop= */ 0, 965 /* lineBaseline= */ view.getHeight(), 966 /* lineBottom= */ view.getHeight(), 967 /* flags= */ 0); 968 } 969 return builder.build(); 970 } 971 972 @Nullable findFirstTextViewDescendent(View view)973 private static TextView findFirstTextViewDescendent(View view) { 974 if (view instanceof ViewGroup viewGroup) { 975 TextView textView; 976 for (int i = 0; i < viewGroup.getChildCount(); ++i) { 977 View child = viewGroup.getChildAt(i); 978 textView = (child instanceof TextView tv) 979 ? tv : findFirstTextViewDescendent(viewGroup.getChildAt(i)); 980 if (textView != null 981 && textView.isAggregatedVisible() 982 && (!TextUtils.isEmpty(textView.getText()) 983 || !TextUtils.isEmpty(textView.getHint()))) { 984 return textView; 985 } 986 } 987 } 988 return null; 989 } 990 991 /** 992 * A class used to track the handwriting areas set by the Views. 993 * 994 * @hide 995 */ 996 @VisibleForTesting 997 public static class HandwritingAreaTracker { 998 private final List<HandwritableViewInfo> mHandwritableViewInfos; 999 HandwritingAreaTracker()1000 public HandwritingAreaTracker() { 1001 mHandwritableViewInfos = new ArrayList<>(); 1002 } 1003 1004 /** 1005 * Notify this tracker that the handwriting area of the given view has been updated. 1006 * This method does three things: 1007 * a) iterate over the all the tracked ViewInfos and remove those already invalid ones. 1008 * b) mark the given view's ViewInfo to be dirty. So that next time when 1009 * {@link #computeViewInfos} is called, this view's handwriting area will be recomputed. 1010 * c) If no the given view is not in the tracked ViewInfo list, a new ViewInfo object will 1011 * be created and added to the list. 1012 * 1013 * @param view the view whose handwriting area is updated. 1014 */ updateHandwritingAreaForView(@onNull View view)1015 public void updateHandwritingAreaForView(@NonNull View view) { 1016 Iterator<HandwritableViewInfo> iterator = mHandwritableViewInfos.iterator(); 1017 boolean found = false; 1018 while (iterator.hasNext()) { 1019 final HandwritableViewInfo handwritableViewInfo = iterator.next(); 1020 final View curView = handwritableViewInfo.getView(); 1021 if (!isViewActive(curView)) { 1022 iterator.remove(); 1023 } 1024 if (curView == view) { 1025 found = true; 1026 handwritableViewInfo.mIsDirty = true; 1027 } 1028 } 1029 if (!found && isViewActive(view)) { 1030 // The given view is not tracked. Create a new HandwritableViewInfo for it and add 1031 // to the list. 1032 mHandwritableViewInfos.add(new HandwritableViewInfo(view)); 1033 } 1034 } 1035 1036 /** 1037 * Update the handwriting areas and return a list of ViewInfos containing the view 1038 * reference and its handwriting area. 1039 */ 1040 @NonNull computeViewInfos()1041 public List<HandwritableViewInfo> computeViewInfos() { 1042 mHandwritableViewInfos.removeIf(viewInfo -> !viewInfo.update()); 1043 return mHandwritableViewInfos; 1044 } 1045 } 1046 1047 /** 1048 * A class that reference to a View and its handwriting area(in the ViewRoot's coordinate.) 1049 * 1050 * @hide 1051 */ 1052 @VisibleForTesting 1053 public static class HandwritableViewInfo { 1054 final WeakReference<View> mViewRef; 1055 Rect mHandwritingArea = null; 1056 @VisibleForTesting 1057 public boolean mIsDirty = true; 1058 1059 @VisibleForTesting HandwritableViewInfo(@onNull View view)1060 public HandwritableViewInfo(@NonNull View view) { 1061 mViewRef = new WeakReference<>(view); 1062 } 1063 1064 /** Return the tracked view. */ 1065 @Nullable getView()1066 public View getView() { 1067 return mViewRef.get(); 1068 } 1069 1070 /** 1071 * Return the tracked handwriting area, represented in the ViewRoot's coordinates. 1072 * Notice, the caller should not modify the returned Rect. 1073 */ 1074 @Nullable getHandwritingArea()1075 public Rect getHandwritingArea() { 1076 return mHandwritingArea; 1077 } 1078 1079 /** 1080 * Update the handwriting area in this ViewInfo. 1081 * 1082 * @return true if this ViewInfo is still valid. Or false if this ViewInfo has become 1083 * invalid due to either view is no longer visible, or the handwriting area set by the 1084 * view is removed. {@link HandwritingAreaTracker} no longer need to keep track of this 1085 * HandwritableViewInfo this method returns false. 1086 */ update()1087 public boolean update() { 1088 final View view = getView(); 1089 if (!isViewActive(view)) { 1090 return false; 1091 } 1092 1093 if (!mIsDirty) { 1094 return true; 1095 } 1096 final Rect handwritingArea = view.getHandwritingArea(); 1097 if (handwritingArea == null) { 1098 return false; 1099 } 1100 1101 ViewParent parent = view.getParent(); 1102 if (parent != null) { 1103 if (mHandwritingArea == null) { 1104 mHandwritingArea = new Rect(); 1105 } 1106 mHandwritingArea.set(handwritingArea); 1107 if (!parent.getChildVisibleRect(view, mHandwritingArea, null /* offset */)) { 1108 mHandwritingArea = null; 1109 } 1110 } 1111 mIsDirty = false; 1112 return true; 1113 } 1114 } 1115 1116 private class DelegationCallback implements ConnectionlessHandwritingCallback { 1117 private final View mView; 1118 private final String mDelegatePackageName; 1119 DelegationCallback(View view, String delegatePackageName)1120 private DelegationCallback(View view, String delegatePackageName) { 1121 mView = view; 1122 mDelegatePackageName = delegatePackageName; 1123 } 1124 1125 @Override onResult(@onNull CharSequence text)1126 public void onResult(@NonNull CharSequence text) { 1127 mView.getHandwritingDelegatorCallback().run(); 1128 } 1129 1130 @Override onError(int errorCode)1131 public void onError(int errorCode) { 1132 switch (errorCode) { 1133 case CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED: 1134 mView.getHandwritingDelegatorCallback().run(); 1135 break; 1136 case CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED: 1137 // Fall back to the old delegation flow 1138 mImm.prepareStylusHandwritingDelegation(mView, mDelegatePackageName); 1139 mView.getHandwritingDelegatorCallback().run(); 1140 break; 1141 } 1142 } 1143 } 1144 } 1145