1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.wm.shell.pip.phone;
17 
18 import static com.android.internal.policy.TaskResizingAlgorithm.CTRL_NONE;
19 
20 import android.annotation.Nullable;
21 import android.content.Context;
22 import android.content.res.Resources;
23 import android.graphics.Point;
24 import android.graphics.PointF;
25 import android.graphics.Rect;
26 import android.hardware.input.InputManager;
27 import android.os.Looper;
28 import android.view.BatchedInputEventReceiver;
29 import android.view.Choreographer;
30 import android.view.InputChannel;
31 import android.view.InputEvent;
32 import android.view.InputEventReceiver;
33 import android.view.InputMonitor;
34 import android.view.MotionEvent;
35 import android.view.ViewConfiguration;
36 
37 import androidx.annotation.VisibleForTesting;
38 
39 import com.android.wm.shell.R;
40 import com.android.wm.shell.common.ShellExecutor;
41 import com.android.wm.shell.common.pip.PipBoundsAlgorithm;
42 import com.android.wm.shell.common.pip.PipBoundsState;
43 import com.android.wm.shell.common.pip.PipPerfHintController;
44 import com.android.wm.shell.common.pip.PipPinchResizingAlgorithm;
45 import com.android.wm.shell.common.pip.PipUiEventLogger;
46 import com.android.wm.shell.pip.PipAnimationController;
47 import com.android.wm.shell.pip.PipTaskOrganizer;
48 
49 import java.io.PrintWriter;
50 import java.util.function.Consumer;
51 
52 /**
53  * Helper on top of PipTouchHandler that handles inputs OUTSIDE of the PIP window, which is used to
54  * trigger dynamic resize.
55  */
56 public class PipResizeGestureHandler {
57 
58     private static final String TAG = "PipResizeGestureHandler";
59     private static final int PINCH_RESIZE_SNAP_DURATION = 250;
60     private static final float PINCH_RESIZE_AUTO_MAX_RATIO = 0.9f;
61 
62     private final Context mContext;
63     private final PipBoundsAlgorithm mPipBoundsAlgorithm;
64     private final PipMotionHelper mMotionHelper;
65     private final PipBoundsState mPipBoundsState;
66     private final PipTouchState mPipTouchState;
67     private final PipTaskOrganizer mPipTaskOrganizer;
68     private final PhonePipMenuController mPhonePipMenuController;
69     private final PipDismissTargetHandler mPipDismissTargetHandler;
70     private final PipUiEventLogger mPipUiEventLogger;
71     private final PipPinchResizingAlgorithm mPinchResizingAlgorithm;
72     private final int mDisplayId;
73     private final ShellExecutor mMainExecutor;
74 
75     private final PointF mDownPoint = new PointF();
76     private final PointF mDownSecondPoint = new PointF();
77     private final PointF mLastPoint = new PointF();
78     private final PointF mLastSecondPoint = new PointF();
79     private final Point mMaxSize = new Point();
80     private final Point mMinSize = new Point();
81     private final Rect mLastResizeBounds = new Rect();
82     private final Rect mUserResizeBounds = new Rect();
83     private final Rect mDownBounds = new Rect();
84     private final Runnable mUpdateMovementBoundsRunnable;
85     private final Consumer<Rect> mUpdateResizeBoundsCallback;
86 
87     private float mTouchSlop;
88 
89     private boolean mAllowGesture;
90     private boolean mIsAttached;
91     private boolean mIsEnabled;
92     private boolean mEnablePinchResize;
93     private boolean mIsSysUiStateValid;
94     private boolean mThresholdCrossed;
95     private boolean mOngoingPinchToResize = false;
96     private float mAngle = 0;
97     int mFirstIndex = -1;
98     int mSecondIndex = -1;
99 
100     private InputMonitor mInputMonitor;
101     private InputEventReceiver mInputEventReceiver;
102 
103     @Nullable
104     private final PipPerfHintController mPipPerfHintController;
105 
106     @Nullable
107     private PipPerfHintController.PipHighPerfSession mPipHighPerfSession;
108 
109     private int mCtrlType;
110     private int mOhmOffset;
111 
PipResizeGestureHandler(Context context, PipBoundsAlgorithm pipBoundsAlgorithm, PipBoundsState pipBoundsState, PipMotionHelper motionHelper, PipTouchState pipTouchState, PipTaskOrganizer pipTaskOrganizer, PipDismissTargetHandler pipDismissTargetHandler, Runnable updateMovementBoundsRunnable, PipUiEventLogger pipUiEventLogger, PhonePipMenuController menuActivityController, ShellExecutor mainExecutor, @Nullable PipPerfHintController pipPerfHintController)112     public PipResizeGestureHandler(Context context, PipBoundsAlgorithm pipBoundsAlgorithm,
113             PipBoundsState pipBoundsState, PipMotionHelper motionHelper,
114             PipTouchState pipTouchState, PipTaskOrganizer pipTaskOrganizer,
115             PipDismissTargetHandler pipDismissTargetHandler,
116             Runnable updateMovementBoundsRunnable,
117             PipUiEventLogger pipUiEventLogger, PhonePipMenuController menuActivityController,
118             ShellExecutor mainExecutor, @Nullable PipPerfHintController pipPerfHintController) {
119         mContext = context;
120         mDisplayId = context.getDisplayId();
121         mMainExecutor = mainExecutor;
122         mPipPerfHintController = pipPerfHintController;
123         mPipBoundsAlgorithm = pipBoundsAlgorithm;
124         mPipBoundsState = pipBoundsState;
125         mMotionHelper = motionHelper;
126         mPipTouchState = pipTouchState;
127         mPipTaskOrganizer = pipTaskOrganizer;
128         mPipDismissTargetHandler = pipDismissTargetHandler;
129         mUpdateMovementBoundsRunnable = updateMovementBoundsRunnable;
130         mPhonePipMenuController = menuActivityController;
131         mPipUiEventLogger = pipUiEventLogger;
132         mPinchResizingAlgorithm = new PipPinchResizingAlgorithm();
133 
134         mUpdateResizeBoundsCallback = (rect) -> {
135             mUserResizeBounds.set(rect);
136             mMotionHelper.synchronizePinnedStackBounds();
137             mUpdateMovementBoundsRunnable.run();
138             resetState();
139         };
140     }
141 
init()142     public void init() {
143         mContext.getDisplay().getRealSize(mMaxSize);
144         reloadResources();
145 
146         final Resources res = mContext.getResources();
147         mEnablePinchResize = res.getBoolean(R.bool.config_pipEnablePinchResize);
148     }
149 
onConfigurationChanged()150     public void onConfigurationChanged() {
151         reloadResources();
152     }
153 
154     /**
155      * Called when SysUI state changed.
156      *
157      * @param isSysUiStateValid Is SysUI valid or not.
158      */
onSystemUiStateChanged(boolean isSysUiStateValid)159     public void onSystemUiStateChanged(boolean isSysUiStateValid) {
160         mIsSysUiStateValid = isSysUiStateValid;
161     }
162 
reloadResources()163     private void reloadResources() {
164         mTouchSlop = ViewConfiguration.get(mContext).getScaledTouchSlop();
165     }
166 
disposeInputChannel()167     private void disposeInputChannel() {
168         if (mInputEventReceiver != null) {
169             mInputEventReceiver.dispose();
170             mInputEventReceiver = null;
171         }
172         if (mInputMonitor != null) {
173             mInputMonitor.dispose();
174             mInputMonitor = null;
175         }
176     }
177 
onActivityPinned()178     void onActivityPinned() {
179         mIsAttached = true;
180         updateIsEnabled();
181     }
182 
onActivityUnpinned()183     void onActivityUnpinned() {
184         mIsAttached = false;
185         mUserResizeBounds.setEmpty();
186         updateIsEnabled();
187     }
188 
updateIsEnabled()189     private void updateIsEnabled() {
190         boolean isEnabled = mIsAttached;
191         if (isEnabled == mIsEnabled) {
192             return;
193         }
194         mIsEnabled = isEnabled;
195         disposeInputChannel();
196 
197         if (mIsEnabled) {
198             // Register input event receiver
199             mInputMonitor = mContext.getSystemService(InputManager.class).monitorGestureInput(
200                     "pip-resize", mDisplayId);
201             try {
202                 mMainExecutor.executeBlocking(() -> {
203                     mInputEventReceiver = new PipResizeInputEventReceiver(
204                             mInputMonitor.getInputChannel(), Looper.myLooper());
205                 });
206             } catch (InterruptedException e) {
207                 throw new RuntimeException("Failed to create input event receiver", e);
208             }
209         }
210     }
211 
212     @VisibleForTesting
onInputEvent(InputEvent ev)213     void onInputEvent(InputEvent ev) {
214         if (!mEnablePinchResize) {
215             // No need to handle anything if neither form of resizing is enabled.
216             return;
217         }
218 
219         if (!mPipTouchState.getAllowInputEvents()) {
220             // No need to handle anything if touches are not enabled
221             return;
222         }
223 
224         // Don't allow resize when PiP is stashed.
225         if (mPipBoundsState.isStashed()) {
226             return;
227         }
228 
229         if (ev instanceof MotionEvent) {
230             MotionEvent mv = (MotionEvent) ev;
231             int action = mv.getActionMasked();
232             final Rect pipBounds = mPipBoundsState.getBounds();
233             if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) {
234                 if (!pipBounds.contains((int) mv.getRawX(), (int) mv.getRawY())
235                         && mPhonePipMenuController.isMenuVisible()) {
236                     mPhonePipMenuController.hideMenu();
237                 }
238             }
239 
240             if (mEnablePinchResize && mOngoingPinchToResize) {
241                 onPinchResize(mv);
242             }
243         }
244     }
245 
246     /**
247      * Checks if there is currently an on-going gesture, either drag-resize or pinch-resize.
248      */
hasOngoingGesture()249     public boolean hasOngoingGesture() {
250         return mCtrlType != CTRL_NONE || mOngoingPinchToResize;
251     }
252 
isUsingPinchToZoom()253     public boolean isUsingPinchToZoom() {
254         return mEnablePinchResize;
255     }
256 
isResizing()257     public boolean isResizing() {
258         return mAllowGesture;
259     }
260 
willStartResizeGesture(MotionEvent ev)261     public boolean willStartResizeGesture(MotionEvent ev) {
262         if (isInValidSysUiState()) {
263             if (ev.getActionMasked() == MotionEvent.ACTION_POINTER_DOWN) {
264                 if (mEnablePinchResize && ev.getPointerCount() == 2) {
265                     onPinchResize(ev);
266                     mOngoingPinchToResize = mAllowGesture;
267                     return mAllowGesture;
268                 }
269             }
270         }
271         return false;
272     }
273 
isInValidSysUiState()274     private boolean isInValidSysUiState() {
275         return mIsSysUiStateValid;
276     }
277 
onHighPerfSessionTimeout(PipPerfHintController.PipHighPerfSession session)278     private void onHighPerfSessionTimeout(PipPerfHintController.PipHighPerfSession session) {}
279 
cleanUpHighPerfSessionMaybe()280     private void cleanUpHighPerfSessionMaybe() {
281         if (mPipHighPerfSession != null) {
282             // Close the high perf session once pointer interactions are over;
283             mPipHighPerfSession.close();
284             mPipHighPerfSession = null;
285         }
286     }
287 
288     @VisibleForTesting
onPinchResize(MotionEvent ev)289     void onPinchResize(MotionEvent ev) {
290         int action = ev.getActionMasked();
291 
292         if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) {
293             mFirstIndex = -1;
294             mSecondIndex = -1;
295             mAllowGesture = false;
296             finishResize();
297             cleanUpHighPerfSessionMaybe();
298         }
299 
300         if (ev.getPointerCount() != 2) {
301             return;
302         }
303 
304         final Rect pipBounds = mPipBoundsState.getBounds();
305         if (action == MotionEvent.ACTION_POINTER_DOWN) {
306             if (mFirstIndex == -1 && mSecondIndex == -1
307                     && pipBounds.contains((int) ev.getRawX(0), (int) ev.getRawY(0))
308                     && pipBounds.contains((int) ev.getRawX(1), (int) ev.getRawY(1))) {
309                 mAllowGesture = true;
310                 mFirstIndex = 0;
311                 mSecondIndex = 1;
312                 mDownPoint.set(ev.getRawX(mFirstIndex), ev.getRawY(mFirstIndex));
313                 mDownSecondPoint.set(ev.getRawX(mSecondIndex), ev.getRawY(mSecondIndex));
314                 mDownBounds.set(pipBounds);
315 
316                 mLastPoint.set(mDownPoint);
317                 mLastSecondPoint.set(mLastSecondPoint);
318                 mLastResizeBounds.set(mDownBounds);
319 
320                 // start the high perf session as the second pointer gets detected
321                 if (mPipPerfHintController != null) {
322                     mPipHighPerfSession = mPipPerfHintController.startSession(
323                             this::onHighPerfSessionTimeout, "onPinchResize");
324                 }
325             }
326         }
327 
328         if (action == MotionEvent.ACTION_MOVE) {
329             if (mFirstIndex == -1 || mSecondIndex == -1) {
330                 return;
331             }
332 
333             float x0 = ev.getRawX(mFirstIndex);
334             float y0 = ev.getRawY(mFirstIndex);
335             float x1 = ev.getRawX(mSecondIndex);
336             float y1 = ev.getRawY(mSecondIndex);
337             mLastPoint.set(x0, y0);
338             mLastSecondPoint.set(x1, y1);
339 
340             // Capture inputs
341             if (!mThresholdCrossed
342                     && (distanceBetween(mDownSecondPoint, mLastSecondPoint) > mTouchSlop
343                             || distanceBetween(mDownPoint, mLastPoint) > mTouchSlop)) {
344                 pilferPointers();
345                 mThresholdCrossed = true;
346                 // Reset the down to begin resizing from this point
347                 mDownPoint.set(mLastPoint);
348                 mDownSecondPoint.set(mLastSecondPoint);
349 
350                 if (mPhonePipMenuController.isMenuVisible()) {
351                     mPhonePipMenuController.hideMenu();
352                 }
353             }
354 
355             if (mThresholdCrossed) {
356                 mAngle = mPinchResizingAlgorithm.calculateBoundsAndAngle(mDownPoint,
357                         mDownSecondPoint, mLastPoint, mLastSecondPoint, mMinSize, mMaxSize,
358                         mDownBounds, mLastResizeBounds);
359 
360                 mPipTaskOrganizer.scheduleUserResizePip(mDownBounds, mLastResizeBounds,
361                         mAngle, null);
362                 mPipBoundsState.setHasUserResizedPip(true);
363             }
364         }
365     }
366 
snapToMovementBoundsEdge(Rect bounds, Rect movementBounds)367     private void snapToMovementBoundsEdge(Rect bounds, Rect movementBounds) {
368         final int leftEdge = bounds.left;
369 
370 
371         final int fromLeft = Math.abs(leftEdge - movementBounds.left);
372         final int fromRight = Math.abs(movementBounds.right - leftEdge);
373 
374         // The PIP will be snapped to either the right or left edge, so calculate which one
375         // is closest to the current position.
376         final int newLeft = fromLeft < fromRight
377                 ? movementBounds.left : movementBounds.right;
378 
379         bounds.offsetTo(newLeft, mLastResizeBounds.top);
380     }
381 
382     /**
383      * Resizes the pip window and updates user-resized bounds.
384      *
385      * @param bounds target bounds to resize to
386      * @param snapFraction snap fraction to apply after resizing
387      */
388     void userResizeTo(Rect bounds, float snapFraction) {
389         Rect finalBounds = new Rect(bounds);
390 
391         // get the current movement bounds
392         final Rect movementBounds = mPipBoundsAlgorithm.getMovementBounds(finalBounds);
393 
394         // snap the target bounds to the either left or right edge, by choosing the closer one
395         snapToMovementBoundsEdge(finalBounds, movementBounds);
396 
397         // apply the requested snap fraction onto the target bounds
398         mPipBoundsAlgorithm.applySnapFraction(finalBounds, snapFraction);
399 
400         // resize from current bounds to target bounds without animation
401         mPipTaskOrganizer.scheduleUserResizePip(mPipBoundsState.getBounds(), finalBounds, null);
402         // set the flag that pip has been resized
403         mPipBoundsState.setHasUserResizedPip(true);
404 
405         // finish the resize operation and update the state of the bounds
406         mPipTaskOrganizer.scheduleFinishResizePip(finalBounds, mUpdateResizeBoundsCallback);
407     }
408 
409     private void finishResize() {
410         if (!mLastResizeBounds.isEmpty()) {
411             // Pinch-to-resize needs to re-calculate snap fraction and animate to the snapped
412             // position correctly. Drag-resize does not need to move, so just finalize resize.
413             if (mOngoingPinchToResize) {
414                 final Rect startBounds = new Rect(mLastResizeBounds);
415                 // If user resize is pretty close to max size, just auto resize to max.
416                 if (mLastResizeBounds.width() >= PINCH_RESIZE_AUTO_MAX_RATIO * mMaxSize.x
417                         || mLastResizeBounds.height() >= PINCH_RESIZE_AUTO_MAX_RATIO * mMaxSize.y) {
418                     resizeRectAboutCenter(mLastResizeBounds, mMaxSize.x, mMaxSize.y);
419                 }
420 
421                 // If user resize is smaller than min size, auto resize to min
422                 if (mLastResizeBounds.width() < mMinSize.x
423                         || mLastResizeBounds.height() < mMinSize.y) {
424                     resizeRectAboutCenter(mLastResizeBounds, mMinSize.x, mMinSize.y);
425                 }
426 
427                 // get the current movement bounds
428                 final Rect movementBounds = mPipBoundsAlgorithm
429                         .getMovementBounds(mLastResizeBounds);
430 
431                 // snap mLastResizeBounds to the correct edge based on movement bounds
432                 snapToMovementBoundsEdge(mLastResizeBounds, movementBounds);
433 
434                 final float snapFraction = mPipBoundsAlgorithm.getSnapFraction(
435                         mLastResizeBounds, movementBounds);
436                 mPipBoundsAlgorithm.applySnapFraction(mLastResizeBounds, snapFraction);
437 
438                 // disable any touch events beyond resizing too
439                 mPipTouchState.setAllowInputEvents(false);
440 
441                 mPipTaskOrganizer.scheduleAnimateResizePip(startBounds, mLastResizeBounds,
442                         PINCH_RESIZE_SNAP_DURATION, mAngle, mUpdateResizeBoundsCallback, () -> {
443                             // enable touch events
444                             mPipTouchState.setAllowInputEvents(true);
445                         });
446             } else {
mPipTaskOrganizer.scheduleFinishResizePip(mLastResizeBounds, PipAnimationController.TRANSITION_DIRECTION_USER_RESIZE, mUpdateResizeBoundsCallback)447                 mPipTaskOrganizer.scheduleFinishResizePip(mLastResizeBounds,
448                         PipAnimationController.TRANSITION_DIRECTION_USER_RESIZE,
449                         mUpdateResizeBoundsCallback);
450             }
451             final float magnetRadiusPercent = (float) mLastResizeBounds.width() / mMinSize.x / 2.f;
452             mPipDismissTargetHandler
setMagneticFieldRadiusPercent(magnetRadiusPercent)453                     .setMagneticFieldRadiusPercent(magnetRadiusPercent);
mPipUiEventLogger.log( PipUiEventLogger.PipUiEventEnum.PICTURE_IN_PICTURE_RESIZE)454             mPipUiEventLogger.log(
455                     PipUiEventLogger.PipUiEventEnum.PICTURE_IN_PICTURE_RESIZE);
456         } else {
resetState()457             resetState();
458         }
459     }
460 
461     private void resetState() {
462         mCtrlType = CTRL_NONE;
463         mAngle = 0;
464         mOngoingPinchToResize = false;
465         mAllowGesture = false;
466         mThresholdCrossed = false;
467     }
468 
469     void setUserResizeBounds(Rect bounds) {
470         mUserResizeBounds.set(bounds);
471     }
472 
473     void invalidateUserResizeBounds() {
474         mUserResizeBounds.setEmpty();
475     }
476 
477     Rect getUserResizeBounds() {
478         return mUserResizeBounds;
479     }
480 
481     @VisibleForTesting
482     Rect getLastResizeBounds() {
483         return mLastResizeBounds;
484     }
485 
486     @VisibleForTesting
487     void pilferPointers() {
488         mInputMonitor.pilferPointers();
489     }
490 
491 
492     @VisibleForTesting public void updateMaxSize(int maxX, int maxY) {
493         mMaxSize.set(maxX, maxY);
494     }
495 
496     @VisibleForTesting public void updateMinSize(int minX, int minY) {
497         mMinSize.set(minX, minY);
498     }
499 
500     void setOhmOffset(int offset) {
501         mOhmOffset = offset;
502     }
503 
504     private float distanceBetween(PointF p1, PointF p2) {
505         return (float) Math.hypot(p2.x - p1.x, p2.y - p1.y);
506     }
507 
508     private void resizeRectAboutCenter(Rect rect, int w, int h) {
509         int cx = rect.centerX();
510         int cy = rect.centerY();
511         int l = cx - w / 2;
512         int r = l + w;
513         int t = cy - h / 2;
514         int b = t + h;
515         rect.set(l, t, r, b);
516     }
517 
518     public void dump(PrintWriter pw, String prefix) {
519         final String innerPrefix = prefix + "  ";
520         pw.println(prefix + TAG);
521         pw.println(innerPrefix + "mAllowGesture=" + mAllowGesture);
522         pw.println(innerPrefix + "mIsAttached=" + mIsAttached);
523         pw.println(innerPrefix + "mIsEnabled=" + mIsEnabled);
524         pw.println(innerPrefix + "mEnablePinchResize=" + mEnablePinchResize);
525         pw.println(innerPrefix + "mThresholdCrossed=" + mThresholdCrossed);
526         pw.println(innerPrefix + "mOhmOffset=" + mOhmOffset);
527         pw.println(innerPrefix + "mMinSize=" + mMinSize);
528         pw.println(innerPrefix + "mMaxSize=" + mMaxSize);
529     }
530 
531     class PipResizeInputEventReceiver extends BatchedInputEventReceiver {
532         PipResizeInputEventReceiver(InputChannel channel, Looper looper) {
533             super(channel, looper, Choreographer.getInstance());
534         }
535 
536         public void onInputEvent(InputEvent event) {
537             PipResizeGestureHandler.this.onInputEvent(event);
538             finishInputEvent(event, true);
539         }
540     }
541 }
542