1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server.autofill.ui;
18 
19 import static com.android.server.autofill.Helper.sDebug;
20 import static com.android.server.autofill.Helper.sVerbose;
21 
22 import android.annotation.NonNull;
23 import android.annotation.Nullable;
24 import android.content.IntentSender;
25 import android.os.Handler;
26 import android.os.IBinder;
27 import android.os.RemoteException;
28 import android.service.autofill.IInlineSuggestionUi;
29 import android.service.autofill.IInlineSuggestionUiCallback;
30 import android.service.autofill.ISurfacePackageResultCallback;
31 import android.util.Slog;
32 import android.view.SurfaceControlViewHost;
33 
34 import com.android.internal.view.inline.IInlineContentCallback;
35 
36 /**
37  * The instance of this class lives in the system server, orchestrating the communication between
38  * the remote process owning embedded view (i.e. ExtServices) and the remote process hosting the
39  * embedded view (i.e. IME). It's also responsible for releasing the embedded view from the owning
40  * process when it's not longer needed in the hosting process.
41  *
42  * <p>An instance of this class may be reused to associate with multiple instances of
43  * {@link InlineContentProviderImpl}s, each of which wraps a callback from the IME. But at any
44  * given time, there is only one active IME callback which this class will callback into.
45  *
46  * <p>This class is thread safe, because all the outside calls are piped into a single handler
47  * thread to be processed.
48  */
49 final class RemoteInlineSuggestionUi {
50 
51     private static final String TAG = RemoteInlineSuggestionUi.class.getSimpleName();
52 
53     // The delay time to release the remote inline suggestion view (in the renderer
54     // process) after receiving a signal about the surface package being released due to being
55     // detached from the window in the host app (in the IME process). The release will be
56     // canceled if the host app reattaches the view to a window within this delay time.
57     // TODO(b/154683107): try out using the Chroreographer to schedule the release right at the
58     // next frame. Basically if the view is not re-attached to the window immediately in the next
59     // frame after it was detached, then it will be released.
60     private static final long RELEASE_REMOTE_VIEW_HOST_DELAY_MS = 200;
61 
62     @NonNull
63     private final Handler mHandler;
64     @NonNull
65     private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector;
66     private final int mWidth;
67     private final int mHeight;
68     @NonNull
69     private final InlineSuggestionUiCallbackImpl mInlineSuggestionUiCallback;
70 
71     @Nullable
72     private IInlineContentCallback mInlineContentCallback; // from IME
73 
74     /**
75      * Remote inline suggestion view, backed by an instance of {@link SurfaceControlViewHost} in
76      * the render service process. We takes care of releasing it when there is no remote
77      * reference to it (from IME), and we will create a new instance of the view when it's needed
78      * by IME again.
79      */
80     @Nullable
81     private IInlineSuggestionUi mInlineSuggestionUi;
82     private int mRefCount = 0;
83     private boolean mWaitingForUiCreation = false;
84     private int mActualWidth;
85     private int mActualHeight;
86 
87     @Nullable
88     private Runnable mDelayedReleaseViewRunnable;
89 
RemoteInlineSuggestionUi( @onNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector, int width, int height, Handler handler)90     RemoteInlineSuggestionUi(
91             @NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector,
92             int width, int height, Handler handler) {
93         mHandler = handler;
94         mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector;
95         mWidth = width;
96         mHeight = height;
97         mInlineSuggestionUiCallback = new InlineSuggestionUiCallbackImpl();
98     }
99 
100     /**
101      * Updates the callback from the IME process. It'll swap out the previous IME callback, and
102      * all the subsequent callback events (onClick, onLongClick, touch event transfer, etc) will
103      * be directed to the new callback.
104      */
setInlineContentCallback(@onNull IInlineContentCallback inlineContentCallback)105     void setInlineContentCallback(@NonNull IInlineContentCallback inlineContentCallback) {
106         mHandler.post(() -> {
107             mInlineContentCallback = inlineContentCallback;
108         });
109     }
110 
111     /**
112      * Handles the request from the IME process to get a new surface package. May create a new
113      * view in the renderer process if the existing view is already released.
114      */
requestSurfacePackage()115     void requestSurfacePackage() {
116         mHandler.post(this::handleRequestSurfacePackage);
117     }
118 
119     /**
120      * Handles the signal from the IME process that the previously sent surface package has been
121      * released.
122      */
surfacePackageReleased()123     void surfacePackageReleased() {
124         mHandler.post(() -> handleUpdateRefCount(-1));
125     }
126 
127     /**
128      * Returns true if the provided size matches the remote view's size.
129      */
match(int width, int height)130     boolean match(int width, int height) {
131         return mWidth == width && mHeight == height;
132     }
133 
handleRequestSurfacePackage()134     private void handleRequestSurfacePackage() {
135         cancelPendingReleaseViewRequest();
136 
137         if (mInlineSuggestionUi == null) {
138             if (mWaitingForUiCreation) {
139                 // This could happen in the following case: the remote embedded view was released
140                 // when previously detached from window. An event after that to re-attached to
141                 // the window will cause us calling the renderSuggestion again. Now, before the
142                 // render call returns a new surface package, if the view is detached and
143                 // re-attached to the window, causing this method to be called again, we will get
144                 // to this state. This request will be ignored and the surface package will still
145                 // be sent back once the view is rendered.
146                 if (sDebug) Slog.d(TAG, "Inline suggestion ui is not ready");
147             } else {
148                 mRemoteInlineSuggestionViewConnector.renderSuggestion(mWidth, mHeight,
149                         mInlineSuggestionUiCallback);
150                 mWaitingForUiCreation = true;
151             }
152         } else {
153             try {
154                 mInlineSuggestionUi.getSurfacePackage(new ISurfacePackageResultCallback.Stub() {
155                     @Override
156                     public void onResult(SurfaceControlViewHost.SurfacePackage result) {
157                         mHandler.post(() -> {
158                             if (sVerbose) Slog.v(TAG, "Sending refreshed SurfacePackage to IME");
159                             try {
160                                 mInlineContentCallback.onContent(result, mActualWidth,
161                                         mActualHeight);
162                                 handleUpdateRefCount(1);
163                             } catch (RemoteException e) {
164                                 Slog.w(TAG, "RemoteException calling onContent");
165                             }
166                         });
167                     }
168                 });
169             } catch (RemoteException e) {
170                 Slog.w(TAG, "RemoteException calling getSurfacePackage.");
171             }
172         }
173     }
174 
handleUpdateRefCount(int delta)175     private void handleUpdateRefCount(int delta) {
176         cancelPendingReleaseViewRequest();
177         mRefCount += delta;
178         if (mRefCount <= 0) {
179             mDelayedReleaseViewRunnable = () -> {
180                 if (mInlineSuggestionUi != null) {
181                     try {
182                         if (sVerbose) Slog.v(TAG, "releasing the host");
183                         mInlineSuggestionUi.releaseSurfaceControlViewHost();
184                         mInlineSuggestionUi = null;
185                     } catch (RemoteException e) {
186                         Slog.w(TAG, "RemoteException calling releaseSurfaceControlViewHost");
187                     }
188                 }
189                 mDelayedReleaseViewRunnable = null;
190             };
191             mHandler.postDelayed(mDelayedReleaseViewRunnable, RELEASE_REMOTE_VIEW_HOST_DELAY_MS);
192         }
193     }
194 
cancelPendingReleaseViewRequest()195     private void cancelPendingReleaseViewRequest() {
196         if (mDelayedReleaseViewRunnable != null) {
197             mHandler.removeCallbacks(mDelayedReleaseViewRunnable);
198             mDelayedReleaseViewRunnable = null;
199         }
200     }
201 
202     /**
203      * This is called when a new inline suggestion UI is inflated from the ext services.
204      */
handleInlineSuggestionUiReady(IInlineSuggestionUi content, SurfaceControlViewHost.SurfacePackage surfacePackage, int width, int height)205     private void handleInlineSuggestionUiReady(IInlineSuggestionUi content,
206             SurfaceControlViewHost.SurfacePackage surfacePackage, int width, int height) {
207         mInlineSuggestionUi = content;
208         mRefCount = 0;
209         mWaitingForUiCreation = false;
210         mActualWidth = width;
211         mActualHeight = height;
212         if (mInlineContentCallback != null) {
213             try {
214                 if (sVerbose) Slog.v(TAG, "Sending new UI content to IME");
215                 handleUpdateRefCount(1);
216                 mInlineContentCallback.onContent(surfacePackage, mActualWidth, mActualHeight);
217             } catch (RemoteException e) {
218                 Slog.w(TAG, "RemoteException calling onContent");
219             }
220         }
221         if (surfacePackage != null) {
222             surfacePackage.release();
223         }
224         mRemoteInlineSuggestionViewConnector.onRender();
225     }
226 
handleOnClick()227     private void handleOnClick() {
228         // Autofill the value
229         mRemoteInlineSuggestionViewConnector.onClick();
230 
231         // Notify the remote process (IME) that hosts the embedded UI that it's clicked
232         if (mInlineContentCallback != null) {
233             try {
234                 mInlineContentCallback.onClick();
235             } catch (RemoteException e) {
236                 Slog.w(TAG, "RemoteException calling onClick");
237             }
238         }
239     }
240 
handleOnLongClick()241     private void handleOnLongClick() {
242         // Notify the remote process (IME) that hosts the embedded UI that it's long clicked
243         if (mInlineContentCallback != null) {
244             try {
245                 mInlineContentCallback.onLongClick();
246             } catch (RemoteException e) {
247                 Slog.w(TAG, "RemoteException calling onLongClick");
248             }
249         }
250     }
251 
handleOnError()252     private void handleOnError() {
253         mRemoteInlineSuggestionViewConnector.onError();
254     }
255 
handleOnTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId)256     private void handleOnTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) {
257         mRemoteInlineSuggestionViewConnector.onTransferTouchFocusToImeWindow(sourceInputToken,
258                 displayId);
259     }
260 
handleOnStartIntentSender(IntentSender intentSender)261     private void handleOnStartIntentSender(IntentSender intentSender) {
262         mRemoteInlineSuggestionViewConnector.onStartIntentSender(intentSender);
263     }
264 
265     /**
266      * Responsible for communicating with the inline suggestion view owning process.
267      */
268     private class InlineSuggestionUiCallbackImpl extends IInlineSuggestionUiCallback.Stub {
269 
270         @Override
onClick()271         public void onClick() {
272             mHandler.post(RemoteInlineSuggestionUi.this::handleOnClick);
273         }
274 
275         @Override
onLongClick()276         public void onLongClick() {
277             mHandler.post(RemoteInlineSuggestionUi.this::handleOnLongClick);
278         }
279 
280         @Override
onContent(IInlineSuggestionUi content, SurfaceControlViewHost.SurfacePackage surface, int width, int height)281         public void onContent(IInlineSuggestionUi content,
282                 SurfaceControlViewHost.SurfacePackage surface, int width, int height) {
283             mHandler.post(() -> handleInlineSuggestionUiReady(content, surface, width, height));
284         }
285 
286         @Override
onError()287         public void onError() {
288             mHandler.post(RemoteInlineSuggestionUi.this::handleOnError);
289         }
290 
291         @Override
onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId)292         public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) {
293             mHandler.post(() -> handleOnTransferTouchFocusToImeWindow(sourceInputToken, displayId));
294         }
295 
296         @Override
onStartIntentSender(IntentSender intentSender)297         public void onStartIntentSender(IntentSender intentSender) {
298             mHandler.post(() -> handleOnStartIntentSender(intentSender));
299         }
300     }
301 }
302