1 /*
2  * Copyright 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server.wm;
18 
19 import static android.graphics.Matrix.MSCALE_X;
20 import static android.graphics.Matrix.MSCALE_Y;
21 import static android.graphics.Matrix.MSKEW_X;
22 import static android.graphics.Matrix.MSKEW_Y;
23 
24 import static com.android.internal.protolog.ProtoLogGroup.WM_DEBUG_TPL;
25 
26 import android.graphics.Matrix;
27 import android.graphics.Rect;
28 import android.graphics.RectF;
29 import android.graphics.Region;
30 import android.os.Handler;
31 import android.os.HandlerThread;
32 import android.os.IBinder;
33 import android.os.RemoteException;
34 import android.util.ArrayMap;
35 import android.util.IntArray;
36 import android.util.Pair;
37 import android.util.Size;
38 import android.view.InputWindowHandle;
39 import android.window.ITrustedPresentationListener;
40 import android.window.TrustedPresentationThresholds;
41 import android.window.WindowInfosListener;
42 
43 import com.android.internal.protolog.common.ProtoLog;
44 import com.android.server.wm.utils.RegionUtils;
45 
46 import java.io.PrintWriter;
47 import java.util.ArrayList;
48 import java.util.Optional;
49 
50 /**
51  * Class to handle TrustedPresentationListener registrations in a thread safe manner. This class
52  * also takes care of cleaning up listeners when the remote process dies.
53  */
54 public class TrustedPresentationListenerController {
55 
56     // Should only be accessed by the posting to the handler
57     private class Listeners {
58         private final class ListenerDeathRecipient implements IBinder.DeathRecipient {
59             IBinder mListenerBinder;
60             int mInstances;
61 
ListenerDeathRecipient(IBinder listenerBinder)62             ListenerDeathRecipient(IBinder listenerBinder) {
63                 mListenerBinder = listenerBinder;
64                 mInstances = 0;
65                 try {
66                     mListenerBinder.linkToDeath(this, 0);
67                 } catch (RemoteException ignore) {
68                 }
69             }
70 
addInstance()71             void addInstance() {
72                 mInstances++;
73             }
74 
75             // return true if there are no instances alive
removeInstance()76             boolean removeInstance() {
77                 mInstances--;
78                 if (mInstances > 0) {
79                     return false;
80                 }
81                 mListenerBinder.unlinkToDeath(this, 0);
82                 return true;
83             }
84 
binderDied()85             public void binderDied() {
86                 mHandler.post(() -> {
87                     mUniqueListeners.remove(mListenerBinder);
88                     removeListeners(mListenerBinder, Optional.empty());
89                 });
90             }
91         }
92 
93         // tracks binder deaths for cleanup
94         ArrayMap<IBinder, ListenerDeathRecipient> mUniqueListeners = new ArrayMap<>();
95         ArrayMap<IBinder /*window*/, ArrayList<TrustedPresentationInfo>> mWindowToListeners =
96                 new ArrayMap<>();
97 
register(IBinder window, ITrustedPresentationListener listener, TrustedPresentationThresholds thresholds, int id)98         void register(IBinder window, ITrustedPresentationListener listener,
99                 TrustedPresentationThresholds thresholds, int id) {
100             var listenersForWindow = mWindowToListeners.computeIfAbsent(window,
101                     iBinder -> new ArrayList<>());
102             listenersForWindow.add(new TrustedPresentationInfo(thresholds, id, listener));
103 
104             // register death listener
105             var listenerBinder = listener.asBinder();
106             var deathRecipient = mUniqueListeners.computeIfAbsent(listenerBinder,
107                     ListenerDeathRecipient::new);
108             deathRecipient.addInstance();
109         }
110 
unregister(ITrustedPresentationListener trustedPresentationListener, int id)111         void unregister(ITrustedPresentationListener trustedPresentationListener, int id) {
112             var listenerBinder = trustedPresentationListener.asBinder();
113             var deathRecipient = mUniqueListeners.get(listenerBinder);
114             if (deathRecipient == null) {
115                 ProtoLog.e(WM_DEBUG_TPL, "unregister failed, couldn't find"
116                         + " deathRecipient for %s with id=%d", trustedPresentationListener, id);
117                 return;
118             }
119 
120             if (deathRecipient.removeInstance()) {
121                 mUniqueListeners.remove(listenerBinder);
122             }
123             removeListeners(listenerBinder, Optional.of(id));
124         }
125 
isEmpty()126         boolean isEmpty() {
127             return mWindowToListeners.isEmpty();
128         }
129 
get(IBinder windowToken)130         ArrayList<TrustedPresentationInfo> get(IBinder windowToken) {
131             return mWindowToListeners.get(windowToken);
132         }
133 
removeListeners(IBinder listenerBinder, Optional<Integer> id)134         private void removeListeners(IBinder listenerBinder, Optional<Integer> id) {
135             for (int i = mWindowToListeners.size() - 1; i >= 0; i--) {
136                 var listeners = mWindowToListeners.valueAt(i);
137                 for (int j = listeners.size() - 1; j >= 0; j--) {
138                     var listener = listeners.get(j);
139                     if (listener.mListener.asBinder() == listenerBinder && (id.isEmpty()
140                             || listener.mId == id.get())) {
141                         listeners.remove(j);
142                     }
143                 }
144                 if (listeners.isEmpty()) {
145                     mWindowToListeners.removeAt(i);
146                 }
147             }
148         }
149     }
150 
151     private final Object mHandlerThreadLock = new Object();
152     private HandlerThread mHandlerThread;
153     private Handler mHandler;
154 
155     private WindowInfosListener mWindowInfosListener;
156 
157     Listeners mRegisteredListeners = new Listeners();
158 
159     private InputWindowHandle[] mLastWindowHandles;
160 
startHandlerThreadIfNeeded()161     private void startHandlerThreadIfNeeded() {
162         synchronized (mHandlerThreadLock) {
163             if (mHandler == null) {
164                 mHandlerThread = new HandlerThread("WindowInfosListenerForTpl");
165                 mHandlerThread.start();
166                 mHandler = new Handler(mHandlerThread.getLooper());
167             }
168         }
169     }
170 
registerListener(IBinder window, ITrustedPresentationListener listener, TrustedPresentationThresholds thresholds, int id)171     void registerListener(IBinder window, ITrustedPresentationListener listener,
172             TrustedPresentationThresholds thresholds, int id) {
173         startHandlerThreadIfNeeded();
174         mHandler.post(() -> {
175             ProtoLog.d(WM_DEBUG_TPL, "Registering listener=%s with id=%d for window=%s with %s",
176                     listener, id, window, thresholds);
177 
178             mRegisteredListeners.register(window, listener, thresholds, id);
179             registerWindowInfosListener();
180             // Update the initial state for the new registered listener
181             computeTpl(mLastWindowHandles);
182         });
183     }
184 
unregisterListener(ITrustedPresentationListener listener, int id)185     void unregisterListener(ITrustedPresentationListener listener, int id) {
186         startHandlerThreadIfNeeded();
187         mHandler.post(() -> {
188             ProtoLog.d(WM_DEBUG_TPL, "Unregistering listener=%s with id=%d",
189                     listener, id);
190 
191             mRegisteredListeners.unregister(listener, id);
192             if (mRegisteredListeners.isEmpty()) {
193                 unregisterWindowInfosListener();
194             }
195         });
196     }
197 
dump(PrintWriter pw)198     void dump(PrintWriter pw) {
199         final String innerPrefix = "  ";
200         pw.println("TrustedPresentationListenerController:");
201         pw.println(innerPrefix + "Active unique listeners ("
202                 + mRegisteredListeners.mUniqueListeners.size() + "):");
203         for (int i = 0; i < mRegisteredListeners.mWindowToListeners.size(); i++) {
204             pw.println(
205                     innerPrefix + "  window=" + mRegisteredListeners.mWindowToListeners.keyAt(i));
206             final var listeners = mRegisteredListeners.mWindowToListeners.valueAt(i);
207             for (int j = 0; j < listeners.size(); j++) {
208                 final var listener = listeners.get(j);
209                 pw.println(innerPrefix + innerPrefix + "  listener=" + listener.mListener.asBinder()
210                         + " id=" + listener.mId
211                         + " thresholds=" + listener.mThresholds);
212             }
213         }
214     }
215 
registerWindowInfosListener()216     private void registerWindowInfosListener() {
217         if (mWindowInfosListener != null) {
218             return;
219         }
220 
221         mWindowInfosListener = new WindowInfosListener() {
222             @Override
223             public void onWindowInfosChanged(InputWindowHandle[] windowHandles,
224                     DisplayInfo[] displayInfos) {
225                 mHandler.post(() -> computeTpl(windowHandles));
226             }
227         };
228         mLastWindowHandles = mWindowInfosListener.register().first;
229     }
230 
unregisterWindowInfosListener()231     private void unregisterWindowInfosListener() {
232         if (mWindowInfosListener == null) {
233             return;
234         }
235 
236         mWindowInfosListener.unregister();
237         mWindowInfosListener = null;
238         mLastWindowHandles = null;
239     }
240 
computeTpl(InputWindowHandle[] windowHandles)241     private void computeTpl(InputWindowHandle[] windowHandles) {
242         mLastWindowHandles = windowHandles;
243         if (mLastWindowHandles == null || mLastWindowHandles.length == 0
244                 || mRegisteredListeners.isEmpty()) {
245             return;
246         }
247 
248         Rect tmpRect = new Rect();
249         Matrix tmpInverseMatrix = new Matrix();
250         float[] tmpMatrix = new float[9];
251         Region coveredRegionsAbove = new Region();
252         long currTimeMs = System.currentTimeMillis();
253         ProtoLog.v(WM_DEBUG_TPL, "Checking %d windows", mLastWindowHandles.length);
254 
255         ArrayMap<ITrustedPresentationListener, Pair<IntArray, IntArray>> listenerUpdates =
256                 new ArrayMap<>();
257         for (var windowHandle : mLastWindowHandles) {
258             if (!windowHandle.canOccludePresentation) {
259                 ProtoLog.v(WM_DEBUG_TPL, "Skipping %s", windowHandle.name);
260                 continue;
261             }
262             tmpRect.set(windowHandle.frame);
263             var listeners = mRegisteredListeners.get(windowHandle.getWindowToken());
264             if (listeners != null) {
265                 Region region = new Region();
266                 region.op(tmpRect, coveredRegionsAbove, Region.Op.DIFFERENCE);
267                 windowHandle.transform.invert(tmpInverseMatrix);
268                 tmpInverseMatrix.getValues(tmpMatrix);
269                 float scaleX = (float) Math.sqrt(tmpMatrix[MSCALE_X] * tmpMatrix[MSCALE_X]
270                         + tmpMatrix[MSKEW_X] * tmpMatrix[MSKEW_X]);
271                 float scaleY = (float) Math.sqrt(tmpMatrix[MSCALE_Y] * tmpMatrix[MSCALE_Y]
272                         + tmpMatrix[MSKEW_Y] * tmpMatrix[MSKEW_Y]);
273 
274                 float fractionRendered = computeFractionRendered(region, new RectF(tmpRect),
275                         windowHandle.contentSize,
276                         scaleX, scaleY);
277 
278                 checkIfInThreshold(listeners, listenerUpdates, fractionRendered, windowHandle.alpha,
279                         currTimeMs);
280             }
281 
282             coveredRegionsAbove.op(tmpRect, Region.Op.UNION);
283             ProtoLog.v(WM_DEBUG_TPL, "coveredRegionsAbove updated with %s frame:%s region:%s",
284                     windowHandle.name, tmpRect.toShortString(), coveredRegionsAbove);
285         }
286 
287         for (int i = 0; i < listenerUpdates.size(); i++) {
288             var updates = listenerUpdates.valueAt(i);
289             var listener = listenerUpdates.keyAt(i);
290             try {
291                 listener.onTrustedPresentationChanged(updates.first.toArray(),
292                         updates.second.toArray());
293             } catch (RemoteException ignore) {
294             }
295         }
296     }
297 
addListenerUpdate( ArrayMap<ITrustedPresentationListener, Pair<IntArray, IntArray>> listenerUpdates, ITrustedPresentationListener listener, int id, boolean presentationState)298     private void addListenerUpdate(
299             ArrayMap<ITrustedPresentationListener, Pair<IntArray, IntArray>> listenerUpdates,
300             ITrustedPresentationListener listener, int id, boolean presentationState) {
301         var updates = listenerUpdates.get(listener);
302         if (updates == null) {
303             updates = new Pair<>(new IntArray(), new IntArray());
304             listenerUpdates.put(listener, updates);
305         }
306         if (presentationState) {
307             updates.first.add(id);
308         } else {
309             updates.second.add(id);
310         }
311     }
312 
313 
checkIfInThreshold( ArrayList<TrustedPresentationInfo> listeners, ArrayMap<ITrustedPresentationListener, Pair<IntArray, IntArray>> listenerUpdates, float fractionRendered, float alpha, long currTimeMs)314     private void checkIfInThreshold(
315             ArrayList<TrustedPresentationInfo> listeners,
316             ArrayMap<ITrustedPresentationListener, Pair<IntArray, IntArray>> listenerUpdates,
317             float fractionRendered, float alpha, long currTimeMs) {
318         ProtoLog.v(WM_DEBUG_TPL, "checkIfInThreshold fractionRendered=%f alpha=%f currTimeMs=%d",
319                 fractionRendered, alpha, currTimeMs);
320         for (int i = 0; i < listeners.size(); i++) {
321             var trustedPresentationInfo = listeners.get(i);
322             var listener = trustedPresentationInfo.mListener;
323             boolean lastState = trustedPresentationInfo.mLastComputedTrustedPresentationState;
324             boolean newState =
325                     (alpha >= trustedPresentationInfo.mThresholds.getMinAlpha())
326                             && (fractionRendered >= trustedPresentationInfo.mThresholds
327                                     .getMinFractionRendered());
328             trustedPresentationInfo.mLastComputedTrustedPresentationState = newState;
329 
330             ProtoLog.v(WM_DEBUG_TPL,
331                     "lastState=%s newState=%s alpha=%f minAlpha=%f fractionRendered=%f "
332                             + "minFractionRendered=%f",
333                     lastState, newState, alpha, trustedPresentationInfo.mThresholds.getMinAlpha(),
334                     fractionRendered, trustedPresentationInfo.mThresholds
335                             .getMinFractionRendered());
336 
337             if (lastState && !newState) {
338                 // We were in the trusted presentation state, but now we left it,
339                 // emit the callback if needed
340                 if (trustedPresentationInfo.mLastReportedTrustedPresentationState) {
341                     trustedPresentationInfo.mLastReportedTrustedPresentationState = false;
342                     addListenerUpdate(listenerUpdates, listener,
343                             trustedPresentationInfo.mId, /*presentationState*/ false);
344                     ProtoLog.d(WM_DEBUG_TPL, "Adding untrusted state listener=%s with id=%d",
345                             listener, trustedPresentationInfo.mId);
346                 }
347                 // Reset the timer
348                 trustedPresentationInfo.mEnteredTrustedPresentationStateTime = -1;
349             } else if (!lastState && newState) {
350                 // We were not in the trusted presentation state, but we entered it, begin the timer
351                 // and make sure this gets called at least once more!
352                 trustedPresentationInfo.mEnteredTrustedPresentationStateTime = currTimeMs;
353                 mHandler.postDelayed(() -> {
354                     computeTpl(mLastWindowHandles);
355                 }, (long) (trustedPresentationInfo.mThresholds
356                             .getStabilityRequirementMillis() * 1.5));
357             }
358 
359             // Has the timer elapsed, but we are still in the state? Emit a callback if needed
360             if (!trustedPresentationInfo.mLastReportedTrustedPresentationState && newState && (
361                     currTimeMs - trustedPresentationInfo.mEnteredTrustedPresentationStateTime
362                             > trustedPresentationInfo.mThresholds
363                                         .getStabilityRequirementMillis())) {
364                 trustedPresentationInfo.mLastReportedTrustedPresentationState = true;
365                 addListenerUpdate(listenerUpdates, listener,
366                         trustedPresentationInfo.mId, /*presentationState*/ true);
367                 ProtoLog.d(WM_DEBUG_TPL, "Adding trusted state listener=%s with id=%d",
368                         listener, trustedPresentationInfo.mId);
369             }
370         }
371     }
372 
computeFractionRendered(Region visibleRegion, RectF screenBounds, Size contentSize, float sx, float sy)373     private float computeFractionRendered(Region visibleRegion, RectF screenBounds,
374             Size contentSize,
375             float sx, float sy) {
376         ProtoLog.v(WM_DEBUG_TPL,
377                 "computeFractionRendered: visibleRegion=%s screenBounds=%s contentSize=%s "
378                         + "scale=%f,%f",
379                 visibleRegion, screenBounds, contentSize, sx, sy);
380 
381         if (contentSize.getWidth() == 0 || contentSize.getHeight() == 0) {
382             return -1;
383         }
384         if (screenBounds.width() == 0 || screenBounds.height() == 0) {
385             return -1;
386         }
387 
388         float fractionRendered = Math.min(sx * sy, 1.0f);
389         ProtoLog.v(WM_DEBUG_TPL, "fractionRendered scale=%f", fractionRendered);
390 
391         float boundsOverSourceW = screenBounds.width() / (float) contentSize.getWidth();
392         float boundsOverSourceH = screenBounds.height() / (float) contentSize.getHeight();
393         fractionRendered *= boundsOverSourceW * boundsOverSourceH;
394         ProtoLog.v(WM_DEBUG_TPL, "fractionRendered boundsOverSource=%f", fractionRendered);
395         // Compute the size of all the rects since they may be disconnected.
396         float[] visibleSize = new float[1];
397         RegionUtils.forEachRect(visibleRegion, rect -> {
398             float size = rect.width() * rect.height();
399             visibleSize[0] += size;
400         });
401 
402         fractionRendered *= visibleSize[0] / (screenBounds.width() * screenBounds.height());
403         return fractionRendered;
404     }
405 
406     private static class TrustedPresentationInfo {
407         boolean mLastComputedTrustedPresentationState = false;
408         boolean mLastReportedTrustedPresentationState = false;
409         long mEnteredTrustedPresentationStateTime = -1;
410         final TrustedPresentationThresholds mThresholds;
411 
412         final ITrustedPresentationListener mListener;
413         final int mId;
414 
TrustedPresentationInfo(TrustedPresentationThresholds thresholds, int id, ITrustedPresentationListener listener)415         private TrustedPresentationInfo(TrustedPresentationThresholds thresholds, int id,
416                 ITrustedPresentationListener listener) {
417             mThresholds = thresholds;
418             mId = id;
419             mListener = listener;
420         }
421     }
422 }
423