1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.wm; 18 19 import static com.android.internal.util.DumpUtils.KeyDumper; 20 import static com.android.internal.util.DumpUtils.ValueDumper; 21 import static com.android.internal.util.DumpUtils.dumpSparseArray; 22 import static com.android.server.wm.utils.RegionUtils.forEachRect; 23 24 import android.annotation.NonNull; 25 import android.annotation.Nullable; 26 import android.graphics.Matrix; 27 import android.graphics.Rect; 28 import android.graphics.RectF; 29 import android.graphics.Region; 30 import android.os.Handler; 31 import android.os.IBinder; 32 import android.os.InputConfig; 33 import android.os.Looper; 34 import android.os.Message; 35 import android.util.Pair; 36 import android.util.Slog; 37 import android.util.SparseArray; 38 import android.view.InputWindowHandle; 39 import android.view.MagnificationSpec; 40 import android.view.WindowInfo; 41 import android.view.WindowManager; 42 import android.window.WindowInfosListener; 43 44 import com.android.internal.annotations.GuardedBy; 45 46 import java.io.PrintWriter; 47 import java.util.ArrayList; 48 import java.util.HashMap; 49 import java.util.List; 50 import java.util.Map; 51 52 /** 53 * This class is the accessibility windows population adapter. 54 */ 55 public final class AccessibilityWindowsPopulator extends WindowInfosListener { 56 57 private static final String TAG = AccessibilityWindowsPopulator.class.getSimpleName(); 58 // If the surface flinger callback is not coming within in 2 frames time, i.e. about 59 // 35ms, then assuming the windows become stable. 60 private static final int SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS = 35; 61 // To avoid the surface flinger callbacks always comes within in 2 frames, then no windows 62 // are reported to the A11y framework, and the animation duration time is 500ms, so setting 63 // this value as the max timeout value to force computing changed windows. However, since 64 // UiAutomator waits 500ms to determine that things are idle. Since we aren't actually idle, 65 // we need to reduce the timeout here a little so that we can deliver an updated state before 66 // UiAutomator reports idle based-on stale information. 67 private static final int WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS = 450; 68 69 private static final float[] sTempFloats = new float[9]; 70 71 private final WindowManagerService mService; 72 private final AccessibilityController mAccessibilityController; 73 @GuardedBy("mLock") 74 private final SparseArray<List<InputWindowHandle>> mInputWindowHandlesOnDisplays = 75 new SparseArray<>(); 76 @GuardedBy("mLock") 77 private final SparseArray<Matrix> mMagnificationSpecInverseMatrix = new SparseArray<>(); 78 @GuardedBy("mLock") 79 private final SparseArray<DisplayInfo> mDisplayInfos = new SparseArray<>(); 80 private final SparseArray<MagnificationSpec> mCurrentMagnificationSpec = new SparseArray<>(); 81 @GuardedBy("mLock") 82 private final SparseArray<MagnificationSpec> mPreviousMagnificationSpec = new SparseArray<>(); 83 @GuardedBy("mLock") 84 private final List<InputWindowHandle> mVisibleWindows = new ArrayList<>(); 85 @GuardedBy("mLock") 86 private boolean mWindowsNotificationEnabled = false; 87 @GuardedBy("mLock") 88 private final Map<IBinder, Matrix> mWindowsTransformMatrixMap = new HashMap<>(); 89 private final Object mLock = new Object(); 90 private final Handler mHandler; 91 92 private final Matrix mTempMatrix1 = new Matrix(); 93 private final Matrix mTempMatrix2 = new Matrix(); 94 private final float[] mTempFloat1 = new float[9]; 95 private final float[] mTempFloat2 = new float[9]; 96 private final float[] mTempFloat3 = new float[9]; 97 AccessibilityWindowsPopulator(WindowManagerService service, AccessibilityController accessibilityController)98 AccessibilityWindowsPopulator(WindowManagerService service, 99 AccessibilityController accessibilityController) { 100 mService = service; 101 mAccessibilityController = accessibilityController; 102 mHandler = new MyHandler(mService.mH.getLooper()); 103 } 104 105 /** 106 * Gets the visible windows list with the window layer on the specified display. 107 * 108 * @param displayId The display. 109 * @param outWindows The visible windows list. The z-order of each window in the list 110 * is from the top to bottom. 111 */ populateVisibleWindowsOnScreenLocked(int displayId, List<AccessibilityWindow> outWindows)112 public void populateVisibleWindowsOnScreenLocked(int displayId, 113 List<AccessibilityWindow> outWindows) { 114 List<InputWindowHandle> inputWindowHandles; 115 final Matrix inverseMatrix = new Matrix(); 116 final Matrix displayMatrix = new Matrix(); 117 118 synchronized (mLock) { 119 inputWindowHandles = mInputWindowHandlesOnDisplays.get(displayId); 120 if (inputWindowHandles == null) { 121 outWindows.clear(); 122 123 return; 124 } 125 inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId)); 126 127 final DisplayInfo displayInfo = mDisplayInfos.get(displayId); 128 if (displayInfo != null) { 129 displayMatrix.set(displayInfo.mTransform); 130 } else { 131 Slog.w(TAG, "The displayInfo of this displayId (" + displayId + ") called " 132 + "back from the surface fligner is null"); 133 } 134 } 135 136 final DisplayContent dc = mService.mRoot.getDisplayContent(displayId); 137 final ShellRoot shellroot = dc.mShellRoots.get(WindowManager.SHELL_ROOT_LAYER_PIP); 138 final IBinder pipMenuIBinder = 139 shellroot != null ? shellroot.getAccessibilityWindowToken() : null; 140 141 for (final InputWindowHandle windowHandle : inputWindowHandles) { 142 final AccessibilityWindow accessibilityWindow = 143 AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix, 144 pipMenuIBinder, displayMatrix); 145 146 outWindows.add(accessibilityWindow); 147 } 148 } 149 150 @Override onWindowInfosChanged(InputWindowHandle[] windowHandles, DisplayInfo[] displayInfos)151 public void onWindowInfosChanged(InputWindowHandle[] windowHandles, 152 DisplayInfo[] displayInfos) { 153 if (com.android.server.accessibility.Flags.removeOnWindowInfosChangedHandler()) { 154 onWindowInfosChangedInternal(windowHandles, displayInfos); 155 } else { 156 mHandler.post(() -> onWindowInfosChangedInternal(windowHandles, displayInfos)); 157 } 158 } 159 onWindowInfosChangedInternal(InputWindowHandle[] windowHandles, DisplayInfo[] displayInfos)160 private void onWindowInfosChangedInternal(InputWindowHandle[] windowHandles, 161 DisplayInfo[] displayInfos) { 162 final List<InputWindowHandle> tempVisibleWindows = new ArrayList<>(); 163 164 for (InputWindowHandle window : windowHandles) { 165 final boolean visible = (window.inputConfig & InputConfig.NOT_VISIBLE) == 0; 166 final boolean isNotClone = (window.inputConfig & InputConfig.CLONE) == 0; 167 final boolean hasTouchableRegion = !window.touchableRegion.isEmpty(); 168 final boolean hasNonEmptyFrame = !window.frame.isEmpty(); 169 if (visible && isNotClone && hasTouchableRegion && hasNonEmptyFrame) { 170 tempVisibleWindows.add(window); 171 } 172 } 173 final HashMap<IBinder, Matrix> windowsTransformMatrixMap = 174 getWindowsTransformMatrix(tempVisibleWindows); 175 176 synchronized (mLock) { 177 mWindowsTransformMatrixMap.clear(); 178 mWindowsTransformMatrixMap.putAll(windowsTransformMatrixMap); 179 180 mVisibleWindows.clear(); 181 mVisibleWindows.addAll(tempVisibleWindows); 182 183 mDisplayInfos.clear(); 184 for (final DisplayInfo displayInfo : displayInfos) { 185 mDisplayInfos.put(displayInfo.mDisplayId, displayInfo); 186 } 187 188 if (mWindowsNotificationEnabled) { 189 if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { 190 mHandler.sendEmptyMessageDelayed( 191 MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT, 192 WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS); 193 } 194 populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded(); 195 } 196 } 197 } 198 getWindowsTransformMatrix(List<InputWindowHandle> windows)199 private HashMap<IBinder, Matrix> getWindowsTransformMatrix(List<InputWindowHandle> windows) { 200 synchronized (mService.mGlobalLock) { 201 final HashMap<IBinder, Matrix> windowsTransformMatrixMap = new HashMap<>(); 202 203 for (InputWindowHandle inputWindowHandle : windows) { 204 final IBinder iWindow = inputWindowHandle.getWindowToken(); 205 final WindowState windowState = iWindow != null ? mService.mWindowMap.get(iWindow) 206 : null; 207 208 if (windowState != null && windowState.shouldMagnify()) { 209 final Matrix transformMatrix = new Matrix(); 210 windowState.getTransformationMatrix(sTempFloats, transformMatrix); 211 windowsTransformMatrixMap.put(iWindow, transformMatrix); 212 } 213 } 214 215 return windowsTransformMatrixMap; 216 } 217 } 218 219 /** 220 * Sets to notify the accessibilityController to compute changed windows on 221 * the display after populating the visible windows if the windows reported 222 * from the surface flinger changes. 223 * 224 * @param register {@code true} means starting windows population. 225 */ setWindowsNotification(boolean register)226 public void setWindowsNotification(boolean register) { 227 synchronized (mLock) { 228 if (mWindowsNotificationEnabled == register) { 229 return; 230 } 231 mWindowsNotificationEnabled = register; 232 if (mWindowsNotificationEnabled) { 233 Pair<InputWindowHandle[], DisplayInfo[]> info = register(); 234 onWindowInfosChangedInternal(info.first, info.second); 235 } else { 236 unregister(); 237 releaseResources(); 238 } 239 } 240 } 241 242 /** 243 * Sets the magnification spec for calculating the window bounds of all windows 244 * reported from the surface flinger in the magnifying. 245 * 246 * @param displayId The display Id. 247 * @param spec THe magnification spec. 248 */ setMagnificationSpec(int displayId, MagnificationSpec spec)249 public void setMagnificationSpec(int displayId, MagnificationSpec spec) { 250 synchronized (mLock) { 251 MagnificationSpec currentMagnificationSpec = mCurrentMagnificationSpec.get(displayId); 252 if (currentMagnificationSpec == null) { 253 currentMagnificationSpec = new MagnificationSpec(); 254 currentMagnificationSpec.setTo(spec); 255 mCurrentMagnificationSpec.put(displayId, currentMagnificationSpec); 256 257 return; 258 } 259 260 MagnificationSpec previousMagnificationSpec = mPreviousMagnificationSpec.get(displayId); 261 if (previousMagnificationSpec == null) { 262 previousMagnificationSpec = new MagnificationSpec(); 263 mPreviousMagnificationSpec.put(displayId, previousMagnificationSpec); 264 } 265 previousMagnificationSpec.setTo(currentMagnificationSpec); 266 currentMagnificationSpec.setTo(spec); 267 } 268 } 269 270 @GuardedBy("mLock") populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded()271 private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded() { 272 final SparseArray<List<InputWindowHandle>> tempWindowHandleList = new SparseArray<>(); 273 274 for (final InputWindowHandle windowHandle : mVisibleWindows) { 275 List<InputWindowHandle> inputWindowHandles = tempWindowHandleList.get( 276 windowHandle.displayId); 277 278 if (inputWindowHandles == null) { 279 inputWindowHandles = new ArrayList<>(); 280 tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles); 281 } 282 inputWindowHandles.add(windowHandle); 283 } 284 findMagnificationSpecInverseMatrixIfNeeded(tempWindowHandleList); 285 286 final List<Integer> displayIdsForWindowsChanged = new ArrayList<>(); 287 getDisplaysForWindowsChanged(displayIdsForWindowsChanged, tempWindowHandleList, 288 mInputWindowHandlesOnDisplays); 289 290 // Clones all windows from the callback of the surface flinger. 291 mInputWindowHandlesOnDisplays.clear(); 292 for (int i = 0; i < tempWindowHandleList.size(); i++) { 293 final int displayId = tempWindowHandleList.keyAt(i); 294 mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId)); 295 } 296 297 if (!displayIdsForWindowsChanged.isEmpty()) { 298 if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { 299 mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, 300 displayIdsForWindowsChanged).sendToTarget(); 301 } 302 303 return; 304 } 305 mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); 306 mHandler.sendEmptyMessageDelayed(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE, 307 SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS); 308 } 309 310 @GuardedBy("mLock") getDisplaysForWindowsChanged(List<Integer> outDisplayIdsForWindowsChanged, SparseArray<List<InputWindowHandle>> newWindowsList, SparseArray<List<InputWindowHandle>> oldWindowsList)311 private static void getDisplaysForWindowsChanged(List<Integer> outDisplayIdsForWindowsChanged, 312 SparseArray<List<InputWindowHandle>> newWindowsList, 313 SparseArray<List<InputWindowHandle>> oldWindowsList) { 314 for (int i = 0; i < newWindowsList.size(); i++) { 315 final int displayId = newWindowsList.keyAt(i); 316 final List<InputWindowHandle> newWindows = newWindowsList.get(displayId); 317 final List<InputWindowHandle> oldWindows = oldWindowsList.get(displayId); 318 319 if (hasWindowsChanged(newWindows, oldWindows)) { 320 outDisplayIdsForWindowsChanged.add(displayId); 321 } 322 } 323 } 324 325 @GuardedBy("mLock") hasWindowsChanged(List<InputWindowHandle> newWindows, List<InputWindowHandle> oldWindows)326 private static boolean hasWindowsChanged(List<InputWindowHandle> newWindows, 327 List<InputWindowHandle> oldWindows) { 328 if (oldWindows == null || oldWindows.size() != newWindows.size()) { 329 return true; 330 } 331 332 final int windowsCount = newWindows.size(); 333 // Since we always traverse windows from high to low layer, 334 // the old and new windows at the same index should be the 335 // same, otherwise something changed. 336 for (int i = 0; i < windowsCount; i++) { 337 final IBinder newWindowToken = newWindows.get(i).getWindowToken(); 338 final IBinder oldWindowToken = oldWindows.get(i).getWindowToken(); 339 final boolean hasNewWindowToken = newWindowToken != null; 340 final boolean hasOldWindowToken = oldWindowToken != null; 341 342 // If window token presence has changed then the windows have changed. 343 if (hasNewWindowToken != hasOldWindowToken) { 344 return true; 345 } 346 347 // If both old and new windows had window tokens, but those tokens differ, 348 // then the windows have changed. 349 if (hasNewWindowToken && hasOldWindowToken && !newWindowToken.equals(oldWindowToken)) { 350 return true; 351 } 352 } 353 354 return false; 355 } 356 357 @GuardedBy("mLock") findMagnificationSpecInverseMatrixIfNeeded(SparseArray<List<InputWindowHandle>> windowHandleList)358 private void findMagnificationSpecInverseMatrixIfNeeded(SparseArray<List<InputWindowHandle>> 359 windowHandleList) { 360 MagnificationSpec currentMagnificationSpec; 361 MagnificationSpec previousMagnificationSpec; 362 for (int i = 0; i < windowHandleList.size(); i++) { 363 final int displayId = windowHandleList.keyAt(i); 364 List<InputWindowHandle> inputWindowHandles = windowHandleList.get(displayId); 365 366 final MagnificationSpec currentSpec = mCurrentMagnificationSpec.get(displayId); 367 if (currentSpec == null) { 368 continue; 369 } 370 currentMagnificationSpec = new MagnificationSpec(); 371 currentMagnificationSpec.setTo(currentSpec); 372 373 final MagnificationSpec previousSpec = mPreviousMagnificationSpec.get(displayId); 374 375 if (previousSpec == null) { 376 final Matrix inverseMatrixForCurrentSpec = new Matrix(); 377 generateInverseMatrix(currentMagnificationSpec, inverseMatrixForCurrentSpec); 378 mMagnificationSpecInverseMatrix.put(displayId, inverseMatrixForCurrentSpec); 379 continue; 380 } 381 previousMagnificationSpec = new MagnificationSpec(); 382 previousMagnificationSpec.setTo(previousSpec); 383 384 generateInverseMatrixBasedOnProperMagnificationSpecForDisplay(inputWindowHandles, 385 currentMagnificationSpec, previousMagnificationSpec); 386 } 387 } 388 389 @GuardedBy("mLock") generateInverseMatrixBasedOnProperMagnificationSpecForDisplay( List<InputWindowHandle> inputWindowHandles, MagnificationSpec currentMagnificationSpec, MagnificationSpec previousMagnificationSpec)390 private void generateInverseMatrixBasedOnProperMagnificationSpecForDisplay( 391 List<InputWindowHandle> inputWindowHandles, MagnificationSpec currentMagnificationSpec, 392 MagnificationSpec previousMagnificationSpec) { 393 // To decrease the counts of holding the WindowManagerService#mGlogalLock in 394 // the method, getWindowTransformMatrix(), this for loop begins from the bottom 395 // to top of the z-order windows. 396 for (int index = inputWindowHandles.size() - 1; index >= 0; index--) { 397 final Matrix windowTransformMatrix = mTempMatrix2; 398 final InputWindowHandle windowHandle = inputWindowHandles.get(index); 399 final IBinder iBinder = windowHandle.getWindowToken(); 400 if (getWindowTransformMatrix(iBinder, windowTransformMatrix)) { 401 generateMagnificationSpecInverseMatrix(windowHandle, currentMagnificationSpec, 402 previousMagnificationSpec, windowTransformMatrix); 403 404 break; 405 } 406 } 407 } 408 409 @GuardedBy("mLock") getWindowTransformMatrix(IBinder iBinder, Matrix outTransform)410 private boolean getWindowTransformMatrix(IBinder iBinder, Matrix outTransform) { 411 final Matrix windowMatrix = iBinder != null 412 ? mWindowsTransformMatrixMap.get(iBinder) : null; 413 414 if (windowMatrix == null) { 415 return false; 416 } 417 outTransform.set(windowMatrix); 418 419 return true; 420 } 421 422 /** 423 * Generates the inverse matrix based on the proper magnification spec. 424 * The magnification spec associated with the InputWindowHandle might not the current 425 * spec set by WM, which might be the previous one. To find the appropriate spec, 426 * we store two consecutive magnification specs, and found out which one is the proper 427 * one closing the identity matrix for generating the inverse matrix. 428 * 429 * @param inputWindowHandle The window from the surface flinger. 430 * @param currentMagnificationSpec The current magnification spec. 431 * @param previousMagnificationSpec The previous magnification spec. 432 * @param transformMatrix The transform matrix of the window doesn't consider the 433 * magnifying effect. 434 */ 435 @GuardedBy("mLock") generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle, @NonNull MagnificationSpec currentMagnificationSpec, @NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix)436 private void generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle, 437 @NonNull MagnificationSpec currentMagnificationSpec, 438 @NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix) { 439 440 final float[] identityMatrixFloatsForCurrentSpec = mTempFloat1; 441 computeIdentityMatrix(inputWindowHandle, currentMagnificationSpec, 442 transformMatrix, identityMatrixFloatsForCurrentSpec); 443 final float[] identityMatrixFloatsForPreviousSpec = mTempFloat2; 444 computeIdentityMatrix(inputWindowHandle, previousMagnificationSpec, 445 transformMatrix, identityMatrixFloatsForPreviousSpec); 446 447 Matrix inverseMatrixForMagnificationSpec = new Matrix(); 448 if (selectProperMagnificationSpecByComparingIdentityDegree( 449 identityMatrixFloatsForCurrentSpec, identityMatrixFloatsForPreviousSpec)) { 450 generateInverseMatrix(currentMagnificationSpec, 451 inverseMatrixForMagnificationSpec); 452 453 // Choosing the current spec means the previous spec is out of date, 454 // so removing it. And if the current spec is no magnifying, meaning 455 // the magnifying is done so removing the inverse matrix of this display. 456 mPreviousMagnificationSpec.remove(inputWindowHandle.displayId); 457 if (currentMagnificationSpec.isNop()) { 458 mCurrentMagnificationSpec.remove(inputWindowHandle.displayId); 459 mMagnificationSpecInverseMatrix.remove(inputWindowHandle.displayId); 460 return; 461 } 462 } else { 463 generateInverseMatrix(previousMagnificationSpec, 464 inverseMatrixForMagnificationSpec); 465 } 466 467 mMagnificationSpecInverseMatrix.put(inputWindowHandle.displayId, 468 inverseMatrixForMagnificationSpec); 469 } 470 471 /** 472 * Computes the identity matrix for generating the 473 * inverse matrix based on below formula under window is at the stable state: 474 * inputWindowHandle#transform * MagnificationSpecMatrix * WindowState#transform 475 * = IdentityMatrix 476 */ 477 @GuardedBy("mLock") computeIdentityMatrix(InputWindowHandle inputWindowHandle, @NonNull MagnificationSpec magnificationSpec, Matrix transformMatrix, float[] magnifyMatrixFloats)478 private void computeIdentityMatrix(InputWindowHandle inputWindowHandle, 479 @NonNull MagnificationSpec magnificationSpec, 480 Matrix transformMatrix, float[] magnifyMatrixFloats) { 481 final Matrix specMatrix = mTempMatrix1; 482 transformMagnificationSpecToMatrix(magnificationSpec, specMatrix); 483 484 final Matrix resultMatrix = new Matrix(inputWindowHandle.transform); 485 resultMatrix.preConcat(specMatrix); 486 resultMatrix.preConcat(transformMatrix); 487 488 resultMatrix.getValues(magnifyMatrixFloats); 489 } 490 491 /** 492 * @return true if selecting the magnification spec one, otherwise selecting the 493 * magnification spec two. 494 */ 495 @GuardedBy("mLock") selectProperMagnificationSpecByComparingIdentityDegree( float[] magnifyMatrixFloatsForSpecOne, float[] magnifyMatrixFloatsForSpecTwo)496 private boolean selectProperMagnificationSpecByComparingIdentityDegree( 497 float[] magnifyMatrixFloatsForSpecOne, 498 float[] magnifyMatrixFloatsForSpecTwo) { 499 final float[] IdentityMatrixValues = mTempFloat3; 500 Matrix.IDENTITY_MATRIX.getValues(IdentityMatrixValues); 501 502 final float scaleDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] 503 - magnifyMatrixFloatsForSpecOne[Matrix.MSCALE_X]); 504 final float scaleDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] 505 - magnifyMatrixFloatsForSpecTwo[Matrix.MSCALE_X]); 506 final float offsetXDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] 507 - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_X]); 508 final float offsetXDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] 509 - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_X]); 510 final float offsetYDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] 511 - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_Y]); 512 final float offsetYDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] 513 - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_Y]); 514 final float offsetDiffForSpecOne = offsetXDiffForSpecOne 515 + offsetYDiffForSpecOne; 516 final float offsetDiffForSpecTwo = offsetXDiffForSpecTwo 517 + offsetYDiffForSpecTwo; 518 519 return Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) > 0 520 || (Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) == 0 521 && Float.compare(offsetDiffForSpecTwo, offsetDiffForSpecOne) > 0); 522 } 523 524 @GuardedBy("mLock") generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix)525 private static void generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix) { 526 outMatrix.reset(); 527 528 final Matrix tempMatrix = new Matrix(); 529 transformMagnificationSpecToMatrix(spec, tempMatrix); 530 531 final boolean result = tempMatrix.invert(outMatrix); 532 if (!result) { 533 Slog.e(TAG, "Can't inverse the magnification spec matrix with the " 534 + "magnification spec = " + spec); 535 outMatrix.reset(); 536 } 537 } 538 539 @GuardedBy("mLock") transformMagnificationSpecToMatrix(MagnificationSpec spec, Matrix outMatrix)540 private static void transformMagnificationSpecToMatrix(MagnificationSpec spec, 541 Matrix outMatrix) { 542 outMatrix.reset(); 543 outMatrix.postScale(spec.scale, spec.scale); 544 outMatrix.postTranslate(spec.offsetX, spec.offsetY); 545 } 546 notifyWindowsChanged(@onNull List<Integer> displayIdsForWindowsChanged)547 private void notifyWindowsChanged(@NonNull List<Integer> displayIdsForWindowsChanged) { 548 mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT); 549 550 for (int i = 0; i < displayIdsForWindowsChanged.size(); i++) { 551 mAccessibilityController.performComputeChangedWindowsNot( 552 displayIdsForWindowsChanged.get(i), false); 553 } 554 } 555 forceUpdateWindows()556 private void forceUpdateWindows() { 557 final List<Integer> displayIdsForWindowsChanged = new ArrayList<>(); 558 559 synchronized (mLock) { 560 for (int i = 0; i < mInputWindowHandlesOnDisplays.size(); i++) { 561 final int displayId = mInputWindowHandlesOnDisplays.keyAt(i); 562 displayIdsForWindowsChanged.add(displayId); 563 } 564 } 565 notifyWindowsChanged(displayIdsForWindowsChanged); 566 } 567 dump(PrintWriter pw, String prefix)568 void dump(PrintWriter pw, String prefix) { 569 synchronized (mLock) { 570 pw.print(prefix); pw.println("AccessibilityWindowsPopulator"); 571 String prefix2 = prefix + " "; 572 573 pw.print(prefix2); pw.print("mWindowsNotificationEnabled: "); 574 pw.println(mWindowsNotificationEnabled); 575 576 if (mVisibleWindows.isEmpty()) { 577 pw.print(prefix2); pw.println("No visible windows"); 578 } else { 579 pw.print(prefix2); pw.print(mVisibleWindows.size()); 580 pw.print(" visible windows: "); pw.println(mVisibleWindows); 581 } 582 KeyDumper noKeyDumper = (i, k) -> {}; // display id is already shown on value; 583 KeyDumper displayDumper = (i, d) -> pw.printf("%sDisplay #%d: ", prefix, d); 584 // Ideally magnificationSpecDumper should use spec.dump(pw), but there is no such method 585 ValueDumper<MagnificationSpec> magnificationSpecDumper = spec -> pw.print(spec); 586 587 dumpSparseArray(pw, prefix2, mDisplayInfos, 588 "display info", noKeyDumper, d -> pw.print(d)); 589 dumpSparseArray(pw, prefix2, mInputWindowHandlesOnDisplays, 590 "window handles on display", displayDumper, list -> pw.print(list)); 591 dumpSparseArray(pw, prefix2, mMagnificationSpecInverseMatrix, 592 "magnification spec matrix", noKeyDumper, matrix -> matrix.dump(pw)); 593 dumpSparseArray(pw, prefix2, mCurrentMagnificationSpec, 594 "current magnification spec", noKeyDumper, magnificationSpecDumper); 595 dumpSparseArray(pw, prefix2, mPreviousMagnificationSpec, 596 "previous magnification spec", noKeyDumper, magnificationSpecDumper); 597 } 598 } 599 600 @GuardedBy("mLock") releaseResources()601 private void releaseResources() { 602 mInputWindowHandlesOnDisplays.clear(); 603 mMagnificationSpecInverseMatrix.clear(); 604 mVisibleWindows.clear(); 605 mDisplayInfos.clear(); 606 mCurrentMagnificationSpec.clear(); 607 mPreviousMagnificationSpec.clear(); 608 mWindowsTransformMatrixMap.clear(); 609 mWindowsNotificationEnabled = false; 610 mHandler.removeCallbacksAndMessages(null); 611 } 612 613 private class MyHandler extends Handler { 614 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED = 1; 615 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE = 2; 616 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT = 3; 617 MyHandler(Looper looper)618 MyHandler(Looper looper) { 619 super(looper, null, false); 620 } 621 622 @Override handleMessage(Message message)623 public void handleMessage(Message message) { 624 switch (message.what) { 625 case MESSAGE_NOTIFY_WINDOWS_CHANGED: { 626 final List<Integer> displayIdsForWindowsChanged = (List<Integer>) message.obj; 627 notifyWindowsChanged(displayIdsForWindowsChanged); 628 } break; 629 630 case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE: { 631 forceUpdateWindows(); 632 } break; 633 634 case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT: { 635 Slog.w(TAG, "Windows change within in 2 frames continuously over 500 ms " 636 + "and notify windows changed immediately"); 637 mHandler.removeMessages( 638 MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); 639 640 forceUpdateWindows(); 641 } break; 642 } 643 } 644 } 645 646 /** 647 * This class represents information about a window from the 648 * surface flinger to the accessibility framework. 649 */ 650 public static class AccessibilityWindow { 651 // Data 652 private IBinder mWindow; 653 private int mDisplayId; 654 @WindowManager.LayoutParams.WindowType 655 private int mType; 656 @InputWindowHandle.InputConfigFlags 657 private int mInputConfig; 658 private int mPrivateFlags; 659 private boolean mIsPIPMenu; 660 private boolean mIsFocused; 661 private boolean mShouldMagnify; 662 private boolean mIgnoreDuetoRecentsAnimation; 663 private final Region mTouchableRegionInScreen = new Region(); 664 private final Region mTouchableRegionInWindow = new Region(); 665 private WindowInfo mWindowInfo; 666 private Rect mSystemBarInsetFrame = null; 667 668 669 /** 670 * Returns the instance after initializing the internal data. 671 * @param service The window manager service. 672 * @param inputWindowHandle The window from the surface flinger. 673 * @param magnificationInverseMatrix The magnification spec inverse matrix. 674 */ initializeData(WindowManagerService service, InputWindowHandle inputWindowHandle, Matrix magnificationInverseMatrix, IBinder pipIBinder, Matrix displayMatrix)675 public static AccessibilityWindow initializeData(WindowManagerService service, 676 InputWindowHandle inputWindowHandle, Matrix magnificationInverseMatrix, 677 IBinder pipIBinder, Matrix displayMatrix) { 678 final IBinder window = inputWindowHandle.getWindowToken(); 679 final WindowState windowState = window != null ? service.mWindowMap.get(window) : null; 680 681 final AccessibilityWindow instance = new AccessibilityWindow(); 682 683 instance.mWindow = window; 684 instance.mDisplayId = inputWindowHandle.displayId; 685 instance.mInputConfig = inputWindowHandle.inputConfig; 686 instance.mType = inputWindowHandle.layoutParamsType; 687 instance.mIsPIPMenu = window != null && window.equals(pipIBinder); 688 689 // TODO (b/199357848): gets the private flag of the window from other way. 690 instance.mPrivateFlags = windowState != null ? windowState.mAttrs.privateFlags : 0; 691 // TODO (b/199358208) : using new way to implement the focused window. 692 instance.mIsFocused = windowState != null && windowState.isFocused(); 693 instance.mShouldMagnify = windowState == null || windowState.shouldMagnify(); 694 695 final RecentsAnimationController controller = service.getRecentsAnimationController(); 696 instance.mIgnoreDuetoRecentsAnimation = windowState != null && controller != null 697 && controller.shouldIgnoreForAccessibility(windowState); 698 699 final Rect windowFrame = new Rect(inputWindowHandle.frame); 700 getTouchableRegionInWindow(instance.mShouldMagnify, inputWindowHandle.touchableRegion, 701 instance.mTouchableRegionInWindow, windowFrame, magnificationInverseMatrix, 702 displayMatrix); 703 getUnMagnifiedTouchableRegion(instance.mShouldMagnify, 704 inputWindowHandle.touchableRegion, instance.mTouchableRegionInScreen, 705 magnificationInverseMatrix, displayMatrix); 706 instance.mWindowInfo = windowState != null 707 ? windowState.getWindowInfo() : getWindowInfoForWindowlessWindows(instance); 708 709 // Compute the transform matrix that will transform bounds from the window 710 // coordinates to screen coordinates. 711 final Matrix inverseTransform = new Matrix(); 712 inputWindowHandle.transform.invert(inverseTransform); 713 inverseTransform.postConcat(displayMatrix); 714 inverseTransform.getValues(instance.mWindowInfo.mTransformMatrix); 715 716 // Compute the magnification spec matrix. 717 final Matrix magnificationSpecMatrix = new Matrix(); 718 if (instance.shouldMagnify() && magnificationInverseMatrix != null 719 && !magnificationInverseMatrix.isIdentity()) { 720 if (magnificationInverseMatrix.invert(magnificationSpecMatrix)) { 721 magnificationSpecMatrix.getValues(sTempFloats); 722 final MagnificationSpec spec = instance.mWindowInfo.mMagnificationSpec; 723 spec.scale = sTempFloats[Matrix.MSCALE_X]; 724 spec.offsetX = sTempFloats[Matrix.MTRANS_X]; 725 spec.offsetY = sTempFloats[Matrix.MTRANS_Y]; 726 } else { 727 Slog.w(TAG, "can't find spec"); 728 } 729 } 730 731 // Compute system bar insets frame if needed. 732 if (com.android.server.accessibility.Flags.computeWindowChangesOnA11yV2() 733 && windowState != null && instance.isUntouchableNavigationBar()) { 734 final InsetsSourceProvider provider = 735 windowState.getControllableInsetProvider(); 736 if (provider != null) { 737 instance.mSystemBarInsetFrame = provider.getSource().getFrame(); 738 } 739 } 740 return instance; 741 } 742 743 /** 744 * Returns the touchable region in the screen. 745 * @param outRegion The touchable region. 746 */ getTouchableRegionInScreen(Region outRegion)747 public void getTouchableRegionInScreen(Region outRegion) { 748 outRegion.set(mTouchableRegionInScreen); 749 } 750 751 /** 752 * Returns the touchable region in the window. 753 * @param outRegion The touchable region. 754 */ getTouchableRegionInWindow(Region outRegion)755 public void getTouchableRegionInWindow(Region outRegion) { 756 outRegion.set(mTouchableRegionInWindow); 757 } 758 759 /** 760 * @return the layout parameter type {@link android.view.WindowManager.LayoutParams#type}. 761 */ getType()762 public int getType() { 763 return mType; 764 } 765 766 /** 767 * @return the layout parameter private flag 768 * {@link android.view.WindowManager.LayoutParams#privateFlags}. 769 */ getPrivateFlag()770 public int getPrivateFlag() { 771 return mPrivateFlags; 772 } 773 774 /** 775 * @return the windowInfo {@link WindowInfo}. 776 */ getWindowInfo()777 public WindowInfo getWindowInfo() { 778 return mWindowInfo; 779 } 780 781 /** 782 * @return true if this window should be magnified. 783 */ shouldMagnify()784 public boolean shouldMagnify() { 785 return mShouldMagnify; 786 } 787 788 /** 789 * @return true if this window is focused. 790 */ isFocused()791 public boolean isFocused() { 792 return mIsFocused; 793 } 794 795 /** 796 * @return true if it's running the recent animation but not the target app. 797 */ ignoreRecentsAnimationForAccessibility()798 public boolean ignoreRecentsAnimationForAccessibility() { 799 return mIgnoreDuetoRecentsAnimation; 800 } 801 802 /** 803 * @return true if this window is the trusted overlay. 804 */ isTrustedOverlay()805 public boolean isTrustedOverlay() { 806 return (mInputConfig & InputConfig.TRUSTED_OVERLAY) != 0; 807 } 808 809 /** 810 * @return true if this window is touchable. 811 */ isTouchable()812 public boolean isTouchable() { 813 return (mInputConfig & InputConfig.NOT_TOUCHABLE) == 0; 814 } 815 816 /** 817 * @return true if this window is the navigation bar with the gesture mode. 818 */ isUntouchableNavigationBar()819 public boolean isUntouchableNavigationBar() { 820 if (mType != WindowManager.LayoutParams.TYPE_NAVIGATION_BAR) { 821 return false; 822 } 823 824 return mTouchableRegionInScreen.isEmpty(); 825 } 826 827 /** 828 * @return true if this window is PIP menu. 829 */ isPIPMenu()830 public boolean isPIPMenu() { 831 return mIsPIPMenu; 832 } 833 834 /** 835 * @return the system inset frame size if the window is untouchable navigation bar. 836 * Returns null otherwise. 837 */ 838 @Nullable getSystemBarInsetsFrame()839 public Rect getSystemBarInsetsFrame() { 840 return mSystemBarInsetFrame; 841 } 842 getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix)843 private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, 844 Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) { 845 // Some modal windows, like the activity with Theme.dialog, has the full screen 846 // as its touchable region, but its window frame is smaller than the touchable 847 // region. The region we report should be the touchable area in the window frame 848 // for the consistency and match developers expectation. 849 // So we need to make the intersection between the frame and touchable region to 850 // obtain the real touch region in the screen. 851 Region touchRegion = new Region(); 852 touchRegion.set(inRegion); 853 touchRegion.op(frame, Region.Op.INTERSECT); 854 855 getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix, 856 displayMatrix); 857 } 858 859 /** 860 * Gets the un-magnified touchable region. If this window can be magnified and magnifying, 861 * we will transform the input touchable region by applying the inverse matrix of the 862 * magnification spec to get the un-magnified touchable region. 863 * @param shouldMagnify The window can be magnified. 864 * @param inRegion The touchable region of this window. 865 * @param outRegion The un-magnified touchable region of this window. 866 * @param inverseMatrix The inverse matrix of the magnification spec. 867 * @param displayMatrix The display transform matrix which takes display coordinates to 868 * logical display coordinates. 869 */ getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, Region outRegion, Matrix inverseMatrix, Matrix displayMatrix)870 private static void getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, 871 Region outRegion, Matrix inverseMatrix, Matrix displayMatrix) { 872 if ((!shouldMagnify || inverseMatrix.isIdentity()) && displayMatrix.isIdentity()) { 873 outRegion.set(inRegion); 874 return; 875 } 876 877 forEachRect(inRegion, rect -> { 878 // Move to origin as all transforms are captured by the matrix. 879 RectF windowFrame = new RectF(rect); 880 881 displayMatrix.mapRect(windowFrame); 882 inverseMatrix.mapRect(windowFrame); 883 // Union all rects. 884 outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top, 885 (int) windowFrame.right, (int) windowFrame.bottom)); 886 }); 887 } 888 getWindowInfoForWindowlessWindows(AccessibilityWindow window)889 private static WindowInfo getWindowInfoForWindowlessWindows(AccessibilityWindow window) { 890 WindowInfo windowInfo = WindowInfo.obtain(); 891 windowInfo.displayId = window.mDisplayId; 892 windowInfo.type = window.mType; 893 windowInfo.token = window.mWindow; 894 windowInfo.hasFlagWatchOutsideTouch = (window.mInputConfig 895 & InputConfig.WATCH_OUTSIDE_TOUCH) != 0; 896 // Set it to true to be consistent with the legacy implementation. 897 windowInfo.inPictureInPicture = window.mIsPIPMenu; 898 return windowInfo; 899 } 900 901 @Override toString()902 public String toString() { 903 String windowToken = 904 mWindow != null ? mWindow.toString() : "(no window token)"; 905 return "A11yWindow=[" + windowToken 906 + ", displayId=" + mDisplayId 907 + ", inputConfig=0x" + Integer.toHexString(mInputConfig) 908 + ", type=" + mType 909 + ", privateFlag=0x" + Integer.toHexString(mPrivateFlags) 910 + ", focused=" + mIsFocused 911 + ", shouldMagnify=" + mShouldMagnify 912 + ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation 913 + ", isTrustedOverlay=" + isTrustedOverlay() 914 + ", regionInScreen=" + mTouchableRegionInScreen 915 + ", touchableRegion=" + mTouchableRegionInWindow 916 + ", isPIPMenu=" + mIsPIPMenu 917 + ", windowInfo=" + mWindowInfo 918 + "]"; 919 } 920 } 921 } 922