1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.systemui.classifier; 18 19 import static com.android.systemui.classifier.Classifier.BACK_GESTURE; 20 import static com.android.systemui.classifier.Classifier.GENERIC; 21 import static com.android.systemui.classifier.Classifier.MEDIA_SEEKBAR; 22 import static com.android.systemui.classifier.FalsingManagerProxy.FALSING_SUCCESS; 23 import static com.android.systemui.classifier.FalsingModule.BRIGHT_LINE_GESTURE_CLASSIFERS; 24 25 import android.net.Uri; 26 import android.os.Build; 27 import android.util.IndentingPrintWriter; 28 import android.util.Log; 29 import android.view.accessibility.AccessibilityManager; 30 31 import androidx.annotation.NonNull; 32 33 import com.android.internal.logging.MetricsLogger; 34 import com.android.systemui.classifier.FalsingDataProvider.SessionListener; 35 import com.android.systemui.classifier.HistoryTracker.BeliefListener; 36 import com.android.systemui.dagger.qualifiers.TestHarness; 37 import com.android.systemui.flags.FeatureFlags; 38 import com.android.systemui.flags.Flags; 39 import com.android.systemui.plugins.FalsingManager; 40 import com.android.systemui.statusbar.policy.KeyguardStateController; 41 42 import java.io.PrintWriter; 43 import java.util.ArrayDeque; 44 import java.util.ArrayList; 45 import java.util.Collection; 46 import java.util.Collections; 47 import java.util.List; 48 import java.util.Queue; 49 import java.util.Set; 50 import java.util.StringJoiner; 51 import java.util.stream.Collectors; 52 53 import javax.inject.Inject; 54 import javax.inject.Named; 55 56 /** 57 * FalsingManager designed to make clear why a touch was rejected. 58 */ 59 public class BrightLineFalsingManager implements FalsingManager { 60 61 private static final String TAG = "FalsingManager"; 62 public static final boolean DEBUG = Log.isLoggable(TAG, Log.DEBUG); 63 64 private static final int RECENT_INFO_LOG_SIZE = 40; 65 private static final int RECENT_SWIPE_LOG_SIZE = 20; 66 private static final double TAP_CONFIDENCE_THRESHOLD = 0.7; 67 private static final double FALSE_BELIEF_THRESHOLD = 0.9; 68 69 private final FalsingDataProvider mDataProvider; 70 private final LongTapClassifier mLongTapClassifier; 71 private final SingleTapClassifier mSingleTapClassifier; 72 private final DoubleTapClassifier mDoubleTapClassifier; 73 private final HistoryTracker mHistoryTracker; 74 private final KeyguardStateController mKeyguardStateController; 75 private AccessibilityManager mAccessibilityManager; 76 private final boolean mTestHarness; 77 private final MetricsLogger mMetricsLogger; 78 private int mIsFalseTouchCalls; 79 private FeatureFlags mFeatureFlags; 80 private static final Queue<String> RECENT_INFO_LOG = 81 new ArrayDeque<>(RECENT_INFO_LOG_SIZE + 1); 82 private static final Queue<DebugSwipeRecord> RECENT_SWIPES = 83 new ArrayDeque<>(RECENT_SWIPE_LOG_SIZE + 1); 84 85 private final Collection<FalsingClassifier> mClassifiers; 86 private final List<FalsingBeliefListener> mFalsingBeliefListeners = new ArrayList<>(); 87 private List<FalsingTapListener> mFalsingTapListeners = new ArrayList<>(); 88 private ProximityEvent mLastProximityEvent; 89 90 private boolean mDestroyed; 91 92 private final SessionListener mSessionListener = new SessionListener() { 93 @Override 94 public void onSessionEnded() { 95 mLastProximityEvent = null; 96 mHistoryTracker.removeBeliefListener(mBeliefListener); 97 mClassifiers.forEach(FalsingClassifier::onSessionEnded); 98 } 99 100 @Override 101 public void onSessionStarted() { 102 mHistoryTracker.addBeliefListener(mBeliefListener); 103 mClassifiers.forEach(FalsingClassifier::onSessionStarted); 104 } 105 }; 106 107 private final BeliefListener mBeliefListener = new BeliefListener() { 108 @Override 109 public void onBeliefChanged(double belief) { 110 logInfo(String.format( 111 "{belief=%s confidence=%s}", 112 mHistoryTracker.falseBelief(), 113 mHistoryTracker.falseConfidence())); 114 if (belief > FALSE_BELIEF_THRESHOLD) { 115 mFalsingBeliefListeners.forEach(FalsingBeliefListener::onFalse); 116 logInfo("Triggering False Event (Threshold: " + FALSE_BELIEF_THRESHOLD + ")"); 117 } 118 } 119 }; 120 121 private final FalsingDataProvider.GestureFinalizedListener mGestureFinalizedListener = 122 new FalsingDataProvider.GestureFinalizedListener() { 123 @Override 124 public void onGestureFinalized(long completionTimeMs) { 125 if (mPriorResults != null) { 126 boolean boolResult = mPriorResults.stream().anyMatch( 127 FalsingClassifier.Result::isFalse); 128 129 mPriorResults.forEach(result -> { 130 if (result.isFalse()) { 131 String reason = result.getReason(); 132 if (reason != null) { 133 logInfo(reason); 134 } 135 } 136 }); 137 138 if (Build.IS_ENG || Build.IS_USERDEBUG) { 139 // Copy motion events, as the results returned by 140 // #getRecentMotionEvents are recycled elsewhere. 141 RECENT_SWIPES.add(new DebugSwipeRecord( 142 boolResult, 143 mPriorInteractionType, 144 mDataProvider.getRecentMotionEvents().stream().map( 145 motionEvent -> new XYDt( 146 (int) motionEvent.getX(), 147 (int) motionEvent.getY(), 148 (int) (motionEvent.getEventTime() 149 - motionEvent.getDownTime()))) 150 .collect(Collectors.toList()))); 151 while (RECENT_SWIPES.size() > RECENT_INFO_LOG_SIZE) { 152 RECENT_SWIPES.remove(); 153 } 154 } 155 156 157 mHistoryTracker.addResults(mPriorResults, completionTimeMs); 158 mPriorResults = null; 159 mPriorInteractionType = Classifier.GENERIC; 160 } else { 161 // Gestures that were not classified get treated as a false. 162 // Gestures that look like simple taps are less likely to be false 163 // than swipes. They may simply be mis-clicks. 164 double penalty = mSingleTapClassifier.isTap( 165 mDataProvider.getRecentMotionEvents(), 0).isFalse() 166 ? 0.7 : 0.8; 167 mHistoryTracker.addResults( 168 Collections.singleton( 169 FalsingClassifier.Result.falsed( 170 penalty, getClass().getSimpleName(), 171 "unclassified")), 172 completionTimeMs); 173 } 174 } 175 }; 176 177 private Collection<FalsingClassifier.Result> mPriorResults; 178 private @Classifier.InteractionType int mPriorInteractionType = Classifier.GENERIC; 179 180 @Inject BrightLineFalsingManager( FalsingDataProvider falsingDataProvider, MetricsLogger metricsLogger, @Named(BRIGHT_LINE_GESTURE_CLASSIFERS) Set<FalsingClassifier> classifiers, SingleTapClassifier singleTapClassifier, LongTapClassifier longTapClassifier, DoubleTapClassifier doubleTapClassifier, HistoryTracker historyTracker, KeyguardStateController keyguardStateController, AccessibilityManager accessibilityManager, @TestHarness boolean testHarness, FeatureFlags featureFlags)181 public BrightLineFalsingManager( 182 FalsingDataProvider falsingDataProvider, 183 MetricsLogger metricsLogger, 184 @Named(BRIGHT_LINE_GESTURE_CLASSIFERS) Set<FalsingClassifier> classifiers, 185 SingleTapClassifier singleTapClassifier, LongTapClassifier longTapClassifier, 186 DoubleTapClassifier doubleTapClassifier, HistoryTracker historyTracker, 187 KeyguardStateController keyguardStateController, 188 AccessibilityManager accessibilityManager, 189 @TestHarness boolean testHarness, 190 FeatureFlags featureFlags) { 191 mDataProvider = falsingDataProvider; 192 mMetricsLogger = metricsLogger; 193 mClassifiers = classifiers; 194 mSingleTapClassifier = singleTapClassifier; 195 mLongTapClassifier = longTapClassifier; 196 mDoubleTapClassifier = doubleTapClassifier; 197 mHistoryTracker = historyTracker; 198 mKeyguardStateController = keyguardStateController; 199 mAccessibilityManager = accessibilityManager; 200 mTestHarness = testHarness; 201 mFeatureFlags = featureFlags; 202 203 mDataProvider.addSessionListener(mSessionListener); 204 mDataProvider.addGestureCompleteListener(mGestureFinalizedListener); 205 } 206 207 @Override isClassifierEnabled()208 public boolean isClassifierEnabled() { 209 return true; 210 } 211 212 @Override isFalseTouch(@lassifier.InteractionType int interactionType)213 public boolean isFalseTouch(@Classifier.InteractionType int interactionType) { 214 checkDestroyed(); 215 216 mPriorInteractionType = interactionType; 217 if (skipFalsing(interactionType)) { 218 mPriorResults = getPassedResult(1); 219 logDebug("Skipped falsing"); 220 return false; 221 } 222 223 final boolean[] localResult = {false}; 224 mPriorResults = mClassifiers.stream().map(falsingClassifier -> { 225 FalsingClassifier.Result r = falsingClassifier.classifyGesture( 226 interactionType, 227 mHistoryTracker.falseBelief(), 228 mHistoryTracker.falseConfidence()); 229 localResult[0] |= r.isFalse(); 230 231 return r; 232 }).collect(Collectors.toList()); 233 234 // check for false tap if it is a seekbar interaction 235 if (interactionType == MEDIA_SEEKBAR) { 236 localResult[0] &= isFalseTap(FalsingManager.MODERATE_PENALTY); 237 } 238 239 logDebug("False Gesture (type: " + interactionType + "): " + localResult[0]); 240 241 return localResult[0]; 242 } 243 244 @Override isSimpleTap()245 public boolean isSimpleTap() { 246 checkDestroyed(); 247 248 FalsingClassifier.Result result = mSingleTapClassifier.isTap( 249 mDataProvider.getRecentMotionEvents(), 0); 250 mPriorResults = Collections.singleton(result); 251 252 return !result.isFalse(); 253 } 254 checkDestroyed()255 private void checkDestroyed() { 256 if (mDestroyed) { 257 Log.wtf(TAG, "Tried to use FalsingManager after being destroyed!"); 258 } 259 } 260 261 @Override isFalseTap(@enalty int penalty)262 public boolean isFalseTap(@Penalty int penalty) { 263 checkDestroyed(); 264 265 if (skipFalsing(GENERIC)) { 266 mPriorResults = getPassedResult(1); 267 logDebug("Skipped falsing"); 268 return false; 269 } 270 271 double falsePenalty = 0; 272 switch(penalty) { 273 case NO_PENALTY: 274 falsePenalty = 0; 275 break; 276 case LOW_PENALTY: 277 falsePenalty = 0.1; 278 break; 279 case MODERATE_PENALTY: 280 falsePenalty = 0.3; 281 break; 282 case HIGH_PENALTY: 283 falsePenalty = 0.6; 284 break; 285 } 286 287 FalsingClassifier.Result singleTapResult = 288 mSingleTapClassifier.isTap(mDataProvider.getRecentMotionEvents().isEmpty() 289 ? mDataProvider.getPriorMotionEvents() 290 : mDataProvider.getRecentMotionEvents(), falsePenalty); 291 mPriorResults = Collections.singleton(singleTapResult); 292 293 if (!singleTapResult.isFalse()) { 294 if (mDataProvider.isJustUnlockedWithFace()) { 295 // Immediately pass if a face is detected. 296 mPriorResults = getPassedResult(1); 297 logDebug("False Single Tap: false (face detected)"); 298 return false; 299 } else if (!isFalseDoubleTap()) { 300 // We must check double tapping before other heuristics. This is because 301 // the double tap will fail if there's only been one tap. We don't want that 302 // failure to be recorded in mPriorResults. 303 logDebug("False Single Tap: false (double tapped)"); 304 return false; 305 } else if (mHistoryTracker.falseBelief() > TAP_CONFIDENCE_THRESHOLD) { 306 mPriorResults = Collections.singleton( 307 FalsingClassifier.Result.falsed( 308 0, getClass().getSimpleName(), "bad history")); 309 logDebug("False Single Tap: true (bad history)"); 310 mFalsingTapListeners.forEach(FalsingTapListener::onAdditionalTapRequired); 311 return true; 312 } else { 313 mPriorResults = getPassedResult(0.1); 314 logDebug("False Single Tap: false (default)"); 315 return false; 316 } 317 318 } else { 319 logDebug("False Single Tap: " + singleTapResult.isFalse() + " (simple)"); 320 return singleTapResult.isFalse(); 321 } 322 323 } 324 325 @Override isFalseLongTap(@enalty int penalty)326 public boolean isFalseLongTap(@Penalty int penalty) { 327 checkDestroyed(); 328 329 if (skipFalsing(GENERIC)) { 330 mPriorResults = getPassedResult(1); 331 logDebug("Skipped falsing"); 332 return false; 333 } 334 335 double falsePenalty = 0; 336 switch(penalty) { 337 case NO_PENALTY: 338 falsePenalty = 0; 339 break; 340 case LOW_PENALTY: 341 falsePenalty = 0.1; 342 break; 343 case MODERATE_PENALTY: 344 falsePenalty = 0.3; 345 break; 346 case HIGH_PENALTY: 347 falsePenalty = 0.6; 348 break; 349 } 350 351 FalsingClassifier.Result longTapResult = 352 mLongTapClassifier.isTap(mDataProvider.getRecentMotionEvents().isEmpty() 353 ? mDataProvider.getPriorMotionEvents() 354 : mDataProvider.getRecentMotionEvents(), falsePenalty); 355 mPriorResults = Collections.singleton(longTapResult); 356 357 if (!longTapResult.isFalse()) { 358 if (mDataProvider.isJustUnlockedWithFace()) { 359 // Immediately pass if a face is detected. 360 mPriorResults = getPassedResult(1); 361 logDebug("False Long Tap: false (face detected)"); 362 } else { 363 mPriorResults = getPassedResult(0.1); 364 logDebug("False Long Tap: false (default)"); 365 } 366 return false; 367 } else { 368 logDebug("False Long Tap: " + longTapResult.isFalse() + " (simple)"); 369 return longTapResult.isFalse(); 370 } 371 } 372 373 @Override isFalseDoubleTap()374 public boolean isFalseDoubleTap() { 375 checkDestroyed(); 376 377 if (skipFalsing(GENERIC)) { 378 mPriorResults = getPassedResult(1); 379 logDebug("Skipped falsing"); 380 return false; 381 } 382 383 FalsingClassifier.Result result = mDoubleTapClassifier.classifyGesture( 384 Classifier.GENERIC, 385 mHistoryTracker.falseBelief(), 386 mHistoryTracker.falseConfidence()); 387 mPriorResults = Collections.singleton(result); 388 logDebug("False Double Tap: " + result.isFalse() + " reason=" + result.getReason()); 389 return result.isFalse(); 390 } 391 skipFalsing(@lassifier.InteractionType int interactionType)392 private boolean skipFalsing(@Classifier.InteractionType int interactionType) { 393 return interactionType == BACK_GESTURE 394 || !mKeyguardStateController.isShowing() 395 || mTestHarness 396 || mDataProvider.isJustUnlockedWithFace() 397 || mDataProvider.isDocked() 398 || mAccessibilityManager.isTouchExplorationEnabled() 399 || mDataProvider.isA11yAction() 400 || mDataProvider.isFromTrackpad() 401 || mDataProvider.isFromKeyboard() 402 || (mFeatureFlags.isEnabled(Flags.FALSING_OFF_FOR_UNFOLDED) 403 && mDataProvider.isUnfolded()); 404 } 405 406 @Override onProximityEvent(ProximityEvent proximityEvent)407 public void onProximityEvent(ProximityEvent proximityEvent) { 408 // TODO: some of these classifiers might allow us to abort early, meaning we don't have to 409 // make these calls. 410 mLastProximityEvent = proximityEvent; 411 mClassifiers.forEach((classifier) -> classifier.onProximityEvent(proximityEvent)); 412 } 413 414 @Override onSuccessfulUnlock()415 public void onSuccessfulUnlock() { 416 if (mIsFalseTouchCalls != 0) { 417 mMetricsLogger.histogram(FALSING_SUCCESS, mIsFalseTouchCalls); 418 mIsFalseTouchCalls = 0; 419 } 420 } 421 422 @Override isProximityNear()423 public boolean isProximityNear() { 424 return mLastProximityEvent != null && mLastProximityEvent.getCovered(); 425 } 426 427 @Override isUnlockingDisabled()428 public boolean isUnlockingDisabled() { 429 return false; 430 } 431 432 @Override shouldEnforceBouncer()433 public boolean shouldEnforceBouncer() { 434 return false; 435 } 436 437 @Override reportRejectedTouch()438 public Uri reportRejectedTouch() { 439 return null; 440 } 441 442 @Override isReportingEnabled()443 public boolean isReportingEnabled() { 444 return false; 445 } 446 447 @Override addFalsingBeliefListener(FalsingBeliefListener listener)448 public void addFalsingBeliefListener(FalsingBeliefListener listener) { 449 mFalsingBeliefListeners.add(listener); 450 } 451 452 @Override removeFalsingBeliefListener(FalsingBeliefListener listener)453 public void removeFalsingBeliefListener(FalsingBeliefListener listener) { 454 mFalsingBeliefListeners.remove(listener); 455 } 456 457 @Override addTapListener(FalsingTapListener listener)458 public void addTapListener(FalsingTapListener listener) { 459 mFalsingTapListeners.add(listener); 460 } 461 462 @Override removeTapListener(FalsingTapListener listener)463 public void removeTapListener(FalsingTapListener listener) { 464 mFalsingTapListeners.remove(listener); 465 } 466 467 @Override dump(@onNull PrintWriter pw, @NonNull String[] args)468 public void dump(@NonNull PrintWriter pw, @NonNull String[] args) { 469 IndentingPrintWriter ipw = new IndentingPrintWriter(pw, " "); 470 ipw.println("BRIGHTLINE FALSING MANAGER"); 471 ipw.print("classifierEnabled="); 472 ipw.println(isClassifierEnabled() ? 1 : 0); 473 ipw.print("mJustUnlockedWithFace="); 474 ipw.println(mDataProvider.isJustUnlockedWithFace() ? 1 : 0); 475 ipw.print("isDocked="); 476 ipw.println(mDataProvider.isDocked() ? 1 : 0); 477 ipw.print("width="); 478 ipw.println(mDataProvider.getWidthPixels()); 479 ipw.print("height="); 480 ipw.println(mDataProvider.getHeightPixels()); 481 ipw.println(); 482 if (RECENT_SWIPES.size() != 0) { 483 ipw.println("Recent swipes:"); 484 ipw.increaseIndent(); 485 for (DebugSwipeRecord record : RECENT_SWIPES) { 486 ipw.println(record.getString()); 487 ipw.println(); 488 } 489 ipw.decreaseIndent(); 490 } else { 491 ipw.println("No recent swipes"); 492 } 493 ipw.println(); 494 ipw.println("Recent falsing info:"); 495 ipw.increaseIndent(); 496 for (String msg : RECENT_INFO_LOG) { 497 ipw.println(msg); 498 } 499 ipw.println(); 500 } 501 502 @Override cleanupInternal()503 public void cleanupInternal() { 504 mDestroyed = true; 505 mDataProvider.removeSessionListener(mSessionListener); 506 mDataProvider.removeGestureCompleteListener(mGestureFinalizedListener); 507 mClassifiers.forEach(FalsingClassifier::cleanup); 508 mFalsingBeliefListeners.clear(); 509 mHistoryTracker.removeBeliefListener(mBeliefListener); 510 } 511 getPassedResult(double confidence)512 private static Collection<FalsingClassifier.Result> getPassedResult(double confidence) { 513 return Collections.singleton(FalsingClassifier.Result.passed(confidence)); 514 } 515 logDebug(String msg)516 static void logDebug(String msg) { 517 logDebug(msg, null); 518 } 519 logDebug(String msg, Throwable throwable)520 static void logDebug(String msg, Throwable throwable) { 521 if (DEBUG) { 522 Log.d(TAG, msg, throwable); 523 } 524 } 525 logVerbose(String msg)526 static void logVerbose(String msg) { 527 if (DEBUG) { 528 Log.v(TAG, msg); 529 } 530 } 531 logInfo(String msg)532 static void logInfo(String msg) { 533 Log.i(TAG, msg); 534 RECENT_INFO_LOG.add(msg); 535 while (RECENT_INFO_LOG.size() > RECENT_INFO_LOG_SIZE) { 536 RECENT_INFO_LOG.remove(); 537 } 538 } 539 logError(String msg)540 static void logError(String msg) { 541 Log.e(TAG, msg); 542 } 543 544 private static class DebugSwipeRecord { 545 private static final byte VERSION = 1; // opaque version number indicating format of data. 546 private final boolean mIsFalse; 547 private final int mInteractionType; 548 private final List<XYDt> mRecentMotionEvents; 549 DebugSwipeRecord(boolean isFalse, int interactionType, List<XYDt> recentMotionEvents)550 DebugSwipeRecord(boolean isFalse, int interactionType, 551 List<XYDt> recentMotionEvents) { 552 mIsFalse = isFalse; 553 mInteractionType = interactionType; 554 mRecentMotionEvents = recentMotionEvents; 555 } 556 getString()557 String getString() { 558 StringJoiner sj = new StringJoiner(","); 559 sj.add(Integer.toString(VERSION)) 560 .add(mIsFalse ? "1" : "0") 561 .add(Integer.toString(mInteractionType)); 562 for (XYDt event : mRecentMotionEvents) { 563 sj.add(event.toString()); 564 } 565 return sj.toString(); 566 } 567 } 568 569 private static class XYDt { 570 private final int mX; 571 private final int mY; 572 private final int mDT; 573 XYDt(int x, int y, int dT)574 XYDt(int x, int y, int dT) { 575 mX = x; 576 mY = y; 577 mDT = dT; 578 } 579 580 @Override toString()581 public String toString() { 582 return mX + "," + mY + "," + mDT; 583 } 584 } 585 } 586