1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.voiceinteraction.cts.services; 18 19 import static android.Manifest.permission.BIND_HOTWORD_DETECTION_SERVICE; 20 import static android.Manifest.permission.BIND_VISUAL_QUERY_DETECTION_SERVICE; 21 import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; 22 import static android.Manifest.permission.MANAGE_HOTWORD_DETECTION; 23 import static android.Manifest.permission.RECORD_AUDIO; 24 import static android.voiceinteraction.cts.testcore.Helper.WAIT_EXPECTED_NO_CALL_TIMEOUT_IN_MS; 25 import static android.voiceinteraction.cts.testcore.Helper.WAIT_LONG_TIMEOUT_IN_MS; 26 import static android.voiceinteraction.cts.testcore.Helper.WAIT_TIMEOUT_IN_MS; 27 28 import static com.android.compatibility.common.util.SystemUtil.callWithShellPermissionIdentity; 29 import static com.android.compatibility.common.util.SystemUtil.runWithShellPermissionIdentity; 30 31 import android.os.Handler; 32 import android.os.HandlerThread; 33 import android.os.Looper; 34 import android.os.PersistableBundle; 35 import android.service.voice.AlwaysOnHotwordDetector; 36 import android.service.voice.HotwordDetectionService; 37 import android.service.voice.HotwordDetectionServiceFailure; 38 import android.service.voice.HotwordDetector; 39 import android.service.voice.HotwordRejectedResult; 40 import android.service.voice.SandboxedDetectionInitializer; 41 import android.service.voice.SoundTriggerFailure; 42 import android.service.voice.VisualQueryDetectedResult; 43 import android.service.voice.VisualQueryDetectionService; 44 import android.service.voice.VisualQueryDetectionServiceFailure; 45 import android.service.voice.VisualQueryDetector; 46 import android.service.voice.VoiceInteractionService; 47 import android.util.Log; 48 49 import androidx.annotation.NonNull; 50 import androidx.annotation.Nullable; 51 52 import java.util.ArrayList; 53 import java.util.Arrays; 54 import java.util.List; 55 import java.util.concurrent.CountDownLatch; 56 import java.util.concurrent.TimeUnit; 57 58 /** 59 * The {@link VoiceInteractionService} included a basic HotwordDetectionService for testing. 60 */ 61 public class CtsBasicVoiceInteractionService extends BaseVoiceInteractionService { 62 private static final String TAG = "CtsBasicVoiceInteractionService"; 63 64 private final Handler mHandler; 65 66 // The CountDownLatch waits for a service Availability change result 67 private CountDownLatch mAvailabilityChangeLatch; 68 // The CountDownLatch waits for a service detect or reject result 69 private CountDownLatch mOnDetectRejectLatch; 70 // The CountDownLatch waits for a service onError called 71 private CountDownLatch mOnErrorLatch; 72 // The CountDownLatch waits for a service onFailure called 73 private CountDownLatch mOnFailureLatch; 74 // The CountDownLatch waits for vqds 75 private CountDownLatch mOnQueryFinishRejectLatch; 76 // The CountDownLatch waits for a service onRecognitionPaused called 77 private CountDownLatch mOnRecognitionPausedLatch; 78 // The CountDownLatch waits for a service onRecognitionResumed called 79 private CountDownLatch mOnRecognitionResumedLatch; 80 // The CountDownLatch waits for a service onHotwordDetectionServiceRestarted called 81 private CountDownLatch mOnHotwordDetectionServiceRestartedLatch; 82 // The CountDownLatch waits for a service onVisualQueryDetectionServiceRestarted called 83 private CountDownLatch mOnVisualQueryDetectionServiceRestartedLatch; 84 85 private AlwaysOnHotwordDetector.EventPayload mDetectedResult; 86 private HotwordRejectedResult mRejectedResult; 87 private ArrayList<String> mStreamedQueries = new ArrayList<>(); 88 private ArrayList<byte[]> mStreamedAccessibilityData = new ArrayList(); 89 private String mCurrentQuery = ""; 90 private byte[] mCurrentAccessibilityData = new byte[0]; 91 private HotwordDetectionServiceFailure mHotwordDetectionServiceFailure = null; 92 private SoundTriggerFailure mSoundTriggerFailure = null; 93 private String mUnknownFailure = null; 94 95 private int mSoftwareOnDetectedCount = 0; 96 private int mDspOnDetectedCount = 0; 97 private int mDspOnRejectedCount = 0; 98 99 private boolean mVoiceActivationPermissionEnabled; 100 CtsBasicVoiceInteractionService()101 public CtsBasicVoiceInteractionService() { 102 HandlerThread handlerThread = new HandlerThread("CtsBasicVoiceInteractionService"); 103 handlerThread.start(); 104 mHandler = Handler.createAsync(handlerThread.getLooper()); 105 } 106 107 @Override resetState()108 public void resetState() { 109 super.resetState(); 110 mAvailabilityChangeLatch = null; 111 mOnDetectRejectLatch = null; 112 mOnErrorLatch = null; 113 mOnFailureLatch = null; 114 mOnQueryFinishRejectLatch = null; 115 mOnRecognitionPausedLatch = null; 116 mOnRecognitionResumedLatch = null; 117 mOnHotwordDetectionServiceRestartedLatch = null; 118 mDetectedResult = null; 119 mRejectedResult = null; 120 mStreamedQueries.clear(); 121 mCurrentQuery = ""; 122 mHotwordDetectionServiceFailure = null; 123 mSoundTriggerFailure = null; 124 mUnknownFailure = null; 125 mSoftwareOnDetectedCount = 0; 126 mDspOnDetectedCount = 0; 127 mDspOnRejectedCount = 0; 128 } 129 130 /** 131 * Returns the onDetected() callback count for the software detector. 132 */ getSoftwareOnDetectedCount()133 public int getSoftwareOnDetectedCount() { 134 return mSoftwareOnDetectedCount; 135 } 136 137 /** 138 * Returns the onDetected() callback count for the dsp detector. 139 */ getDspOnDetectedCount()140 public int getDspOnDetectedCount() { 141 return mDspOnDetectedCount; 142 } 143 144 /** 145 * Returns the onRejected() callback count for the dsp detector. 146 */ getDspOnRejectedCount()147 public int getDspOnRejectedCount() { 148 return mDspOnRejectedCount; 149 } 150 createAlwaysOnHotwordDetectorNoHotwordDetectionService(boolean useExecutor, boolean runOnMainThread)151 public void createAlwaysOnHotwordDetectorNoHotwordDetectionService(boolean useExecutor, 152 boolean runOnMainThread) { 153 Log.i(TAG, "createAlwaysOnHotwordDetectorNoHotwordDetectionService"); 154 mDetectorInitializedLatch = new CountDownLatch(1); 155 156 AlwaysOnHotwordDetector.Callback callback = 157 createAlwaysOnHotwordDetectorCallbackWithListeners(); 158 final Handler handler = runOnMainThread ? new Handler(Looper.getMainLooper()) : mHandler; 159 handler.post(() -> runWithShellPermissionIdentity(() -> { 160 mAlwaysOnHotwordDetector = callCreateAlwaysOnHotwordDetectorNoHotwordDetectionService( 161 callback, useExecutor); 162 if (mDetectorInitializedLatch != null) { 163 mDetectorInitializedLatch.countDown(); 164 } 165 }, MANAGE_HOTWORD_DETECTION, RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD)); 166 } 167 168 /** 169 * Create an AlwaysOnHotwordDetector, but it will not implement the onFailure method of 170 * AlwaysOnHotwordDetector.Callback. It will implement the onFailure method by using 171 * createAlwaysOnHotwordDetectorWithOnFailureCallback method. 172 */ createAlwaysOnHotwordDetector()173 public void createAlwaysOnHotwordDetector() { 174 createAlwaysOnHotwordDetector(/* useExecutor= */ false, /* runOnMainThread= */ false); 175 } 176 177 /** 178 * Create an AlwaysOnHotwordDetector, but it will not implement the onFailure method of 179 * AlwaysOnHotwordDetector.Callback. It will implement the onFailure method by using 180 * createAlwaysOnHotwordDetectorWithOnFailureCallback method. 181 */ createAlwaysOnHotwordDetector(boolean useExecutor, boolean runOnMainThread)182 public void createAlwaysOnHotwordDetector(boolean useExecutor, boolean runOnMainThread) { 183 createAlwaysOnHotwordDetector(useExecutor, runOnMainThread, /* options= */ null); 184 } 185 186 /** 187 * Create an AlwaysOnHotwordDetector, but it will not implement the onFailure method of 188 * AlwaysOnHotwordDetector.Callback. It will implement the onFailure method by using 189 * createAlwaysOnHotwordDetectorWithOnFailureCallback method. 190 */ createAlwaysOnHotwordDetector(boolean useExecutor, boolean runOnMainThread, @Nullable PersistableBundle options)191 public void createAlwaysOnHotwordDetector(boolean useExecutor, boolean runOnMainThread, 192 @Nullable PersistableBundle options) { 193 Log.i(TAG, "createAlwaysOnHotwordDetector!!!!"); 194 mDetectorInitializedLatch = new CountDownLatch(1); 195 196 final AlwaysOnHotwordDetector.Callback callback = new AlwaysOnHotwordDetector.Callback() { 197 @Override 198 public void onAvailabilityChanged(int status) { 199 Log.i(TAG, "onAvailabilityChanged(" + status + ")"); 200 mAvailabilityStatus = status; 201 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 202 if (mAvailabilityChangeLatch != null) { 203 mAvailabilityChangeLatch.countDown(); 204 } 205 } 206 207 @Override 208 public void onDetected(AlwaysOnHotwordDetector.EventPayload eventPayload) { 209 Log.i(TAG, "onDetected"); 210 mDetectedResult = eventPayload; 211 mDspOnDetectedCount++; 212 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 213 if (mOnDetectRejectLatch != null) { 214 mOnDetectRejectLatch.countDown(); 215 } 216 } 217 218 @Override 219 public void onRejected(@NonNull HotwordRejectedResult result) { 220 Log.i(TAG, "onRejected"); 221 mRejectedResult = result; 222 mDspOnRejectedCount++; 223 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 224 if (mOnDetectRejectLatch != null) { 225 mOnDetectRejectLatch.countDown(); 226 } 227 } 228 229 @Override 230 public void onError() { 231 Log.i(TAG, "onError"); 232 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 233 if (mOnErrorLatch != null) { 234 mOnErrorLatch.countDown(); 235 } 236 } 237 238 @Override 239 public void onRecognitionPaused() { 240 Log.i(TAG, "onRecognitionPaused"); 241 } 242 243 @Override 244 public void onRecognitionResumed() { 245 Log.i(TAG, "onRecognitionResumed"); 246 } 247 248 @Override 249 public void onHotwordDetectionServiceInitialized(int status) { 250 Log.i(TAG, "onHotwordDetectionServiceInitialized status = " + status); 251 mInitializedStatus = status; 252 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 253 if (mDetectorInitializedLatch != null) { 254 mDetectorInitializedLatch.countDown(); 255 } 256 } 257 258 @Override 259 public void onHotwordDetectionServiceRestarted() { 260 Log.i(TAG, "onHotwordDetectionServiceRestarted"); 261 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 262 if (mOnHotwordDetectionServiceRestartedLatch != null) { 263 mOnHotwordDetectionServiceRestartedLatch.countDown(); 264 } 265 } 266 }; 267 268 final Handler handler = runOnMainThread ? new Handler(Looper.getMainLooper()) : mHandler; 269 handler.post(() -> { 270 mAlwaysOnHotwordDetector = callCreateAlwaysOnHotwordDetectorWithNecessaryPerm(callback, 271 useExecutor, options, MANAGE_HOTWORD_DETECTION, RECORD_AUDIO, 272 CAPTURE_AUDIO_HOTWORD); 273 }); 274 } 275 276 /** 277 * Create an AlwaysOnHotwordDetector but doesn't hold MANAGE_HOTWORD_DETECTION 278 */ createAlwaysOnHotwordDetectorWithoutManageHotwordDetectionPermission()279 public void createAlwaysOnHotwordDetectorWithoutManageHotwordDetectionPermission() { 280 mDetectorInitializedLatch = new CountDownLatch(1); 281 mHandler.post(() -> callCreateAlwaysOnHotwordDetectorWithNecessaryPerm( 282 mNoOpHotwordDetectorCallback, /* useExecutor= */ false, null, RECORD_AUDIO, 283 CAPTURE_AUDIO_HOTWORD)); 284 } 285 286 /** 287 * Create a SoftwareHotwordDetector but doesn't hold MANAGE_HOTWORD_DETECTION 288 */ createSoftwareHotwordDetectorWithoutManageHotwordDetectionPermission()289 public void createSoftwareHotwordDetectorWithoutManageHotwordDetectionPermission() { 290 mDetectorInitializedLatch = new CountDownLatch(1); 291 mHandler.post(() -> callCreateSoftwareDetectorWithNecessaryPerm( 292 mNoOpSoftwareDetectorCallback, /* useExecutor= */ false, 293 null, CAPTURE_AUDIO_HOTWORD)); 294 } 295 296 /** 297 * Create an SoftwareHotwordDetector holds MANAGE_HOTWORD_DETECTION and 298 * BIND_HOTWORD_DETECTION_SERVICE. The client should have MANAGE_HOTWORD_DETECTION to make the 299 * API call to the system to do BIND_HOTWORD_DETECTION_SERVICE permission checking. 300 */ createSoftwareHotwordDetectorHoldBindHotwordDetectionPermission()301 public void createSoftwareHotwordDetectorHoldBindHotwordDetectionPermission() { 302 mDetectorInitializedLatch = new CountDownLatch(1); 303 mHandler.post(() -> callCreateSoftwareDetectorWithNecessaryPerm( 304 mNoOpSoftwareDetectorCallback, /* useExecutor= */ false, null, 305 MANAGE_HOTWORD_DETECTION, BIND_HOTWORD_DETECTION_SERVICE)); 306 } 307 308 /** 309 * Create an AlwaysOnHotwordDetector holds MANAGE_HOTWORD_DETECTION and 310 * BIND_HOTWORD_DETECTION_SERVICE. The client should have MANAGE_HOTWORD_DETECTION to make the 311 * API call to the system to do BIND_HOTWORD_DETECTION_SERVICE permission checking. 312 */ createAlwaysOnHotwordDetectorHoldBindHotwordDetectionPermission()313 public void createAlwaysOnHotwordDetectorHoldBindHotwordDetectionPermission() { 314 mDetectorInitializedLatch = new CountDownLatch(1); 315 mHandler.post(() -> 316 callCreateAlwaysOnHotwordDetectorWithNecessaryPerm( 317 mNoOpHotwordDetectorCallback, /* useExecutor= */ false, null, 318 RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD, MANAGE_HOTWORD_DETECTION, 319 BIND_HOTWORD_DETECTION_SERVICE)); 320 } 321 322 /** 323 * Create an AlwaysOnHotwordDetector with onFailure callback. The onFailure provides the error 324 * code, error message and suggested action the assistant application should take. 325 */ createAlwaysOnHotwordDetectorWithOnFailureCallback(boolean useExecutor, boolean runOnMainThread)326 public void createAlwaysOnHotwordDetectorWithOnFailureCallback(boolean useExecutor, 327 boolean runOnMainThread) { 328 createAlwaysOnHotwordDetectorWithOnFailureCallback(useExecutor, runOnMainThread, 329 null /* options */); 330 } 331 332 /** 333 * Create an AlwaysOnHotwordDetector with onFailure callback. The onFailure provides the error 334 * code, error message and suggested action the assistant application should take. 335 */ createAlwaysOnHotwordDetectorWithOnFailureCallback(boolean useExecutor, boolean runOnMainThread, @Nullable PersistableBundle options)336 public void createAlwaysOnHotwordDetectorWithOnFailureCallback(boolean useExecutor, 337 boolean runOnMainThread, @Nullable PersistableBundle options) { 338 Log.i(TAG, "createAlwaysOnHotwordDetectorWithOnFailureCallback"); 339 mDetectorInitializedLatch = new CountDownLatch(1); 340 341 final Handler handler = runOnMainThread ? new Handler(Looper.getMainLooper()) : mHandler; 342 handler.post(() -> { 343 mAlwaysOnHotwordDetector = 344 callCreateAlwaysOnHotwordDetectorWithNecessaryPerm( 345 createAlwaysOnHotwordDetectorCallbackWithListeners(), useExecutor, 346 options, MANAGE_HOTWORD_DETECTION, RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD); 347 }); 348 } 349 350 /** 351 * Creates a callback which is set up to listen and log all events. 352 */ createAlwaysOnHotwordDetectorCallbackWithListeners()353 public AlwaysOnHotwordDetector.Callback createAlwaysOnHotwordDetectorCallbackWithListeners() { 354 return new AlwaysOnHotwordDetector.Callback() { 355 @Override 356 public void onAvailabilityChanged(int status) { 357 Log.i(TAG, "onAvailabilityChanged(" + status + ")"); 358 mAvailabilityStatus = status; 359 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 360 if (mAvailabilityChangeLatch != null) { 361 mAvailabilityChangeLatch.countDown(); 362 } 363 } 364 365 @Override 366 public void onDetected(AlwaysOnHotwordDetector.EventPayload eventPayload) { 367 Log.i(TAG, "onDetected"); 368 mDetectedResult = eventPayload; 369 mDspOnDetectedCount++; 370 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 371 if (mOnDetectRejectLatch != null) { 372 mOnDetectRejectLatch.countDown(); 373 } 374 } 375 376 @Override 377 public void onRejected(@NonNull HotwordRejectedResult result) { 378 Log.i(TAG, "onRejected"); 379 mRejectedResult = result; 380 mDspOnRejectedCount++; 381 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 382 if (mOnDetectRejectLatch != null) { 383 mOnDetectRejectLatch.countDown(); 384 } 385 } 386 387 @Override 388 public void onError() { 389 Log.i(TAG, "onError"); 390 } 391 392 @Override 393 public void onFailure(HotwordDetectionServiceFailure hotwordDetectionServiceFailure) { 394 Log.i(TAG, "onFailure hotwordDetectionServiceFailure=" 395 + hotwordDetectionServiceFailure); 396 mHotwordDetectionServiceFailure = hotwordDetectionServiceFailure; 397 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 398 if (mOnFailureLatch != null) { 399 mOnFailureLatch.countDown(); 400 } 401 } 402 403 @Override 404 public void onFailure(SoundTriggerFailure soundTriggerFailure) { 405 Log.i(TAG, "onFailure soundTriggerFailure=" + soundTriggerFailure); 406 mSoundTriggerFailure = soundTriggerFailure; 407 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 408 if (mOnFailureLatch != null) { 409 mOnFailureLatch.countDown(); 410 } 411 } 412 413 @Override 414 public void onUnknownFailure(String errorMessage) { 415 Log.i(TAG, "onUnknownFailure errorMessage=" + errorMessage); 416 mUnknownFailure = errorMessage; 417 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 418 if (mOnFailureLatch != null) { 419 mOnFailureLatch.countDown(); 420 } 421 } 422 423 @Override 424 public void onRecognitionPaused() { 425 Log.i(TAG, "onRecognitionPaused"); 426 if (mOnRecognitionPausedLatch != null) { 427 mOnRecognitionPausedLatch.countDown(); 428 } 429 } 430 431 @Override 432 public void onRecognitionResumed() { 433 Log.i(TAG, "onRecognitionResumed"); 434 if (mOnRecognitionResumedLatch != null) { 435 mOnRecognitionResumedLatch.countDown(); 436 } 437 } 438 439 @Override 440 public void onHotwordDetectionServiceInitialized(int status) { 441 Log.i(TAG, "onHotwordDetectionServiceInitialized status = " + status); 442 mInitializedStatus = status; 443 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 444 if (mDetectorInitializedLatch != null) { 445 mDetectorInitializedLatch.countDown(); 446 } 447 } 448 449 @Override 450 public void onHotwordDetectionServiceRestarted() { 451 Log.i(TAG, "onHotwordDetectionServiceRestarted"); 452 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 453 if (mOnHotwordDetectionServiceRestartedLatch != null) { 454 mOnHotwordDetectionServiceRestartedLatch.countDown(); 455 } 456 } 457 }; 458 } 459 460 /** 461 * Create a SoftwareHotwordDetector, but it will not implement the onFailure method of 462 * HotwordDetector.Callback. It will implement the onFailure method by using 463 * createSoftwareHotwordDetectorWithOnFailureCallback method. 464 */ 465 public void createSoftwareHotwordDetector() { 466 createSoftwareHotwordDetector(/* useExecutor= */ false, /* runOnMainThread= */ false); 467 } 468 469 /** 470 * Create a SoftwareHotwordDetector, but it will not implement the onFailure method of 471 * HotwordDetector.Callback. It will implement the onFailure method by using 472 * createSoftwareHotwordDetectorWithOnFailureCallback method. 473 */ 474 public void createSoftwareHotwordDetector(boolean useExecutor, boolean runOnMainThread) { 475 createSoftwareHotwordDetector(useExecutor, runOnMainThread, /* options= */ null); 476 } 477 478 /** 479 * Create a SoftwareHotwordDetector, but it will not implement the onFailure method of 480 * HotwordDetector.Callback. It will implement the onFailure method by using 481 * createSoftwareHotwordDetectorWithOnFailureCallback method. 482 */ 483 public void createSoftwareHotwordDetector(boolean useExecutor, boolean runOnMainThread, 484 @Nullable PersistableBundle options) { 485 mDetectorInitializedLatch = new CountDownLatch(1); 486 487 final HotwordDetector.Callback callback = new HotwordDetector.Callback() { 488 @Override 489 public void onDetected(AlwaysOnHotwordDetector.EventPayload eventPayload) { 490 Log.i(TAG, "onDetected"); 491 mDetectedResult = eventPayload; 492 mSoftwareOnDetectedCount++; 493 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 494 if (mOnDetectRejectLatch != null) { 495 mOnDetectRejectLatch.countDown(); 496 } 497 } 498 499 @Override 500 public void onError() { 501 Log.i(TAG, "onError"); 502 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 503 if (mOnErrorLatch != null) { 504 mOnErrorLatch.countDown(); 505 } 506 } 507 508 @Override 509 public void onRecognitionPaused() { 510 Log.i(TAG, "onRecognitionPaused"); 511 } 512 513 @Override 514 public void onRecognitionResumed() { 515 Log.i(TAG, "onRecognitionResumed"); 516 } 517 518 @Override 519 public void onRejected(HotwordRejectedResult result) { 520 Log.i(TAG, "onRejected"); 521 mRejectedResult = result; 522 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 523 if (mOnDetectRejectLatch != null) { 524 mOnDetectRejectLatch.countDown(); 525 } 526 } 527 528 @Override 529 public void onHotwordDetectionServiceInitialized(int status) { 530 Log.i(TAG, "onHotwordDetectionServiceInitialized status = " + status); 531 mInitializedStatus = status; 532 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 533 if (mDetectorInitializedLatch != null) { 534 mDetectorInitializedLatch.countDown(); 535 } 536 } 537 538 @Override 539 public void onHotwordDetectionServiceRestarted() { 540 Log.i(TAG, "onHotwordDetectionServiceRestarted"); 541 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 542 if (mOnHotwordDetectionServiceRestartedLatch != null) { 543 mOnHotwordDetectionServiceRestartedLatch.countDown(); 544 } 545 } 546 }; 547 548 final Handler handler = runOnMainThread ? new Handler(Looper.getMainLooper()) : mHandler; 549 handler.post(() -> { 550 mSoftwareHotwordDetector = 551 callCreateSoftwareDetectorWithNecessaryPerm(callback, useExecutor, options, 552 MANAGE_HOTWORD_DETECTION); 553 }); 554 } 555 556 /** 557 * Create a SoftwareHotwordDetector with onFailure callback. The onFailure provides the error 558 * code, error message and suggested action the assistant application should take. 559 */ 560 public void createSoftwareHotwordDetectorWithOnFailureCallback(boolean useExecutor, 561 boolean runOnMainThread) { 562 mDetectorInitializedLatch = new CountDownLatch(1); 563 564 final HotwordDetector.Callback callback = new HotwordDetector.Callback() { 565 @Override 566 public void onDetected(AlwaysOnHotwordDetector.EventPayload eventPayload) { 567 Log.i(TAG, "onDetected"); 568 mDetectedResult = eventPayload; 569 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 570 if (mOnDetectRejectLatch != null) { 571 mOnDetectRejectLatch.countDown(); 572 } 573 } 574 575 @Override 576 public void onError() { 577 Log.i(TAG, "onError"); 578 } 579 580 @Override 581 public void onFailure(HotwordDetectionServiceFailure hotwordDetectionServiceFailure) { 582 Log.i(TAG, "onFailure hotwordDetectionServiceFailure=" 583 + hotwordDetectionServiceFailure); 584 mHotwordDetectionServiceFailure = hotwordDetectionServiceFailure; 585 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 586 if (mOnFailureLatch != null) { 587 mOnFailureLatch.countDown(); 588 } 589 } 590 591 @Override 592 public void onUnknownFailure(String errorMessage) { 593 Log.i(TAG, "onUnknownFailure errorMessage=" + errorMessage); 594 mUnknownFailure = errorMessage; 595 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 596 if (mOnFailureLatch != null) { 597 mOnFailureLatch.countDown(); 598 } 599 } 600 601 @Override 602 public void onRecognitionPaused() { 603 Log.i(TAG, "onRecognitionPaused"); 604 } 605 606 @Override 607 public void onRecognitionResumed() { 608 Log.i(TAG, "onRecognitionResumed"); 609 } 610 611 @Override 612 public void onRejected(HotwordRejectedResult result) { 613 Log.i(TAG, "onRejected"); 614 mRejectedResult = result; 615 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 616 if (mOnDetectRejectLatch != null) { 617 mOnDetectRejectLatch.countDown(); 618 } 619 } 620 621 @Override 622 public void onHotwordDetectionServiceInitialized(int status) { 623 Log.i(TAG, "onHotwordDetectionServiceInitialized status = " + status); 624 mInitializedStatus = status; 625 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 626 if (mDetectorInitializedLatch != null) { 627 mDetectorInitializedLatch.countDown(); 628 } 629 } 630 631 @Override 632 public void onHotwordDetectionServiceRestarted() { 633 Log.i(TAG, "onHotwordDetectionServiceRestarted"); 634 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 635 if (mOnHotwordDetectionServiceRestartedLatch != null) { 636 mOnHotwordDetectionServiceRestartedLatch.countDown(); 637 } 638 } 639 }; 640 641 final Handler handler = runOnMainThread ? new Handler(Looper.getMainLooper()) : mHandler; 642 handler.post(() -> { 643 mSoftwareHotwordDetector = callCreateSoftwareDetectorWithNecessaryPerm(callback, 644 useExecutor, null, MANAGE_HOTWORD_DETECTION); 645 }); 646 } 647 648 /** 649 * Create a VisualQueryDetector but doesn't hold MANAGE_HOTWORD_DETECTION 650 */ 651 public void createVisualQueryDetectorWithoutManageHotwordDetectionPermission() { 652 mDetectorInitializedLatch = new CountDownLatch(1); 653 mHandler.post(() -> runWithShellPermissionIdentity( 654 () -> callCreateVisualQueryDetector(mNoOpVisualQueryDetectorCallback))); 655 } 656 657 /** 658 * Create a VisualQueryDetector but doesn't hold MANAGE_HOTWORD_DETECTION but hold 659 * BIND_VISUAL_QUERY_DETECTION_SERVICE. 660 */ 661 public void createVisualQueryDetectorHoldBindVisualQueryDetectionPermission() { 662 mDetectorInitializedLatch = new CountDownLatch(1); 663 mHandler.post(() -> runWithShellPermissionIdentity( 664 () -> callCreateVisualQueryDetector(mNoOpVisualQueryDetectorCallback), 665 BIND_VISUAL_QUERY_DETECTION_SERVICE)); 666 } 667 668 public void createVisualQueryDetector() { 669 mDetectorInitializedLatch = new CountDownLatch(1); 670 mHandler.post(() -> runWithShellPermissionIdentity(() -> { 671 VisualQueryDetector.Callback callback = new VisualQueryDetector.Callback() { 672 @Override 673 public void onQueryDetected(@NonNull String transcribedText) { 674 Log.i(TAG, "onQueryDetected"); 675 mCurrentQuery += transcribedText; 676 } 677 678 @Override 679 public void onQueryDetected(@NonNull VisualQueryDetectedResult partialResult) { 680 Log.i(TAG, "onQueryDetected with VisualQueryDetectedResult"); 681 mCurrentQuery += partialResult.getPartialQuery(); 682 mCurrentAccessibilityData = accumulateAccessibilityStreamedData( 683 mCurrentAccessibilityData, 684 partialResult.getAccessibilityDetectionData()); 685 } 686 687 private byte[] accumulateAccessibilityStreamedData(byte[] streamedData, 688 byte[] newData) { 689 if (newData == null) { 690 return streamedData; 691 } 692 byte[] newStreamedData = new byte[streamedData.length + newData.length]; 693 System.arraycopy(streamedData, 0, newStreamedData, 0, 694 streamedData.length); 695 System.arraycopy(newData, 0, newStreamedData, streamedData.length, 696 newData.length); 697 return newStreamedData; 698 } 699 700 @Override 701 public void onQueryRejected() { 702 Log.i(TAG, "onQueryRejected"); 703 // mStreamedQueries are used to store previously streamed queries for testing 704 // reason, regardless of the queries being rejected or finished. 705 mStreamedQueries.add(mCurrentQuery); 706 mCurrentQuery = ""; 707 mCurrentAccessibilityData = new byte[0]; 708 if (mOnQueryFinishRejectLatch != null) { 709 mOnQueryFinishRejectLatch.countDown(); 710 } 711 } 712 713 @Override 714 public void onQueryFinished() { 715 Log.i(TAG, "onQueryFinished"); 716 mStreamedQueries.add(mCurrentQuery); 717 mStreamedAccessibilityData.add(mCurrentAccessibilityData); 718 mCurrentQuery = ""; 719 mCurrentAccessibilityData = new byte[0]; 720 if (mOnQueryFinishRejectLatch != null) { 721 mOnQueryFinishRejectLatch.countDown(); 722 } 723 } 724 725 @Override 726 public void onVisualQueryDetectionServiceInitialized(int status) { 727 Log.i(TAG, "onVisualQueryDetectionServiceInitialized status = " + status); 728 if (status != SandboxedDetectionInitializer.INITIALIZATION_STATUS_SUCCESS) { 729 return; 730 } 731 mInitializedStatus = status; 732 if (mDetectorInitializedLatch != null) { 733 mDetectorInitializedLatch.countDown(); 734 } 735 } 736 737 @Override 738 public void onVisualQueryDetectionServiceRestarted() { 739 Log.i(TAG, "onVisualQueryDetectionServiceRestarted"); 740 setIsDetectorCallbackRunningOnMainThread(isRunningOnMainThread()); 741 if (mOnVisualQueryDetectionServiceRestartedLatch != null) { 742 mOnVisualQueryDetectionServiceRestartedLatch.countDown(); 743 } 744 } 745 746 @Override 747 public void onFailure( 748 VisualQueryDetectionServiceFailure visualQueryDetectionServiceFailure) { 749 Log.i(TAG, "onFailure visualQueryDetectionServiceFailure: " 750 + visualQueryDetectionServiceFailure); 751 if (mOnFailureLatch != null) { 752 mOnFailureLatch.countDown(); 753 } 754 } 755 756 @Override 757 public void onUnknownFailure(String errorMessage) { 758 Log.i(TAG, "onUnknownFailure errorMessage: " + errorMessage); 759 if (mOnFailureLatch != null) { 760 mOnFailureLatch.countDown(); 761 } 762 } 763 }; 764 mVisualQueryDetector = callCreateVisualQueryDetector(callback); 765 }, MANAGE_HOTWORD_DETECTION)); 766 } 767 768 /** 769 * Create a CountDownLatch that is used to wait for a HotwordDetector to be fully initialized 770 * after calling create. 771 * 772 * <p>For detector create requests which do not use an isolated service like 773 * {@link HotwordDetectionService} or {@link VisualQueryDetectionService}, this latch 774 * is counted down as soon as the create detector API returns. 775 * 776 * <p>For detector create requests which use {@link HotwordDetectionService}, this latch is 777 * counted down when {@link HotwordDetector.Callback#onHotwordDetectionServiceInitialized} is 778 * received. 779 * 780 * <p>For detector create requests which use {@link VisualQueryDetectionService}, this latch is 781 * counted down when 782 * {@link VisualQueryDetector.Callback#onVisualQueryDetectionServiceInitialized} is received. 783 */ 784 public void initDetectorInitializedLatch() { 785 mDetectorInitializedLatch = new CountDownLatch(1); 786 } 787 788 /** 789 * Create a CountDownLatch that is used to wait for onHotwordDetectionServiceRestarted() called 790 */ 791 public void initOnHotwordDetectionServiceRestartedLatch() { 792 mOnHotwordDetectionServiceRestartedLatch = new CountDownLatch(1); 793 } 794 795 public void initOnVisualQueryDetectionServiceRestartedLatch() { 796 mOnVisualQueryDetectionServiceRestartedLatch = new CountDownLatch(1); 797 } 798 799 /** 800 * Create a CountDownLatch that is used to wait for availability change 801 */ 802 public void initAvailabilityChangeLatch() { 803 mAvailabilityChangeLatch = new CountDownLatch(1); 804 } 805 806 /** 807 * Create a CountDownLatch that is used to wait for onDetected() or onRejected() result 808 */ 809 public void initDetectRejectLatch() { 810 mOnDetectRejectLatch = new CountDownLatch(1); 811 } 812 813 /** 814 * Create a CountDownLatch that wait for onQueryFinished() or onQueryRejected() result 815 */ 816 public void initQueryFinishRejectLatch(int numQueries) { 817 mOnQueryFinishRejectLatch = new CountDownLatch(numQueries); 818 } 819 820 /** 821 * Create a CountDownLatch that is used to wait for onError() 822 */ 823 public void initOnErrorLatch() { 824 Log.d(TAG, "initOnErrorLatch()"); 825 mOnErrorLatch = new CountDownLatch(1); 826 } 827 828 /** 829 * Create a CountDownLatch that is used to wait for onFailure() 830 */ 831 public void initOnFailureLatch() { 832 mOnFailureLatch = new CountDownLatch(1); 833 } 834 835 public boolean isOnFailureLatchOpen() { 836 return mOnFailureLatch != null && mOnFailureLatch.getCount() > 0; 837 } 838 839 /** 840 * Create a CountDownLatch that is used to wait for onRecognitionPaused() 841 */ 842 public void initOnRecognitionPausedLatch() { 843 mOnRecognitionPausedLatch = new CountDownLatch(1); 844 } 845 846 /** 847 * Create a CountDownLatch that is used to wait for OnRecognitionResumed() 848 */ 849 public void initOnRecognitionResumedLatch() { 850 mOnRecognitionResumedLatch = new CountDownLatch(1); 851 } 852 853 /** 854 * Returns the onDetected() result. 855 */ 856 public AlwaysOnHotwordDetector.EventPayload getHotwordServiceOnDetectedResult() { 857 return mDetectedResult; 858 } 859 860 /** 861 * Returns the OnRejected() result. 862 */ 863 public HotwordRejectedResult getHotwordServiceOnRejectedResult() { 864 return mRejectedResult; 865 } 866 867 /** 868 * Resets the onDetected() and OnRejected() result. 869 */ 870 public void resetHotwordServiceOnDetectedAndOnRejectedResult() { 871 mDetectedResult = null; 872 mRejectedResult = null; 873 mDspOnDetectedCount = 0; 874 mDspOnRejectedCount = 0; 875 mSoftwareOnDetectedCount = 0; 876 } 877 878 /** 879 * Returns the OnQueryDetected() result. 880 */ 881 public ArrayList<String> getStreamedQueriesResult() { 882 return mStreamedQueries; 883 } 884 885 /** 886 * Returns the OnQueryDetected() result. 887 */ 888 public ArrayList<byte[]> getAccessibilityDataResult() { 889 return mStreamedAccessibilityData; 890 } 891 892 /** 893 * Wait for onHotwordDetectionServiceRestarted() callback called. 894 */ 895 public void waitOnHotwordDetectionServiceRestartedCalled() throws InterruptedException { 896 Log.d(TAG, "waitOnHotwordDetectionServiceRestartedCalled(), latch=" 897 + mOnHotwordDetectionServiceRestartedLatch); 898 if (mOnHotwordDetectionServiceRestartedLatch == null 899 || !mOnHotwordDetectionServiceRestartedLatch.await(WAIT_LONG_TIMEOUT_IN_MS, 900 TimeUnit.MILLISECONDS)) { 901 mOnHotwordDetectionServiceRestartedLatch = null; 902 throw new AssertionError( 903 "HotwordDetectionService onHotwordDetectionServiceRestarted not called."); 904 } 905 mOnHotwordDetectionServiceRestartedLatch = null; 906 } 907 908 public void waitOnVisualQueryDetectionServiceRestartedCalled() throws InterruptedException { 909 Log.d(TAG, "waitOnVisualQueryDetectionServiceRestartedCalled(), latch=" 910 + mOnVisualQueryDetectionServiceRestartedLatch); 911 if (mOnVisualQueryDetectionServiceRestartedLatch == null 912 || !mOnVisualQueryDetectionServiceRestartedLatch.await(WAIT_TIMEOUT_IN_MS, 913 TimeUnit.MILLISECONDS)) { 914 mOnVisualQueryDetectionServiceRestartedLatch = null; 915 throw new AssertionError( 916 "VisualQueryDetectionService onVisualQueryDetectionServiceRestarted not called."); 917 } 918 mOnVisualQueryDetectionServiceRestartedLatch = null; 919 } 920 921 922 /** 923 * Returns the OnFailure() with HotwordDetectionServiceFailure result. 924 */ 925 public HotwordDetectionServiceFailure getHotwordDetectionServiceFailure() { 926 return mHotwordDetectionServiceFailure; 927 } 928 929 /** 930 * Returns the OnFailure() with SoundTriggerFailure result. 931 */ 932 public SoundTriggerFailure getSoundTriggerFailure() { 933 return mSoundTriggerFailure; 934 } 935 936 /** 937 * Returns the onUnknownFailure() with error message. 938 */ 939 public String getUnknownFailure() { 940 return mUnknownFailure; 941 } 942 943 /** 944 * Wait for onAvailabilityChanged() callback called. 945 */ 946 public void waitAvailabilityChangedCalled() throws InterruptedException { 947 Log.d(TAG, "waitAvailabilityChangedCalled(), latch=" + mAvailabilityChangeLatch); 948 if (mAvailabilityChangeLatch == null 949 || !mAvailabilityChangeLatch.await(WAIT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS)) { 950 mAvailabilityChangeLatch = null; 951 throw new AssertionError("HotwordDetectionService get ability fail."); 952 } 953 mAvailabilityChangeLatch = null; 954 } 955 956 /** 957 * Return the result for onAvailabilityChanged(). 958 */ 959 public int getHotwordDetectionServiceAvailabilityResult() { 960 return mAvailabilityStatus; 961 } 962 963 /** 964 * Wait for onDetected() or OnRejected() callback called. 965 */ 966 public void waitOnDetectOrRejectCalled() throws InterruptedException { 967 Log.d(TAG, "waitOnDetectOrRejectCalled(), latch=" + mOnDetectRejectLatch); 968 if (mOnDetectRejectLatch == null 969 || !mOnDetectRejectLatch.await(WAIT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS)) { 970 mOnDetectRejectLatch = null; 971 throw new AssertionError("onDetected or OnRejected() fail."); 972 } 973 mOnDetectRejectLatch = null; 974 } 975 976 /** 977 * Wait for onQueryFinished() or OnQueryRejected() callback called. 978 */ 979 public void waitOnQueryFinishedRejectCalled() throws InterruptedException { 980 Log.d(TAG, "waitOnQueryFinishedRejectCalled(), latch=" + mOnQueryFinishRejectLatch); 981 if (mOnQueryFinishRejectLatch == null 982 || !mOnQueryFinishRejectLatch.await(WAIT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS)) { 983 mOnQueryFinishRejectLatch = null; 984 throw new AssertionError("onDetected or OnRejected() fail."); 985 } 986 mOnQueryFinishRejectLatch = null; 987 } 988 989 /** 990 * Wait for onError() callback called. 991 */ 992 public void waitOnErrorCalled() throws InterruptedException { 993 Log.d(TAG, "waitOnErrorCalled(), latch=" + mOnErrorLatch); 994 if (mOnErrorLatch == null 995 || !mOnErrorLatch.await(WAIT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS)) { 996 mOnErrorLatch = null; 997 throw new AssertionError("OnError() fail."); 998 } 999 mOnErrorLatch = null; 1000 } 1001 1002 /** 1003 * Wait for onFailure() callback called. 1004 */ 1005 public void waitOnFailureCalled() throws InterruptedException { 1006 Log.d(TAG, "waitOnFailureCalled(), latch=" + mOnFailureLatch); 1007 if (mOnFailureLatch == null 1008 || !mOnFailureLatch.await(WAIT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS)) { 1009 mOnFailureLatch = null; 1010 throw new AssertionError("OnFailure() fail."); 1011 } 1012 mOnFailureLatch = null; 1013 } 1014 1015 /** 1016 * Wait for onRecognitionPaused() callback called. 1017 */ 1018 public void waitOnRecognitionPausedCalled() throws InterruptedException { 1019 Log.d(TAG, "waitOnRecognitionPausedCalled(), latch=" + mOnRecognitionPausedLatch); 1020 if (mOnRecognitionPausedLatch == null 1021 || !mOnRecognitionPausedLatch.await(WAIT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS)) { 1022 mOnRecognitionPausedLatch = null; 1023 throw new AssertionError("onRecognitionPaused() fail."); 1024 } 1025 mOnRecognitionPausedLatch = null; 1026 } 1027 1028 /** 1029 * Wait for onRecognitionResumed() callback called. 1030 */ 1031 public void waitOnRecognitionResumedCalled() throws InterruptedException { 1032 Log.d(TAG, "waitOnRecognitionResumedCalled(), latch=" + mOnRecognitionResumedLatch); 1033 if (mOnRecognitionResumedLatch == null 1034 || !mOnRecognitionResumedLatch.await(WAIT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS)) { 1035 mOnRecognitionResumedLatch = null; 1036 throw new AssertionError("onRecognitionResumed() fail."); 1037 } 1038 mOnRecognitionResumedLatch = null; 1039 } 1040 1041 /** 1042 * Wait for no onRecognitionPaused() callback called. 1043 */ 1044 public boolean waitNoOnRecognitionPausedCalled() throws InterruptedException { 1045 Log.d(TAG, "waitNoOnRecognitionPausedCalled(), latch=" + mOnRecognitionPausedLatch); 1046 if (mOnRecognitionPausedLatch == null) { 1047 throw new AssertionError("mOnRecognitionPausedLatch is not initialized."); 1048 } 1049 boolean result = mOnRecognitionPausedLatch.await(WAIT_EXPECTED_NO_CALL_TIMEOUT_IN_MS, 1050 TimeUnit.MILLISECONDS); 1051 mOnRecognitionPausedLatch = null; 1052 return !result; 1053 } 1054 1055 /** 1056 * Creates always on hotword detector. 1057 * 1058 */ 1059 private AlwaysOnHotwordDetector callCreateAlwaysOnHotwordDetectorWithNecessaryPerm( 1060 AlwaysOnHotwordDetector.Callback callback, boolean useExecutor, 1061 @Nullable PersistableBundle options, String... permissions) { 1062 List<String> requestedPermissions = new ArrayList<String>(Arrays.asList(permissions)); 1063 1064 try { 1065 return callWithShellPermissionIdentity(() -> 1066 callCreateAlwaysOnHotwordDetector(callback, useExecutor, options), 1067 requestedPermissions.toArray(new String[0])); 1068 } catch (Exception e) { 1069 return null; 1070 } 1071 } 1072 1073 /** 1074 * Creates software hotword detector. 1075 * 1076 */ 1077 private HotwordDetector callCreateSoftwareDetectorWithNecessaryPerm( 1078 HotwordDetector.Callback callback, boolean useExecutor, 1079 @Nullable PersistableBundle options, String... permissions) { 1080 List<String> requestedPermissions = new ArrayList<String>(Arrays.asList(permissions)); 1081 1082 try { 1083 return callWithShellPermissionIdentity(() -> 1084 callCreateSoftwareHotwordDetector(callback, useExecutor, options), 1085 requestedPermissions.toArray(new String[0])); 1086 } catch (Exception e) { 1087 return null; 1088 } 1089 } 1090 } 1091