1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import static java.util.Objects.requireNonNull; 20 21 import android.annotation.DurationMillisLong; 22 import android.annotation.FlaggedApi; 23 import android.annotation.IntDef; 24 import android.annotation.NonNull; 25 import android.annotation.Nullable; 26 import android.annotation.SdkConstant; 27 import android.annotation.SuppressLint; 28 import android.annotation.SystemApi; 29 import android.annotation.TestApi; 30 import android.app.Service; 31 import android.content.ContentCaptureOptions; 32 import android.content.Context; 33 import android.content.Intent; 34 import android.hardware.soundtrigger.SoundTrigger; 35 import android.media.AudioFormat; 36 import android.media.AudioSystem; 37 import android.os.IBinder; 38 import android.os.IRemoteCallback; 39 import android.os.ParcelFileDescriptor; 40 import android.os.PersistableBundle; 41 import android.os.RemoteException; 42 import android.os.SharedMemory; 43 import android.speech.IRecognitionServiceManager; 44 import android.util.Log; 45 import android.view.contentcapture.ContentCaptureManager; 46 import android.view.contentcapture.IContentCaptureManager; 47 48 import java.lang.annotation.Documented; 49 import java.lang.annotation.Retention; 50 import java.lang.annotation.RetentionPolicy; 51 import java.util.Locale; 52 import java.util.function.IntConsumer; 53 54 /** 55 * Implemented by an application that wants to offer detection for hotword. The service can be used 56 * for both DSP and non-DSP detectors. 57 * 58 * The system will bind an application's {@link VoiceInteractionService} first. When {@link 59 * VoiceInteractionService#createHotwordDetector(PersistableBundle, SharedMemory, 60 * HotwordDetector.Callback)} or {@link VoiceInteractionService#createAlwaysOnHotwordDetector( 61 * String, Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} is called, 62 * the system will bind application's {@link HotwordDetectionService}. Either on a hardware 63 * trigger or on request from the {@link VoiceInteractionService}, the system calls into the 64 * {@link HotwordDetectionService} to request detection. The {@link HotwordDetectionService} then 65 * uses {@link Callback#onDetected(HotwordDetectedResult)} to inform the system that a relevant 66 * keyphrase was detected, or if applicable uses {@link Callback#onRejected(HotwordRejectedResult)} 67 * to inform the system that a keyphrase was not detected. The system then relays this result to 68 * the {@link VoiceInteractionService} through {@link HotwordDetector.Callback}. 69 * 70 * Note: Methods in this class may be called concurrently 71 * 72 * @hide 73 */ 74 @SystemApi 75 public abstract class HotwordDetectionService extends Service 76 implements SandboxedDetectionInitializer { 77 private static final String TAG = "HotwordDetectionService"; 78 private static final boolean DBG = false; 79 80 private static final long UPDATE_TIMEOUT_MILLIS = 20000; 81 82 /** 83 * The PersistableBundle options key used in {@link #onDetect(ParcelFileDescriptor, AudioFormat, 84 * PersistableBundle, Callback)} to indicate whether the system will close the audio stream 85 * after {@code Callback} is invoked. 86 */ 87 @FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API) 88 public static final String KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK = 89 "android.service.voice.HotwordDetectionService." 90 + "KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK"; 91 92 /** 93 * Feature flag for Attention Service. 94 * 95 * @hide 96 */ 97 @TestApi 98 public static final boolean ENABLE_PROXIMITY_RESULT = true; 99 100 /** 101 * Indicates that the updated status is successful. 102 * 103 * @deprecated Replaced with 104 * {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_SUCCESS} 105 */ 106 @Deprecated 107 public static final int INITIALIZATION_STATUS_SUCCESS = 108 SandboxedDetectionInitializer.INITIALIZATION_STATUS_SUCCESS; 109 110 /** 111 * Indicates that the callback wasn’t invoked within the timeout. 112 * This is used by system. 113 * 114 * @deprecated Replaced with 115 * {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_UNKNOWN} 116 */ 117 @Deprecated 118 public static final int INITIALIZATION_STATUS_UNKNOWN = 119 SandboxedDetectionInitializer.INITIALIZATION_STATUS_UNKNOWN; 120 121 /** 122 * Source for the given audio stream. 123 * 124 * @hide 125 */ 126 @Documented 127 @Retention(RetentionPolicy.SOURCE) 128 @IntDef({ 129 AUDIO_SOURCE_MICROPHONE, 130 AUDIO_SOURCE_EXTERNAL 131 }) 132 @interface AudioSource {} 133 134 /** @hide */ 135 public static final int AUDIO_SOURCE_MICROPHONE = 1; 136 /** @hide */ 137 public static final int AUDIO_SOURCE_EXTERNAL = 2; 138 139 /** 140 * The {@link Intent} that must be declared as handled by the service. 141 * To be supported, the service must also require the 142 * {@link android.Manifest.permission#BIND_HOTWORD_DETECTION_SERVICE} permission so 143 * that other applications can not abuse it. 144 */ 145 @SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION) 146 public static final String SERVICE_INTERFACE = 147 "android.service.voice.HotwordDetectionService"; 148 149 @Nullable 150 private ContentCaptureManager mContentCaptureManager; 151 @Nullable 152 private IRecognitionServiceManager mIRecognitionServiceManager; 153 154 private final ISandboxedDetectionService mInterface = new ISandboxedDetectionService.Stub() { 155 @Override 156 public void detectFromDspSource( 157 SoundTrigger.KeyphraseRecognitionEvent event, 158 AudioFormat audioFormat, 159 long timeoutMillis, 160 IDspHotwordDetectionCallback callback) 161 throws RemoteException { 162 if (DBG) { 163 Log.d(TAG, "#detectFromDspSource"); 164 } 165 HotwordDetectionService.this.onDetect( 166 new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(), 167 timeoutMillis, 168 new Callback(callback)); 169 } 170 171 @Override 172 public void updateState(PersistableBundle options, SharedMemory sharedMemory, 173 IRemoteCallback callback) throws RemoteException { 174 Log.v(TAG, "#updateState" + (callback != null ? " with callback" : "")); 175 HotwordDetectionService.this.onUpdateStateInternal( 176 options, 177 sharedMemory, 178 callback); 179 } 180 181 @Override 182 public void detectFromMicrophoneSource( 183 ParcelFileDescriptor audioStream, 184 @AudioSource int audioSource, 185 AudioFormat audioFormat, 186 PersistableBundle options, 187 IDspHotwordDetectionCallback callback) 188 throws RemoteException { 189 if (DBG) { 190 Log.d(TAG, "#detectFromMicrophoneSource"); 191 } 192 switch (audioSource) { 193 case AUDIO_SOURCE_MICROPHONE: 194 HotwordDetectionService.this.onDetect( 195 new Callback(callback)); 196 break; 197 case AUDIO_SOURCE_EXTERNAL: 198 HotwordDetectionService.this.onDetect( 199 audioStream, 200 audioFormat, 201 options, 202 new Callback(callback)); 203 break; 204 default: 205 Log.i(TAG, "Unsupported audio source " + audioSource); 206 } 207 } 208 209 @Override 210 public void detectWithVisualSignals( 211 IDetectorSessionVisualQueryDetectionCallback callback) { 212 throw new UnsupportedOperationException("Not supported by HotwordDetectionService"); 213 } 214 215 @Override 216 public void updateAudioFlinger(IBinder audioFlinger) { 217 AudioSystem.setAudioFlingerBinder(audioFlinger); 218 } 219 220 @Override 221 public void updateContentCaptureManager(IContentCaptureManager manager, 222 ContentCaptureOptions options) { 223 mContentCaptureManager = new ContentCaptureManager( 224 HotwordDetectionService.this, manager, options); 225 } 226 227 @Override 228 public void updateRecognitionServiceManager(IRecognitionServiceManager manager) { 229 mIRecognitionServiceManager = manager; 230 } 231 232 @Override 233 public void ping(IRemoteCallback callback) throws RemoteException { 234 callback.sendResult(null); 235 } 236 237 @Override 238 public void stopDetection() { 239 HotwordDetectionService.this.onStopDetection(); 240 } 241 242 @Override 243 public void registerRemoteStorageService(IDetectorSessionStorageService 244 detectorSessionStorageService) { 245 throw new UnsupportedOperationException("Hotword cannot access files from the disk."); 246 } 247 }; 248 249 @Override 250 @Nullable onBind(@onNull Intent intent)251 public final IBinder onBind(@NonNull Intent intent) { 252 if (SERVICE_INTERFACE.equals(intent.getAction())) { 253 return mInterface.asBinder(); 254 } 255 Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": " 256 + intent); 257 return null; 258 } 259 260 @Override 261 @SuppressLint("OnNameExpected") getSystemService(@erviceName @onNull String name)262 public @Nullable Object getSystemService(@ServiceName @NonNull String name) { 263 if (Context.CONTENT_CAPTURE_MANAGER_SERVICE.equals(name)) { 264 return mContentCaptureManager; 265 } else if (Context.SPEECH_RECOGNITION_SERVICE.equals(name) 266 && mIRecognitionServiceManager != null) { 267 return mIRecognitionServiceManager.asBinder(); 268 } else { 269 return super.getSystemService(name); 270 } 271 } 272 273 /** 274 * Returns the maximum number of initialization status for some application specific failed 275 * reasons. 276 * 277 * Note: The value 0 is reserved for success. 278 * 279 * @hide 280 * @deprecated Replaced with 281 * {@link SandboxedDetectionInitializer#getMaxCustomInitializationStatus()} 282 */ 283 @SystemApi 284 @Deprecated getMaxCustomInitializationStatus()285 public static int getMaxCustomInitializationStatus() { 286 return MAXIMUM_NUMBER_OF_INITIALIZATION_STATUS_CUSTOM_ERROR; 287 } 288 289 /** 290 * Called when the device hardware (such as a DSP) detected the hotword, to request second stage 291 * validation before handing over the audio to the {@link AlwaysOnHotwordDetector}. 292 * 293 * <p>After {@code callback} is invoked or {@code timeoutMillis} has passed, and invokes the 294 * appropriate {@link AlwaysOnHotwordDetector.Callback callback}. 295 * 296 * <p>When responding to a detection event, the 297 * {@link HotwordDetectedResult#getHotwordPhraseId()} must match a keyphrase ID listed 298 * in the eventPayload's 299 * {@link AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras()} list. This is 300 * forcing the intention of the {@link HotwordDetectionService} to validate an event from the 301 * voice engine and not augment its result. 302 * 303 * @param eventPayload Payload data for the hardware detection event. This may contain the 304 * trigger audio, if requested when calling 305 * {@link AlwaysOnHotwordDetector#startRecognition(int)}. 306 * Each {@link AlwaysOnHotwordDetector} will be associated with at minimum a unique 307 * keyphrase ID indicated by 308 * {@link AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras()}[0]. 309 * Any extra 310 * {@link android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra}'s 311 * in the eventPayload represent additional phrases detected by the voice engine. 312 * @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If 313 * the application fails to abide by the timeout, system will close the 314 * microphone and cancel the operation. 315 * @param callback The callback to use for responding to the detection request. 316 * 317 * @hide 318 */ 319 @SystemApi onDetect( @onNull AlwaysOnHotwordDetector.EventPayload eventPayload, @DurationMillisLong long timeoutMillis, @NonNull Callback callback)320 public void onDetect( 321 @NonNull AlwaysOnHotwordDetector.EventPayload eventPayload, 322 @DurationMillisLong long timeoutMillis, 323 @NonNull Callback callback) { 324 // TODO: Add a helpful error message. 325 throw new UnsupportedOperationException(); 326 } 327 328 /** 329 * Called when the {@link VoiceInteractionService#createAlwaysOnHotwordDetector(String, Locale, 330 * PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} or 331 * {@link AlwaysOnHotwordDetector#updateState(PersistableBundle, SharedMemory)} requests an 332 * update of the hotword detection parameters. 333 * 334 * {@inheritDoc} 335 * @hide 336 */ 337 @Override 338 @SystemApi onUpdateState( @ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @DurationMillisLong long callbackTimeoutMillis, @Nullable IntConsumer statusCallback)339 public void onUpdateState( 340 @Nullable PersistableBundle options, 341 @Nullable SharedMemory sharedMemory, 342 @DurationMillisLong long callbackTimeoutMillis, 343 @Nullable IntConsumer statusCallback) {} 344 345 /** 346 * Called when the {@link VoiceInteractionService} requests that this service 347 * {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly 348 * from the device microphone. 349 * <p> 350 * On successful detection of a hotword, call 351 * {@link Callback#onDetected(HotwordDetectedResult)}. 352 * 353 * @param callback The callback to use for responding to the detection request. 354 * {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here. 355 */ onDetect(@onNull Callback callback)356 public void onDetect(@NonNull Callback callback) { 357 // TODO: Add a helpful error message. 358 throw new UnsupportedOperationException(); 359 } 360 361 /** 362 * Called when the {@link VoiceInteractionService} requests that this service 363 * {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat, 364 * PersistableBundle)} run} hotword recognition on audio coming from an external connected 365 * microphone. 366 * 367 * <p>Upon invoking the {@code callback}, the system will send the detection result to 368 * the {@link HotwordDetector}'s callback. If {@code 369 * options.getBoolean(KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK, true)} returns true, 370 * the system will also close the {@code audioStream} after {@code callback} is invoked. 371 * 372 * @param audioStream Stream containing audio bytes returned from a microphone 373 * @param audioFormat Format of the supplied audio 374 * @param options Options supporting detection, such as configuration specific to the source of 375 * the audio, provided through 376 * {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat, 377 * PersistableBundle)}. 378 * @param callback The callback to use for responding to the detection request. 379 */ onDetect( @onNull ParcelFileDescriptor audioStream, @NonNull AudioFormat audioFormat, @Nullable PersistableBundle options, @NonNull Callback callback)380 public void onDetect( 381 @NonNull ParcelFileDescriptor audioStream, 382 @NonNull AudioFormat audioFormat, 383 @Nullable PersistableBundle options, 384 @NonNull Callback callback) { 385 // TODO: Add a helpful error message. 386 throw new UnsupportedOperationException(); 387 } 388 onUpdateStateInternal(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, IRemoteCallback callback)389 private void onUpdateStateInternal(@Nullable PersistableBundle options, 390 @Nullable SharedMemory sharedMemory, IRemoteCallback callback) { 391 IntConsumer intConsumer = 392 SandboxedDetectionInitializer.createInitializationStatusConsumer(callback); 393 onUpdateState(options, sharedMemory, UPDATE_TIMEOUT_MILLIS, intConsumer); 394 } 395 396 /** 397 * Called when the {@link VoiceInteractionService} 398 * {@link HotwordDetector#stopRecognition() requests} that hotword recognition be stopped. 399 * <p> 400 * Any open {@link android.media.AudioRecord} should be closed here. 401 */ onStopDetection()402 public void onStopDetection() { 403 } 404 405 /** 406 * Callback for returning the detection result. 407 * 408 * @hide 409 */ 410 @SystemApi 411 public static final class Callback { 412 // TODO: consider making the constructor a test api for testing purpose 413 private final IDspHotwordDetectionCallback mRemoteCallback; 414 Callback(IDspHotwordDetectionCallback remoteCallback)415 private Callback(IDspHotwordDetectionCallback remoteCallback) { 416 mRemoteCallback = remoteCallback; 417 } 418 419 /** 420 * Informs the {@link HotwordDetector} that the keyphrase was detected. 421 * 422 * @param result Info about the detection result. This is provided to the 423 * {@link HotwordDetector}. 424 */ onDetected(@onNull HotwordDetectedResult result)425 public void onDetected(@NonNull HotwordDetectedResult result) { 426 requireNonNull(result); 427 final PersistableBundle persistableBundle = result.getExtras(); 428 if (!persistableBundle.isEmpty() && HotwordDetectedResult.getParcelableSize( 429 persistableBundle) > HotwordDetectedResult.getMaxBundleSize()) { 430 throw new IllegalArgumentException( 431 "The bundle size of result is larger than max bundle size (" 432 + HotwordDetectedResult.getMaxBundleSize() 433 + ") of HotwordDetectedResult"); 434 } 435 try { 436 mRemoteCallback.onDetected(result); 437 } catch (RemoteException e) { 438 throw e.rethrowFromSystemServer(); 439 } 440 } 441 442 /** 443 * Informs the {@link HotwordDetector} that the keyphrase was not detected. 444 * <p> 445 * This cannot not be used when recognition is done through 446 * {@link #onDetect(ParcelFileDescriptor, AudioFormat, Callback)}. 447 * 448 * @param result Info about the second stage detection result. This is provided to 449 * the {@link HotwordDetector}. 450 */ onRejected(@onNull HotwordRejectedResult result)451 public void onRejected(@NonNull HotwordRejectedResult result) { 452 requireNonNull(result); 453 try { 454 mRemoteCallback.onRejected(result); 455 } catch (RemoteException e) { 456 throw e.rethrowFromSystemServer(); 457 } 458 } 459 } 460 } 461