1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; 20 import static android.Manifest.permission.RECORD_AUDIO; 21 22 import android.annotation.NonNull; 23 import android.annotation.Nullable; 24 import android.annotation.RequiresPermission; 25 import android.annotation.SystemApi; 26 import android.media.AudioFormat; 27 import android.os.ParcelFileDescriptor; 28 import android.os.PersistableBundle; 29 import android.os.SharedMemory; 30 31 import java.io.PrintWriter; 32 33 /** 34 * Basic functionality for sandboxed detectors. This interface will be used by detectors that 35 * manages their service lifecycle. 36 * 37 * @hide 38 */ 39 @SystemApi 40 public interface HotwordDetector { 41 42 /** 43 * Indicates that it is a non-trusted hotword detector. 44 * 45 * @hide 46 */ 47 int DETECTOR_TYPE_NORMAL = 0; 48 49 /** 50 * Indicates that it is a DSP trusted hotword detector. 51 * 52 * @hide 53 */ 54 int DETECTOR_TYPE_TRUSTED_HOTWORD_DSP = 1; 55 56 /** 57 * Indicates that it is a software trusted hotword detector. 58 * 59 * @hide 60 */ 61 int DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE = 2; 62 63 /** 64 * Indicates that it is a visual query detector. 65 * 66 * @hide 67 */ 68 int DETECTOR_TYPE_VISUAL_QUERY_DETECTOR = 3; 69 70 /** 71 * Starts sandboxed detection recognition. 72 * <p> 73 * If a {@link VisualQueryDetector} calls this method, {@link VisualQueryDetectionService 74 * #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start detection. 75 * <p> 76 * Otherwise if a {@link AlwaysOnHotwordDetector} or {@link SoftwareHotwordDetector} calls this, 77 * the system streams audio from the device microphone to this application's 78 * {@link HotwordDetectionService}. Audio is streamed until {@link #stopRecognition()} is 79 * called. 80 * <p> 81 * On detection of a hotword, 82 * {@link AlwaysOnHotwordDetector.Callback#onDetected(AlwaysOnHotwordDetector.EventPayload)} 83 * is called on the callback provided when creating this {@link HotwordDetector}. 84 * <p> 85 * There is a noticeable impact on battery while recognition is active, so make sure to call 86 * {@link #stopRecognition()} when detection isn't needed. 87 * <p> 88 * Calling this again while recognition is active does nothing. 89 * 90 * @return {@code true} if the request to start recognition succeeded 91 */ 92 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) startRecognition()93 boolean startRecognition(); 94 95 /** 96 * Stops sandboxed detection recognition. 97 * 98 * @return {@code true} if the request to stop recognition succeeded 99 */ stopRecognition()100 boolean stopRecognition(); 101 102 /** 103 * Starts hotword recognition on audio coming from an external connected microphone. 104 * <p> 105 * {@link #stopRecognition()} must be called before {@code audioStream} is closed. 106 * 107 * @param audioStream stream containing the audio bytes to run detection on 108 * @param audioFormat format of the encoded audio 109 * @param options options supporting detection, such as configuration specific to the 110 * source of the audio. This will be provided to the {@link HotwordDetectionService}. 111 * PersistableBundle does not allow any remotable objects or other contents that can be 112 * used to communicate with other processes. 113 * @return {@code true} if the request to start recognition succeeded 114 */ startRecognition( @onNull ParcelFileDescriptor audioStream, @NonNull AudioFormat audioFormat, @Nullable PersistableBundle options)115 boolean startRecognition( 116 @NonNull ParcelFileDescriptor audioStream, 117 @NonNull AudioFormat audioFormat, 118 @Nullable PersistableBundle options); 119 120 /** 121 * Set configuration and pass read-only data to sandboxed detection service. 122 * 123 * @param options Application configuration data to provide to sandboxed detection services. 124 * PersistableBundle does not allow any remotable objects or other contents that can be used to 125 * communicate with other processes. 126 * @param sharedMemory The unrestricted data blob to provide to sandboxed detection services. 127 * Use this to provide model data or other such data to the trusted process. 128 * @throws IllegalStateException if this HotwordDetector wasn't specified to use a 129 * sandboxed detection service when it was created. 130 */ updateState(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)131 void updateState(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory); 132 133 /** 134 * Invalidates this detector so that any future calls to this result 135 * in an {@link IllegalStateException} when a caller has a target SDK below API level 33 136 * or an {@link IllegalDetectorStateException} when a caller has a target SDK of API level 33 137 * or above. 138 * 139 * <p>If there are no other {@link HotwordDetector} instances linked to the 140 * sandboxed detection service, the service will be shutdown. 141 */ destroy()142 default void destroy() { 143 throw new UnsupportedOperationException("Not implemented. Must override in a subclass."); 144 } 145 146 /** 147 * @hide 148 */ isUsingSandboxedDetectionService()149 default boolean isUsingSandboxedDetectionService() { 150 throw new UnsupportedOperationException("Not implemented. Must override in a subclass."); 151 } 152 153 /** 154 * @hide 155 */ detectorTypeToString(int detectorType)156 static String detectorTypeToString(int detectorType) { 157 switch (detectorType) { 158 case DETECTOR_TYPE_NORMAL: 159 return "normal"; 160 case DETECTOR_TYPE_TRUSTED_HOTWORD_DSP: 161 return "trusted_hotword_dsp"; 162 case DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE: 163 return "trusted_hotword_software"; 164 case DETECTOR_TYPE_VISUAL_QUERY_DETECTOR: 165 return "visual_query_detector"; 166 default: 167 return Integer.toString(detectorType); 168 } 169 } 170 171 /** @hide */ dump(String prefix, PrintWriter pw)172 default void dump(String prefix, PrintWriter pw) { 173 throw new UnsupportedOperationException("Not implemented. Must override in a subclass."); 174 } 175 176 /** 177 * The callback to notify of detection events. 178 */ 179 interface Callback { 180 181 /** 182 * Called when the keyphrase is spoken. 183 * 184 * @param eventPayload Payload data for the detection event. 185 */ 186 // TODO: Consider creating a new EventPayload that the AOHD one subclasses. onDetected(@onNull AlwaysOnHotwordDetector.EventPayload eventPayload)187 void onDetected(@NonNull AlwaysOnHotwordDetector.EventPayload eventPayload); 188 189 /** 190 * Called when the detection fails due to an error. 191 * 192 * @deprecated On {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE} and above, 193 * implement {@link HotwordDetector.Callback#onFailure(HotwordDetectionServiceFailure)}, 194 * {@link AlwaysOnHotwordDetector.Callback#onFailure(SoundTriggerFailure)}, 195 * {@link HotwordDetector.Callback#onUnknownFailure(String)} instead. 196 */ 197 @Deprecated onError()198 void onError(); 199 200 /** 201 * Called when the detection fails due to an error occurs in the 202 * {@link HotwordDetectionService}, {@link HotwordDetectionServiceFailure} will be reported 203 * to the detector. 204 * 205 * @param hotwordDetectionServiceFailure It provides the error code, error message and 206 * suggested action. 207 */ onFailure( @onNull HotwordDetectionServiceFailure hotwordDetectionServiceFailure)208 default void onFailure( 209 @NonNull HotwordDetectionServiceFailure hotwordDetectionServiceFailure) { 210 onError(); 211 } 212 213 /** 214 * Called when the detection fails due to an unknown error occurs, an error message 215 * will be reported to the detector. 216 * 217 * @param errorMessage It provides the error message. 218 */ onUnknownFailure(@onNull String errorMessage)219 default void onUnknownFailure(@NonNull String errorMessage) { 220 onError(); 221 } 222 223 /** 224 * Called when the recognition is paused temporarily for some reason. 225 * This is an informational callback, and the clients shouldn't be doing anything here 226 * except showing an indication on their UI if they have to. 227 */ onRecognitionPaused()228 void onRecognitionPaused(); 229 230 /** 231 * Called when the recognition is resumed after it was temporarily paused. 232 * This is an informational callback, and the clients shouldn't be doing anything here 233 * except showing an indication on their UI if they have to. 234 */ onRecognitionResumed()235 void onRecognitionResumed(); 236 237 /** 238 * Called when the {@link HotwordDetectionService} second stage detection did not detect the 239 * keyphrase. 240 * 241 * @param result Info about the second stage detection result, provided by the 242 * {@link HotwordDetectionService}. 243 */ onRejected(@onNull HotwordRejectedResult result)244 void onRejected(@NonNull HotwordRejectedResult result); 245 246 /** 247 * Called when the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is 248 * created by the system and given a short amount of time to report their initialization 249 * state. 250 * 251 * @param status Info about initialization state of {@link HotwordDetectionService} or 252 * {@link VisualQueryDetectionService}; allowed values are 253 * {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_SUCCESS}, 254 * 1<->{@link SandboxedDetectionInitializer#getMaxCustomInitializationStatus()}, 255 * {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_UNKNOWN}. 256 */ onHotwordDetectionServiceInitialized(int status)257 void onHotwordDetectionServiceInitialized(int status); 258 259 /** 260 * Called with the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is 261 * restarted. 262 * 263 * Clients are expected to call {@link HotwordDetector#updateState} to share the state with 264 * the newly created service. 265 */ onHotwordDetectionServiceRestarted()266 void onHotwordDetectionServiceRestarted(); 267 } 268 } 269