1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import android.annotation.CallSuper; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.app.ActivityThread; 23 import android.media.AudioFormat; 24 import android.media.permission.Identity; 25 import android.os.Binder; 26 import android.os.Handler; 27 import android.os.HandlerExecutor; 28 import android.os.IBinder; 29 import android.os.Looper; 30 import android.os.ParcelFileDescriptor; 31 import android.os.PersistableBundle; 32 import android.os.RemoteException; 33 import android.os.SharedMemory; 34 import android.util.Slog; 35 36 import com.android.internal.app.IHotwordRecognitionStatusCallback; 37 import com.android.internal.app.IVoiceInteractionManagerService; 38 39 import java.util.concurrent.Executor; 40 import java.util.concurrent.atomic.AtomicBoolean; 41 import java.util.function.Consumer; 42 43 /** Base implementation of {@link HotwordDetector}. 44 * 45 * This class provides methods to manage the detector lifecycle for both 46 * {@link HotwordDetectionService} and {@link VisualQueryDetectionService}. We keep the name of the 47 * interface {@link HotwordDetector} since {@link VisualQueryDetectionService} can be logically 48 * treated as a visual activation hotword detection and also because of the existing public 49 * interface. To avoid confusion on the naming between the trusted hotword framework and the actual 50 * isolated {@link HotwordDetectionService}, the hotword from the names is removed. 51 */ 52 abstract class AbstractDetector implements HotwordDetector { 53 private static final String TAG = AbstractDetector.class.getSimpleName(); 54 private static final boolean DEBUG = false; 55 56 protected final Object mLock = new Object(); 57 58 private final IVoiceInteractionManagerService mManagerService; 59 private final Executor mExecutor; 60 private final HotwordDetector.Callback mCallback; 61 private Consumer<AbstractDetector> mOnDestroyListener; 62 private final AtomicBoolean mIsDetectorActive; 63 /** 64 * A token which is used by voice interaction system service to identify different detectors. 65 */ 66 private final IBinder mToken = new Binder(); 67 68 /** 69 * A flag controls whether attributionTag will be passed into the Identity. 70 * TODO(b/289087412): This flag will be converted and confirm to the trunk stable flag 71 * configuration. 72 */ 73 static final boolean IS_IDENTITY_WITH_ATTRIBUTION_TAG = false; 74 AbstractDetector( IVoiceInteractionManagerService managerService, Executor executor, HotwordDetector.Callback callback)75 AbstractDetector( 76 IVoiceInteractionManagerService managerService, 77 Executor executor, 78 HotwordDetector.Callback callback) { 79 mManagerService = managerService; 80 mCallback = callback; 81 mExecutor = executor != null ? executor : new HandlerExecutor( 82 new Handler(Looper.getMainLooper())); 83 mIsDetectorActive = new AtomicBoolean(true); 84 } 85 isSameToken(IBinder token)86 boolean isSameToken(IBinder token) { 87 if (token == null) { 88 return false; 89 } 90 return mToken == token; 91 } 92 93 /** 94 * Method to be called for the detector to ready/register itself with underlying system 95 * services. 96 */ initialize(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)97 abstract void initialize(@Nullable PersistableBundle options, 98 @Nullable SharedMemory sharedMemory); 99 100 /** 101 * Detect from an externally supplied stream of data. 102 * 103 * @return {@code true} if the request to start recognition succeeded 104 */ 105 @Override startRecognition( @onNull ParcelFileDescriptor audioStream, @NonNull AudioFormat audioFormat, @Nullable PersistableBundle options)106 public boolean startRecognition( 107 @NonNull ParcelFileDescriptor audioStream, 108 @NonNull AudioFormat audioFormat, 109 @Nullable PersistableBundle options) { 110 if (DEBUG) { 111 Slog.i(TAG, "#recognizeHotword"); 112 } 113 throwIfDetectorIsNoLongerActive(); 114 115 // TODO: consider closing existing session. 116 117 try { 118 mManagerService.startListeningFromExternalSource( 119 audioStream, 120 audioFormat, 121 options, 122 mToken, 123 new BinderCallback(mExecutor, mCallback)); 124 } catch (RemoteException e) { 125 e.rethrowFromSystemServer(); 126 } 127 128 return true; 129 } 130 131 /** 132 * Set configuration and pass read-only data to trusted detection service. 133 * 134 * @param options Application configuration data to provide to the 135 * {@link VisualQueryDetectionService} and {@link HotwordDetectionService}. 136 * PersistableBundle does not allow any remotable objects or other contents that can be 137 * used to communicate with other processes. 138 * @param sharedMemory The unrestricted data blob to provide to the 139 * {@link VisualQueryDetectionService} and {@link HotwordDetectionService}. Use this to 140 * provide the hotword models data or other such data to the trusted process. 141 * @throws IllegalStateException if this {@link HotwordDetector} wasn't specified to use a 142 * {@link HotwordDetectionService} or {@link VisualQueryDetectionService} when it was 143 * created. 144 */ 145 @Override updateState(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)146 public void updateState(@Nullable PersistableBundle options, 147 @Nullable SharedMemory sharedMemory) { 148 if (DEBUG) { 149 Slog.d(TAG, "updateState()"); 150 } 151 throwIfDetectorIsNoLongerActive(); 152 try { 153 mManagerService.updateState(options, sharedMemory, mToken); 154 } catch (RemoteException e) { 155 throw e.rethrowFromSystemServer(); 156 } 157 } 158 initAndVerifyDetector( @ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @NonNull IHotwordRecognitionStatusCallback callback, int detectorType, @Nullable String attributionTag)159 protected void initAndVerifyDetector( 160 @Nullable PersistableBundle options, 161 @Nullable SharedMemory sharedMemory, 162 @NonNull IHotwordRecognitionStatusCallback callback, 163 int detectorType, 164 @Nullable String attributionTag) { 165 if (DEBUG) { 166 Slog.d(TAG, "initAndVerifyDetector()"); 167 } 168 Identity identity = new Identity(); 169 identity.packageName = ActivityThread.currentOpPackageName(); 170 if (IS_IDENTITY_WITH_ATTRIBUTION_TAG) { 171 identity.attributionTag = attributionTag; 172 } 173 try { 174 mManagerService.initAndVerifyDetector(identity, options, sharedMemory, mToken, callback, 175 detectorType); 176 } catch (RemoteException e) { 177 throw e.rethrowFromSystemServer(); 178 } 179 } 180 registerOnDestroyListener(Consumer<AbstractDetector> onDestroyListener)181 void registerOnDestroyListener(Consumer<AbstractDetector> onDestroyListener) { 182 synchronized (mLock) { 183 if (mOnDestroyListener != null) { 184 throw new IllegalStateException("only one destroy listener can be registered"); 185 } 186 mOnDestroyListener = onDestroyListener; 187 } 188 } 189 190 @CallSuper 191 @Override destroy()192 public void destroy() { 193 if (!mIsDetectorActive.get()) { 194 return; 195 } 196 mIsDetectorActive.set(false); 197 try { 198 mManagerService.destroyDetector(mToken); 199 } catch (RemoteException e) { 200 throw e.rethrowFromSystemServer(); 201 } 202 Consumer<AbstractDetector> onDestroyListener; 203 synchronized (mLock) { 204 onDestroyListener = mOnDestroyListener; 205 } 206 if (onDestroyListener != null) { 207 onDestroyListener.accept(this); 208 } 209 } 210 throwIfDetectorIsNoLongerActive()211 protected void throwIfDetectorIsNoLongerActive() { 212 if (!mIsDetectorActive.get()) { 213 Slog.e(TAG, "attempting to use a destroyed detector which is no longer active"); 214 throw new IllegalStateException( 215 "attempting to use a destroyed detector which is no longer active"); 216 } 217 } 218 219 private static class BinderCallback 220 extends IMicrophoneHotwordDetectionVoiceInteractionCallback.Stub { 221 // TODO: these need to be weak references. 222 private final HotwordDetector.Callback mCallback; 223 private final Executor mExecutor; 224 BinderCallback(Executor executor, HotwordDetector.Callback callback)225 BinderCallback(Executor executor, HotwordDetector.Callback callback) { 226 this.mCallback = callback; 227 this.mExecutor = executor; 228 } 229 230 /** TODO: onDetected */ 231 @Override onDetected( @ullable HotwordDetectedResult hotwordDetectedResult, @Nullable AudioFormat audioFormat, @Nullable ParcelFileDescriptor audioStreamIgnored)232 public void onDetected( 233 @Nullable HotwordDetectedResult hotwordDetectedResult, 234 @Nullable AudioFormat audioFormat, 235 @Nullable ParcelFileDescriptor audioStreamIgnored) { 236 Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> { 237 mCallback.onDetected(new AlwaysOnHotwordDetector.EventPayload.Builder() 238 .setCaptureAudioFormat(audioFormat) 239 .setHotwordDetectedResult(hotwordDetectedResult) 240 .build()); 241 })); 242 } 243 244 /** Called when the detection fails due to an error. */ 245 @Override onHotwordDetectionServiceFailure( HotwordDetectionServiceFailure hotwordDetectionServiceFailure)246 public void onHotwordDetectionServiceFailure( 247 HotwordDetectionServiceFailure hotwordDetectionServiceFailure) { 248 Slog.v(TAG, "BinderCallback#onHotwordDetectionServiceFailure: " 249 + hotwordDetectionServiceFailure); 250 Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> { 251 if (hotwordDetectionServiceFailure != null) { 252 mCallback.onFailure(hotwordDetectionServiceFailure); 253 } else { 254 mCallback.onUnknownFailure("Error data is null"); 255 } 256 })); 257 } 258 259 @Override onRejected(@ullable HotwordRejectedResult result)260 public void onRejected(@Nullable HotwordRejectedResult result) { 261 Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> { 262 mCallback.onRejected( 263 result != null ? result : new HotwordRejectedResult.Builder().build()); 264 })); 265 } 266 } 267 } 268