1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.voiceinteraction.service; 18 19 import static com.google.common.truth.Truth.assertThat; 20 21 import android.graphics.ImageFormat; 22 import android.hardware.camera2.CameraAccessException; 23 import android.hardware.camera2.CameraCaptureSession; 24 import android.hardware.camera2.CameraCharacteristics; 25 import android.hardware.camera2.CameraDevice; 26 import android.hardware.camera2.CameraManager; 27 import android.hardware.camera2.CameraMetadata; 28 import android.hardware.camera2.CaptureRequest; 29 import android.media.Image; 30 import android.media.ImageReader; 31 import android.os.Handler; 32 import android.os.HandlerThread; 33 import android.os.Looper; 34 import android.os.PersistableBundle; 35 import android.os.Process; 36 import android.os.SharedMemory; 37 import android.service.voice.VisualQueryAttentionResult; 38 import android.service.voice.VisualQueryDetectedResult; 39 import android.service.voice.VisualQueryDetectionService; 40 import android.system.ErrnoException; 41 import android.util.Log; 42 import android.util.Size; 43 import android.view.Surface; 44 import android.voiceinteraction.common.Utils; 45 46 import androidx.annotation.Nullable; 47 48 import java.io.FileInputStream; 49 import java.io.FileNotFoundException; 50 import java.io.IOException; 51 import java.nio.ByteBuffer; 52 import java.nio.MappedByteBuffer; 53 import java.nio.channels.FileChannel; 54 import java.nio.channels.NonWritableChannelException; 55 import java.nio.charset.StandardCharsets; 56 import java.util.ArrayList; 57 import java.util.List; 58 import java.util.function.IntConsumer; 59 60 import javax.annotation.concurrent.GuardedBy; 61 62 public class MainVisualQueryDetectionService extends VisualQueryDetectionService { 63 static final String TAG = "MainVisualQueryDetectionService"; 64 65 public static final byte[] TEST_BYTES = new byte[] {0, 1, 2, 3}; 66 public static final String PERCEPTION_MODULE_SUCCESS = "Perception module working"; 67 public static final String FAKE_QUERY_FIRST = "What is "; 68 public static final String FAKE_QUERY_SECOND = "the weather today?"; 69 public static final String MSG_FILE_NOT_WRITABLE = "files does not have writable channel"; 70 public static final String MSG_FILE_NOT_FOUND = "files does not exist in the test directory"; 71 72 public static final String KEY_VQDS_TEST_SCENARIO = "test scenario"; 73 74 public static final int SCENARIO_TEST_PERCEPTION_MODULES = 0; 75 public static final int SCENARIO_ATTENTION_LEAVE = 1; 76 public static final int SCENARIO_ATTENTION_QUERY_REJECTED_LEAVE = 2; 77 public static final int SCENARIO_ATTENTION_QUERY_FINISHED_LEAVE = 3; 78 public static final int SCENARIO_ATTENTION_DOUBLE_QUERY_FINISHED_LEAVE = 4; 79 public static final int SCENARIO_QUERY_NO_ATTENTION = 5; 80 public static final int SCENARIO_QUERY_NO_QUERY_FINISH = 6; 81 public static final int SCENARIO_MULTIPLE_QUERIES_FINISHED = 7; 82 public static final int SCENARIO_COMPLEX_RESULT_STREAM_QUERY_ONLY = 8; 83 public static final int SCENARIO_AUDIO_VISUAL_ATTENTION_STREAM = 9; 84 public static final int SCENARIO_ACCESSIBILITY_ATTENTION_STREAM = 10; 85 public static final int SCENARIO_STREAM_WITH_ACCESSIBILITY_DATA = 11; 86 public static final int SCENARIO_READ_FILE_MMAP_READ_ONLY = 100; 87 public static final int SCENARIO_READ_FILE_MMAP_WRITE = 101; 88 public static final int SCENARIO_READ_FILE_MMAP_MULTIPLE = 102; 89 public static final int SCENARIO_READ_FILE_FILE_NOT_EXIST = 103; 90 91 private static final int TEST_ENGAGEMENT_LEVEL = 100; 92 93 // stores the content of a file for isolated process to perform disk read 94 private ArrayList<String> mResourceContents = new ArrayList<>(); 95 96 private int mScenario = -1; 97 98 private final Object mLock = new Object(); 99 private Handler mHandler; 100 101 private ImageReader mImageReader; 102 private Handler mCameraBackgroundHandler; 103 private HandlerThread mCameraBackgroundThread; 104 105 @GuardedBy("mLock") 106 private boolean mStopDetectionCalled; 107 @GuardedBy("mLock") 108 private int mDetectionDelayMs = 0; 109 110 @GuardedBy("mLock") 111 @Nullable 112 private Runnable mDetectionJob; 113 114 @Override onCreate()115 public void onCreate() { 116 super.onCreate(); 117 mHandler = Handler.createAsync(Looper.getMainLooper()); 118 Log.d(TAG, "onCreate"); 119 } 120 121 @Override onStartDetection()122 public void onStartDetection() { 123 Log.d(TAG, "onStartDetection"); 124 125 startCameraBackgroundThread(); 126 127 synchronized (mLock) { 128 if (mDetectionJob != null) { 129 throw new IllegalStateException("onStartDetection called while already detecting"); 130 } 131 if (!mStopDetectionCalled) { 132 // Delaying this allows us to test other flows, such as stopping detection. It's 133 // also more realistic to schedule it onto another thread. 134 135 // Try different combinations of attention/query permutations with different 136 // detection jobs. 137 mDetectionJob = createTestDetectionJob(mScenario); 138 mHandler.postDelayed(mDetectionJob, 1500); 139 } else { 140 Log.d(TAG, "Sending detected result after stop detection"); 141 // We can't store and use this callback in onStopDetection (not valid anymore 142 // there), so we shut down the service. 143 gainedAttention(); 144 streamQuery(FAKE_QUERY_SECOND); 145 rejectQuery(); 146 lostAttention(); 147 } 148 } 149 } 150 151 @Override onStopDetection()152 public void onStopDetection() { 153 super.onStopDetection(); 154 stopCameraBackgroundThread(); 155 Log.d(TAG, "onStopDetection"); 156 synchronized (mLock) { 157 mHandler.removeCallbacks(mDetectionJob); 158 mDetectionJob = null; 159 mStopDetectionCalled = true; 160 mResourceContents = null; 161 } 162 } 163 164 @Override onUpdateState( @ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, long callbackTimeoutMillis, @Nullable IntConsumer statusCallback)165 public void onUpdateState( 166 @Nullable PersistableBundle options, 167 @Nullable SharedMemory sharedMemory, 168 long callbackTimeoutMillis, 169 @Nullable IntConsumer statusCallback) { 170 super.onUpdateState(options, sharedMemory, callbackTimeoutMillis, statusCallback); 171 Log.d(TAG, "onUpdateState"); 172 173 // Reset mDetectionJob and mStopDetectionCalled when service is initializing. 174 synchronized (mLock) { 175 if (statusCallback != null) { 176 if (mDetectionJob != null) { 177 Log.d(TAG, "onUpdateState mDetectionJob is not null"); 178 mHandler.removeCallbacks(mDetectionJob); 179 mDetectionJob = null; 180 } 181 mStopDetectionCalled = false; 182 } 183 184 if (options != null) { 185 mDetectionDelayMs = options.getInt(Utils.KEY_DETECTION_DELAY_MS, 0); 186 } 187 } 188 189 if (options != null) { 190 if (options.getInt(Utils.KEY_TEST_SCENARIO, -1) 191 == Utils.EXTRA_HOTWORD_DETECTION_SERVICE_ON_UPDATE_STATE_CRASH) { 192 Log.d(TAG, "Crash itself. Pid: " + Process.myPid()); 193 Process.killProcess(Process.myPid()); 194 return; 195 } 196 maybeReadTargetFiles(options); 197 mScenario = options.getInt(KEY_VQDS_TEST_SCENARIO); 198 } 199 200 if (sharedMemory != null) { 201 try { 202 sharedMemory.mapReadWrite(); 203 Log.d(TAG, "sharedMemory : is not read-only"); 204 return; 205 } catch (ErrnoException e) { 206 // For read-only case 207 } finally { 208 sharedMemory.close(); 209 } 210 } 211 212 // Report success 213 Log.d(TAG, "onUpdateState success"); 214 if (statusCallback != null) { 215 statusCallback.accept(INITIALIZATION_STATUS_SUCCESS); 216 } 217 } 218 createTestDetectionJob(int scenario)219 private Runnable createTestDetectionJob(int scenario) { 220 Runnable detectionJob; 221 222 if (scenario == SCENARIO_TEST_PERCEPTION_MODULES) { 223 detectionJob = this::openCamera; 224 } else if (scenario == SCENARIO_ATTENTION_LEAVE) { 225 detectionJob = () -> { 226 gainedAttention(); 227 lostAttention(); 228 }; 229 } else if (scenario == SCENARIO_AUDIO_VISUAL_ATTENTION_STREAM) { 230 detectionJob = () -> { 231 gainedAttention(buildNewVisualQueryAttentionResult( 232 VisualQueryAttentionResult.INTERACTION_INTENTION_AUDIO_VISUAL, 233 TEST_ENGAGEMENT_LEVEL)); 234 streamQuery(FAKE_QUERY_FIRST); 235 finishQuery(); 236 lostAttention(VisualQueryAttentionResult.INTERACTION_INTENTION_AUDIO_VISUAL); 237 }; 238 } else if (scenario == SCENARIO_ACCESSIBILITY_ATTENTION_STREAM) { 239 detectionJob = () -> { 240 gainedAttention(buildNewVisualQueryAttentionResult( 241 VisualQueryAttentionResult.INTERACTION_INTENTION_VISUAL_ACCESSIBILITY, 242 TEST_ENGAGEMENT_LEVEL)); 243 streamQuery(FAKE_QUERY_FIRST); 244 finishQuery(); 245 lostAttention( 246 VisualQueryAttentionResult.INTERACTION_INTENTION_VISUAL_ACCESSIBILITY); 247 }; 248 } else if (scenario == SCENARIO_ATTENTION_QUERY_FINISHED_LEAVE) { 249 detectionJob = () -> { 250 gainedAttention(); 251 streamQuery(FAKE_QUERY_FIRST); 252 streamQuery(FAKE_QUERY_SECOND); 253 finishQuery(); 254 lostAttention(); 255 }; 256 } else if (scenario == SCENARIO_ATTENTION_QUERY_REJECTED_LEAVE) { 257 detectionJob = () -> { 258 gainedAttention(); 259 streamQuery(FAKE_QUERY_FIRST); 260 rejectQuery(); 261 lostAttention(); 262 }; 263 } else if (scenario == SCENARIO_ATTENTION_DOUBLE_QUERY_FINISHED_LEAVE) { 264 detectionJob = () -> { 265 gainedAttention(); 266 streamQuery(FAKE_QUERY_FIRST); 267 finishQuery(); 268 streamQuery(FAKE_QUERY_SECOND); 269 finishQuery(); 270 lostAttention(); 271 }; 272 } else if (scenario == SCENARIO_QUERY_NO_ATTENTION) { 273 detectionJob = () -> { 274 streamQuery(FAKE_QUERY_FIRST); 275 finishQuery(); 276 rejectQuery(); 277 }; 278 } else if (scenario == SCENARIO_QUERY_NO_QUERY_FINISH) { 279 detectionJob = () -> { 280 gainedAttention(); 281 finishQuery(); 282 lostAttention(); 283 }; 284 } else if (scenario == SCENARIO_MULTIPLE_QUERIES_FINISHED) { 285 detectionJob = () -> { 286 gainedAttention(); 287 for (int i = 0; i < Utils.NUM_TEST_QUERY_SESSION_MULTIPLE; i++) { 288 if ((i & 1) == 0) { 289 streamQuery(FAKE_QUERY_FIRST); 290 } else { 291 streamQuery(FAKE_QUERY_SECOND); 292 } 293 finishQuery(); 294 } 295 lostAttention(); 296 }; 297 } else if (scenario == SCENARIO_COMPLEX_RESULT_STREAM_QUERY_ONLY) { 298 detectionJob = () -> { 299 gainedAttention(); 300 streamQuery( 301 new VisualQueryDetectedResult.Builder().setPartialQuery(FAKE_QUERY_FIRST) 302 .build()); 303 streamQuery( 304 new VisualQueryDetectedResult.Builder().setPartialQuery(FAKE_QUERY_SECOND) 305 .build()); 306 finishQuery(); 307 lostAttention(); 308 }; 309 } else if (scenario == SCENARIO_STREAM_WITH_ACCESSIBILITY_DATA) { 310 detectionJob = () -> { 311 gainedAttention(); 312 streamQuery( 313 new VisualQueryDetectedResult.Builder() 314 .setAccessibilityDetectionData(TEST_BYTES) 315 .build()); 316 finishQuery(); 317 lostAttention(); 318 }; 319 } else if (scenario == SCENARIO_READ_FILE_MMAP_READ_ONLY 320 || scenario == SCENARIO_READ_FILE_MMAP_WRITE 321 || scenario == SCENARIO_READ_FILE_FILE_NOT_EXIST) { 322 // leverages the detection API to verify if the content read from the file is correct 323 detectionJob = () -> { 324 gainedAttention(); 325 streamQuery(mResourceContents.get(0)); 326 finishQuery(); 327 lostAttention(); 328 }; 329 } else if (scenario == SCENARIO_READ_FILE_MMAP_MULTIPLE) { 330 // leverages the detection API to verify if the content read from the file is correct 331 detectionJob = () -> { 332 gainedAttention(); 333 for (String content : mResourceContents) { 334 streamQuery(content); 335 finishQuery(); 336 } 337 lostAttention(); 338 }; 339 } else { 340 Log.i(TAG, "Do nothing..."); 341 return null; 342 } 343 return detectionJob; 344 } 345 buildNewVisualQueryAttentionResult( int interactionIntention, int engagementLevel)346 private VisualQueryAttentionResult buildNewVisualQueryAttentionResult( 347 int interactionIntention, int engagementLevel) { 348 return new VisualQueryAttentionResult.Builder() 349 .setInteractionIntention(interactionIntention) 350 .setEngagementLevel(engagementLevel).build(); 351 } 352 sendCameraOpenSuccessSignals()353 private void sendCameraOpenSuccessSignals() { 354 gainedAttention(); 355 streamQuery(PERCEPTION_MODULE_SUCCESS); 356 finishQuery(); 357 lostAttention(); 358 } 359 startCameraBackgroundThread()360 private void startCameraBackgroundThread() { 361 mCameraBackgroundThread = new HandlerThread("Camera Background Thread"); 362 mCameraBackgroundThread.start(); 363 mCameraBackgroundHandler = new Handler(mCameraBackgroundThread.getLooper()); 364 } 365 stopCameraBackgroundThread()366 private void stopCameraBackgroundThread() { 367 mCameraBackgroundThread.quitSafely(); 368 try { 369 mCameraBackgroundThread.join(); 370 mCameraBackgroundThread = null; 371 mCameraBackgroundHandler = null; 372 } catch (InterruptedException e) { 373 Log.e(TAG, "Failed to stop camera thread."); 374 } 375 } 376 onReceiveImage(ImageReader reader)377 private void onReceiveImage(ImageReader reader) { 378 Log.i(TAG, "Image received."); 379 try (Image image = reader.acquireLatestImage()) { 380 ByteBuffer buffer = image.getPlanes()[0].getBuffer(); 381 assertThat(buffer.capacity()).isGreaterThan(0); 382 sendCameraOpenSuccessSignals(); 383 } catch (Exception e) { 384 e.printStackTrace(); 385 } 386 } 387 openCamera()388 private void openCamera() { 389 CameraManager manager = getSystemService(CameraManager.class); 390 assert manager != null; 391 try { 392 // Check camera can be seen 393 String cameraId = manager.getCameraIdList()[0]; //get front facing camera 394 CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); 395 Size imageSize = characteristics.get( 396 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) 397 .getOutputSizes(ImageFormat.JPEG)[0]; 398 initializeImageReader(imageSize.getWidth(), imageSize.getHeight()); 399 manager.openCamera(cameraId, new CameraDevice.StateCallback() { 400 @Override 401 public void onOpened(CameraDevice camera) { 402 // This is called when the camera is open 403 Log.i(TAG, "onCameraOpened"); 404 // The camera data will be zero on virtual device, so it would be better to skip 405 // to check the camera data. 406 if (Utils.isVirtualDevice()) { 407 sendCameraOpenSuccessSignals(); 408 } 409 createCameraPreview(camera); 410 } 411 412 @Override 413 public void onDisconnected(CameraDevice camera) { 414 camera.close(); 415 } 416 417 @Override 418 public void onError(CameraDevice camera, int error) { 419 camera.close(); 420 } 421 }, mCameraBackgroundHandler); 422 } catch (CameraAccessException e) { 423 throw new IllegalStateException("Missing Camera access."); 424 } 425 } 426 initializeImageReader(int width, int height)427 private void initializeImageReader(int width, int height) { 428 // Initialize image reader 429 mImageReader = ImageReader.newInstance(width, height, ImageFormat.JPEG, 2); 430 ImageReader.OnImageAvailableListener readerListener = this::onReceiveImage; 431 mImageReader.setOnImageAvailableListener(readerListener, mCameraBackgroundHandler); 432 } 433 createCameraPreview(CameraDevice cameraDevice)434 private void createCameraPreview(CameraDevice cameraDevice) { 435 try { 436 CaptureRequest.Builder captureRequestBuilder = 437 cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE); 438 Surface imageSurface = mImageReader.getSurface(); 439 captureRequestBuilder.addTarget(imageSurface); 440 cameraDevice.createCaptureSession(List.of(imageSurface), 441 new CameraCaptureSession.StateCallback() { 442 @Override 443 public void onConfigured(CameraCaptureSession cameraCaptureSession) { 444 updatePreview(captureRequestBuilder, cameraCaptureSession); 445 Log.i(TAG, "Capture session configured."); 446 } 447 448 @Override 449 public void onConfigureFailed(CameraCaptureSession cameraCaptureSession) { 450 //No-op 451 } 452 }, null); 453 } catch (CameraAccessException e) { 454 e.printStackTrace(); 455 } 456 Log.i(TAG, "Camera preview created."); 457 } 458 updatePreview(CaptureRequest.Builder captureRequestBuilder, CameraCaptureSession cameraCaptureSession)459 private void updatePreview(CaptureRequest.Builder captureRequestBuilder, 460 CameraCaptureSession cameraCaptureSession) { 461 captureRequestBuilder.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO); 462 try { 463 cameraCaptureSession.capture(captureRequestBuilder.build(), null, 464 mCameraBackgroundHandler); 465 } catch (Exception e) { 466 e.printStackTrace(); 467 } 468 } 469 maybeReadTargetFiles(PersistableBundle options)470 private void maybeReadTargetFiles(PersistableBundle options) { 471 switch (options.getInt(KEY_VQDS_TEST_SCENARIO)) { 472 case SCENARIO_READ_FILE_MMAP_READ_ONLY: 473 case SCENARIO_READ_FILE_FILE_NOT_EXIST: 474 readFileWithMMap(Utils.TEST_RESOURCE_FILE_NAME, FileChannel.MapMode.READ_ONLY); 475 break; 476 case SCENARIO_READ_FILE_MMAP_WRITE: 477 readFileWithMMap(Utils.TEST_RESOURCE_FILE_NAME, FileChannel.MapMode.READ_WRITE); 478 break; 479 case SCENARIO_READ_FILE_MMAP_MULTIPLE: 480 for (int i = 0; i < Utils.NUM_TEST_RESOURCE_FILE_MULTIPLE; i++) { 481 readFileWithMMap(Utils.TEST_RESOURCE_FILE_NAME + i, 482 FileChannel.MapMode.READ_ONLY); 483 } 484 break; 485 } // end switch 486 } 487 readFileWithMMap(String filename, FileChannel.MapMode mode)488 private void readFileWithMMap(String filename, FileChannel.MapMode mode) { 489 try (FileInputStream fis = openFileInput(filename)) { 490 Log.d(TAG, "Reading test file in mode: " + mode); 491 FileChannel fc = fis.getChannel(); 492 MappedByteBuffer buffer = fc.map(mode, 0, fc.size()); 493 byte[] data = new byte[(int) fc.size()]; 494 buffer.get(data); 495 mResourceContents.add(new String(data, StandardCharsets.UTF_8)); 496 } catch (FileNotFoundException e) { 497 Log.d(TAG, "Target file to read does not exist. Filename: " + filename); 498 mResourceContents.add(MSG_FILE_NOT_FOUND); 499 } catch (NonWritableChannelException e) { 500 Log.d(TAG, "Only read-only mode is permitted."); 501 mResourceContents.add(MSG_FILE_NOT_WRITABLE); 502 } catch (IOException e) { 503 Log.e(TAG, "Unexpected IO error: Cannot mmap read from opened file: " 504 + e.getMessage()); 505 } 506 } 507 } 508 509