1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.cts.verifier.camera.its;
17 
18 import android.graphics.Bitmap;
19 import android.graphics.BitmapFactory;
20 import android.graphics.Color;
21 import android.graphics.ImageFormat;
22 import android.graphics.SurfaceTexture;
23 import android.hardware.camera2.CameraAccessException;
24 import android.hardware.camera2.CameraCaptureSession;
25 import android.hardware.camera2.CameraCharacteristics;
26 import android.hardware.camera2.CameraDevice;
27 import android.hardware.camera2.CameraManager;
28 import android.hardware.camera2.CaptureRequest;
29 import android.hardware.camera2.CaptureResult;
30 import android.hardware.camera2.TotalCaptureResult;
31 import android.hardware.camera2.params.StreamConfigurationMap;
32 import android.media.Image;
33 import android.media.ImageReader;
34 import android.os.Bundle;
35 import android.os.Handler;
36 import android.os.HandlerThread;
37 import android.util.Log;
38 import android.util.Size;
39 import android.view.Surface;
40 import android.view.TextureView;
41 import android.widget.Button;
42 import android.widget.ImageButton;
43 import android.widget.ImageView;
44 import android.widget.TextView;
45 
46 import com.android.compatibility.common.util.ResultType;
47 import com.android.compatibility.common.util.ResultUnit;
48 import com.android.cts.verifier.CtsVerifierReportLog;
49 import com.android.cts.verifier.PassFailButtons;
50 import com.android.cts.verifier.R;
51 import com.android.ex.camera2.blocking.BlockingCameraManager;
52 import com.android.ex.camera2.blocking.BlockingCameraManager.BlockingOpenException;
53 import com.android.ex.camera2.blocking.BlockingSessionCallback;
54 import com.android.ex.camera2.blocking.BlockingStateCallback;
55 
56 import java.nio.ByteBuffer;
57 import java.util.ArrayList;
58 import java.util.Arrays;
59 import java.util.Comparator;
60 import java.util.List;
61 
62 /**
63  * Test for manual verification of camera privacy hardware switches
64  * This test verifies that devices which implement camera hardware
65  * privacy toggles enforce sensor privacy when toggles are enabled.
66  * - The video stream should be muted:
67  * - camera preview & capture should be blank
68  * - A dialog or notification should be shown that informs
69  * the user that the sensor privacy is enabled.
70  */
71 public class CameraMuteToggleActivity extends PassFailButtons.Activity
72         implements TextureView.SurfaceTextureListener,
73         ImageReader.OnImageAvailableListener {
74 
75     private static final String TAG = "CameraMuteToggleActivity";
76     private static final int SESSION_READY_TIMEOUT_MS = 5000;
77     private static final int DEFAULT_CAMERA_IDX = 0;
78 
79     private TextureView mPreviewView;
80     private SurfaceTexture mPreviewTexture;
81     private Surface mPreviewSurface;
82 
83     private ImageView mImageView;
84 
85     private CameraManager mCameraManager;
86     private HandlerThread mCameraThread;
87     private Handler mCameraHandler;
88     private BlockingCameraManager mBlockingCameraManager;
89     private CameraCharacteristics mCameraCharacteristics;
90     private BlockingStateCallback mCameraListener;
91 
92     private BlockingSessionCallback mSessionListener;
93     private CaptureRequest.Builder mPreviewRequestBuilder;
94     private CaptureRequest mPreviewRequest;
95     private CaptureRequest.Builder mStillCaptureRequestBuilder;
96     private CaptureRequest mStillCaptureRequest;
97 
98     private CameraCaptureSession mCaptureSession;
99     private CameraDevice mCameraDevice;
100 
101     SizeComparator mSizeComparator = new SizeComparator();
102 
103     private Size mPreviewSize;
104     private Size mJpegSize;
105     private ImageReader mJpegImageReader;
106 
107     private CameraCaptureSession.CaptureCallback mCaptureCallback =
108             new CameraCaptureSession.CaptureCallback() {
109             };
110 
111     @Override
onCreate(Bundle savedInstanceState)112     public void onCreate(Bundle savedInstanceState) {
113         super.onCreate(savedInstanceState);
114 
115         setContentView(R.layout.cam_hw_toggle);
116 
117         setPassFailButtonClickListeners();
118 
119         mPreviewView = findViewById(R.id.preview_view);
120         mImageView = findViewById(R.id.image_view);
121 
122         mPreviewView.setSurfaceTextureListener(this);
123 
124         mCameraManager = getSystemService(CameraManager.class);
125 
126         setInfoResources(R.string.camera_hw_toggle_test, R.string.camera_hw_toggle_test_info, -1);
127 
128         // Enable Pass button only after taking photo
129         setPassButtonEnabled(false);
130         setTakePictureButtonEnabled(false);
131 
132         mBlockingCameraManager = new BlockingCameraManager(mCameraManager);
133         mCameraListener = new BlockingStateCallback();
134     }
135 
136     @Override
onResume()137     public void onResume() {
138         super.onResume();
139 
140         startBackgroundThread();
141 
142         Exception cameraSetupException = null;
143         boolean enablePassButton = false;
144         try {
145             final String[] camerasList = mCameraManager.getCameraIdList();
146             if (camerasList.length > 0) {
147                 String cameraId = mCameraManager.getCameraIdList()[DEFAULT_CAMERA_IDX];
148                 setUpCamera(cameraId);
149             } else {
150                 showCameraErrorText("");
151             }
152         } catch (CameraAccessException e) {
153             cameraSetupException = e;
154             // Enable Pass button for cameras that do not support mute patterns
155             // and will disconnect clients if sensor privacy is enabled
156             enablePassButton = (e.getReason() == CameraAccessException.CAMERA_DISABLED);
157         } catch (BlockingOpenException e) {
158             cameraSetupException = e;
159             enablePassButton = e.wasDisconnected();
160         } finally {
161             if (cameraSetupException != null) {
162                 cameraSetupException.printStackTrace();
163                 showCameraErrorText(cameraSetupException.getMessage());
164                 setPassButtonEnabled(enablePassButton);
165             }
166         }
167     }
168 
showCameraErrorText(String errorMsg)169     private void showCameraErrorText(String errorMsg) {
170         TextView instructionsText = findViewById(R.id.instruction_text);
171         instructionsText.setText(R.string.camera_hw_toggle_test_no_camera);
172         instructionsText.append(errorMsg);
173         setTakePictureButtonEnabled(false);
174     }
175 
176     @Override
onPause()177     public void onPause() {
178         shutdownCamera();
179         stopBackgroundThread();
180 
181         super.onPause();
182     }
183 
184     @Override
onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int width, int height)185     public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture,
186             int width, int height) {
187         mPreviewTexture = surfaceTexture;
188 
189         mPreviewSurface = new Surface(mPreviewTexture);
190 
191         if (mCameraDevice != null) {
192             startPreview();
193         }
194     }
195 
196     @Override
onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height)197     public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) {
198         // Ignored, Camera does all the work for us
199         Log.v(TAG, "onSurfaceTextureSizeChanged: " + width + " x " + height);
200     }
201 
202     @Override
onSurfaceTextureDestroyed(SurfaceTexture surface)203     public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) {
204         mPreviewTexture = null;
205         return true;
206     }
207 
208     @Override
onSurfaceTextureUpdated(SurfaceTexture surface)209     public void onSurfaceTextureUpdated(SurfaceTexture surface) {
210         // Invoked every time there's a new Camera preview frame
211     }
212 
213     @Override
onImageAvailable(ImageReader reader)214     public void onImageAvailable(ImageReader reader) {
215         Image img = null;
216         try {
217             img = reader.acquireNextImage();
218             if (img == null) {
219                 Log.d(TAG, "Invalid image!");
220                 return;
221             }
222             final int format = img.getFormat();
223 
224             Bitmap imgBitmap = null;
225             if (format == ImageFormat.JPEG) {
226                 ByteBuffer jpegBuffer = img.getPlanes()[0].getBuffer();
227                 jpegBuffer.rewind();
228                 byte[] jpegData = new byte[jpegBuffer.limit()];
229                 jpegBuffer.get(jpegData);
230                 imgBitmap = BitmapFactory.decodeByteArray(jpegData, 0, jpegData.length);
231                 img.close();
232             } else {
233                 Log.i(TAG, "Unsupported image format: " + format);
234             }
235             if (imgBitmap != null) {
236                 final Bitmap bitmap = imgBitmap;
237                 final boolean isMuted = isBitmapMuted(imgBitmap);
238                 runOnUiThread(new Runnable() {
239                     @Override
240                     public void run() {
241                         mImageView.setImageBitmap(bitmap);
242                         // enable pass button if image is muted (black)
243                         setPassButtonEnabled(isMuted);
244                     }
245                 });
246             }
247         } catch (java.lang.IllegalStateException e) {
248             // Swallow exceptions
249             e.printStackTrace();
250         } finally {
251             if (img != null) {
252                 img.close();
253             }
254         }
255     }
256 
isBitmapMuted(final Bitmap imgBitmap)257     private boolean isBitmapMuted(final Bitmap imgBitmap) {
258         // black images may have pixels with values > 0
259         // because of JPEG compression artifacts
260         final float COLOR_THRESHOLD = 0.02f;
261         for (int y = 0; y < imgBitmap.getHeight(); y++) {
262             for (int x = 0; x < imgBitmap.getWidth(); x++) {
263                 Color pixelColor = Color.valueOf(imgBitmap.getPixel(x, y));
264                 if (pixelColor.red() > COLOR_THRESHOLD || pixelColor.green() > COLOR_THRESHOLD
265                         || pixelColor.blue() > COLOR_THRESHOLD) {
266                     return false;
267                 }
268             }
269         }
270         return true;
271     }
272 
273     private class SizeComparator implements Comparator<Size> {
274         @Override
compare(Size lhs, Size rhs)275         public int compare(Size lhs, Size rhs) {
276             long lha = lhs.getWidth() * lhs.getHeight();
277             long rha = rhs.getWidth() * rhs.getHeight();
278             if (lha == rha) {
279                 lha = lhs.getWidth();
280                 rha = rhs.getWidth();
281             }
282             return (lha < rha) ? -1 : (lha > rha ? 1 : 0);
283         }
284     }
285 
setUpCamera(String cameraId)286     private void setUpCamera(String cameraId) throws CameraAccessException, BlockingOpenException {
287         shutdownCamera();
288 
289         mCameraCharacteristics = mCameraManager.getCameraCharacteristics(cameraId);
290         mCameraDevice = mBlockingCameraManager.openCamera(cameraId,
291                 mCameraListener, mCameraHandler);
292 
293         StreamConfigurationMap config =
294                 mCameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
295         Size[] jpegSizes = config.getOutputSizes(ImageFormat.JPEG);
296         Arrays.sort(jpegSizes, mSizeComparator);
297         // choose smallest image size, image capture is not the point of this test
298         mJpegSize = jpegSizes[0];
299 
300         mJpegImageReader = ImageReader.newInstance(
301                 mJpegSize.getWidth(), mJpegSize.getHeight(), ImageFormat.JPEG, 1);
302         mJpegImageReader.setOnImageAvailableListener(this, mCameraHandler);
303 
304         if (mPreviewTexture != null) {
305             startPreview();
306         }
307     }
308 
shutdownCamera()309     private void shutdownCamera() {
310         if (null != mCaptureSession) {
311             mCaptureSession.close();
312             mCaptureSession = null;
313         }
314         if (null != mCameraDevice) {
315             mCameraDevice.close();
316             mCameraDevice = null;
317         }
318         if (null != mJpegImageReader) {
319             mJpegImageReader.close();
320             mJpegImageReader = null;
321         }
322     }
323 
324     /**
325      * Starts a background thread and its {@link Handler}.
326      */
startBackgroundThread()327     private void startBackgroundThread() {
328         mCameraThread = new HandlerThread("CameraThreadBackground");
329         mCameraThread.start();
330         mCameraHandler = new Handler(mCameraThread.getLooper());
331     }
332 
333     /**
334      * Stops the background thread and its {@link Handler}.
335      */
stopBackgroundThread()336     private void stopBackgroundThread() {
337         mCameraThread.quitSafely();
338         try {
339             mCameraThread.join();
340             mCameraThread = null;
341             mCameraHandler = null;
342         } catch (InterruptedException e) {
343             e.printStackTrace();
344         }
345     }
346 
getPreviewSize(int minWidth)347     private Size getPreviewSize(int minWidth) {
348         StreamConfigurationMap config =
349                 mCameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
350         Size[] outputSizes = config.getOutputSizes(SurfaceTexture.class);
351         Arrays.sort(outputSizes, mSizeComparator);
352         // choose smallest image size that's at least minWidth
353         // image capture is not the point of this test
354         for (Size outputSize : outputSizes) {
355             if (outputSize.getWidth() > minWidth) {
356                 return outputSize;
357             }
358         }
359         return outputSizes[0];
360     }
361 
startPreview()362     private void startPreview() {
363         try {
364             mPreviewSize = getPreviewSize(256);
365 
366             mPreviewTexture.setDefaultBufferSize(mPreviewSize.getWidth(), mPreviewSize.getHeight());
367             mPreviewRequestBuilder =
368                     mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
369             mPreviewRequestBuilder.addTarget(mPreviewSurface);
370 
371             mStillCaptureRequestBuilder =
372                     mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE);
373             mStillCaptureRequestBuilder.addTarget(mPreviewSurface);
374             mStillCaptureRequestBuilder.addTarget(mJpegImageReader.getSurface());
375 
376             mSessionListener = new BlockingSessionCallback();
377             List<Surface> outputSurfaces = new ArrayList<Surface>(/*capacity*/3);
378             outputSurfaces.add(mPreviewSurface);
379             outputSurfaces.add(mJpegImageReader.getSurface());
380             mCameraDevice.createCaptureSession(outputSurfaces, mSessionListener, mCameraHandler);
381             mCaptureSession = mSessionListener.waitAndGetSession(/*timeoutMs*/3000);
382 
383             mPreviewRequest = mPreviewRequestBuilder.build();
384             mStillCaptureRequest = mStillCaptureRequestBuilder.build();
385 
386             mCaptureSession.setRepeatingRequest(mPreviewRequest, mCaptureCallback, mCameraHandler);
387 
388             setTakePictureButtonEnabled(true);
389         } catch (CameraAccessException e) {
390             e.printStackTrace();
391         }
392     }
393 
takePicture()394     private void takePicture() {
395         try {
396             mCaptureSession.stopRepeating();
397             mSessionListener.getStateWaiter().waitForState(
398                     BlockingSessionCallback.SESSION_READY, SESSION_READY_TIMEOUT_MS);
399 
400             mCaptureSession.capture(mStillCaptureRequest, mCaptureCallback, mCameraHandler);
401         } catch (CameraAccessException e) {
402             e.printStackTrace();
403         }
404     }
405 
setPassButtonEnabled(boolean enabled)406     private void setPassButtonEnabled(boolean enabled) {
407         ImageButton pass_button = findViewById(R.id.pass_button);
408         pass_button.setEnabled(enabled);
409     }
410 
setTakePictureButtonEnabled(boolean enabled)411     private void setTakePictureButtonEnabled(boolean enabled) {
412         Button takePhoto = findViewById(R.id.take_picture_button);
413         takePhoto.setOnClickListener(v -> takePicture());
414         takePhoto.setEnabled(enabled);
415     }
416 }
417