1 /*
2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.hardware.camera2.cts;
18 
19 import static android.hardware.camera2.cts.CameraTestUtils.SimpleImageReaderListener;
20 import static android.hardware.camera2.cts.CameraTestUtils.assertNotNull;
21 import static android.hardware.camera2.cts.CameraTestUtils.configureCameraSessionWithConfig;
22 
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 
26 import android.graphics.ImageFormat;
27 import android.hardware.camera2.CameraCaptureSession;
28 import android.hardware.camera2.CameraCharacteristics;
29 import android.hardware.camera2.CameraDevice;
30 import android.hardware.camera2.CameraMetadata;
31 import android.hardware.camera2.CaptureRequest;
32 import android.hardware.camera2.CaptureResult;
33 import android.hardware.camera2.TotalCaptureResult;
34 import android.hardware.camera2.cts.testcases.Camera2SurfaceViewTestCase;
35 import android.hardware.camera2.params.OutputConfiguration;
36 import android.media.Image;
37 import android.media.ImageReader;
38 import android.util.Log;
39 import android.util.Size;
40 
41 import com.android.ex.camera2.blocking.BlockingSessionCallback;
42 
43 import java.util.ArrayList;
44 import java.util.List;
45 import java.util.concurrent.LinkedBlockingQueue;
46 import java.util.concurrent.TimeUnit;
47 
48 import org.junit.Test;
49 
50 public class ReadoutTimestampTest extends Camera2SurfaceViewTestCase {
51     private static final String TAG = "ReadoutTimestampTest";
52 
53     @Override
setUp()54     public void setUp() throws Exception {
55         super.setUp();
56     }
57 
58     @Override
tearDown()59     public void tearDown() throws Exception {
60         super.tearDown();
61     }
62 
63     /**
64      * Test the camera readout timestamp work properly:
65      *
66      * If SENSOR_READOUT_TIMESTAMP is supported:
67      * - Each onCaptureStarted() callback has a corresponding onReadoutStarted() callback.
68      * - The readout timestamp is at least startOfExposure timestamp + exposure time.
69      * - Image timestamp matches the onReadoutStarted timestamp
70      */
71     @Test
testReadoutTimestamp()72     public void testReadoutTimestamp() throws Exception {
73         int[] timestampBases = new int[]{
74                 OutputConfiguration.TIMESTAMP_BASE_DEFAULT,
75                 OutputConfiguration.TIMESTAMP_BASE_SENSOR,
76                 OutputConfiguration.TIMESTAMP_BASE_MONOTONIC,
77                 OutputConfiguration.TIMESTAMP_BASE_REALTIME,
78                 OutputConfiguration.TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED};
79 
80         for (String cameraId : getCameraIdsUnderTest()) {
81             if (!mAllStaticInfo.get(cameraId).isColorOutputSupported()) {
82                 continue;
83             }
84 
85             try {
86                 openDevice(cameraId);
87                 for (int timestampBase : timestampBases) {
88                     testReadoutTimestamp(cameraId, timestampBase);
89                 }
90             } finally {
91                 closeDevice();
92             }
93         }
94     }
95 
testReadoutTimestamp(String cameraId, int timestampBase)96     private void testReadoutTimestamp(String cameraId, int timestampBase) throws Exception {
97         Log.i(TAG, "testReadoutTimestamp for camera " + cameraId +
98                 " with timestampBase " + timestampBase);
99         Integer sensorReadoutTimestamp = mStaticInfo.getCharacteristics().get(
100                 CameraCharacteristics.SENSOR_READOUT_TIMESTAMP);
101         assertNotNull("Camera " + cameraId + ": READOUT_TIMESTAMP "
102                 + "must not be null", sensorReadoutTimestamp);
103         if (CameraMetadata.SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED
104                 == sensorReadoutTimestamp) {
105             return;
106         }
107 
108         // Camera device supports readout timestamp
109         final int NUM_CAPTURE = 5;
110         final int WAIT_FOR_RESULT_TIMEOUT_MS = 3000;
111         Integer timestampSource = mStaticInfo.getCharacteristics().get(
112                 CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE);
113 
114         // Create ImageReader as output
115         Size maxPreviewSize = mOrderedPreviewSizes.get(0);
116         SimpleImageReaderListener readerListener = new SimpleImageReaderListener();
117         ImageReader reader = ImageReader.newInstance(maxPreviewSize.getWidth(),
118                 maxPreviewSize.getHeight(), ImageFormat.YUV_420_888, /*maxImage*/ NUM_CAPTURE);
119         reader.setOnImageAvailableListener(readerListener, mHandler);
120 
121         // Create session
122         List<OutputConfiguration> outputConfigs = new ArrayList<>();
123         OutputConfiguration configuration = new OutputConfiguration(reader.getSurface());
124         assertFalse("OutputConfiguration: Readout timestamp should be false by default",
125                 configuration.isReadoutTimestampEnabled());
126         configuration.setTimestampBase(timestampBase);
127         configuration.setReadoutTimestampEnabled(true);
128         assertTrue("OutputConfiguration: Readout timestamp should be true",
129                 configuration.isReadoutTimestampEnabled());
130         outputConfigs.add(configuration);
131 
132         BlockingSessionCallback sessionCallback = new BlockingSessionCallback();
133         mSession = configureCameraSessionWithConfig(mCamera, outputConfigs,
134                 sessionCallback, mHandler);
135 
136         // Capture frames as well as shutter/result callbacks
137         CaptureRequest.Builder previewRequest =
138                 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
139         previewRequest.addTarget(reader.getSurface());
140         ReadoutCaptureCallback resultListener = new ReadoutCaptureCallback();
141         List<CaptureRequest> burst = new ArrayList<>();
142         for (int i = 0; i < NUM_CAPTURE; i++) {
143             burst.add(previewRequest.build());
144         }
145         mSession.captureBurst(burst, resultListener, mHandler);
146 
147         ArrayList<CaptureResult> results = new ArrayList<>();
148         for (int i = 0; i < NUM_CAPTURE; i++) {
149             CaptureResult result = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
150             assertNotNull("Camera " + cameraId + ": Capture result must not be null", result);
151             results.add(result);
152         }
153 
154         ReadoutCaptureCallback.TimestampTuple[] timestamps = resultListener.getTimestamps();
155         assertNotNull("Camera " + cameraId + ": No timestamps received by resultListener",
156                 timestamps);
157         assertTrue("Camera " + cameraId + " timestampBase " + timestampBase
158                 + ": Not enough onCaptureStarted/onReadoutStarted "
159                 + "callbacks. Expected at least " + 2 * NUM_CAPTURE + ", actual "
160                 + timestamps.length, timestamps.length >= 2 * NUM_CAPTURE);
161         for (int i = 0; i < NUM_CAPTURE; i++) {
162             // Each capture result has corresponding onCaptureStarted and onReadoutStarted callbacks
163             mCollector.expectTrue(String.format("Camera %s timestampBase %d: timestamps[%d] should "
164                     + "be from onCaptureStarted", cameraId, timestampBase, i * 2),
165                     timestamps[i * 2].mType == ReadoutCaptureCallback.CAPTURE_TIMESTAMP);
166             mCollector.expectTrue(String.format("Camera %s timestampBase %d: timestamps[%d] should "
167                     + "be from onReadoutStarted", cameraId, timestampBase, i * 2 + 1),
168                     timestamps[i * 2 + 1].mType == ReadoutCaptureCallback.READOUT_TIMESTAMP);
169 
170 
171             if (timestampBase == OutputConfiguration.TIMESTAMP_BASE_DEFAULT ||
172                     timestampBase == OutputConfiguration.TIMESTAMP_BASE_SENSOR ||
173                     (timestampBase == OutputConfiguration.TIMESTAMP_BASE_MONOTONIC &&
174                     timestampSource == CameraMetadata.SENSOR_INFO_TIMESTAMP_SOURCE_UNKNOWN) ||
175                     (timestampBase == OutputConfiguration.TIMESTAMP_BASE_REALTIME &&
176                     timestampSource == CameraMetadata.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME)) {
177                 // The readoutTime in onReadoutStarted must match that of the images
178                 Image image = readerListener.getImage(CameraTestUtils.CAPTURE_IMAGE_TIMEOUT_MS);
179                 Long imageTime = image.getTimestamp();
180                 mCollector.expectEquals("Camera " + cameraId + " timestampBase " + timestampBase
181                         + " readoutTimestamp (" + timestamps[i * 2 + 1].mTimestamp
182                         + ") should be equal to image timestamp (" + imageTime,
183                         imageTime, timestamps[i * 2 + 1].mTimestamp);
184                 image.close();
185             }
186 
187             // The exposure time must be at least readoutTime - captureTime
188             Long exposureTime = results.get(i).get(CaptureResult.SENSOR_EXPOSURE_TIME);
189             if (exposureTime == null) {
190                 // If exposureTime is null in CaptureResult, the camera device doesn't support
191                 // READ_SENOSR_SETTINGS capability.
192                 boolean hasReadSensorSettings = mStaticInfo.isCapabilitySupported(
193                         CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_READ_SENSOR_SETTINGS);
194                 assertFalse("Camera " + cameraId + ": ExposureTime must not be null if "
195                         + "READ_SENSOR_SETTINGS is supported", hasReadSensorSettings);
196 
197                 mCollector.expectTrue("Camera " + cameraId + " timestampBase " + timestampBase
198                         + ": readoutTime (" + timestamps[i * 2 + 1].mTimestamp
199                         + ") - captureStart time (" + timestamps[i * 2].mTimestamp
200                         + ") should be > 0",
201                         timestamps[i * 2 + 1].mTimestamp > timestamps[i * 2].mTimestamp);
202             } else {
203                 mCollector.expectTrue("Camera " + cameraId + "timestampBase " + timestampBase
204                         + ": readoutTime (" + timestamps[i * 2 + 1].mTimestamp
205                         + ") - captureStart time (" + timestamps[i * 2].mTimestamp
206                         + ") should be >= exposureTime (" + exposureTime + ")",
207                         timestamps[i * 2 + 1].mTimestamp - timestamps[i * 2].mTimestamp >=
208                         exposureTime);
209             }
210         }
211     }
212 
213     public static class ReadoutCaptureCallback extends CameraCaptureSession.CaptureCallback {
214         public static final int CAPTURE_TIMESTAMP = 0;
215         public static final int READOUT_TIMESTAMP = 1;
216 
217         public static class TimestampTuple {
218             public int mType;
219             public Long mTimestamp;
TimestampTuple(int type, Long timestamp)220             public TimestampTuple(int type, Long timestamp) {
221                 mType = type;
222                 mTimestamp = timestamp;
223             }
224         }
225 
226         private final LinkedBlockingQueue<TotalCaptureResult> mQueue =
227                 new LinkedBlockingQueue<TotalCaptureResult>();
228         // TimestampTuple is a pair of CAPTURE/READOUT flag and timestamp.
229         private final LinkedBlockingQueue<TimestampTuple> mTimestampQueue =
230                 new LinkedBlockingQueue<>();
231 
232         @Override
onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber)233         public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request,
234                 long timestamp, long frameNumber) {
235             try {
236                 mTimestampQueue.put(new TimestampTuple(CAPTURE_TIMESTAMP, timestamp));
237             } catch (InterruptedException e) {
238                 throw new UnsupportedOperationException(
239                         "Can't handle InterruptedException in onCaptureStarted");
240             }
241         }
242 
onReadoutStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber)243         public void onReadoutStarted(CameraCaptureSession session, CaptureRequest request,
244                 long timestamp, long frameNumber) {
245             try {
246                 mTimestampQueue.put(new TimestampTuple(READOUT_TIMESTAMP, timestamp));
247             } catch (InterruptedException e) {
248                 throw new UnsupportedOperationException(
249                         "Can't handle InterruptedException in onReadoutStarted");
250             }
251         }
252 
253         @Override
onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result)254         public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request,
255                 TotalCaptureResult result) {
256             try {
257                 mQueue.put(result);
258             } catch (InterruptedException e) {
259                 throw new UnsupportedOperationException(
260                         "Can't handle InterruptedException in onCaptureCompleted");
261             }
262         }
263 
getCaptureResult(long timeout)264         public CaptureResult getCaptureResult(long timeout) {
265             try {
266                 TotalCaptureResult result = mQueue.poll(timeout, TimeUnit.MILLISECONDS);
267                 assertNotNull("Wait for a capture result timed out in " + timeout + "ms", result);
268                 return result;
269             } catch (InterruptedException e) {
270                 throw new UnsupportedOperationException("Unhandled InterruptedException", e);
271             }
272         }
273 
getTimestamps()274         public TimestampTuple[] getTimestamps() {
275             return mTimestampQueue.toArray(new TimestampTuple[0]);
276         }
277     }
278 }
279