1# Copyright 2022 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Validate preview aspect ratio, crop and FoV vs format."""
15
16import logging
17import os
18
19from mobly import test_runner
20
21import its_base_test
22import camera_properties_utils
23import capture_request_utils
24import image_fov_utils
25import image_processing_utils
26import its_session_utils
27import opencv_processing_utils
28import preview_processing_utils
29import video_processing_utils
30
31
32_NAME = os.path.splitext(os.path.basename(__file__))[0]
33_VIDEO_DURATION = 3  # seconds
34_MAX_8BIT_IMGS = 255
35
36
37def _collect_data(cam, preview_size):
38  """Capture a preview video from the device.
39
40  Captures camera preview frames from the passed device.
41
42  Args:
43    cam: camera object
44    preview_size: str; preview resolution. ex. '1920x1080'
45
46  Returns:
47    recording object as described by cam.do_preview_recording
48  """
49
50  recording_obj = cam.do_preview_recording(preview_size, _VIDEO_DURATION, False)
51  logging.debug('Recorded output path: %s', recording_obj['recordedOutputPath'])
52  logging.debug('Tested quality: %s', recording_obj['quality'])
53
54  return recording_obj
55
56
57def _print_failed_test_results(failed_ar, failed_fov, failed_crop):
58  """Print failed test results."""
59  if failed_ar:
60    logging.error('Aspect ratio test summary')
61    logging.error('Images failed in the aspect ratio test:')
62    logging.error('Aspect ratio value: width / height')
63    for fa in failed_ar:
64      logging.error('%s', fa)
65
66  if failed_fov:
67    logging.error('FoV test summary')
68    logging.error('Images failed in the FoV test:')
69    for fov in failed_fov:
70      logging.error('%s', str(fov))
71
72  if failed_crop:
73    logging.error('Crop test summary')
74    logging.error('Images failed in the crop test:')
75    logging.error('Circle center (H x V) relative to the image center.')
76    for fc in failed_crop:
77      logging.error('%s', fc)
78
79
80class PreviewAspectRatioAndCropTest(its_base_test.ItsBaseTest):
81  """Test preview aspect ratio/field of view/cropping for each tested fmt.
82
83    This test checks for:
84    1. Aspect ratio: images are not stretched
85    2. Crop: center of images is not shifted
86    3. FOV: images cropped to keep maximum possible FOV with only 1 dimension
87       (horizontal or veritical) cropped.
88
89
90
91  The test preview is a black circle on a white background.
92
93  When RAW capture is available, set the height vs. width ratio of the circle in
94  the full-frame RAW as ground truth. In an ideal setup such ratio should be
95  very close to 1.0, but here we just use the value derived from full resolution
96  RAW as ground truth to account for the possibility that the chart is not well
97  positioned to be precisely parallel to image sensor plane.
98  The test then compares the ground truth ratio with the same ratio measured
99  on previews captured using different formats.
100
101  If RAW capture is unavailable, a full resolution JPEG image is used to setup
102  ground truth. In this case, the ground truth aspect ratio is defined as 1.0
103  and it is the tester's responsibility to make sure the test chart is
104  properly positioned so the detected circles indeed have aspect ratio close
105  to 1.0 assuming no bugs causing image stretched.
106
107  The aspect ratio test checks the aspect ratio of the detected circle and
108  it will fail if the aspect ratio differs too much from the ground truth
109  aspect ratio mentioned above.
110
111  The FOV test examines the ratio between the detected circle area and the
112  image size. When the aspect ratio of the test image is the same as the
113  ground truth image, the ratio should be very close to the ground truth
114  value. When the aspect ratio is different, the difference is factored in
115  per the expectation of the Camera2 API specification, which mandates the
116  FOV reduction from full sensor area must only occur in one dimension:
117  horizontally or vertically, and never both. For example, let's say a sensor
118  has a 16:10 full sensor FOV. For all 16:10 output images there should be no
119  FOV reduction on them. For 16:9 output images the FOV should be vertically
120  cropped by 9/10. For 4:3 output images the FOV should be cropped
121  horizontally instead and the ratio (r) can be calculated as follows:
122      (16 * r) / 10 = 4 / 3 => r = 40 / 48 = 0.8333
123  Say the circle is covering x percent of the 16:10 sensor on the full 16:10
124  FOV, and assume the circle in the center will never be cut in any output
125  sizes (this can be achieved by picking the right size and position of the
126  test circle), the from above cropping expectation we can derive on a 16:9
127  output image the circle will cover (x / 0.9) percent of the 16:9 image; on
128  a 4:3 output image the circle will cover (x / 0.8333) percent of the 4:3
129  image.
130
131  The crop test checks that the center of any output image remains aligned
132  with center of sensor's active area, no matter what kind of cropping or
133  scaling is applied. The test verifies that by checking the relative vector
134  from the image center to the center of detected circle remains unchanged.
135  The relative part is normalized by the detected circle size to account for
136  scaling effect.
137  """
138
139  def test_preview_aspect_ratio_and_crop(self):
140    log_path = self.log_path
141    video_processing_utils.log_ffmpeg_version()
142
143    with its_session_utils.ItsSession(
144        device_id=self.dut.serial,
145        camera_id=self.camera_id,
146        hidden_physical_id=self.hidden_physical_id) as cam:
147      failed_ar = []  # Streams failed the aspect ratio test
148      failed_crop = []  # Streams failed the crop test
149      failed_fov = []  # Streams that fail FoV test
150      props = cam.get_camera_properties()
151      fls_logical = props['android.lens.info.availableFocalLengths']
152      logging.debug('logical available focal lengths: %s', str(fls_logical))
153      props = cam.override_with_hidden_physical_camera_props(props)
154      fls_physical = props['android.lens.info.availableFocalLengths']
155      logging.debug('physical available focal lengths: %s', str(fls_physical))
156      name_with_log_path = f'{os.path.join(self.log_path, _NAME)}'
157
158      # Check SKIP conditions
159      first_api_level = its_session_utils.get_first_api_level(self.dut.serial)
160      camera_properties_utils.skip_unless(
161          first_api_level >= its_session_utils.ANDROID14_API_LEVEL)
162
163      # Load scene
164      its_session_utils.load_scene(cam, props, self.scene,
165                                   self.tablet, self.chart_distance)
166      # Raise error if not FRONT or REAR facing camera
167      camera_properties_utils.check_front_or_rear_camera(props)
168
169      # List of preview resolutions to test
170      supported_preview_sizes = cam.get_supported_preview_sizes(self.camera_id)
171      for size in video_processing_utils.LOW_RESOLUTION_SIZES:
172        if size in supported_preview_sizes:
173          supported_preview_sizes.remove(size)
174      logging.debug('Supported preview resolutions: %s',
175                    supported_preview_sizes)
176      raw_avlb = camera_properties_utils.raw16(props)
177      full_or_better = camera_properties_utils.full_or_better(props)
178
179      # Converge 3A
180      cam.do_3a()
181      req = capture_request_utils.auto_capture_request()
182      if raw_avlb and (fls_physical == fls_logical):
183        logging.debug('RAW')
184        raw_bool = True
185      else:
186        logging.debug('JPEG')
187        raw_bool = False
188      ref_fov, cc_ct_gt, aspect_ratio_gt = image_fov_utils.find_fov_reference(
189          cam, req, props, raw_bool, name_with_log_path)
190
191      run_crop_test = full_or_better and raw_avlb
192
193      # Check if we support testing this preview size
194      for preview_size in supported_preview_sizes:
195        logging.debug('Testing preview recording for size: %s', preview_size)
196        # recording preview
197        preview_rec_obj = _collect_data(cam, preview_size)
198
199        # Grab the recording from DUT
200        self.dut.adb.pull([preview_rec_obj['recordedOutputPath'], log_path])
201        preview_file_name = (
202            preview_rec_obj['recordedOutputPath'].split('/')[-1])
203        logging.debug('preview_file_name: %s', preview_file_name)
204        preview_size = preview_rec_obj['videoSize']
205        width = int(preview_size.split('x')[0])
206        height = int(preview_size.split('x')[-1])
207
208        # Extract last key frame as numpy image
209        last_key_frame = (
210            video_processing_utils.extract_last_key_frame_from_recording(
211                self.log_path, preview_file_name)
212        )
213
214        # If front camera, flip preview image to match camera capture
215        if (props['android.lens.facing'] ==
216            camera_properties_utils.LENS_FACING['FRONT']):
217          last_key_frame = (
218              preview_processing_utils.mirror_preview_image_by_sensor_orientation(
219                  props['android.sensor.orientation'], last_key_frame))
220
221        # Check FoV
222        ref_img_name = (f'{name_with_log_path}_{preview_size}_circle.png')
223        circle = opencv_processing_utils.find_circle(
224            last_key_frame, ref_img_name, image_fov_utils.CIRCLE_MIN_AREA,
225            image_fov_utils.CIRCLE_COLOR)
226
227        opencv_processing_utils.append_circle_center_to_img(
228            circle, last_key_frame, ref_img_name)
229
230        max_img_value = _MAX_8BIT_IMGS
231
232        # Check pass/fail for fov coverage for all fmts in AR_CHECKED
233        img_name_stem = f'{name_with_log_path}_{preview_size}'
234        fov_chk_msg = image_fov_utils.check_fov(
235            circle, ref_fov, width, height)
236        if fov_chk_msg:
237          img_name = f'{img_name_stem}_fov.png'
238          fov_chk_preview_msg = f'Preview Size: {preview_size} {fov_chk_msg}'
239          failed_fov.append(fov_chk_preview_msg)
240          image_processing_utils.write_image(
241              last_key_frame/max_img_value, img_name, True)
242
243        # Check pass/fail for aspect ratio
244        ar_chk_msg = image_fov_utils.check_ar(
245            circle, aspect_ratio_gt, width, height,
246            f'{preview_size}')
247        if ar_chk_msg:
248          img_name = f'{img_name_stem}_ar.png'
249          failed_ar.append(ar_chk_msg)
250          image_processing_utils.write_image(
251              last_key_frame/max_img_value, img_name, True)
252
253        # Check pass/fail for crop
254        if run_crop_test:
255          # Normalize the circle size to 1/4 of the image size, so that
256          # circle size won't affect the crop test result
257          crop_thresh_factor = ((min(ref_fov['w'], ref_fov['h']) / 4.0) /
258                                max(ref_fov['circle_w'],
259                                    ref_fov['circle_h']))
260          crop_chk_msg = image_fov_utils.check_crop(
261              circle, cc_ct_gt, width, height,
262              f'{preview_size}', crop_thresh_factor)
263          if crop_chk_msg:
264            crop_img_name = f'{img_name_stem}_crop.png'
265            failed_crop.append(crop_chk_msg)
266            image_processing_utils.write_image(last_key_frame/max_img_value,
267                                               crop_img_name, True)
268        else:
269          logging.debug('Crop test skipped')
270
271    # Print any failed test results
272    _print_failed_test_results(failed_ar, failed_fov, failed_crop)
273
274    e_msg = ''
275    if failed_ar:
276      e_msg = 'Aspect ratio '
277    if failed_fov:
278      e_msg += 'FoV '
279    if failed_crop:
280      e_msg += 'Crop '
281    if e_msg:
282      raise AssertionError(f'{e_msg}check failed.')
283
284if __name__ == '__main__':
285  test_runner.main()
286