1# Copyright 2022 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Image Field-of-View utilities for aspect ratio, crop, and FoV tests."""
15
16
17import logging
18import math
19
20import cv2
21import numpy as np
22
23import camera_properties_utils
24import capture_request_utils
25import image_processing_utils
26import opencv_processing_utils
27
28CIRCLE_COLOR = 0  # [0: black, 255: white]
29CIRCLE_MIN_AREA = 0.01  # 1% of image size
30FOV_PERCENT_RTOL = 0.15  # Relative tolerance on circle FoV % to expected.
31LARGE_SIZE_IMAGE = 2000  # Size of a large image (compared against max(w, h))
32THRESH_AR_L = 0.02  # Aspect ratio test threshold of large images
33THRESH_AR_S = 0.075  # Aspect ratio test threshold of mini images
34THRESH_CROP_L = 0.02  # Crop test threshold of large images
35THRESH_CROP_S = 0.075  # Crop test threshold of mini images
36THRESH_MIN_PIXEL = 4  # Crop test allowed offset
37
38
39def calc_scaler_crop_region_ratio(scaler_crop_region, props):
40  """Calculate ratio of scaler crop region area over active array area.
41
42  Args:
43    scaler_crop_region: Rect(left, top, right, bottom)
44    props: camera properties
45
46  Returns:
47    ratio of scaler crop region area over active array area
48  """
49  a = props['android.sensor.info.activeArraySize']
50  s = scaler_crop_region
51  logging.debug('Active array size: %s', a)
52  active_array_area = (a['right'] - a['left']) * (a['bottom'] - a['top'])
53  scaler_crop_region_area = (s['right'] - s['left']) * (s['bottom'] - s['top'])
54  crop_region_active_array_ratio = scaler_crop_region_area / active_array_area
55  return crop_region_active_array_ratio
56
57
58def check_fov(circle, ref_fov, w, h):
59  """Check the FoV for correct size."""
60  fov_percent = calc_circle_image_ratio(circle['r'], w, h)
61  chk_percent = calc_expected_circle_image_ratio(ref_fov, w, h)
62  if not math.isclose(fov_percent, chk_percent, rel_tol=FOV_PERCENT_RTOL):
63    e_msg = (f'FoV %: {fov_percent:.2f}, Ref FoV %: {chk_percent:.2f}, '
64             f'TOL={FOV_PERCENT_RTOL*100}%, img: {w}x{h}, ref: '
65             f"{ref_fov['w']}x{ref_fov['h']}")
66    return e_msg
67
68
69def check_ar(circle, ar_gt, w, h, e_msg_stem):
70  """Check the aspect ratio of the circle.
71
72  size is the larger of w or h.
73  if size >= LARGE_SIZE_IMAGE: use THRESH_AR_L
74  elif size == 0 (extreme case): THRESH_AR_S
75  elif 0 < image size < LARGE_SIZE_IMAGE: scale between THRESH_AR_S & AR_L
76
77  Args:
78    circle: dict with circle parameters
79    ar_gt: aspect ratio ground truth to compare against
80    w: width of image
81    h: height of image
82    e_msg_stem: customized string for error message
83
84  Returns:
85    error string if check fails
86  """
87  thresh_ar = max(THRESH_AR_L, THRESH_AR_S +
88                  max(w, h) * (THRESH_AR_L-THRESH_AR_S) / LARGE_SIZE_IMAGE)
89  ar = circle['w'] / circle['h']
90  if not math.isclose(ar, ar_gt, abs_tol=thresh_ar):
91    e_msg = (f'{e_msg_stem} {w}x{h}: aspect_ratio {ar:.3f}, '
92             f'thresh {thresh_ar:.3f}')
93    return e_msg
94
95
96def check_crop(circle, cc_gt, w, h, e_msg_stem, crop_thresh_factor):
97  """Check cropping.
98
99  if size >= LARGE_SIZE_IMAGE: use thresh_crop_l
100  elif size == 0 (extreme case): thresh_crop_s
101  elif 0 < size < LARGE_SIZE_IMAGE: scale between thresh_crop_s & thresh_crop_l
102  Also allow at least THRESH_MIN_PIXEL to prevent threshold being too tight
103  for very small circle.
104
105  Args:
106    circle: dict of circle values
107    cc_gt: circle center {'hori', 'vert'}  ground truth (ref'd to img center)
108    w: width of image
109    h: height of image
110    e_msg_stem: text to customize error message
111    crop_thresh_factor: scaling factor for crop thresholds
112
113  Returns:
114    error string if check fails
115  """
116  thresh_crop_l = THRESH_CROP_L * crop_thresh_factor
117  thresh_crop_s = THRESH_CROP_S * crop_thresh_factor
118  thresh_crop_hori = max(
119      [thresh_crop_l,
120       thresh_crop_s + w * (thresh_crop_l - thresh_crop_s) / LARGE_SIZE_IMAGE,
121       THRESH_MIN_PIXEL / circle['w']])
122  thresh_crop_vert = max(
123      [thresh_crop_l,
124       thresh_crop_s + h * (thresh_crop_l - thresh_crop_s) / LARGE_SIZE_IMAGE,
125       THRESH_MIN_PIXEL / circle['h']])
126
127  if (not math.isclose(circle['x_offset'], cc_gt['hori'],
128                       abs_tol=thresh_crop_hori) or
129      not math.isclose(circle['y_offset'], cc_gt['vert'],
130                       abs_tol=thresh_crop_vert)):
131    valid_x_range = (cc_gt['hori'] - thresh_crop_hori,
132                     cc_gt['hori'] + thresh_crop_hori)
133    valid_y_range = (cc_gt['vert'] - thresh_crop_vert,
134                     cc_gt['vert'] + thresh_crop_vert)
135    e_msg = (f'{e_msg_stem} {w}x{h} '
136             f"offset X {circle['x_offset']:.3f}, Y {circle['y_offset']:.3f}, "
137             f'valid X range: {valid_x_range[0]:.3f} ~ {valid_x_range[1]:.3f}, '
138             f'valid Y range: {valid_y_range[0]:.3f} ~ {valid_y_range[1]:.3f}')
139    return e_msg
140
141
142def calc_expected_circle_image_ratio(ref_fov, img_w, img_h):
143  """Determine the circle image area ratio in percentage for a given image size.
144
145  Cropping happens either horizontally or vertically. In both cases crop results
146  in the visble area reduced by a ratio r (r < 1) and the circle will in turn
147  occupy ref_pct/r (percent) on the target image size.
148
149  Args:
150    ref_fov: dict with {fmt, % coverage, w, h, circle_w, circle_h}
151    img_w: the image width
152    img_h: the image height
153
154  Returns:
155    chk_percent: the expected circle image area ratio in percentage
156  """
157  ar_ref = ref_fov['w'] / ref_fov['h']
158  ar_target = img_w / img_h
159
160  r = ar_ref / ar_target
161  if r < 1.0:
162    r = 1.0 / r
163  return ref_fov['percent'] * r
164
165
166def calc_circle_image_ratio(radius, img_w, img_h):
167  """Calculate the percent of area the input circle covers in input image.
168
169  Args:
170    radius: radius of circle
171    img_w: int width of image
172    img_h: int height of image
173  Returns:
174    fov_percent: float % of image covered by circle
175  """
176  return 100 * math.pi * math.pow(radius, 2) / (img_w * img_h)
177
178
179def find_fov_reference(cam, req, props, raw_bool, ref_img_name_stem):
180  """Determine the circle coverage of the image in reference image.
181
182  Captures a full-frame RAW or JPEG and uses its aspect ratio and circle center
183  location as ground truth for the other jpeg or yuv images.
184
185  The intrinsics and distortion coefficients are meant for full-sized RAW,
186  so convert_capture_to_rgb_image returns a 2x downsampled version, so resizes
187  RGB back to full size.
188
189  If the device supports lens distortion correction, applies the coefficients on
190  the RAW image so it can be compared to YUV/JPEG outputs which are subject
191  to the same correction via ISP.
192
193  Finds circle size and location for reference values in calculations for other
194  formats.
195
196  Args:
197    cam: camera object
198    req: camera request
199    props: camera properties
200    raw_bool: True if RAW available
201    ref_img_name_stem: test _NAME + location to save data
202
203  Returns:
204    ref_fov: dict with {fmt, % coverage, w, h, circle_w, circle_h}
205    cc_ct_gt: circle center position relative to the center of image.
206    aspect_ratio_gt: aspect ratio of the detected circle in float.
207  """
208  logging.debug('Creating references for fov_coverage')
209  if raw_bool:
210    logging.debug('Using RAW for reference')
211    fmt_type = 'RAW'
212    out_surface = {'format': 'raw'}
213    cap = cam.do_capture(req, out_surface)
214    logging.debug('Captured RAW %dx%d', cap['width'], cap['height'])
215    img = image_processing_utils.convert_capture_to_rgb_image(
216        cap, props=props)
217    # Resize back up to full scale.
218    img = cv2.resize(img, (0, 0), fx=2.0, fy=2.0)
219
220    fd = float(cap['metadata']['android.lens.focalLength'])
221    k = camera_properties_utils.get_intrinsic_calibration(
222        props, cap['metadata'], True, fd
223    )
224    if (camera_properties_utils.distortion_correction(props) and
225        isinstance(k, np.ndarray)):
226      logging.debug('Applying intrinsic calibration and distortion params')
227      opencv_dist = camera_properties_utils.get_distortion_matrix(props)
228      k_new = cv2.getOptimalNewCameraMatrix(
229          k, opencv_dist, (img.shape[1], img.shape[0]), 0)[0]
230      scale = max(k_new[0][0] / k[0][0], k_new[1][1] / k[1][1])
231      if scale > 1:
232        k_new[0][0] = k[0][0] * scale
233        k_new[1][1] = k[1][1] * scale
234        img = cv2.undistort(img, k, opencv_dist, None, k_new)
235      else:
236        img = cv2.undistort(img, k, opencv_dist)
237    size = img.shape
238
239  else:
240    logging.debug('Using JPEG for reference')
241    fmt_type = 'JPEG'
242    ref_fov = {}
243    fmt = capture_request_utils.get_largest_jpeg_format(props)
244    cap = cam.do_capture(req, fmt)
245    logging.debug('Captured JPEG %dx%d', cap['width'], cap['height'])
246    img = image_processing_utils.convert_capture_to_rgb_image(cap, props)
247    size = (cap['height'], cap['width'])
248
249  # Get image size.
250  w = size[1]
251  h = size[0]
252  img_name = f'{ref_img_name_stem}_{fmt_type}_w{w}_h{h}.png'
253  image_processing_utils.write_image(img, img_name, True)
254
255  # Find circle.
256  img *= 255  # cv2 needs images between [0,255].
257  circle = opencv_processing_utils.find_circle(
258      img, img_name, CIRCLE_MIN_AREA, CIRCLE_COLOR)
259  opencv_processing_utils.append_circle_center_to_img(circle, img, img_name)
260
261  # Determine final return values.
262  if fmt_type == 'RAW':
263    aspect_ratio_gt = circle['w'] / circle['h']
264  else:
265    aspect_ratio_gt = 1.0
266  cc_ct_gt = {'hori': circle['x_offset'], 'vert': circle['y_offset']}
267  fov_percent = calc_circle_image_ratio(circle['r'], w, h)
268  ref_fov = {}
269  ref_fov['fmt'] = fmt_type
270  ref_fov['percent'] = fov_percent
271  ref_fov['w'] = w
272  ref_fov['h'] = h
273  ref_fov['circle_w'] = circle['w']
274  ref_fov['circle_h'] = circle['h']
275  logging.debug('Using %s reference: %s', fmt_type, str(ref_fov))
276  return ref_fov, cc_ct_gt, aspect_ratio_gt
277