1# Copyright 2013 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verifies YUV & JPEG image captures have similar brightness."""
15
16
17import logging
18import os.path
19import matplotlib
20from matplotlib import pylab
21import matplotlib.lines as mlines
22from matplotlib.ticker import MaxNLocator
23from mobly import test_runner
24
25import its_base_test
26import camera_properties_utils
27import capture_request_utils
28import image_processing_utils
29import its_session_utils
30import target_exposure_utils
31
32_JPG_STR = 'jpg'
33_NAME = os.path.splitext(os.path.basename(__file__))[0]
34_PATCH_H = 0.1  # center 10%
35_PATCH_W = 0.1
36_PATCH_X = 0.5 - _PATCH_W/2
37_PATCH_Y = 0.5 - _PATCH_H/2
38_PLOT_ALPHA = 0.5
39_PLOT_MARKER_SIZE = 8
40_PLOT_LEGEND_CIRCLE_SIZE = 10
41_PLOT_LEGEND_TRIANGLE_SIZE = 6
42_THRESHOLD_MAX_RMS_DIFF = 0.03
43_YUV_STR = 'yuv'
44
45
46def do_capture_and_extract_rgb_means(
47    req, cam, props, size, img_type, index, name_with_log_path, debug):
48  """Do capture and extra rgb_means of center patch.
49
50  Args:
51    req: capture request
52    cam: camera object
53    props: camera properties dict
54    size: [width, height]
55    img_type: string of 'yuv' or 'jpeg'
56    index: index to track capture number of img_type
57    name_with_log_path: file name and location for saving image
58    debug: boolean to flag saving captured images
59
60  Returns:
61    rgb: center patch RGB means
62    img: RGB image array
63  """
64  out_surface = {'width': size[0], 'height': size[1], 'format': img_type}
65  if camera_properties_utils.stream_use_case(props):
66    out_surface['useCase'] = camera_properties_utils.USE_CASE_STILL_CAPTURE
67  logging.debug('output surface: %s', str(out_surface))
68  if debug and camera_properties_utils.raw(props):
69    out_surfaces = [{'format': 'raw'}, out_surface]
70    cap_raw, cap = cam.do_capture(req, out_surfaces)
71    img_raw = image_processing_utils.convert_capture_to_rgb_image(
72        cap_raw, props=props)
73    image_processing_utils.write_image(
74        img_raw,
75        f'{name_with_log_path}_raw_{img_type}_w{size[0]}_h{size[1]}.png', True)
76  else:
77    cap = cam.do_capture(req, out_surface)
78  logging.debug('e_cap: %d, s_cap: %d, f_distance: %s',
79                cap['metadata']['android.sensor.exposureTime'],
80                cap['metadata']['android.sensor.sensitivity'],
81                cap['metadata']['android.lens.focusDistance'])
82  if img_type == _JPG_STR:
83    if cap['format'] != 'jpeg':
84      raise AssertionError(f"{cap['format']} != jpeg")
85    img = image_processing_utils.decompress_jpeg_to_rgb_image(cap['data'])
86  else:
87    if cap['format'] != img_type:
88      raise AssertionError(f"{cap['format']} != {img_type}")
89    img = image_processing_utils.convert_capture_to_rgb_image(cap)
90  if cap['width'] != size[0]:
91    raise AssertionError(f"{cap['width']} != {size[0]}")
92  if cap['height'] != size[1]:
93    raise AssertionError(f"{cap['height']} != {size[1]}")
94
95  if img_type == _JPG_STR:
96    if img.shape[0] != size[1]:
97      raise AssertionError(f'{img.shape[0]} != {size[1]}')
98    if img.shape[1] != size[0]:
99      raise AssertionError(f'{img.shape[1]} != {size[0]}')
100    if img.shape[2] != 3:
101      raise AssertionError(f'{img.shape[2]} != 3')
102  patch = image_processing_utils.get_image_patch(
103      img, _PATCH_X, _PATCH_Y, _PATCH_W, _PATCH_H)
104  rgb = image_processing_utils.compute_image_means(patch)
105  logging.debug('Captured %s %dx%d rgb = %s, format number = %d',
106                img_type, cap['width'], cap['height'], str(rgb), index)
107  return rgb, img
108
109
110class YuvJpegAllTest(its_base_test.ItsBaseTest):
111  """Test reported sizes & fmts for YUV & JPEG caps return similar images."""
112
113  def test_yuv_jpeg_all(self):
114    with its_session_utils.ItsSession(
115        device_id=self.dut.serial,
116        camera_id=self.camera_id,
117        hidden_physical_id=self.hidden_physical_id) as cam:
118      props = cam.get_camera_properties()
119      props = cam.override_with_hidden_physical_camera_props(props)
120
121      log_path = self.log_path
122      debug = self.debug_mode
123      name_with_log_path = os.path.join(log_path, _NAME)
124
125      # Check SKIP conditions
126      camera_properties_utils.skip_unless(
127          camera_properties_utils.linear_tonemap(props))
128
129      # Load chart for scene
130      its_session_utils.load_scene(
131          cam, props, self.scene, self.tablet,
132          its_session_utils.CHART_DISTANCE_NO_SCALING)
133
134      # If device supports target exposure computation, use manual capture.
135      # Otherwise, do 3A, then use an auto request.
136      # Both requests use a linear tonemap and focus distance of 0.0
137      # so that the YUV and JPEG should look the same
138      # (once converted by the image_processing_utils).
139      if camera_properties_utils.compute_target_exposure(props):
140        logging.debug('Using manual capture request')
141        e, s = target_exposure_utils.get_target_exposure_combos(
142            log_path, cam)['midExposureTime']
143        logging.debug('e_req: %d, s_req: %d', e, s)
144        req = capture_request_utils.manual_capture_request(
145            s, e, 0.0, True, props)
146        match_ar = None
147      else:
148        logging.debug('Using auto capture request')
149        cam.do_3a(do_af=False)
150        req = capture_request_utils.auto_capture_request(
151            linear_tonemap=True, props=props, do_af=False)
152        largest_yuv = capture_request_utils.get_largest_yuv_format(props)
153        match_ar = (largest_yuv['width'], largest_yuv['height'])
154
155      yuv_rgbs = []
156      yuv_imgs = []
157      for i, size in enumerate(
158          capture_request_utils.get_available_output_sizes(
159              _YUV_STR, props, match_ar_size=match_ar)):
160        yuv_rgb, yuv_img = do_capture_and_extract_rgb_means(
161            req, cam, props, size, _YUV_STR, i, name_with_log_path, debug)
162        yuv_rgbs.append(yuv_rgb)
163        yuv_imgs.append(yuv_img)
164
165      jpg_rgbs = []
166      jpg_imgs = []
167      for i, size in enumerate(
168          capture_request_utils.get_available_output_sizes(
169              _JPG_STR, props, match_ar_size=match_ar)):
170        jpg_rgb, jpg_img = do_capture_and_extract_rgb_means(
171            req, cam, props, size, _JPG_STR, i, name_with_log_path, debug)
172        jpg_rgbs.append(jpg_rgb)
173        jpg_imgs.append(jpg_img)
174
175      # Plot means vs format
176      pylab.figure(_NAME)
177      pylab.title(_NAME)
178      yuv_index = range(len(yuv_rgbs))
179      jpg_index = range(len(jpg_rgbs))
180      pylab.plot(yuv_index, [rgb[0] for rgb in yuv_rgbs],
181                 '-ro', alpha=_PLOT_ALPHA, markersize=_PLOT_MARKER_SIZE)
182      pylab.plot(yuv_index, [rgb[1] for rgb in yuv_rgbs],
183                 '-go', alpha=_PLOT_ALPHA, markersize=_PLOT_MARKER_SIZE)
184      pylab.plot(yuv_index, [rgb[2] for rgb in yuv_rgbs],
185                 '-bo', alpha=_PLOT_ALPHA, markersize=_PLOT_MARKER_SIZE)
186      pylab.plot(jpg_index, [rgb[0] for rgb in jpg_rgbs],
187                 '-r^', alpha=_PLOT_ALPHA, markersize=_PLOT_MARKER_SIZE)
188      pylab.plot(jpg_index, [rgb[1] for rgb in jpg_rgbs],
189                 '-g^', alpha=_PLOT_ALPHA, markersize=_PLOT_MARKER_SIZE)
190      pylab.plot(jpg_index, [rgb[2] for rgb in jpg_rgbs],
191                 '-b^', alpha=_PLOT_ALPHA, markersize=_PLOT_MARKER_SIZE)
192      pylab.ylim([0, 1])
193      ax = pylab.gca()
194      ax.xaxis.set_major_locator(MaxNLocator(integer=True))  # x-axis integers
195      yuv_marker = mlines.Line2D([], [], linestyle='None',
196                                 color='black', marker='.',
197                                 markersize=_PLOT_LEGEND_CIRCLE_SIZE,
198                                 label='YUV')
199      jpg_marker = mlines.Line2D([], [], linestyle='None',
200                                 color='black', marker='^',
201                                 markersize=_PLOT_LEGEND_TRIANGLE_SIZE,
202                                 label='JPEG')
203      ax.legend(handles=[yuv_marker, jpg_marker])
204      pylab.xlabel('format number')
205      pylab.ylabel('RGB avg [0, 1]')
206      matplotlib.pyplot.savefig(f'{name_with_log_path}_plot_means.png')
207
208      # Assert all captures are similar in RGB space using rgbs[0] as ref.
209      rgbs = yuv_rgbs + jpg_rgbs
210      max_diff = 0
211      for rgb_i in rgbs[1:]:
212        rms_diff = image_processing_utils.compute_image_rms_difference_1d(
213            rgbs[0], rgb_i)  # use first capture as reference
214        max_diff = max(max_diff, rms_diff)
215      msg = f'Max RMS difference: {max_diff:.4f}'
216      logging.debug('%s', msg)
217      if max_diff >= _THRESHOLD_MAX_RMS_DIFF:
218        for img in yuv_imgs:
219          image_processing_utils.write_image(
220              img,
221              f'{name_with_log_path}_yuv_{img.shape[1]}x{img.shape[0]}.png'
222          )
223        for img in jpg_imgs:
224          image_processing_utils.write_image(
225              img,
226              f'{name_with_log_path}_jpg_{img.shape[1]}x{img.shape[0]}.png'
227          )
228        raise AssertionError(f'{msg} spec: {_THRESHOLD_MAX_RMS_DIFF}')
229
230if __name__ == '__main__':
231  test_runner.main()
232