1 /*
2  * Copyright 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.media;
18 
19 import android.graphics.ImageFormat;
20 import android.graphics.PixelFormat;
21 import android.hardware.HardwareBuffer;
22 import android.media.Image.Plane;
23 import android.util.Log;
24 import android.util.Size;
25 
26 import libcore.io.Memory;
27 
28 import java.nio.ByteBuffer;
29 
30 /**
31  * Package private utility class for hosting commonly used Image related methods.
32  */
33 class ImageUtils {
34     private static final String IMAGEUTILS_LOG_TAG = "ImageUtils";
35 
36     /**
37      * Only a subset of the formats defined in
38      * {@link android.graphics.ImageFormat ImageFormat} and
39      * {@link android.graphics.PixelFormat PixelFormat} are supported by
40      * ImageReader. When reading RGB data from a surface, the formats defined in
41      * {@link android.graphics.PixelFormat PixelFormat} can be used; when
42      * reading YUV, JPEG, HEIC or raw sensor data (for example, from the camera
43      * or video decoder), formats from {@link android.graphics.ImageFormat ImageFormat}
44      * are used.
45      */
getNumPlanesForFormat(int format)46     public static int getNumPlanesForFormat(int format) {
47         switch (format) {
48             case ImageFormat.YV12:
49             case ImageFormat.YUV_420_888:
50             case ImageFormat.NV21:
51             case ImageFormat.YCBCR_P010:
52                 return 3;
53             case ImageFormat.NV16:
54                 return 2;
55             case PixelFormat.RGB_565:
56             case PixelFormat.RGBA_8888:
57             case PixelFormat.RGBX_8888:
58             case PixelFormat.RGB_888:
59             case ImageFormat.JPEG:
60             case ImageFormat.YUY2:
61             case ImageFormat.Y8:
62             case ImageFormat.Y16:
63             case ImageFormat.RAW_SENSOR:
64             case ImageFormat.RAW_PRIVATE:
65             case ImageFormat.RAW10:
66             case ImageFormat.RAW12:
67             case ImageFormat.DEPTH16:
68             case ImageFormat.DEPTH_POINT_CLOUD:
69             case ImageFormat.RAW_DEPTH:
70             case ImageFormat.RAW_DEPTH10:
71             case ImageFormat.DEPTH_JPEG:
72             case ImageFormat.HEIC:
73             case ImageFormat.JPEG_R:
74                 return 1;
75             case ImageFormat.PRIVATE:
76                 return 0;
77             default:
78                 throw new UnsupportedOperationException(
79                         String.format("Invalid format specified %d", format));
80         }
81     }
82 
83     /**
84      * Only a subset of the formats defined in
85      * {@link android.graphics.HardwareBuffer.Format} constants are supported by ImageReader.
86      */
getNumPlanesForHardwareBufferFormat(int hardwareBufferFormat)87     public static int getNumPlanesForHardwareBufferFormat(int hardwareBufferFormat) {
88         switch(hardwareBufferFormat) {
89             case HardwareBuffer.YCBCR_420_888:
90             case HardwareBuffer.YCBCR_P010:
91                 return 3;
92             case HardwareBuffer.RGBA_8888:
93             case HardwareBuffer.RGBX_8888:
94             case HardwareBuffer.RGB_888:
95             case HardwareBuffer.RGB_565:
96             case HardwareBuffer.RGBA_FP16:
97             case HardwareBuffer.RGBA_1010102:
98             case HardwareBuffer.BLOB:
99             case HardwareBuffer.D_16:
100             case HardwareBuffer.D_24:
101             case HardwareBuffer.DS_24UI8:
102             case HardwareBuffer.D_FP32:
103             case HardwareBuffer.DS_FP32UI8:
104             case HardwareBuffer.S_UI8:
105                 return 1;
106             default:
107                 throw new UnsupportedOperationException(
108                     String.format("Invalid hardwareBuffer format specified %d",
109                             hardwareBufferFormat));
110         }
111     }
112     /**
113      * <p>
114      * Copy source image data to destination Image.
115      * </p>
116      * <p>
117      * Only support the copy between two non-{@link ImageFormat#PRIVATE PRIVATE} format
118      * images with same properties (format, size, etc.). The data from the
119      * source image will be copied to the byteBuffers from the destination Image
120      * starting from position zero, and the destination image will be rewound to
121      * zero after copy is done.
122      * </p>
123      *
124      * @param src The source image to be copied from.
125      * @param dst The destination image to be copied to.
126      * @throws IllegalArgumentException If the source and destination images
127      *             have different format, or one of the images is not copyable.
128      */
imageCopy(Image src, Image dst)129     public static void imageCopy(Image src, Image dst) {
130         if (src == null || dst == null) {
131             throw new IllegalArgumentException("Images should be non-null");
132         }
133         if (src.getFormat() != dst.getFormat()) {
134             throw new IllegalArgumentException("Src and dst images should have the same format");
135         }
136         if (src.getFormat() == ImageFormat.PRIVATE ||
137                 dst.getFormat() == ImageFormat.PRIVATE) {
138             throw new IllegalArgumentException("PRIVATE format images are not copyable");
139         }
140         if (src.getFormat() == ImageFormat.RAW_PRIVATE) {
141             throw new IllegalArgumentException(
142                     "Copy of RAW_OPAQUE format has not been implemented");
143         }
144         if (src.getFormat() == ImageFormat.RAW_DEPTH) {
145             throw new IllegalArgumentException(
146                     "Copy of RAW_DEPTH format has not been implemented");
147         }
148         if (src.getFormat() == ImageFormat.RAW_DEPTH10) {
149             throw new IllegalArgumentException(
150                     "Copy of RAW_DEPTH10 format has not been implemented");
151         }
152         if (!(dst.getOwner() instanceof ImageWriter)) {
153             throw new IllegalArgumentException("Destination image is not from ImageWriter. Only"
154                     + " the images from ImageWriter are writable");
155         }
156         Size srcSize = new Size(src.getWidth(), src.getHeight());
157         Size dstSize = new Size(dst.getWidth(), dst.getHeight());
158         if (!srcSize.equals(dstSize)) {
159             throw new IllegalArgumentException("source image size " + srcSize + " is different"
160                     + " with " + "destination image size " + dstSize);
161         }
162 
163         Plane[] srcPlanes = src.getPlanes();
164         Plane[] dstPlanes = dst.getPlanes();
165         ByteBuffer srcBuffer = null;
166         ByteBuffer dstBuffer = null;
167         for (int i = 0; i < srcPlanes.length; i++) {
168             int srcRowStride = srcPlanes[i].getRowStride();
169             int dstRowStride = dstPlanes[i].getRowStride();
170             srcBuffer = srcPlanes[i].getBuffer();
171             dstBuffer = dstPlanes[i].getBuffer();
172             if (!(srcBuffer.isDirect() && dstBuffer.isDirect())) {
173                 throw new IllegalArgumentException("Source and destination ByteBuffers must be"
174                         + " direct byteBuffer!");
175             }
176             if (srcPlanes[i].getPixelStride() != dstPlanes[i].getPixelStride()) {
177                 throw new IllegalArgumentException("Source plane image pixel stride " +
178                         srcPlanes[i].getPixelStride() +
179                         " must be same as destination image pixel stride " +
180                         dstPlanes[i].getPixelStride());
181             }
182 
183             int srcPos = srcBuffer.position();
184             srcBuffer.rewind();
185             dstBuffer.rewind();
186             if (srcRowStride == dstRowStride) {
187                 // Fast path, just copy the content if the byteBuffer all together.
188                 dstBuffer.put(srcBuffer);
189             } else {
190                 // Source and destination images may have different alignment requirements,
191                 // therefore may have different strides. Copy row by row for such case.
192                 int srcOffset = srcBuffer.position();
193                 int dstOffset = dstBuffer.position();
194                 Size effectivePlaneSize = getEffectivePlaneSizeForImage(src, i);
195                 int srcByteCount = effectivePlaneSize.getWidth() * srcPlanes[i].getPixelStride();
196                 for (int row = 0; row < effectivePlaneSize.getHeight(); row++) {
197                     if (row == effectivePlaneSize.getHeight() - 1) {
198                         // Special case for NV21 backed YUV420_888: need handle the last row
199                         // carefully to avoid memory corruption. Check if we have enough bytes to
200                         // copy.
201                         int remainingBytes = srcBuffer.remaining() - srcOffset;
202                         if (srcByteCount > remainingBytes) {
203                             srcByteCount = remainingBytes;
204                         }
205                     }
206                     directByteBufferCopy(srcBuffer, srcOffset, dstBuffer, dstOffset, srcByteCount);
207                     srcOffset += srcRowStride;
208                     dstOffset += dstRowStride;
209                 }
210             }
211 
212             srcBuffer.position(srcPos);
213             dstBuffer.rewind();
214         }
215     }
216 
217     /**
218      * Return the estimated native allocation size in bytes based on width, height, format,
219      * and number of images.
220      *
221      * <p>This is a very rough estimation and should only be used for native allocation
222      * registration in VM so it can be accounted for during GC.</p>
223      *
224      * @param width The width of the images.
225      * @param height The height of the images.
226      * @param format The format of the images.
227      * @param numImages The number of the images.
228      */
getEstimatedNativeAllocBytes(int width, int height, int format, int numImages)229     public static int getEstimatedNativeAllocBytes(int width, int height, int format,
230             int numImages) {
231         double estimatedBytePerPixel;
232         switch (format) {
233             // 10x compression from RGB_888
234             case ImageFormat.JPEG:
235             case ImageFormat.DEPTH_POINT_CLOUD:
236             case ImageFormat.DEPTH_JPEG:
237             case ImageFormat.HEIC:
238             case ImageFormat.JPEG_R:
239                 estimatedBytePerPixel = 0.3;
240                 break;
241             case ImageFormat.Y8:
242                 estimatedBytePerPixel = 1.0;
243                 break;
244             case ImageFormat.RAW10:
245             case ImageFormat.RAW_DEPTH10:
246                 estimatedBytePerPixel = 1.25;
247                 break;
248             case ImageFormat.YV12:
249             case ImageFormat.YUV_420_888:
250             case ImageFormat.NV21:
251             case ImageFormat.RAW12:
252             case ImageFormat.PRIVATE: // A rough estimate because the real size is unknown.
253                 estimatedBytePerPixel = 1.5;
254                 break;
255             case ImageFormat.NV16:
256             case PixelFormat.RGB_565:
257             case ImageFormat.YUY2:
258             case ImageFormat.Y16:
259             case ImageFormat.RAW_DEPTH:
260             case ImageFormat.RAW_SENSOR:
261             case ImageFormat.RAW_PRIVATE: // round estimate, real size is unknown
262             case ImageFormat.DEPTH16:
263                 estimatedBytePerPixel = 2.0;
264                 break;
265             case PixelFormat.RGB_888:
266             case ImageFormat.YCBCR_P010:
267                 estimatedBytePerPixel = 3.0;
268                 break;
269             case PixelFormat.RGBA_8888:
270             case PixelFormat.RGBX_8888:
271             case PixelFormat.RGBA_1010102:
272                 estimatedBytePerPixel = 4.0;
273                 break;
274             default:
275                 if (Log.isLoggable(IMAGEUTILS_LOG_TAG, Log.VERBOSE)) {
276                     Log.v(IMAGEUTILS_LOG_TAG, "getEstimatedNativeAllocBytes() uses default"
277                             + "estimated native allocation size.");
278                 }
279                 estimatedBytePerPixel = 1.0;
280         }
281 
282         return (int)(width * height * estimatedBytePerPixel * numImages);
283     }
284 
getEffectivePlaneSizeForImage(Image image, int planeIdx)285     private static Size getEffectivePlaneSizeForImage(Image image, int planeIdx) {
286         switch (image.getFormat()) {
287             case ImageFormat.YCBCR_P010:
288             case ImageFormat.YV12:
289             case ImageFormat.YUV_420_888:
290             case ImageFormat.NV21:
291                 if (planeIdx == 0) {
292                     return new Size(image.getWidth(), image.getHeight());
293                 } else {
294                     return new Size(image.getWidth() / 2, image.getHeight() / 2);
295                 }
296             case ImageFormat.NV16:
297                 if (planeIdx == 0) {
298                     return new Size(image.getWidth(), image.getHeight());
299                 } else {
300                     return new Size(image.getWidth(), image.getHeight() / 2);
301                 }
302             case PixelFormat.RGB_565:
303             case PixelFormat.RGBA_8888:
304             case PixelFormat.RGBA_1010102:
305             case PixelFormat.RGBX_8888:
306             case PixelFormat.RGB_888:
307             case ImageFormat.JPEG:
308             case ImageFormat.YUY2:
309             case ImageFormat.Y8:
310             case ImageFormat.Y16:
311             case ImageFormat.RAW_SENSOR:
312             case ImageFormat.RAW10:
313             case ImageFormat.RAW12:
314             case ImageFormat.RAW_DEPTH:
315             case ImageFormat.RAW_DEPTH10:
316             case ImageFormat.HEIC:
317             case ImageFormat.JPEG_R:
318                 return new Size(image.getWidth(), image.getHeight());
319             case ImageFormat.PRIVATE:
320                 return new Size(0, 0);
321             default:
322                 if (Log.isLoggable(IMAGEUTILS_LOG_TAG, Log.VERBOSE)) {
323                     Log.v(IMAGEUTILS_LOG_TAG, "getEffectivePlaneSizeForImage() uses"
324                             + "image's width and height for plane size.");
325                 }
326                 return new Size(image.getWidth(), image.getHeight());
327         }
328     }
329 
directByteBufferCopy(ByteBuffer srcBuffer, int srcOffset, ByteBuffer dstBuffer, int dstOffset, int srcByteCount)330     private static void directByteBufferCopy(ByteBuffer srcBuffer, int srcOffset,
331             ByteBuffer dstBuffer, int dstOffset, int srcByteCount) {
332         Memory.memmove(dstBuffer, dstOffset, srcBuffer, srcOffset, srcByteCount);
333     }
334 }
335