1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "ExtCamUtils"
18 // #define LOG_NDEBUG 0
19 
20 #include "ExternalCameraUtils.h"
21 
22 #include <aidlcommonsupport/NativeHandle.h>
23 #include <jpeglib.h>
24 #include <linux/videodev2.h>
25 #include <log/log.h>
26 #include <algorithm>
27 #include <cinttypes>
28 #include <cmath>
29 
30 #define HAVE_JPEG  // required for libyuv.h to export MJPEG decode APIs
31 #include <libyuv.h>
32 
33 namespace android {
34 namespace hardware {
35 namespace camera {
36 
37 namespace external {
38 namespace common {
39 
40 namespace {
41 const int kDefaultCameraIdOffset = 100;
42 const int kDefaultJpegBufSize = 5 << 20;  // 5MB
43 const int kDefaultNumVideoBuffer = 4;
44 const int kDefaultNumStillBuffer = 2;
45 const int kDefaultOrientation = 0;  // suitable for natural landscape displays like tablet/TV
46                                     // For phone devices 270 is better
47 }  // anonymous namespace
48 
49 const char* ExternalCameraConfig::kDefaultCfgPath = "/vendor/etc/external_camera_config.xml";
50 
loadFromCfg(const char * cfgPath)51 ExternalCameraConfig ExternalCameraConfig::loadFromCfg(const char* cfgPath) {
52     using namespace tinyxml2;
53     ExternalCameraConfig ret;
54 
55     XMLDocument configXml;
56     XMLError err = configXml.LoadFile(cfgPath);
57     if (err != XML_SUCCESS) {
58         ALOGE("%s: Unable to load external camera config file '%s'. Error: %s", __FUNCTION__,
59               cfgPath, XMLDocument::ErrorIDToName(err));
60         return ret;
61     } else {
62         ALOGI("%s: load external camera config succeeded!", __FUNCTION__);
63     }
64 
65     XMLElement* extCam = configXml.FirstChildElement("ExternalCamera");
66     if (extCam == nullptr) {
67         ALOGI("%s: no external camera config specified", __FUNCTION__);
68         return ret;
69     }
70 
71     XMLElement* providerCfg = extCam->FirstChildElement("Provider");
72     if (providerCfg == nullptr) {
73         ALOGI("%s: no external camera provider config specified", __FUNCTION__);
74         return ret;
75     }
76 
77     XMLElement* cameraIdOffset = providerCfg->FirstChildElement("CameraIdOffset");
78     if (cameraIdOffset != nullptr) {
79         ret.cameraIdOffset = std::atoi(cameraIdOffset->GetText());
80     }
81 
82     XMLElement* ignore = providerCfg->FirstChildElement("ignore");
83     if (ignore == nullptr) {
84         ALOGI("%s: no internal ignored device specified", __FUNCTION__);
85         return ret;
86     }
87 
88     XMLElement* id = ignore->FirstChildElement("id");
89     while (id != nullptr) {
90         const char* text = id->GetText();
91         if (text != nullptr) {
92             ret.mInternalDevices.insert(text);
93             ALOGI("%s: device %s will be ignored by external camera provider", __FUNCTION__, text);
94         }
95         id = id->NextSiblingElement("id");
96     }
97 
98     XMLElement* deviceCfg = extCam->FirstChildElement("Device");
99     if (deviceCfg == nullptr) {
100         ALOGI("%s: no external camera device config specified", __FUNCTION__);
101         return ret;
102     }
103 
104     XMLElement* jpegBufSz = deviceCfg->FirstChildElement("MaxJpegBufferSize");
105     if (jpegBufSz == nullptr) {
106         ALOGI("%s: no max jpeg buffer size specified", __FUNCTION__);
107     } else {
108         ret.maxJpegBufSize = jpegBufSz->UnsignedAttribute("bytes", /*Default*/ kDefaultJpegBufSize);
109     }
110 
111     XMLElement* numVideoBuf = deviceCfg->FirstChildElement("NumVideoBuffers");
112     if (numVideoBuf == nullptr) {
113         ALOGI("%s: no num video buffers specified", __FUNCTION__);
114     } else {
115         ret.numVideoBuffers =
116                 numVideoBuf->UnsignedAttribute("count", /*Default*/ kDefaultNumVideoBuffer);
117     }
118 
119     XMLElement* numStillBuf = deviceCfg->FirstChildElement("NumStillBuffers");
120     if (numStillBuf == nullptr) {
121         ALOGI("%s: no num still buffers specified", __FUNCTION__);
122     } else {
123         ret.numStillBuffers =
124                 numStillBuf->UnsignedAttribute("count", /*Default*/ kDefaultNumStillBuffer);
125     }
126 
127     XMLElement* fpsList = deviceCfg->FirstChildElement("FpsList");
128     if (fpsList == nullptr) {
129         ALOGI("%s: no fps list specified", __FUNCTION__);
130     } else {
131         if (!updateFpsList(fpsList, ret.fpsLimits)) {
132             return ret;
133         }
134     }
135 
136     XMLElement* depth = deviceCfg->FirstChildElement("Depth16Supported");
137     if (depth == nullptr) {
138         ret.depthEnabled = false;
139         ALOGI("%s: depth output is not enabled", __FUNCTION__);
140     } else {
141         ret.depthEnabled = depth->BoolAttribute("enabled", false);
142     }
143 
144     if (ret.depthEnabled) {
145         XMLElement* depthFpsList = deviceCfg->FirstChildElement("DepthFpsList");
146         if (depthFpsList == nullptr) {
147             ALOGW("%s: no depth fps list specified", __FUNCTION__);
148         } else {
149             if (!updateFpsList(depthFpsList, ret.depthFpsLimits)) {
150                 return ret;
151             }
152         }
153     }
154 
155     XMLElement* minStreamSize = deviceCfg->FirstChildElement("MinimumStreamSize");
156     if (minStreamSize == nullptr) {
157         ALOGI("%s: no minimum stream size specified", __FUNCTION__);
158     } else {
159         ret.minStreamSize = {
160                 static_cast<int32_t>(minStreamSize->UnsignedAttribute("width", /*Default*/ 0)),
161                 static_cast<int32_t>(minStreamSize->UnsignedAttribute("height", /*Default*/ 0))};
162     }
163 
164     XMLElement* orientation = deviceCfg->FirstChildElement("Orientation");
165     if (orientation == nullptr) {
166         ALOGI("%s: no sensor orientation specified", __FUNCTION__);
167     } else {
168         ret.orientation = orientation->IntAttribute("degree", /*Default*/ kDefaultOrientation);
169     }
170 
171     ALOGI("%s: external camera cfg loaded: maxJpgBufSize %d,"
172           " num video buffers %d, num still buffers %d, orientation %d",
173           __FUNCTION__, ret.maxJpegBufSize, ret.numVideoBuffers, ret.numStillBuffers,
174           ret.orientation);
175     for (const auto& limit : ret.fpsLimits) {
176         ALOGI("%s: fpsLimitList: %dx%d@%f", __FUNCTION__, limit.size.width, limit.size.height,
177               limit.fpsUpperBound);
178     }
179     for (const auto& limit : ret.depthFpsLimits) {
180         ALOGI("%s: depthFpsLimitList: %dx%d@%f", __FUNCTION__, limit.size.width, limit.size.height,
181               limit.fpsUpperBound);
182     }
183     ALOGI("%s: minStreamSize: %dx%d", __FUNCTION__, ret.minStreamSize.width,
184           ret.minStreamSize.height);
185     return ret;
186 }
187 
updateFpsList(tinyxml2::XMLElement * fpsList,std::vector<FpsLimitation> & fpsLimits)188 bool ExternalCameraConfig::updateFpsList(tinyxml2::XMLElement* fpsList,
189                                          std::vector<FpsLimitation>& fpsLimits) {
190     using namespace tinyxml2;
191     std::vector<FpsLimitation> limits;
192     XMLElement* row = fpsList->FirstChildElement("Limit");
193     while (row != nullptr) {
194         FpsLimitation prevLimit{{0, 0}, 1000.0};
195         FpsLimitation limit = {
196                 {/* width */ static_cast<int32_t>(row->UnsignedAttribute("width", /*Default*/ 0)),
197                  /* height */ static_cast<int32_t>(
198                          row->UnsignedAttribute("height", /*Default*/ 0))},
199                 /* fpsUpperBound */ row->DoubleAttribute("fpsBound", /*Default*/ 1000.0)};
200         if (limit.size.width <= prevLimit.size.width ||
201             limit.size.height <= prevLimit.size.height ||
202             limit.fpsUpperBound >= prevLimit.fpsUpperBound) {
203             ALOGE("%s: FPS limit list must have increasing size and decreasing fps!"
204                   " Prev %dx%d@%f, Current %dx%d@%f",
205                   __FUNCTION__, prevLimit.size.width, prevLimit.size.height,
206                   prevLimit.fpsUpperBound, limit.size.width, limit.size.height,
207                   limit.fpsUpperBound);
208             return false;
209         }
210         limits.push_back(limit);
211         row = row->NextSiblingElement("Limit");
212     }
213     fpsLimits = limits;
214     return true;
215 }
216 
ExternalCameraConfig()217 ExternalCameraConfig::ExternalCameraConfig()
218     : cameraIdOffset(kDefaultCameraIdOffset),
219       maxJpegBufSize(kDefaultJpegBufSize),
220       numVideoBuffers(kDefaultNumVideoBuffer),
221       numStillBuffers(kDefaultNumStillBuffer),
222       depthEnabled(false),
223       orientation(kDefaultOrientation) {
224     fpsLimits.push_back({/* size */ {/* width */ 640, /* height */ 480}, /* fpsUpperBound */ 30.0});
225     fpsLimits.push_back({/* size */ {/* width */ 1280, /* height */ 720}, /* fpsUpperBound */ 7.5});
226     fpsLimits.push_back(
227             {/* size */ {/* width */ 1920, /* height */ 1080}, /* fpsUpperBound */ 5.0});
228     minStreamSize = {0, 0};
229 }
230 
231 }  // namespace common
232 }  // namespace external
233 
234 namespace device {
235 namespace implementation {
236 
getFramesPerSecond() const237 double SupportedV4L2Format::FrameRate::getFramesPerSecond() const {
238     return static_cast<double>(durationDenominator) / durationNumerator;
239 }
240 
Frame(uint32_t width,uint32_t height,uint32_t fourcc)241 Frame::Frame(uint32_t width, uint32_t height, uint32_t fourcc)
242     : mWidth(width), mHeight(height), mFourcc(fourcc) {}
~Frame()243 Frame::~Frame() {}
244 
V4L2Frame(uint32_t w,uint32_t h,uint32_t fourcc,int bufIdx,int fd,uint32_t dataSize,uint64_t offset)245 V4L2Frame::V4L2Frame(uint32_t w, uint32_t h, uint32_t fourcc, int bufIdx, int fd, uint32_t dataSize,
246                      uint64_t offset)
247     : Frame(w, h, fourcc), mBufferIndex(bufIdx), mFd(fd), mDataSize(dataSize), mOffset(offset) {}
248 
~V4L2Frame()249 V4L2Frame::~V4L2Frame() {
250     unmap();
251 }
252 
getData(uint8_t ** outData,size_t * dataSize)253 int V4L2Frame::getData(uint8_t** outData, size_t* dataSize) {
254     return map(outData, dataSize);
255 }
256 
map(uint8_t ** data,size_t * dataSize)257 int V4L2Frame::map(uint8_t** data, size_t* dataSize) {
258     if (data == nullptr || dataSize == nullptr) {
259         ALOGI("%s: V4L2 buffer map bad argument: data %p, dataSize %p", __FUNCTION__, data,
260               dataSize);
261         return -EINVAL;
262     }
263 
264     std::lock_guard<std::mutex> lk(mLock);
265     if (!mMapped) {
266         void* addr = mmap(nullptr, mDataSize, PROT_READ, MAP_SHARED, mFd, mOffset);
267         if (addr == MAP_FAILED) {
268             ALOGE("%s: V4L2 buffer map failed: %s", __FUNCTION__, strerror(errno));
269             return -EINVAL;
270         }
271         mData = static_cast<uint8_t*>(addr);
272         mMapped = true;
273     }
274     *data = mData;
275     *dataSize = mDataSize;
276     ALOGV("%s: V4L map FD %d, data %p size %zu", __FUNCTION__, mFd, mData, mDataSize);
277     return 0;
278 }
279 
unmap()280 int V4L2Frame::unmap() {
281     std::lock_guard<std::mutex> lk(mLock);
282     if (mMapped) {
283         ALOGV("%s: V4L unmap data %p size %zu", __FUNCTION__, mData, mDataSize);
284         if (munmap(mData, mDataSize) != 0) {
285             ALOGE("%s: V4L2 buffer unmap failed: %s", __FUNCTION__, strerror(errno));
286             return -EINVAL;
287         }
288         mMapped = false;
289     }
290     return 0;
291 }
292 
AllocatedFrame(uint32_t w,uint32_t h)293 AllocatedFrame::AllocatedFrame(uint32_t w, uint32_t h) : Frame(w, h, V4L2_PIX_FMT_YUV420) {}
~AllocatedFrame()294 AllocatedFrame::~AllocatedFrame() {}
295 
getData(uint8_t ** outData,size_t * dataSize)296 int AllocatedFrame::getData(uint8_t** outData, size_t* dataSize) {
297     YCbCrLayout layout;
298     int ret = allocate(&layout);
299     if (ret != 0) {
300         return ret;
301     }
302     *outData = mData.data();
303     *dataSize = mBufferSize;
304     return 0;
305 }
306 
allocate(YCbCrLayout * out)307 int AllocatedFrame::allocate(YCbCrLayout* out) {
308     std::lock_guard<std::mutex> lk(mLock);
309     if ((mWidth % 2) || (mHeight % 2)) {
310         ALOGE("%s: bad dimension %dx%d (not multiple of 2)", __FUNCTION__, mWidth, mHeight);
311         return -EINVAL;
312     }
313 
314     // This frame might be sent to jpeglib to be encoded. Since AllocatedFrame only contains YUV420,
315     // jpeglib expects height and width of Y component to be an integral multiple of 2*DCTSIZE,
316     // and heights and widths of Cb and Cr components to be an integral multiple of DCTSIZE. If the
317     // image size does not meet this requirement, libjpeg expects its input to be padded to meet the
318     // constraints. This padding is removed from the final encoded image so the content in the
319     // padding doesn't matter. What matters is that the memory is accessible to jpeglib at the time
320     // of encoding.
321     // For example, if the image size is 1500x844 and DCTSIZE is 8, jpeglib expects a YUV 420
322     // frame with components of following sizes:
323     //   Y:      1504x848 because 1504 and 848 are the next smallest multiples of 2*8
324     //   Cb/Cr:  752x424 which are the next smallest multiples of 8
325 
326     // jpeglib takes an array of row pointers which makes vertical padding trivial when setting up
327     // the pointers. Padding horizontally is a bit more complicated. AllocatedFrame holds the data
328     // in a flattened buffer, which means memory accesses past a row will flow into the next logical
329     // row. For any row of a component, we can consider the first few bytes of the next row as
330     // padding for the current one. This is true for Y and Cb components and all but last row of the
331     // Cr component. Reading past the last row of Cr component will lead to undefined behavior as
332     // libjpeg attempts to read memory past the allocated buffer. To prevent undefined behavior,
333     // the buffer allocated here is padded such that libjpeg never accesses unallocated memory when
334     // reading the last row. Effectively, we only need to ensure that the last row of Cr component
335     // has width that is an integral multiple of DCTSIZE.
336 
337     size_t dataSize = mWidth * mHeight * 3 / 2;  // YUV420
338 
339     size_t cbWidth = mWidth / 2;
340     size_t requiredCbWidth = DCTSIZE * ((cbWidth + DCTSIZE - 1) / DCTSIZE);
341     size_t padding = requiredCbWidth - cbWidth;
342     size_t finalSize = dataSize + padding;
343 
344     if (mData.size() != finalSize) {
345         mData.resize(finalSize);
346         mBufferSize = dataSize;
347     }
348 
349     if (out != nullptr) {
350         out->y = mData.data();
351         out->yStride = mWidth;
352         uint8_t* cbStart = mData.data() + mWidth * mHeight;
353         uint8_t* crStart = cbStart + mWidth * mHeight / 4;
354         out->cb = cbStart;
355         out->cr = crStart;
356         out->cStride = mWidth / 2;
357         out->chromaStep = 1;
358     }
359     return 0;
360 }
361 
getLayout(YCbCrLayout * out)362 int AllocatedFrame::getLayout(YCbCrLayout* out) {
363     IMapper::Rect noCrop = {0, 0, static_cast<int32_t>(mWidth), static_cast<int32_t>(mHeight)};
364     return getCroppedLayout(noCrop, out);
365 }
366 
getCroppedLayout(const IMapper::Rect & rect,YCbCrLayout * out)367 int AllocatedFrame::getCroppedLayout(const IMapper::Rect& rect, YCbCrLayout* out) {
368     if (out == nullptr) {
369         ALOGE("%s: null out", __FUNCTION__);
370         return -1;
371     }
372 
373     std::lock_guard<std::mutex> lk(mLock);
374     if ((rect.left + rect.width) > static_cast<int>(mWidth) ||
375         (rect.top + rect.height) > static_cast<int>(mHeight) || (rect.left % 2) || (rect.top % 2) ||
376         (rect.width % 2) || (rect.height % 2)) {
377         ALOGE("%s: bad rect left %d top %d w %d h %d", __FUNCTION__, rect.left, rect.top,
378               rect.width, rect.height);
379         return -1;
380     }
381 
382     out->y = mData.data() + mWidth * rect.top + rect.left;
383     out->yStride = mWidth;
384     uint8_t* cbStart = mData.data() + mWidth * mHeight;
385     uint8_t* crStart = cbStart + mWidth * mHeight / 4;
386     out->cb = cbStart + mWidth * rect.top / 4 + rect.left / 2;
387     out->cr = crStart + mWidth * rect.top / 4 + rect.left / 2;
388     out->cStride = mWidth / 2;
389     out->chromaStep = 1;
390     return 0;
391 }
392 
isAspectRatioClose(float ar1,float ar2)393 bool isAspectRatioClose(float ar1, float ar2) {
394     constexpr float kAspectRatioMatchThres = 0.025f;  // This threshold is good enough to
395                                                       // distinguish 4:3/16:9/20:9 1.33/1.78/2
396     return std::abs(ar1 - ar2) < kAspectRatioMatchThres;
397 }
398 
importBufferImpl(std::map<int,CirculatingBuffers> & circulatingBuffers,HandleImporter & handleImporter,int32_t streamId,uint64_t bufId,buffer_handle_t buf,buffer_handle_t ** outBufPtr)399 aidl::android::hardware::camera::common::Status importBufferImpl(
400         /*inout*/ std::map<int, CirculatingBuffers>& circulatingBuffers,
401         /*inout*/ HandleImporter& handleImporter, int32_t streamId, uint64_t bufId,
402         buffer_handle_t buf,
403         /*out*/ buffer_handle_t** outBufPtr) {
404     using ::aidl::android::hardware::camera::common::Status;
405     // AIDL does not have null NativeHandles. It sends empty handles instead.
406     // We check for when the buf is empty instead of when buf is null.
407     bool isBufEmpty = buf == nullptr || (buf->numFds == 0 && buf->numInts == 0);
408     if (isBufEmpty && bufId == BUFFER_ID_NO_BUFFER) {
409         ALOGE("%s: bufferId %" PRIu64 " has null buffer handle!", __FUNCTION__, bufId);
410         return Status::ILLEGAL_ARGUMENT;
411     }
412 
413     CirculatingBuffers& cbs = circulatingBuffers[streamId];
414     if (cbs.count(bufId) == 0) {
415         if (buf == nullptr) {
416             ALOGE("%s: bufferId %" PRIu64 " has null buffer handle!", __FUNCTION__, bufId);
417             return Status::ILLEGAL_ARGUMENT;
418         }
419         // Register a newly seen buffer
420         buffer_handle_t importedBuf = buf;
421         handleImporter.importBuffer(importedBuf);
422         if (importedBuf == nullptr) {
423             ALOGE("%s: output buffer for stream %d is invalid!", __FUNCTION__, streamId);
424             return Status::INTERNAL_ERROR;
425         } else {
426             cbs[bufId] = importedBuf;
427         }
428     }
429     *outBufPtr = &cbs[bufId];
430     return Status::OK;
431 }
432 
getFourCcFromLayout(const YCbCrLayout & layout)433 uint32_t getFourCcFromLayout(const YCbCrLayout& layout) {
434     intptr_t cb = reinterpret_cast<intptr_t>(layout.cb);
435     intptr_t cr = reinterpret_cast<intptr_t>(layout.cr);
436     if (std::abs(cb - cr) == 1 && layout.chromaStep == 2) {
437         // Interleaved format
438         if (layout.cb > layout.cr) {
439             return V4L2_PIX_FMT_NV21;
440         } else {
441             return V4L2_PIX_FMT_NV12;
442         }
443     } else if (layout.chromaStep == 1) {
444         // Planar format
445         if (layout.cb > layout.cr) {
446             return V4L2_PIX_FMT_YVU420;  // YV12
447         } else {
448             return V4L2_PIX_FMT_YUV420;  // YU12
449         }
450     } else {
451         return FLEX_YUV_GENERIC;
452     }
453 }
454 
getCropRect(CroppingType ct,const Size & inSize,const Size & outSize,IMapper::Rect * out)455 int getCropRect(CroppingType ct, const Size& inSize, const Size& outSize, IMapper::Rect* out) {
456     if (out == nullptr) {
457         ALOGE("%s: out is null", __FUNCTION__);
458         return -1;
459     }
460 
461     uint32_t inW = inSize.width;
462     uint32_t inH = inSize.height;
463     uint32_t outW = outSize.width;
464     uint32_t outH = outSize.height;
465 
466     // Handle special case where aspect ratio is close to input but scaled
467     // dimension is slightly larger than input
468     float arIn = ASPECT_RATIO(inSize);
469     float arOut = ASPECT_RATIO(outSize);
470     if (isAspectRatioClose(arIn, arOut)) {
471         out->left = 0;
472         out->top = 0;
473         out->width = static_cast<int32_t>(inW);
474         out->height = static_cast<int32_t>(inH);
475         return 0;
476     }
477 
478     if (ct == VERTICAL) {
479         uint64_t scaledOutH = static_cast<uint64_t>(outH) * inW / outW;
480         if (scaledOutH > inH) {
481             ALOGE("%s: Output size %dx%d cannot be vertically cropped from input size %dx%d",
482                   __FUNCTION__, outW, outH, inW, inH);
483             return -1;
484         }
485         scaledOutH = scaledOutH & ~0x1;  // make it multiple of 2
486 
487         out->left = 0;
488         out->top = static_cast<int32_t>((inH - scaledOutH) / 2) & ~0x1;
489         out->width = static_cast<int32_t>(inW);
490         out->height = static_cast<int32_t>(scaledOutH);
491         ALOGV("%s: crop %dx%d to %dx%d: top %d, scaledH %d", __FUNCTION__, inW, inH, outW, outH,
492               out->top, static_cast<int32_t>(scaledOutH));
493     } else {
494         uint64_t scaledOutW = static_cast<uint64_t>(outW) * inH / outH;
495         if (scaledOutW > inW) {
496             ALOGE("%s: Output size %dx%d cannot be horizontally cropped from input size %dx%d",
497                   __FUNCTION__, outW, outH, inW, inH);
498             return -1;
499         }
500         scaledOutW = scaledOutW & ~0x1;  // make it multiple of 2
501 
502         out->left = static_cast<int32_t>((inW - scaledOutW) / 2) & ~0x1;
503         out->top = 0;
504         out->width = static_cast<int32_t>(scaledOutW);
505         out->height = static_cast<int32_t>(inH);
506         ALOGV("%s: crop %dx%d to %dx%d: top %d, scaledW %d", __FUNCTION__, inW, inH, outW, outH,
507               out->top, static_cast<int32_t>(scaledOutW));
508     }
509 
510     return 0;
511 }
512 
formatConvert(const YCbCrLayout & in,const YCbCrLayout & out,Size sz,uint32_t format)513 int formatConvert(const YCbCrLayout& in, const YCbCrLayout& out, Size sz, uint32_t format) {
514     int ret = 0;
515     switch (format) {
516         case V4L2_PIX_FMT_NV21:
517             ret = libyuv::I420ToNV21(
518                     static_cast<uint8_t*>(in.y), static_cast<int32_t>(in.yStride),
519                     static_cast<uint8_t*>(in.cb), static_cast<int32_t>(in.cStride),
520                     static_cast<uint8_t*>(in.cr), static_cast<int32_t>(in.cStride),
521                     static_cast<uint8_t*>(out.y), static_cast<int32_t>(out.yStride),
522                     static_cast<uint8_t*>(out.cr), static_cast<int32_t>(out.cStride),
523                     static_cast<int32_t>(sz.width), static_cast<int32_t>(sz.height));
524             if (ret != 0) {
525                 ALOGE("%s: convert to NV21 buffer failed! ret %d", __FUNCTION__, ret);
526                 return ret;
527             }
528             break;
529         case V4L2_PIX_FMT_NV12:
530             ret = libyuv::I420ToNV12(
531                     static_cast<uint8_t*>(in.y), static_cast<int32_t>(in.yStride),
532                     static_cast<uint8_t*>(in.cb), static_cast<int32_t>(in.cStride),
533                     static_cast<uint8_t*>(in.cr), static_cast<int32_t>(in.cStride),
534                     static_cast<uint8_t*>(out.y), static_cast<int32_t>(out.yStride),
535                     static_cast<uint8_t*>(out.cb), static_cast<int32_t>(out.cStride),
536                     static_cast<int32_t>(sz.width), static_cast<int32_t>(sz.height));
537             if (ret != 0) {
538                 ALOGE("%s: convert to NV12 buffer failed! ret %d", __FUNCTION__, ret);
539                 return ret;
540             }
541             break;
542         case V4L2_PIX_FMT_YVU420:  // YV12
543         case V4L2_PIX_FMT_YUV420:  // YU12
544             // TODO: maybe we can speed up here by somehow save this copy?
545             ret = libyuv::I420Copy(static_cast<uint8_t*>(in.y), static_cast<int32_t>(in.yStride),
546                                    static_cast<uint8_t*>(in.cb), static_cast<int32_t>(in.cStride),
547                                    static_cast<uint8_t*>(in.cr), static_cast<int32_t>(in.cStride),
548                                    static_cast<uint8_t*>(out.y), static_cast<int32_t>(out.yStride),
549                                    static_cast<uint8_t*>(out.cb), static_cast<int32_t>(out.cStride),
550                                    static_cast<uint8_t*>(out.cr), static_cast<int32_t>(out.cStride),
551                                    static_cast<int32_t>(sz.width), static_cast<int32_t>(sz.height));
552             if (ret != 0) {
553                 ALOGE("%s: copy to YV12 or YU12 buffer failed! ret %d", __FUNCTION__, ret);
554                 return ret;
555             }
556             break;
557         case FLEX_YUV_GENERIC:
558             // TODO: b/72261744 write to arbitrary flexible YUV layout. Slow.
559             ALOGE("%s: unsupported flexible yuv layout"
560                   " y %p cb %p cr %p y_str %d c_str %d c_step %d",
561                   __FUNCTION__, out.y, out.cb, out.cr, out.yStride, out.cStride, out.chromaStep);
562             return -1;
563         default:
564             ALOGE("%s: unknown YUV format 0x%x!", __FUNCTION__, format);
565             return -1;
566     }
567     return 0;
568 }
569 
encodeJpegYU12(const Size & inSz,const YCbCrLayout & inLayout,int jpegQuality,const void * app1Buffer,size_t app1Size,void * out,size_t maxOutSize,size_t & actualCodeSize)570 int encodeJpegYU12(const Size& inSz, const YCbCrLayout& inLayout, int jpegQuality,
571                    const void* app1Buffer, size_t app1Size, void* out, size_t maxOutSize,
572                    size_t& actualCodeSize) {
573     /* libjpeg is a C library so we use C-style "inheritance" by
574      * putting libjpeg's jpeg_destination_mgr first in our custom
575      * struct. This allows us to cast jpeg_destination_mgr* to
576      * CustomJpegDestMgr* when we get it passed to us in a callback */
577     struct CustomJpegDestMgr {
578         struct jpeg_destination_mgr mgr;
579         JOCTET* mBuffer;
580         size_t mBufferSize;
581         size_t mEncodedSize;
582         bool mSuccess;
583     } dmgr;
584 
585     jpeg_compress_struct cinfo = {};
586     jpeg_error_mgr jerr;
587 
588     /* Initialize error handling with standard callbacks, but
589      * then override output_message (to print to ALOG) and
590      * error_exit to set a flag and print a message instead
591      * of killing the whole process */
592     cinfo.err = jpeg_std_error(&jerr);
593 
594     cinfo.err->output_message = [](j_common_ptr cinfo) {
595         char buffer[JMSG_LENGTH_MAX];
596 
597         /* Create the message */
598         (*cinfo->err->format_message)(cinfo, buffer);
599         ALOGE("libjpeg error: %s", buffer);
600     };
601     cinfo.err->error_exit = [](j_common_ptr cinfo) {
602         (*cinfo->err->output_message)(cinfo);
603         if (cinfo->client_data) {
604             auto& dmgr = *reinterpret_cast<CustomJpegDestMgr*>(cinfo->client_data);
605             dmgr.mSuccess = false;
606         }
607     };
608 
609     /* Now that we initialized some callbacks, let's create our compressor */
610     jpeg_create_compress(&cinfo);
611 
612     /* Initialize our destination manager */
613     dmgr.mBuffer = static_cast<JOCTET*>(out);
614     dmgr.mBufferSize = maxOutSize;
615     dmgr.mEncodedSize = 0;
616     dmgr.mSuccess = true;
617     cinfo.client_data = static_cast<void*>(&dmgr);
618 
619     /* These lambdas become C-style function pointers and as per C++11 spec
620      * may not capture anything */
621     dmgr.mgr.init_destination = [](j_compress_ptr cinfo) {
622         auto& dmgr = reinterpret_cast<CustomJpegDestMgr&>(*cinfo->dest);
623         dmgr.mgr.next_output_byte = dmgr.mBuffer;
624         dmgr.mgr.free_in_buffer = dmgr.mBufferSize;
625         ALOGV("%s:%d jpeg start: %p [%zu]", __FUNCTION__, __LINE__, dmgr.mBuffer, dmgr.mBufferSize);
626     };
627 
628     dmgr.mgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) {
629         ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__);
630         return 0;
631     };
632 
633     dmgr.mgr.term_destination = [](j_compress_ptr cinfo) {
634         auto& dmgr = reinterpret_cast<CustomJpegDestMgr&>(*cinfo->dest);
635         dmgr.mEncodedSize = dmgr.mBufferSize - dmgr.mgr.free_in_buffer;
636         ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, dmgr.mEncodedSize);
637     };
638     cinfo.dest = reinterpret_cast<struct jpeg_destination_mgr*>(&dmgr);
639 
640     /* We are going to be using JPEG in raw data mode, so we are passing
641      * straight subsampled planar YCbCr and it will not touch our pixel
642      * data or do any scaling or anything */
643     cinfo.image_width = inSz.width;
644     cinfo.image_height = inSz.height;
645     cinfo.input_components = 3;
646     cinfo.in_color_space = JCS_YCbCr;
647 
648     /* Initialize defaults and then override what we want */
649     jpeg_set_defaults(&cinfo);
650 
651     jpeg_set_quality(&cinfo, jpegQuality, 1);
652     jpeg_set_colorspace(&cinfo, JCS_YCbCr);
653     cinfo.raw_data_in = 1;
654     cinfo.dct_method = JDCT_IFAST;
655 
656     /* Configure sampling factors. The sampling factor is JPEG subsampling 420
657      * because the source format is YUV420. Note that libjpeg sampling factors
658      * are... a little weird. Sampling of Y=2,U=1,V=1 means there is 1 U and
659      * 1 V value for each 2 Y values */
660     cinfo.comp_info[0].h_samp_factor = 2;
661     cinfo.comp_info[0].v_samp_factor = 2;
662     cinfo.comp_info[1].h_samp_factor = 1;
663     cinfo.comp_info[1].v_samp_factor = 1;
664     cinfo.comp_info[2].h_samp_factor = 1;
665     cinfo.comp_info[2].v_samp_factor = 1;
666 
667     /* Start the compressor */
668     jpeg_start_compress(&cinfo, TRUE);
669 
670     /* Let's not hardcode YUV420 in 6 places... 5 was enough */
671     int maxVSampFactor = cinfo.max_v_samp_factor;
672     int cVSubSampling = cinfo.comp_info[0].v_samp_factor / cinfo.comp_info[1].v_samp_factor;
673 
674     /* Compute our macroblock height, so we can pad our input to be vertically
675      * macroblock aligned. No need to for horizontal alignment since AllocatedFrame already
676      * pads horizontally */
677 
678     size_t mcuV = DCTSIZE * maxVSampFactor;
679     size_t paddedHeight = mcuV * ((inSz.height + mcuV - 1) / mcuV);
680 
681     /* libjpeg uses arrays of row pointers, which makes it really easy to pad
682      * data vertically (unfortunately doesn't help horizontally) */
683     std::vector<JSAMPROW> yLines(paddedHeight);
684     std::vector<JSAMPROW> cbLines(paddedHeight / cVSubSampling);
685     std::vector<JSAMPROW> crLines(paddedHeight / cVSubSampling);
686 
687     uint8_t* py = static_cast<uint8_t*>(inLayout.y);
688     uint8_t* pcb = static_cast<uint8_t*>(inLayout.cb);
689     uint8_t* pcr = static_cast<uint8_t*>(inLayout.cr);
690 
691     for (int32_t i = 0; i < paddedHeight; i++) {
692         /* Once we are in the padding territory we still point to the last line
693          * effectively replicating it several times ~ CLAMP_TO_EDGE */
694         int li = std::min(i, inSz.height - 1);
695         yLines[i] = static_cast<JSAMPROW>(py + li * inLayout.yStride);
696         if (i < paddedHeight / cVSubSampling) {
697             li = std::min(i, (inSz.height - 1) / cVSubSampling);
698             cbLines[i] = static_cast<JSAMPROW>(pcb + li * inLayout.cStride);
699             crLines[i] = static_cast<JSAMPROW>(pcr + li * inLayout.cStride);
700         }
701     }
702 
703     /* If APP1 data was passed in, use it */
704     if (app1Buffer && app1Size) {
705         jpeg_write_marker(&cinfo, JPEG_APP0 + 1, static_cast<const JOCTET*>(app1Buffer), app1Size);
706     }
707 
708     /* While we still have padded height left to go, keep giving it one
709      * macroblock at a time. */
710     while (cinfo.next_scanline < cinfo.image_height) {
711         const uint32_t batchSize = DCTSIZE * maxVSampFactor;
712         const uint32_t nl = cinfo.next_scanline;
713         JSAMPARRAY planes[3]{&yLines[nl], &cbLines[nl / cVSubSampling],
714                              &crLines[nl / cVSubSampling]};
715 
716         uint32_t done = jpeg_write_raw_data(&cinfo, planes, batchSize);
717 
718         if (done != batchSize) {
719             ALOGE("%s: compressed %u lines, expected %u (total %u/%u)", __FUNCTION__, done,
720                   batchSize, cinfo.next_scanline, cinfo.image_height);
721             return -1;
722         }
723     }
724 
725     /* This will flush everything */
726     jpeg_finish_compress(&cinfo);
727 
728     /* Grab the actual code size and set it */
729     actualCodeSize = dmgr.mEncodedSize;
730 
731     return 0;
732 }
733 
getMaxThumbnailResolution(const common::V1_0::helper::CameraMetadata & chars)734 Size getMaxThumbnailResolution(const common::V1_0::helper::CameraMetadata& chars) {
735     Size thumbSize{0, 0};
736     camera_metadata_ro_entry entry = chars.find(ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES);
737     for (uint32_t i = 0; i < entry.count; i += 2) {
738         Size sz{.width = entry.data.i32[i], .height = entry.data.i32[i + 1]};
739         if (sz.width * sz.height > thumbSize.width * thumbSize.height) {
740             thumbSize = sz;
741         }
742     }
743 
744     if (thumbSize.width * thumbSize.height == 0) {
745         ALOGW("%s: non-zero thumbnail size not available", __FUNCTION__);
746     }
747 
748     return thumbSize;
749 }
750 
freeReleaseFences(std::vector<CaptureResult> & results)751 void freeReleaseFences(std::vector<CaptureResult>& results) {
752     for (auto& result : results) {
753         // NativeHandles free fd's on desctruction. Simply delete the objects!
754         result.inputBuffer.releaseFence.fds.clear();  // Implicitly closes fds
755         result.inputBuffer.releaseFence.ints.clear();
756         for (auto& buf : result.outputBuffers) {
757             buf.releaseFence.fds.clear();  // Implicitly closes fds
758             buf.releaseFence.ints.clear();
759         }
760     }
761 }
762 
763 #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
764 #define UPDATE(md, tag, data, size)               \
765     do {                                          \
766         if ((md).update((tag), (data), (size))) { \
767             ALOGE("Update " #tag " failed!");     \
768             return BAD_VALUE;                     \
769         }                                         \
770     } while (0)
771 
fillCaptureResultCommon(CameraMetadata & md,nsecs_t timestamp,camera_metadata_ro_entry & activeArraySize)772 status_t fillCaptureResultCommon(CameraMetadata& md, nsecs_t timestamp,
773                                  camera_metadata_ro_entry& activeArraySize) {
774     if (activeArraySize.count < 4) {
775         ALOGE("%s: cannot find active array size!", __FUNCTION__);
776         return -EINVAL;
777     }
778     // android.control
779     // For USB camera, we don't know the AE state. Set the state to converged to
780     // indicate the frame should be good to use. Then apps don't have to wait the
781     // AE state.
782     const uint8_t aeState = ANDROID_CONTROL_AE_STATE_CONVERGED;
783     UPDATE(md, ANDROID_CONTROL_AE_STATE, &aeState, 1);
784 
785     const uint8_t ae_lock = ANDROID_CONTROL_AE_LOCK_OFF;
786     UPDATE(md, ANDROID_CONTROL_AE_LOCK, &ae_lock, 1);
787 
788     // Set AWB state to converged to indicate the frame should be good to use.
789     const uint8_t awbState = ANDROID_CONTROL_AWB_STATE_CONVERGED;
790     UPDATE(md, ANDROID_CONTROL_AWB_STATE, &awbState, 1);
791 
792     const uint8_t awbLock = ANDROID_CONTROL_AWB_LOCK_OFF;
793     UPDATE(md, ANDROID_CONTROL_AWB_LOCK, &awbLock, 1);
794 
795     const uint8_t flashState = ANDROID_FLASH_STATE_UNAVAILABLE;
796     UPDATE(md, ANDROID_FLASH_STATE, &flashState, 1);
797 
798     // This means pipeline latency of X frame intervals. The maximum number is 4.
799     const uint8_t requestPipelineMaxDepth = 4;
800     UPDATE(md, ANDROID_REQUEST_PIPELINE_DEPTH, &requestPipelineMaxDepth, 1);
801 
802     // android.scaler
803     const int32_t crop_region[] = {
804             activeArraySize.data.i32[0],
805             activeArraySize.data.i32[1],
806             activeArraySize.data.i32[2],
807             activeArraySize.data.i32[3],
808     };
809     UPDATE(md, ANDROID_SCALER_CROP_REGION, crop_region, ARRAY_SIZE(crop_region));
810 
811     // android.sensor
812     UPDATE(md, ANDROID_SENSOR_TIMESTAMP, &timestamp, 1);
813 
814     // android.statistics
815     const uint8_t lensShadingMapMode = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
816     UPDATE(md, ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, &lensShadingMapMode, 1);
817 
818     const uint8_t sceneFlicker = ANDROID_STATISTICS_SCENE_FLICKER_NONE;
819     UPDATE(md, ANDROID_STATISTICS_SCENE_FLICKER, &sceneFlicker, 1);
820 
821     return OK;
822 }
823 
824 #undef ARRAY_SIZE
825 #undef UPDATE
826 
AllocatedV4L2Frame(std::shared_ptr<V4L2Frame> frameIn)827 AllocatedV4L2Frame::AllocatedV4L2Frame(std::shared_ptr<V4L2Frame> frameIn)
828     : Frame(frameIn->mWidth, frameIn->mHeight, frameIn->mFourcc) {
829     uint8_t* dataIn;
830     size_t dataSize;
831     if (frameIn->getData(&dataIn, &dataSize) != 0) {
832         ALOGE("%s: map input V4L2 frame failed!", __FUNCTION__);
833         return;
834     }
835 
836     mData.resize(dataSize);
837     std::memcpy(mData.data(), dataIn, dataSize);
838 }
839 
~AllocatedV4L2Frame()840 AllocatedV4L2Frame::~AllocatedV4L2Frame() {}
841 
getData(uint8_t ** outData,size_t * dataSize)842 int AllocatedV4L2Frame::getData(uint8_t** outData, size_t* dataSize) {
843     if (outData == nullptr || dataSize == nullptr) {
844         ALOGE("%s: outData(%p)/dataSize(%p) must not be null", __FUNCTION__, outData, dataSize);
845         return -1;
846     }
847 
848     *outData = mData.data();
849     *dataSize = mData.size();
850     return 0;
851 }
852 
853 }  // namespace implementation
854 }  // namespace device
855 }  // namespace camera
856 }  // namespace hardware
857 }  // namespace android
858