1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICESESSION_H
18 #define ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICESESSION_H
19 
20 #include <android/hardware/camera/device/3.2/ICameraDevice.h>
21 #include <android/hardware/camera/device/3.4/ICameraDeviceSession.h>
22 #include <fmq/MessageQueue.h>
23 #include <hidl/MQDescriptor.h>
24 #include <hidl/Status.h>
25 #include <include/convert.h>
26 #include <chrono>
27 #include <condition_variable>
28 #include <list>
29 #include <unordered_map>
30 #include <unordered_set>
31 #include "CameraMetadata.h"
32 #include "HandleImporter.h"
33 #include "Exif.h"
34 #include "utils/KeyedVector.h"
35 #include "utils/Mutex.h"
36 #include "utils/Thread.h"
37 #include "android-base/unique_fd.h"
38 #include "ExternalCameraUtils.h"
39 
40 namespace android {
41 namespace hardware {
42 namespace camera {
43 namespace device {
44 namespace V3_4 {
45 namespace implementation {
46 
47 using ::android::hardware::camera::device::V3_2::BufferCache;
48 using ::android::hardware::camera::device::V3_2::BufferStatus;
49 using ::android::hardware::camera::device::V3_2::CameraMetadata;
50 using ::android::hardware::camera::device::V3_2::CaptureRequest;
51 using ::android::hardware::camera::device::V3_2::CaptureResult;
52 using ::android::hardware::camera::device::V3_2::ErrorCode;
53 using ::android::hardware::camera::device::V3_2::ICameraDeviceCallback;
54 using ::android::hardware::camera::device::V3_2::MsgType;
55 using ::android::hardware::camera::device::V3_2::NotifyMsg;
56 using ::android::hardware::camera::device::V3_2::RequestTemplate;
57 using ::android::hardware::camera::device::V3_2::Stream;
58 using ::android::hardware::camera::device::V3_4::StreamConfiguration;
59 using ::android::hardware::camera::device::V3_2::StreamConfigurationMode;
60 using ::android::hardware::camera::device::V3_2::StreamRotation;
61 using ::android::hardware::camera::device::V3_2::StreamType;
62 using ::android::hardware::camera::device::V3_2::DataspaceFlags;
63 using ::android::hardware::camera::device::V3_2::CameraBlob;
64 using ::android::hardware::camera::device::V3_2::CameraBlobId;
65 using ::android::hardware::camera::device::V3_4::HalStreamConfiguration;
66 using ::android::hardware::camera::device::V3_4::ICameraDeviceSession;
67 using ::android::hardware::camera::common::V1_0::Status;
68 using ::android::hardware::camera::common::V1_0::helper::HandleImporter;
69 using ::android::hardware::camera::common::V1_0::helper::ExifUtils;
70 using ::android::hardware::camera::external::common::ExternalCameraConfig;
71 using ::android::hardware::camera::external::common::Size;
72 using ::android::hardware::camera::external::common::SizeHasher;
73 using ::android::hardware::graphics::common::V1_0::BufferUsage;
74 using ::android::hardware::graphics::common::V1_0::Dataspace;
75 using ::android::hardware::graphics::common::V1_0::PixelFormat;
76 using ::android::hardware::kSynchronizedReadWrite;
77 using ::android::hardware::MessageQueue;
78 using ::android::hardware::MQDescriptorSync;
79 using ::android::hardware::Return;
80 using ::android::hardware::Void;
81 using ::android::hardware::hidl_vec;
82 using ::android::hardware::hidl_string;
83 using ::android::sp;
84 using ::android::Mutex;
85 using ::android::base::unique_fd;
86 
87 struct ExternalCameraDeviceSession : public virtual RefBase,
88         public virtual OutputThreadInterface {
89 
90     ExternalCameraDeviceSession(const sp<ICameraDeviceCallback>&,
91             const ExternalCameraConfig& cfg,
92             const std::vector<SupportedV4L2Format>& sortedFormats,
93             const CroppingType& croppingType,
94             const common::V1_0::helper::CameraMetadata& chars,
95             const std::string& cameraId,
96             unique_fd v4l2Fd);
97     virtual ~ExternalCameraDeviceSession();
98     // Call by CameraDevice to dump active device states
99     void dumpState(const native_handle_t*);
100     // Caller must use this method to check if CameraDeviceSession ctor failed
101     bool isInitFailed();
102     bool isClosed();
103 
104     // Retrieve the HIDL interface, split into its own class to avoid inheritance issues when
105     // dealing with minor version revs and simultaneous implementation and interface inheritance
getInterfaceExternalCameraDeviceSession106     virtual sp<ICameraDeviceSession> getInterface() {
107         return new TrampolineSessionInterface_3_4(this);
108     }
109 
110     static const int kMaxProcessedStream = 2;
111     static const int kMaxStallStream = 1;
112     static const uint32_t kMaxBytesPerPixel = 2;
113 
114     class OutputThread : public android::Thread {
115     public:
116         OutputThread(wp<OutputThreadInterface> parent, CroppingType,
117                 const common::V1_0::helper::CameraMetadata&);
118         virtual ~OutputThread();
119 
120         Status allocateIntermediateBuffers(
121                 const Size& v4lSize, const Size& thumbSize,
122                 const hidl_vec<Stream>& streams,
123                 uint32_t blobBufferSize);
124         Status submitRequest(const std::shared_ptr<HalRequest>&);
125         void flush();
126         void dump(int fd);
127         virtual bool threadLoop() override;
128 
129         void setExifMakeModel(const std::string& make, const std::string& model);
130 
131         // The remaining request list is returned for offline processing
132         std::list<std::shared_ptr<HalRequest>> switchToOffline();
133 
134     protected:
135         // Methods to request output buffer in parallel
136         // No-op for device@3.4. Implemented in device@3.5
requestBufferStartExternalCameraDeviceSession137         virtual int requestBufferStart(const std::vector<HalStreamBuffer>&) { return 0; }
waitForBufferRequestDoneExternalCameraDeviceSession138         virtual int waitForBufferRequestDone(
139                 /*out*/std::vector<HalStreamBuffer>*) { return 0; }
140 
141         static const int kFlushWaitTimeoutSec = 3; // 3 sec
142         static const int kReqWaitTimeoutMs = 33;   // 33ms
143         static const int kReqWaitTimesMax = 90;    // 33ms * 90 ~= 3 sec
144 
145         void waitForNextRequest(std::shared_ptr<HalRequest>* out);
146         void signalRequestDone();
147 
148         int cropAndScaleLocked(
149                 sp<AllocatedFrame>& in, const Size& outSize,
150                 YCbCrLayout* out);
151 
152         int cropAndScaleThumbLocked(
153                 sp<AllocatedFrame>& in, const Size& outSize,
154                 YCbCrLayout* out);
155 
156         int createJpegLocked(HalStreamBuffer &halBuf,
157                 const common::V1_0::helper::CameraMetadata& settings);
158 
159         void clearIntermediateBuffers();
160 
161         const wp<OutputThreadInterface> mParent;
162         const CroppingType mCroppingType;
163         const common::V1_0::helper::CameraMetadata mCameraCharacteristics;
164 
165         mutable std::mutex mRequestListLock;      // Protect acccess to mRequestList,
166                                                   // mProcessingRequest and mProcessingFrameNumer
167         std::condition_variable mRequestCond;     // signaled when a new request is submitted
168         std::condition_variable mRequestDoneCond; // signaled when a request is done processing
169         std::list<std::shared_ptr<HalRequest>> mRequestList;
170         bool mProcessingRequest = false;
171         uint32_t mProcessingFrameNumer = 0;
172 
173         // V4L2 frameIn
174         // (MJPG decode)-> mYu12Frame
175         // (Scale)-> mScaledYu12Frames
176         // (Format convert) -> output gralloc frames
177         mutable std::mutex mBufferLock; // Protect access to intermediate buffers
178         sp<AllocatedFrame> mYu12Frame;
179         sp<AllocatedFrame> mYu12ThumbFrame;
180         std::unordered_map<Size, sp<AllocatedFrame>, SizeHasher> mIntermediateBuffers;
181         std::unordered_map<Size, sp<AllocatedFrame>, SizeHasher> mScaledYu12Frames;
182         YCbCrLayout mYu12FrameLayout;
183         YCbCrLayout mYu12ThumbFrameLayout;
184         std::vector<uint8_t> mMuteTestPatternFrame;
185         uint32_t mTestPatternData[4] = {0, 0, 0, 0};
186         bool mCameraMuted = false;
187         uint32_t mBlobBufferSize = 0; // 0 -> HAL derive buffer size, else: use given size
188 
189         std::string mExifMake;
190         std::string mExifModel;
191     };
192 
193 protected:
194 
195     // Methods from ::android::hardware::camera::device::V3_2::ICameraDeviceSession follow
196 
197     Return<void> constructDefaultRequestSettings(
198             RequestTemplate,
199             ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb);
200 
201     Return<void> configureStreams(
202             const V3_2::StreamConfiguration&,
203             ICameraDeviceSession::configureStreams_cb);
204 
205     Return<void> getCaptureRequestMetadataQueue(
206         ICameraDeviceSession::getCaptureRequestMetadataQueue_cb);
207 
208     Return<void> getCaptureResultMetadataQueue(
209         ICameraDeviceSession::getCaptureResultMetadataQueue_cb);
210 
211     Return<void> processCaptureRequest(
212             const hidl_vec<CaptureRequest>&,
213             const hidl_vec<BufferCache>&,
214             ICameraDeviceSession::processCaptureRequest_cb);
215 
216     Return<Status> flush();
217     Return<void> close(bool callerIsDtor = false);
218 
219     Return<void> configureStreams_3_3(
220             const V3_2::StreamConfiguration&,
221             ICameraDeviceSession::configureStreams_3_3_cb);
222 
223     Return<void> configureStreams_3_4(
224             const V3_4::StreamConfiguration& requestedConfiguration,
225             ICameraDeviceSession::configureStreams_3_4_cb _hidl_cb);
226 
227     Return<void> processCaptureRequest_3_4(
228             const hidl_vec<V3_4::CaptureRequest>& requests,
229             const hidl_vec<V3_2::BufferCache>& cachesToRemove,
230             ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb);
231 
232 protected:
233     // Methods from OutputThreadInterface
234     virtual Status importBuffer(int32_t streamId,
235             uint64_t bufId, buffer_handle_t buf,
236             /*out*/buffer_handle_t** outBufPtr,
237             bool allowEmptyBuf) override;
238 
239     virtual Status processCaptureResult(std::shared_ptr<HalRequest>&) override;
240 
241     virtual Status processCaptureRequestError(const std::shared_ptr<HalRequest>&,
242         /*out*/std::vector<NotifyMsg>* msgs = nullptr,
243         /*out*/std::vector<CaptureResult>* results = nullptr) override;
244 
245     virtual ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const override;
246 
247     virtual void notifyError(uint32_t frameNumber, int32_t streamId, ErrorCode ec) override;
248     // End of OutputThreadInterface methods
249 
250     Status constructDefaultRequestSettingsRaw(RequestTemplate type,
251             V3_2::CameraMetadata *outMetadata);
252 
253     bool initialize();
254     // To init/close different version of output thread
255     virtual void initOutputThread();
256     virtual void closeOutputThread();
257     void closeOutputThreadImpl();
258 
259     Status initStatus() const;
260     status_t initDefaultRequests();
261     status_t fillCaptureResult(common::V1_0::helper::CameraMetadata& md, nsecs_t timestamp);
262     Status configureStreams(const V3_2::StreamConfiguration&,
263             V3_3::HalStreamConfiguration* out,
264             // Only filled by configureStreams_3_4, and only one blob stream supported
265             uint32_t blobBufferSize = 0);
266     // fps = 0.0 means default, which is
267     // slowest fps that is at least 30, or fastest fps if 30 is not supported
268     int configureV4l2StreamLocked(const SupportedV4L2Format& fmt, double fps = 0.0);
269     int v4l2StreamOffLocked();
270     int setV4l2FpsLocked(double fps);
271     static Status isStreamCombinationSupported(const V3_2::StreamConfiguration& config,
272             const std::vector<SupportedV4L2Format>& supportedFormats,
273             const ExternalCameraConfig& devCfg);
274 
275     // TODO: change to unique_ptr for better tracking
276     sp<V4L2Frame> dequeueV4l2FrameLocked(/*out*/nsecs_t* shutterTs); // Called with mLock hold
277     void enqueueV4l2Frame(const sp<V4L2Frame>&);
278 
279     // Check if input Stream is one of supported stream setting on this device
280     static bool isSupported(const Stream& stream,
281             const std::vector<SupportedV4L2Format>& supportedFormats,
282             const ExternalCameraConfig& cfg);
283 
284     // Validate and import request's output buffers and acquire fence
285     virtual Status importRequestLocked(
286             const CaptureRequest& request,
287             hidl_vec<buffer_handle_t*>& allBufPtrs,
288             hidl_vec<int>& allFences);
289 
290     Status importRequestLockedImpl(
291             const CaptureRequest& request,
292             hidl_vec<buffer_handle_t*>& allBufPtrs,
293             hidl_vec<int>& allFences,
294             // Optional argument for ICameraDeviceSession@3.5 impl
295             bool allowEmptyBuf = false);
296 
297     Status importBufferLocked(int32_t streamId,
298             uint64_t bufId, buffer_handle_t buf,
299             /*out*/buffer_handle_t** outBufPtr,
300             bool allowEmptyBuf);
301 
302     static void cleanupInflightFences(
303             hidl_vec<int>& allFences, size_t numFences);
304     void cleanupBuffersLocked(int id);
305     void updateBufferCaches(const hidl_vec<BufferCache>& cachesToRemove);
306 
307     Status processOneCaptureRequest(const CaptureRequest& request);
308 
309     void notifyShutter(uint32_t frameNumber, nsecs_t shutterTs);
310     void invokeProcessCaptureResultCallback(
311             hidl_vec<CaptureResult> &results, bool tryWriteFmq);
312 
313     Size getMaxJpegResolution() const;
314     Size getMaxThumbResolution() const;
315 
316     int waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex>& lk);
317 
318     // Protect (most of) HIDL interface methods from synchronized-entering
319     mutable Mutex mInterfaceLock;
320 
321     mutable Mutex mLock; // Protect all private members except otherwise noted
322     const sp<ICameraDeviceCallback> mCallback;
323     const ExternalCameraConfig& mCfg;
324     const common::V1_0::helper::CameraMetadata mCameraCharacteristics;
325     const std::vector<SupportedV4L2Format> mSupportedFormats;
326     const CroppingType mCroppingType;
327     const std::string mCameraId;
328 
329     // Not protected by mLock, this is almost a const.
330     // Setup in constructor, reset in close() after OutputThread is joined
331     unique_fd mV4l2Fd;
332 
333     // device is closed either
334     //    - closed by user
335     //    - init failed
336     //    - camera disconnected
337     bool mClosed = false;
338     bool mInitialized = false;
339     bool mInitFail = false;
340     bool mFirstRequest = false;
341     common::V1_0::helper::CameraMetadata mLatestReqSetting;
342 
343     bool mV4l2Streaming = false;
344     SupportedV4L2Format mV4l2StreamingFmt;
345     double mV4l2StreamingFps = 0.0;
346     size_t mV4L2BufferCount = 0;
347 
348     static const int kBufferWaitTimeoutSec = 3; // TODO: handle long exposure (or not allowing)
349     std::mutex mV4l2BufferLock; // protect the buffer count and condition below
350     std::condition_variable mV4L2BufferReturned;
351     size_t mNumDequeuedV4l2Buffers = 0;
352     uint32_t mMaxV4L2BufferSize = 0;
353 
354     // Not protected by mLock (but might be used when mLock is locked)
355     sp<OutputThread> mOutputThread;
356 
357     // Stream ID -> Camera3Stream cache
358     std::unordered_map<int, Stream> mStreamMap;
359 
360     std::mutex mInflightFramesLock; // protect mInflightFrames
361     std::unordered_set<uint32_t>  mInflightFrames;
362 
363     // Stream ID -> circulating buffers map
364     std::map<int, CirculatingBuffers> mCirculatingBuffers;
365     // Protect mCirculatingBuffers, must not lock mLock after acquiring this lock
366     mutable Mutex mCbsLock;
367 
368     std::mutex mAfTriggerLock; // protect mAfTrigger
369     bool mAfTrigger = false;
370 
371     uint32_t mBlobBufferSize = 0;
372 
373     static HandleImporter sHandleImporter;
374 
375     /* Beginning of members not changed after initialize() */
376     using RequestMetadataQueue = MessageQueue<uint8_t, kSynchronizedReadWrite>;
377     std::unique_ptr<RequestMetadataQueue> mRequestMetadataQueue;
378     using ResultMetadataQueue = MessageQueue<uint8_t, kSynchronizedReadWrite>;
379     std::shared_ptr<ResultMetadataQueue> mResultMetadataQueue;
380 
381     // Protect against invokeProcessCaptureResultCallback()
382     Mutex mProcessCaptureResultLock;
383 
384     std::unordered_map<RequestTemplate, CameraMetadata> mDefaultRequests;
385 
386     const Size mMaxThumbResolution;
387     const Size mMaxJpegResolution;
388 
389     std::string mExifMake;
390     std::string mExifModel;
391     /* End of members not changed after initialize() */
392 
393 private:
394 
395     struct TrampolineSessionInterface_3_4 : public ICameraDeviceSession {
TrampolineSessionInterface_3_4ExternalCameraDeviceSession::TrampolineSessionInterface_3_4396         TrampolineSessionInterface_3_4(sp<ExternalCameraDeviceSession> parent) :
397                 mParent(parent) {}
398 
constructDefaultRequestSettingsExternalCameraDeviceSession::TrampolineSessionInterface_3_4399         virtual Return<void> constructDefaultRequestSettings(
400                 RequestTemplate type,
401                 V3_3::ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb) override {
402             return mParent->constructDefaultRequestSettings(type, _hidl_cb);
403         }
404 
configureStreamsExternalCameraDeviceSession::TrampolineSessionInterface_3_4405         virtual Return<void> configureStreams(
406                 const V3_2::StreamConfiguration& requestedConfiguration,
407                 V3_3::ICameraDeviceSession::configureStreams_cb _hidl_cb) override {
408             return mParent->configureStreams(requestedConfiguration, _hidl_cb);
409         }
410 
processCaptureRequestExternalCameraDeviceSession::TrampolineSessionInterface_3_4411         virtual Return<void> processCaptureRequest(const hidl_vec<V3_2::CaptureRequest>& requests,
412                 const hidl_vec<V3_2::BufferCache>& cachesToRemove,
413                 V3_3::ICameraDeviceSession::processCaptureRequest_cb _hidl_cb) override {
414             return mParent->processCaptureRequest(requests, cachesToRemove, _hidl_cb);
415         }
416 
getCaptureRequestMetadataQueueExternalCameraDeviceSession::TrampolineSessionInterface_3_4417         virtual Return<void> getCaptureRequestMetadataQueue(
418                 V3_3::ICameraDeviceSession::getCaptureRequestMetadataQueue_cb _hidl_cb) override  {
419             return mParent->getCaptureRequestMetadataQueue(_hidl_cb);
420         }
421 
getCaptureResultMetadataQueueExternalCameraDeviceSession::TrampolineSessionInterface_3_4422         virtual Return<void> getCaptureResultMetadataQueue(
423                 V3_3::ICameraDeviceSession::getCaptureResultMetadataQueue_cb _hidl_cb) override  {
424             return mParent->getCaptureResultMetadataQueue(_hidl_cb);
425         }
426 
flushExternalCameraDeviceSession::TrampolineSessionInterface_3_4427         virtual Return<Status> flush() override {
428             return mParent->flush();
429         }
430 
closeExternalCameraDeviceSession::TrampolineSessionInterface_3_4431         virtual Return<void> close() override {
432             return mParent->close();
433         }
434 
configureStreams_3_3ExternalCameraDeviceSession::TrampolineSessionInterface_3_4435         virtual Return<void> configureStreams_3_3(
436                 const V3_2::StreamConfiguration& requestedConfiguration,
437                 configureStreams_3_3_cb _hidl_cb) override {
438             return mParent->configureStreams_3_3(requestedConfiguration, _hidl_cb);
439         }
440 
configureStreams_3_4ExternalCameraDeviceSession::TrampolineSessionInterface_3_4441         virtual Return<void> configureStreams_3_4(
442                 const V3_4::StreamConfiguration& requestedConfiguration,
443                 configureStreams_3_4_cb _hidl_cb) override {
444             return mParent->configureStreams_3_4(requestedConfiguration, _hidl_cb);
445         }
446 
processCaptureRequest_3_4ExternalCameraDeviceSession::TrampolineSessionInterface_3_4447         virtual Return<void> processCaptureRequest_3_4(const hidl_vec<V3_4::CaptureRequest>& requests,
448                 const hidl_vec<V3_2::BufferCache>& cachesToRemove,
449                 ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb) override {
450             return mParent->processCaptureRequest_3_4(requests, cachesToRemove, _hidl_cb);
451         }
452 
453     private:
454         sp<ExternalCameraDeviceSession> mParent;
455     };
456 };
457 
458 }  // namespace implementation
459 }  // namespace V3_4
460 }  // namespace device
461 }  // namespace camera
462 }  // namespace hardware
463 }  // namespace android
464 
465 #endif  // ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICESESSION_H
466