1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #include "vsock_frame_provider.h"
17 #include <hardware/camera3.h>
18 #include <libyuv.h>
19 #include <cstring>
20 #define LOG_TAG "VsockFrameProvider"
21 #include <log/log.h>
22 
23 namespace cuttlefish {
24 
25 namespace {
writeJsonEventMessage(std::shared_ptr<cuttlefish::VsockConnection> connection,const std::string & message)26 bool writeJsonEventMessage(
27     std::shared_ptr<cuttlefish::VsockConnection> connection,
28     const std::string& message) {
29   Json::Value json_message;
30   json_message["event"] = message;
31   return connection && connection->WriteMessage(json_message);
32 }
33 }  // namespace
34 
~VsockFrameProvider()35 VsockFrameProvider::~VsockFrameProvider() { stop(); }
36 
start(std::shared_ptr<cuttlefish::VsockConnection> connection,uint32_t width,uint32_t height)37 void VsockFrameProvider::start(
38     std::shared_ptr<cuttlefish::VsockConnection> connection, uint32_t width,
39     uint32_t height) {
40   stop();
41   running_ = true;
42   connection_ = connection;
43   writeJsonEventMessage(connection, "VIRTUAL_DEVICE_START_CAMERA_SESSION");
44   reader_thread_ =
45       std::thread([this, width, height] { VsockReadLoop(width, height); });
46 }
47 
stop()48 void VsockFrameProvider::stop() {
49   running_ = false;
50   jpeg_pending_ = false;
51   if (reader_thread_.joinable()) {
52     reader_thread_.join();
53   }
54   writeJsonEventMessage(connection_, "VIRTUAL_DEVICE_STOP_CAMERA_SESSION");
55   connection_ = nullptr;
56 }
57 
waitYUVFrame(unsigned int max_wait_ms)58 bool VsockFrameProvider::waitYUVFrame(unsigned int max_wait_ms) {
59   auto timeout = std::chrono::milliseconds(max_wait_ms);
60   nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC);
61   std::unique_lock<std::mutex> lock(frame_mutex_);
62   return yuv_frame_updated_.wait_for(
63       lock, timeout, [this, now] { return timestamp_.load() > now; });
64 }
65 
requestJpeg()66 void VsockFrameProvider::requestJpeg() {
67   jpeg_pending_ = true;
68   writeJsonEventMessage(connection_, "VIRTUAL_DEVICE_CAPTURE_IMAGE");
69 }
70 
cancelJpegRequest()71 void VsockFrameProvider::cancelJpegRequest() { jpeg_pending_ = false; }
72 
copyYUVFrame(uint32_t w,uint32_t h,YCbCrLayout dst)73 bool VsockFrameProvider::copyYUVFrame(uint32_t w, uint32_t h, YCbCrLayout dst) {
74   size_t y_size = w * h;
75   size_t cbcr_size = (w / 2) * (h / 2);
76   size_t total_size = y_size + 2 * cbcr_size;
77   std::lock_guard<std::mutex> lock(frame_mutex_);
78   if (frame_.size() < total_size) {
79     ALOGE("%s: %zu is too little for %ux%u frame", __FUNCTION__, frame_.size(),
80           w, h);
81     return false;
82   }
83   if (dst.y == nullptr) {
84     ALOGE("%s: Destination is nullptr!", __FUNCTION__);
85     return false;
86   }
87   YCbCrLayout src{.y = static_cast<void*>(frame_.data()),
88                   .cb = static_cast<void*>(frame_.data() + y_size),
89                   .cr = static_cast<void*>(frame_.data() + y_size + cbcr_size),
90                   .yStride = w,
91                   .cStride = w / 2,
92                   .chromaStep = 1};
93   uint8_t* src_y = static_cast<uint8_t*>(src.y);
94   uint8_t* dst_y = static_cast<uint8_t*>(dst.y);
95   uint8_t* src_cb = static_cast<uint8_t*>(src.cb);
96   uint8_t* dst_cb = static_cast<uint8_t*>(dst.cb);
97   uint8_t* src_cr = static_cast<uint8_t*>(src.cr);
98   uint8_t* dst_cr = static_cast<uint8_t*>(dst.cr);
99   libyuv::CopyPlane(src_y, src.yStride, dst_y, dst.yStride, w, h);
100   if (dst.chromaStep == 1) {
101     // Planar
102     libyuv::CopyPlane(src_cb, src.cStride, dst_cb, dst.cStride, w / 2, h / 2);
103     libyuv::CopyPlane(src_cr, src.cStride, dst_cr, dst.cStride, w / 2, h / 2);
104   } else if (dst.chromaStep == 2 && dst_cr - dst_cb == 1) {
105     // Interleaved cb/cr planes starting with cb
106     libyuv::MergeUVPlane(src_cb, src.cStride, src_cr, src.cStride, dst_cb,
107                          dst.cStride, w / 2, h / 2);
108   } else if (dst.chromaStep == 2 && dst_cb - dst_cr == 1) {
109     // Interleaved cb/cr planes starting with cr
110     libyuv::MergeUVPlane(src_cr, src.cStride, src_cb, src.cStride, dst_cr,
111                          dst.cStride, w / 2, h / 2);
112   } else {
113     ALOGE("%s: Unsupported interleaved U/V layout", __FUNCTION__);
114     return false;
115   }
116   return true;
117 }
118 
copyJpegData(uint32_t size,void * dst)119 bool VsockFrameProvider::copyJpegData(uint32_t size, void* dst) {
120   std::lock_guard<std::mutex> lock(jpeg_mutex_);
121   auto jpeg_header_offset = size - sizeof(struct camera3_jpeg_blob);
122   if (cached_jpeg_.empty()) {
123     ALOGE("%s: No source data", __FUNCTION__);
124     return false;
125   } else if (dst == nullptr) {
126     ALOGE("%s: Destination is nullptr", __FUNCTION__);
127     return false;
128   } else if (jpeg_header_offset <= cached_jpeg_.size()) {
129     ALOGE("%s: %ubyte target buffer too small", __FUNCTION__, size);
130     return false;
131   }
132   std::memcpy(dst, cached_jpeg_.data(), cached_jpeg_.size());
133   struct camera3_jpeg_blob* blob = reinterpret_cast<struct camera3_jpeg_blob*>(
134       static_cast<char*>(dst) + jpeg_header_offset);
135   blob->jpeg_blob_id = CAMERA3_JPEG_BLOB_ID;
136   blob->jpeg_size = cached_jpeg_.size();
137   cached_jpeg_.clear();
138   return true;
139 }
140 
isBlob(const std::vector<char> & blob)141 bool VsockFrameProvider::isBlob(const std::vector<char>& blob) {
142   bool is_png = blob.size() > 4 && (blob[0] & 0xff) == 0x89 &&
143                 (blob[1] & 0xff) == 0x50 && (blob[2] & 0xff) == 0x4e &&
144                 (blob[3] & 0xff) == 0x47;
145   bool is_jpeg =
146       blob.size() > 2 && (blob[0] & 0xff) == 0xff && (blob[1] & 0xff) == 0xd8;
147   return is_png || is_jpeg;
148 }
149 
framesizeMatches(uint32_t width,uint32_t height,const std::vector<char> & data)150 bool VsockFrameProvider::framesizeMatches(uint32_t width, uint32_t height,
151                                           const std::vector<char>& data) {
152   return data.size() == 3 * width * height / 2;
153 }
154 
VsockReadLoop(uint32_t width,uint32_t height)155 void VsockFrameProvider::VsockReadLoop(uint32_t width, uint32_t height) {
156   jpeg_pending_ = false;
157   while (running_.load() && connection_->ReadMessage(next_frame_)) {
158     if (framesizeMatches(width, height, next_frame_)) {
159       std::lock_guard<std::mutex> lock(frame_mutex_);
160       timestamp_ = systemTime();
161       frame_.swap(next_frame_);
162       yuv_frame_updated_.notify_one();
163     } else if (isBlob(next_frame_)) {
164       std::lock_guard<std::mutex> lock(jpeg_mutex_);
165       bool was_pending = jpeg_pending_.exchange(false);
166       if (was_pending) {
167         cached_jpeg_.swap(next_frame_);
168       }
169     } else {
170       ALOGE("%s: Unexpected data of %zu bytes", __FUNCTION__,
171             next_frame_.size());
172     }
173   }
174   if (!connection_->IsConnected()) {
175     ALOGE("%s: Connection closed - exiting", __FUNCTION__);
176     running_ = false;
177   }
178 }
179 
180 }  // namespace cuttlefish
181