1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "host/frontend/webrtc/libdevice/local_recorder.h"
18 
19 #include <atomic>
20 #include <chrono>
21 #include <list>
22 #include <mutex>
23 #include <thread>
24 #include <vector>
25 
26 #include <android-base/logging.h>
27 #include <api/media_stream_interface.h>
28 #include <api/rtp_parameters.h>
29 #include <api/task_queue/default_task_queue_factory.h>
30 #include <api/video/builtin_video_bitrate_allocator_factory.h>
31 #include <api/video_codecs/builtin_video_encoder_factory.h>
32 #include <api/video_codecs/video_encoder.h>
33 #include <mkvmuxer/mkvmuxer.h>
34 #include <mkvmuxer/mkvwriter.h>
35 #include <system_wrappers/include/clock.h>
36 
37 namespace cuttlefish {
38 namespace webrtc_streaming {
39 
40 constexpr double kRtpTicksPerSecond = 90000.;
41 constexpr double kRtpTicksPerMs = kRtpTicksPerSecond / 1000.;
42 constexpr double kRtpTicksPerUs = kRtpTicksPerMs / 1000.;
43 constexpr double kRtpTicksPerNs = kRtpTicksPerUs / 1000.;
44 
45 class LocalRecorder::Display
46     : public webrtc::EncodedImageCallback
47     , public rtc::VideoSinkInterface<webrtc::VideoFrame> {
48 public:
49   Display(LocalRecorder::Impl& impl);
~Display()50   ~Display() {
51     CHECK(!encoder_running_) << "LocalRecorder::Display destroyed before calling Stop()";
52   }
53 
54   void EncoderLoop();
55   void Stop();
56 
57   // VideoSinkInterface
58   virtual void OnFrame(const webrtc::VideoFrame& frame) override;
59 
60   // EncodedImageCallback
61   virtual webrtc::EncodedImageCallback::Result OnEncodedImage(
62       const webrtc::EncodedImage& encoded_image,
63       const webrtc::CodecSpecificInfo* codec_specific_info) override;
64 
65   LocalRecorder::Impl& impl_;
66   std::shared_ptr<webrtc::VideoTrackSourceInterface> source_;
67   std::unique_ptr<webrtc::VideoEncoder> video_encoder_;
68   uint64_t video_track_number_;
69 
70   // TODO(schuffelen): Use a WebRTC task queue?
71   std::thread encoder_thread_;
72   std::condition_variable encoder_queue_signal_;
73   std::mutex encode_queue_mutex_;
74   std::list<webrtc::VideoFrame> encode_queue_;
75   std::atomic_bool encoder_running_ = true;
76 };
77 
78 class LocalRecorder::Impl {
79 public:
80   mkvmuxer::MkvWriter file_writer_;
81   mkvmuxer::Segment segment_;
82   std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
83   std::mutex mkv_mutex_;
84   std::map<std::string, std::unique_ptr<Display>> displays_;
85 };
86 
87 /* static */
Create(const std::string & filename)88 std::unique_ptr<LocalRecorder> LocalRecorder::Create(
89     const std::string& filename) {
90   std::unique_ptr<Impl> impl(new Impl());
91 
92   if (!impl->file_writer_.Open(filename.c_str())) {
93     LOG(ERROR) << "Failed to open \"" << filename << "\" to write a webm";
94     return {};
95   }
96 
97   if (!impl->segment_.Init(&impl->file_writer_)) {
98     LOG(ERROR) << "Failed to initialize the mkvkmuxer segment";
99     return {};
100   }
101 
102   impl->segment_.AccurateClusterDuration(true);
103   impl->segment_.set_estimate_file_duration(true);
104 
105   impl->encoder_factory_ = webrtc::CreateBuiltinVideoEncoderFactory();
106   if (!impl->encoder_factory_) {
107     LOG(ERROR) << "Failed to create webRTC built-in video encoder factory";
108     return {};
109   }
110 
111   return std::unique_ptr<LocalRecorder>(new LocalRecorder(std::move(impl)));
112 }
113 
LocalRecorder(std::unique_ptr<LocalRecorder::Impl> impl)114 LocalRecorder::LocalRecorder(std::unique_ptr<LocalRecorder::Impl> impl)
115     : impl_(std::move(impl)) {
116 }
117 
118 LocalRecorder::~LocalRecorder() = default;
119 
AddDisplay(const std::string & label,size_t width,size_t height,std::shared_ptr<webrtc::VideoTrackSourceInterface> source)120 void LocalRecorder::AddDisplay(
121     const std::string& label, size_t width, size_t height,
122     std::shared_ptr<webrtc::VideoTrackSourceInterface> source) {
123   LOG(ERROR) << "Display added with label '" << label << "'";
124 
125   std::lock_guard lock(impl_->mkv_mutex_);
126 
127   auto existing_display = impl_->displays_.find(label);
128   if (existing_display != impl_->displays_.end()) {
129     auto display = existing_display->second.get();
130     CHECK(display);
131     display->source_ = source;
132     source->AddOrUpdateSink(display, rtc::VideoSinkWants{});
133     return;
134   }
135 
136   std::unique_ptr<Display> display(new Display(*impl_));
137   display->source_ = source;
138   display->video_track_number_ =
139       impl_->segment_.AddVideoTrack(width, height, 0);
140   if (display->video_track_number_ == 0) {
141     LOG(ERROR) << "Failed to add video track to webm muxer";
142     return;
143   }
144 
145   display->video_encoder_ =
146       impl_->encoder_factory_->CreateVideoEncoder(webrtc::SdpVideoFormat("VP8"));
147   if (!display->video_encoder_) {
148     LOG(ERROR) << "Could not create vp8 video encoder";
149     return;
150   }
151   auto rc =
152       display->video_encoder_->RegisterEncodeCompleteCallback(display.get());
153   if (rc != 0) {
154     LOG(ERROR) << "Could not register encode complete callback";
155     return;
156   }
157   source->AddOrUpdateSink(display.get(), rtc::VideoSinkWants{});
158 
159   webrtc::VideoCodec codec {};
160   memset(&codec, 0, sizeof(codec));
161   codec.codecType = webrtc::kVideoCodecVP8;
162   codec.width = width;
163   codec.height = height;
164   codec.startBitrate = 1000; // kilobits/sec
165   codec.maxBitrate = 2000;
166   codec.minBitrate = 0;
167   codec.maxFramerate = 60;
168   codec.active = true;
169   codec.qpMax = 56; // kDefaultMaxQp from simulcast_encoder_adapter.cc
170   codec.mode = webrtc::VideoCodecMode::kScreensharing;
171   codec.expect_encode_from_texture = false;
172   *codec.VP8() = webrtc::VideoEncoder::GetDefaultVp8Settings();
173 
174   webrtc::VideoEncoder::Capabilities capabilities(false);
175   webrtc::VideoEncoder::Settings settings(capabilities, 1, 1 << 20);
176 
177   rc = display->video_encoder_->InitEncode(&codec, settings);
178   if (rc != 0) {
179     LOG(ERROR) << "Failed to InitEncode";
180     return;
181   }
182 
183   display->encoder_running_ = true;
184   display->encoder_thread_ = std::thread([](Display* display) {
185     display->EncoderLoop();
186   }, display.get());
187 
188   impl_->displays_[label] = std::move(display);
189 }
190 
Stop()191 void LocalRecorder::Stop() {
192   for (auto& [label, display] : impl_->displays_) {
193     display->Stop();
194   }
195   std::lock_guard lock(impl_->mkv_mutex_);
196   impl_->segment_.Finalize();
197 }
198 
Display(LocalRecorder::Impl & impl)199 LocalRecorder::Display::Display(LocalRecorder::Impl& impl) : impl_(impl) {
200 }
201 
OnFrame(const webrtc::VideoFrame & frame)202 void LocalRecorder::Display::OnFrame(const webrtc::VideoFrame& frame) {
203   std::lock_guard queue_lock(encode_queue_mutex_);
204   static int kMaxQueuedFrames = 10;
205   if (encode_queue_.size() >= kMaxQueuedFrames) {
206     LOG(VERBOSE) << "Dropped frame, encoder queue too long";
207     return;
208   }
209   encode_queue_.push_back(frame);
210   encoder_queue_signal_.notify_one();
211 }
212 
EncoderLoop()213 void LocalRecorder::Display::EncoderLoop() {
214   int frames_since_keyframe = 0;
215   std::chrono::time_point<std::chrono::steady_clock> start_timestamp;
216   auto last_keyframe_time = std::chrono::steady_clock::now();
217   while (encoder_running_) {
218     std::unique_ptr<webrtc::VideoFrame> frame;
219     {
220       std::unique_lock queue_lock(encode_queue_mutex_);
221       while (encode_queue_.size() == 0 && encoder_running_) {
222         encoder_queue_signal_.wait(queue_lock);
223       }
224       if (!encoder_running_) {
225         break;
226       }
227       frame = std::make_unique<webrtc::VideoFrame>(
228           std::move(encode_queue_.front()));
229       encode_queue_.pop_front();
230     }
231 
232     auto now = std::chrono::steady_clock::now();
233     if (start_timestamp.time_since_epoch().count() == 0) {
234       start_timestamp = now;
235     }
236     auto timestamp_diff =
237         std::chrono::duration_cast<std::chrono::microseconds>(
238               now - start_timestamp);
239     frame->set_timestamp_us(timestamp_diff.count());
240     frame->set_timestamp(timestamp_diff.count() * kRtpTicksPerUs);
241 
242     std::vector<webrtc::VideoFrameType> types;
243     auto time_since_keyframe = now - last_keyframe_time;
244     const auto min_keyframe_time = std::chrono::seconds(10);
245     if (frames_since_keyframe > 60 || time_since_keyframe > min_keyframe_time) {
246       last_keyframe_time = now;
247       frames_since_keyframe = 0;
248       types.push_back(webrtc::VideoFrameType::kVideoFrameKey);
249     } else {
250       types.push_back(webrtc::VideoFrameType::kVideoFrameDelta);
251     }
252     auto rc = video_encoder_->Encode(*frame, &types);
253     if (rc != 0) {
254       LOG(ERROR) << "Failed to encode frame";
255     }
256   }
257 }
258 
Stop()259 void LocalRecorder::Display::Stop() {
260   source_->RemoveSink(this);
261   encoder_running_ = false;
262   encoder_queue_signal_.notify_all();
263   if (encoder_thread_.joinable()) {
264     encoder_thread_.join();
265   }
266 }
267 
OnEncodedImage(const webrtc::EncodedImage & encoded_image,const webrtc::CodecSpecificInfo * codec_specific_info)268 webrtc::EncodedImageCallback::Result LocalRecorder::Display::OnEncodedImage(
269     const webrtc::EncodedImage& encoded_image,
270     const webrtc::CodecSpecificInfo* codec_specific_info) {
271   uint64_t timestamp = encoded_image.Timestamp() / kRtpTicksPerNs;
272 
273   std::lock_guard lock(impl_.mkv_mutex_);
274 
275   bool is_key =
276       encoded_image._frameType == webrtc::VideoFrameType::kVideoFrameKey;
277   bool success = impl_.segment_.AddFrame(
278       encoded_image.data(),
279       encoded_image.size(),
280       video_track_number_,
281       timestamp,
282       is_key);
283 
284   webrtc::EncodedImageCallback::Result result(
285       success
286           ? webrtc::EncodedImageCallback::Result::Error::OK
287           : webrtc::EncodedImageCallback::Result::Error::ERROR_SEND_FAILED);
288 
289   if (success) {
290     result.frame_id = encoded_image.Timestamp();
291   }
292   return result;
293 }
294 
295 } // namespace webrtc_streaming
296 } // namespace cuttlefish
297 
298