1 /**
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <ImsMediaVideoRenderer.h>
18 #include <ImsMediaTrace.h>
19 #include <ImsMediaTimer.h>
20 #include <ImsMediaVideoUtil.h>
21 #include <thread>
22 
23 #define CODEC_TIMEOUT_NANO 100000
24 #define INTERVAL_MILLIS    10
25 
ImsMediaVideoRenderer()26 ImsMediaVideoRenderer::ImsMediaVideoRenderer()
27 {
28     mCallback = nullptr;
29     mWindow = nullptr;
30     mCodec = nullptr;
31     mFormat = nullptr;
32     mCodecType = -1;
33     mWidth = 0;
34     mHeight = 0;
35     mFarOrientationDegree = 0;
36     mNearOrientationDegree = 0;
37     mStopped = false;
38 }
39 
~ImsMediaVideoRenderer()40 ImsMediaVideoRenderer::~ImsMediaVideoRenderer()
41 {
42     while (!mFrameDatas.empty())
43     {
44         FrameData* frame = mFrameDatas.front();
45         delete frame;
46         mFrameDatas.pop_front();
47     }
48 }
49 
SetSessionCallback(BaseSessionCallback * callback)50 void ImsMediaVideoRenderer::SetSessionCallback(BaseSessionCallback* callback)
51 {
52     mCallback = callback;
53 }
54 
SetCodec(int32_t codecType)55 void ImsMediaVideoRenderer::SetCodec(int32_t codecType)
56 {
57     IMLOGD1("[SetCodec] codec[%d]", codecType);
58     mCodecType = codecType;
59 }
60 
SetResolution(uint32_t width,uint32_t height)61 void ImsMediaVideoRenderer::SetResolution(uint32_t width, uint32_t height)
62 {
63     IMLOGD2("[SetResolution] width[%d], height[%d]", width, height);
64     mWidth = width;
65     mHeight = height;
66 }
67 
SetDeviceOrientation(uint32_t orientation)68 void ImsMediaVideoRenderer::SetDeviceOrientation(uint32_t orientation)
69 {
70     IMLOGD1("[SetDeviceOrientation] orientation[%d]", orientation);
71     mNearOrientationDegree = orientation;
72 }
73 
SetSurface(ANativeWindow * window)74 void ImsMediaVideoRenderer::SetSurface(ANativeWindow* window)
75 {
76     IMLOGD1("[SetSurface] surface[%p]", window);
77     mWindow = window;
78 }
79 
Start()80 bool ImsMediaVideoRenderer::Start()
81 {
82     IMLOGD0("[Start]");
83     mFormat = AMediaFormat_new();
84     AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_WIDTH, mWidth);
85     AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_HEIGHT, mHeight);
86 
87     char kMimeType[128] = {'\0'};
88     sprintf(kMimeType, "video/avc");
89 
90     if (mCodecType == kVideoCodecHevc)
91     {
92         sprintf(kMimeType, "video/hevc");
93     }
94 
95     AMediaFormat_setString(mFormat, AMEDIAFORMAT_KEY_MIME, kMimeType);
96     AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT,
97             21);  // #21 : COLOR_FormatYUV420SemiPlanar
98     AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, mWidth * mHeight * 10);
99     AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_ROTATION, mFarOrientationDegree);
100 
101     mCodec = AMediaCodec_createDecoderByType(kMimeType);
102 
103     if (mCodec == nullptr)
104     {
105         IMLOGE0("[Start] Unable to create decoder");
106         return false;
107     }
108 
109     if (mWindow != nullptr)
110     {
111         ANativeWindow_acquire(mWindow);
112     }
113 
114     media_status_t err = AMediaCodec_configure(mCodec, mFormat, mWindow, nullptr, 0);
115 
116     if (err != AMEDIA_OK)
117     {
118         IMLOGE1("[Start] configure error[%d]", err);
119         AMediaCodec_delete(mCodec);
120         mCodec = nullptr;
121         AMediaFormat_delete(mFormat);
122         mFormat = nullptr;
123         return false;
124     }
125 
126     err = AMediaCodec_start(mCodec);
127 
128     if (err != AMEDIA_OK)
129     {
130         IMLOGE1("[Start] codec start[%d]", err);
131         AMediaCodec_delete(mCodec);
132         mCodec = nullptr;
133         AMediaFormat_delete(mFormat);
134         mFormat = nullptr;
135         return false;
136     }
137 
138     mStopped = false;
139     std::thread t1(&ImsMediaVideoRenderer::processBuffers, this);
140     t1.detach();
141     return true;
142 }
143 
Stop()144 void ImsMediaVideoRenderer::Stop()
145 {
146     IMLOGD0("[Stop]");
147 
148     mMutex.lock();
149     mStopped = true;
150     mMutex.unlock();
151     mConditionExit.wait_timeout(MAX_WAIT_RESTART);
152 
153     if (mCodec != nullptr)
154     {
155         AMediaCodec_signalEndOfInputStream(mCodec);
156         AMediaCodec_stop(mCodec);
157         AMediaCodec_delete(mCodec);
158         mCodec = nullptr;
159     }
160 
161     if (mWindow != nullptr)
162     {
163         ANativeWindow_release(mWindow);
164     }
165 
166     if (mFormat != nullptr)
167     {
168         AMediaFormat_delete(mFormat);
169         mFormat = nullptr;
170     }
171 }
172 
OnDataFrame(uint8_t * buffer,uint32_t size,uint32_t timestamp,const bool isConfigFrame)173 void ImsMediaVideoRenderer::OnDataFrame(
174         uint8_t* buffer, uint32_t size, uint32_t timestamp, const bool isConfigFrame)
175 {
176     if (size == 0 || buffer == nullptr)
177     {
178         return;
179     }
180 
181     IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO, "[OnDataFrame] frame size[%u], list[%d]", size,
182             mFrameDatas.size());
183 
184     mMutex.lock();
185 
186     if (mStopped)
187     {
188         mMutex.unlock();
189         return;
190     }
191 
192     mMutex.unlock();
193 
194     mFrameDatas.push_back(new FrameData(buffer, size, timestamp, isConfigFrame));
195 }
196 
processBuffers()197 void ImsMediaVideoRenderer::processBuffers()
198 {
199     uint32_t nextTime = ImsMediaTimer::GetTimeInMilliSeconds();
200     uint32_t timeDiff = 0;
201 
202     IMLOGD1("[processBuffers] enter time[%u]", nextTime);
203 
204     while (true)
205     {
206         mMutex.lock();
207 
208         if (mStopped)
209         {
210             mMutex.unlock();
211             break;
212         }
213 
214         mMutex.unlock();
215 
216         if (mFrameDatas.size() == 0)
217         {
218             continue;
219         }
220 
221         auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO);
222 
223         if (index >= 0)
224         {
225             size_t bufferSize = 0;
226             uint8_t* inputBuffer = AMediaCodec_getInputBuffer(mCodec, index, &bufferSize);
227 
228             if (inputBuffer != nullptr)
229             {
230                 FrameData* frame = mFrameDatas.front();
231                 memcpy(inputBuffer, frame->data, frame->size);
232                 IMLOGD_PACKET4(IM_PACKET_LOG_VIDEO,
233                         "[processBuffers] queue input buffer index[%d], size[%d], TS[%d], "
234                         "config[%d]",
235                         index, frame->size, frame->timestamp, frame->isConfig);
236 
237                 media_status_t err = AMediaCodec_queueInputBuffer(
238                         mCodec, index, 0, frame->size, frame->timestamp * 1000, 0);
239 
240                 if (err != AMEDIA_OK)
241                 {
242                     IMLOGE1("[processBuffers] Unable to queue input buffers - err[%d]", err);
243                 }
244 
245                 delete frame;
246                 mFrameDatas.pop_front();
247             }
248         }
249 
250         AMediaCodecBufferInfo info;
251         index = AMediaCodec_dequeueOutputBuffer(mCodec, &info, CODEC_TIMEOUT_NANO);
252 
253         if (index >= 0)
254         {
255             IMLOGD_PACKET5(IM_PACKET_LOG_VIDEO,
256                     "[processBuffers] index[%d], size[%d], offset[%d], time[%ld], flags[%d]", index,
257                     info.size, info.offset, info.presentationTimeUs, info.flags);
258 
259             AMediaCodec_releaseOutputBuffer(mCodec, index, true);
260         }
261         else if (index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
262         {
263             IMLOGD0("[processBuffers] output buffer changed");
264         }
265         else if (index == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
266         {
267             if (mFormat != nullptr)
268             {
269                 AMediaFormat_delete(mFormat);
270             }
271             mFormat = AMediaCodec_getOutputFormat(mCodec);
272             IMLOGD1("[processBuffers] format changed, format[%s]", AMediaFormat_toString(mFormat));
273         }
274         else if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
275         {
276             IMLOGD0("[processBuffers] no output buffer");
277         }
278         else
279         {
280             IMLOGD1("[processBuffers] unexpected index[%d]", index);
281         }
282 
283         nextTime += INTERVAL_MILLIS;
284         uint32_t nCurrTime = ImsMediaTimer::GetTimeInMilliSeconds();
285 
286         if (nextTime > nCurrTime)
287         {
288             timeDiff = nextTime - nCurrTime;
289             IMLOGD_PACKET1(IM_PACKET_LOG_VIDEO, "[processBuffers] timeDiff[%u]", timeDiff);
290             ImsMediaTimer::Sleep(timeDiff);
291         }
292     }
293 
294     mConditionExit.signal();
295     IMLOGD0("[processBuffers] exit");
296 }
297 
UpdateDeviceOrientation(uint32_t degree)298 void ImsMediaVideoRenderer::UpdateDeviceOrientation(uint32_t degree)
299 {
300     IMLOGD1("[UpdateDeviceOrientation] orientation[%d]", degree);
301     mNearOrientationDegree = degree;
302 }
303 
UpdatePeerOrientation(uint32_t degree)304 void ImsMediaVideoRenderer::UpdatePeerOrientation(uint32_t degree)
305 {
306     IMLOGD1("[UpdatePeerOrientation] orientation[%d]", degree);
307 
308     if (mFarOrientationDegree != degree)
309     {
310         Stop();
311         mFarOrientationDegree = degree;
312         Start();
313     }
314 }