1 /**
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <ImsMediaVideoSource.h>
18 #include <ImsMediaTrace.h>
19 #include <ImsMediaTimer.h>
20 #include <ImsMediaImageRotate.h>
21 #include <thread>
22 #include <list>
23 #include <time.h>
24
25 #define CODEC_TIMEOUT_NANO 100000
26
ImsMediaVideoSource()27 ImsMediaVideoSource::ImsMediaVideoSource()
28 {
29 mCamera = nullptr;
30 mWindow = nullptr;
31 mCodec = nullptr;
32 mFormat = nullptr;
33 mImageReaderSurface = nullptr;
34 mImageReader = nullptr;
35 mCodecType = -1;
36 mVideoMode = -1;
37 mCodecProfile = 0;
38 mCodecLevel = 0;
39 mCameraId = 0;
40 mCameraZoom = 0;
41 mWidth = 0;
42 mHeight = 0;
43 mFramerate = 0;
44 mBitrate = 0;
45 mIntraInterval = 1;
46 mImagePath = android::String8("");
47 mDeviceOrientation = -1;
48 mTimestamp = 0;
49 mPrevTimestamp = 0;
50 mStopped = false;
51 }
52
~ImsMediaVideoSource()53 ImsMediaVideoSource::~ImsMediaVideoSource() {}
54
SetListener(IVideoSourceCallback * listener)55 void ImsMediaVideoSource::SetListener(IVideoSourceCallback* listener)
56 {
57 std::lock_guard<std::mutex> guard(mMutex);
58 mListener = listener;
59 }
60
SetVideoMode(const int32_t mode)61 void ImsMediaVideoSource::SetVideoMode(const int32_t mode)
62 {
63 IMLOGD1("[SetVideoMode] mode[%d]", mode);
64 mVideoMode = mode;
65 }
66
SetCameraConfig(const uint32_t cameraId,const uint32_t cameraZoom)67 void ImsMediaVideoSource::SetCameraConfig(const uint32_t cameraId, const uint32_t cameraZoom)
68 {
69 IMLOGD2("[SetCameraConfig] id[%d], zoom[%d]", cameraId, cameraZoom);
70 mCameraId = cameraId;
71 mCameraZoom = cameraZoom;
72 }
73
SetImagePath(const android::String8 & path)74 void ImsMediaVideoSource::SetImagePath(const android::String8& path)
75 {
76 IMLOGD1("[SetImagePath] path[%s]", path.c_str());
77 mImagePath = path;
78 }
79
SetCodecConfig(int32_t codecType,const uint32_t profile,const uint32_t level,const uint32_t bitrate,const uint32_t framerate,const uint32_t interval)80 void ImsMediaVideoSource::SetCodecConfig(int32_t codecType, const uint32_t profile,
81 const uint32_t level, const uint32_t bitrate, const uint32_t framerate,
82 const uint32_t interval)
83 {
84 IMLOGD6("[SetCodecConfig] type[%d], profile[%d], level[%d], bitrate[%d], FPS[%d], interval[%d]",
85 codecType, profile, level, bitrate, framerate, interval);
86 mCodecType = codecType;
87 mCodecProfile = profile;
88 mCodecLevel = level;
89 mBitrate = bitrate;
90 mFramerate = framerate;
91 mIntraInterval = interval;
92 }
93
SetResolution(const uint32_t width,const uint32_t height)94 void ImsMediaVideoSource::SetResolution(const uint32_t width, const uint32_t height)
95 {
96 IMLOGD2("[SetResolution] width[%d], height[%d]", width, height);
97 mWidth = width;
98 mHeight = height;
99 }
100
SetSurface(ANativeWindow * window)101 void ImsMediaVideoSource::SetSurface(ANativeWindow* window)
102 {
103 IMLOGD1("[SetSurface] surface[%p]", window);
104 mWindow = window;
105 }
106
SetDeviceOrientation(const uint32_t degree)107 void ImsMediaVideoSource::SetDeviceOrientation(const uint32_t degree)
108 {
109 IMLOGD1("[SetDeviceOrientation] degree[%d]", degree);
110
111 if (mDeviceOrientation != degree)
112 {
113 if (mVideoMode == kVideoModeRecording)
114 {
115 int32_t facing = kCameraFacingFront;
116 int32_t sensorOrientation = 0;
117 int32_t rotateDegree = 0;
118
119 if (mCamera != nullptr)
120 {
121 mCamera->GetSensorOrientation(mCameraId, &facing, &sensorOrientation);
122 IMLOGD2("[SetDeviceOrientation] camera facing[%d], sensorOrientation[%d]", facing,
123 sensorOrientation);
124 }
125
126 // assume device is always portrait
127 if (mWidth > mHeight)
128 {
129 if (facing == kCameraFacingFront)
130 {
131 sensorOrientation = (sensorOrientation + 180) % 360;
132 }
133
134 rotateDegree = sensorOrientation - degree;
135
136 if (rotateDegree < 0)
137 {
138 rotateDegree += 360;
139 }
140 }
141 else
142 {
143 if (degree == 90 || degree == 270)
144 {
145 rotateDegree = (degree + 180) % 360;
146 }
147 else
148 {
149 rotateDegree = degree;
150 }
151 }
152
153 if (mListener != nullptr)
154 {
155 mListener->OnEvent(kVideoSourceEventUpdateOrientation, facing, rotateDegree);
156 }
157 }
158
159 mDeviceOrientation = degree;
160 }
161 }
162
Start()163 bool ImsMediaVideoSource::Start()
164 {
165 IMLOGD1("[Start], VideoMode[%d]", mVideoMode);
166
167 if (mVideoMode == kVideoModeRecording || mVideoMode == kVideoModePauseImage)
168 {
169 mFormat = AMediaFormat_new();
170 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_WIDTH, mWidth);
171 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_HEIGHT, mHeight);
172
173 char kMimeType[128] = {'\0'};
174 sprintf(kMimeType, "video/avc");
175
176 if (mCodecType == kVideoCodecHevc)
177 {
178 sprintf(kMimeType, "video/hevc");
179 }
180
181 AMediaFormat_setString(mFormat, AMEDIAFORMAT_KEY_MIME, kMimeType);
182
183 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT,
184 0x00000015); // COLOR_FormatYUV420SemiPlanar
185 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_BIT_RATE, mBitrate * 1000);
186 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_PROFILE, mCodecProfile);
187 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_LEVEL, mCodecLevel);
188 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_BITRATE_MODE,
189 2); // #2 : BITRATE_MODE_CBR
190 AMediaFormat_setFloat(mFormat, AMEDIAFORMAT_KEY_FRAME_RATE, mFramerate);
191 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL, mIntraInterval);
192 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, mWidth * mHeight * 10);
193
194 mCodec = AMediaCodec_createEncoderByType(kMimeType);
195
196 if (mCodec == nullptr)
197 {
198 IMLOGE0("[Start] Unable to create encoder");
199 return false;
200 }
201
202 media_status_t err = AMediaCodec_configure(
203 mCodec, mFormat, nullptr, nullptr, AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
204
205 if (err != AMEDIA_OK)
206 {
207 IMLOGE1("[Start] configure error[%d]", err);
208 AMediaCodec_delete(mCodec);
209 mCodec = nullptr;
210 AMediaFormat_delete(mFormat);
211 mFormat = nullptr;
212 return false;
213 }
214
215 mImageReaderSurface = CreateImageReader(mWidth, mHeight);
216
217 if (mImageReaderSurface == nullptr)
218 {
219 IMLOGE0("[Start] create image reader failed");
220 return false;
221 }
222
223 mCodecStride = mWidth;
224 AMediaFormat* encoderInputFormat = AMediaCodec_getInputFormat(mCodec);
225
226 if (encoderInputFormat != nullptr)
227 {
228 // Check if encoder is initialized with the expected configurations.
229 int32_t width = 0, height = 0;
230 AMediaFormat_getInt32(encoderInputFormat, AMEDIAFORMAT_KEY_WIDTH, &width);
231 AMediaFormat_getInt32(encoderInputFormat, AMEDIAFORMAT_KEY_HEIGHT, &height);
232 AMediaFormat_getInt32(encoderInputFormat, AMEDIAFORMAT_KEY_STRIDE, &mCodecStride);
233 AMediaFormat_delete(encoderInputFormat);
234
235 // TODO: More configuration checks should be added
236 if (mWidth != width || mHeight != height || width > mCodecStride)
237 {
238 IMLOGE0("Encoder doesn't support requested configuration.");
239 return false;
240 }
241 }
242
243 err = AMediaCodec_start(mCodec);
244
245 if (err != AMEDIA_OK)
246 {
247 IMLOGE1("[Start] codec start[%d]", err);
248 AMediaCodec_delete(mCodec);
249 mCodec = nullptr;
250 AMediaFormat_delete(mFormat);
251 mFormat = nullptr;
252 return false;
253 }
254 }
255
256 if (mCameraId != -1 && (mVideoMode == kVideoModePreview || mVideoMode == kVideoModeRecording))
257 {
258 mCamera = ImsMediaCamera::getInstance();
259 mCamera->Initialize();
260 mCamera->SetCameraConfig(mCameraId, mCameraZoom, mFramerate);
261
262 if (!mCamera->OpenCamera())
263 {
264 IMLOGE0("[Start] error open camera");
265 AMediaCodec_delete(mCodec);
266 mCodec = nullptr;
267 AMediaFormat_delete(mFormat);
268 mFormat = nullptr;
269 return false;
270 }
271
272 if (mCamera->CreateSession(mWindow, mImageReaderSurface) == false)
273 {
274 IMLOGE0("[Start] error create camera session");
275 AMediaCodec_delete(mCodec);
276 mCodec = nullptr;
277 AMediaFormat_delete(mFormat);
278 mFormat = nullptr;
279 mCamera->DeleteSession();
280 mCamera->DeInitialize();
281 return false;
282 }
283
284 if (mCamera->StartSession(mVideoMode == kVideoModeRecording) == false)
285 {
286 IMLOGE0("[Start] error camera start");
287 AMediaCodec_delete(mCodec);
288 mCodec = nullptr;
289 AMediaFormat_delete(mFormat);
290 mFormat = nullptr;
291 return false;
292 }
293 }
294 else if (mVideoMode == kVideoModePauseImage)
295 {
296 mPauseImageSource.Initialize(mWidth, mHeight, mCodecStride);
297 // start encoder output thread
298 if (mCodec != nullptr)
299 {
300 mStopped = false;
301 std::thread t1(&ImsMediaVideoSource::EncodePauseImage, this);
302 t1.detach();
303 }
304 }
305
306 mDeviceOrientation = -1;
307 IMLOGD0("[Start] exit");
308 return true;
309 }
310
Stop()311 void ImsMediaVideoSource::Stop()
312 {
313 IMLOGD0("[Stop]");
314
315 std::lock_guard<std::mutex> guard(mMutex);
316 mStopped = true;
317
318 if (mImageReader != nullptr)
319 {
320 AImageReader_delete(mImageReader);
321 mImageReader = nullptr;
322 mImageReaderSurface = nullptr;
323 }
324
325 if (mCamera != nullptr)
326 {
327 mCamera->StopSession();
328 mCamera->DeleteSession();
329 mCamera->DeInitialize();
330 mCamera = nullptr;
331 }
332
333 if (mCodec != nullptr)
334 {
335 if (mVideoMode == kVideoModePauseImage)
336 {
337 mConditionExit.wait_timeout(mFramerate != 0 ? 1000 / mFramerate : 66);
338 }
339
340 AMediaCodec_stop(mCodec);
341 AMediaCodec_delete(mCodec);
342 mCodec = nullptr;
343 }
344
345 if (mFormat != nullptr)
346 {
347 AMediaFormat_delete(mFormat);
348 mFormat = nullptr;
349 }
350
351 if (mVideoMode == kVideoModePauseImage)
352 {
353 mPauseImageSource.Uninitialize();
354 }
355 }
356
IsStopped()357 bool ImsMediaVideoSource::IsStopped()
358 {
359 std::lock_guard<std::mutex> guard(mMutex);
360 return mStopped;
361 }
362
onCameraFrame(AImage * pImage)363 void ImsMediaVideoSource::onCameraFrame(AImage* pImage)
364 {
365 std::lock_guard<std::mutex> guard(mMutex);
366
367 if (mImageReader == nullptr || pImage == nullptr)
368 {
369 return;
370 }
371
372 auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO);
373
374 if (index >= 0)
375 {
376 size_t buffCapacity = 0;
377 uint8_t* encoderBuf = AMediaCodec_getInputBuffer(mCodec, index, &buffCapacity);
378 if (!encoderBuf || !buffCapacity)
379 {
380 IMLOGE1("[onCameraFrame] returned null buffer pointer or buffCapacity[%d]",
381 buffCapacity);
382 return;
383 }
384
385 int32_t width, height, ylen, uvlen, result = 0;
386 uint8_t *yPlane, *uvPlane;
387 AImage_getWidth(pImage, &width);
388 AImage_getHeight(pImage, &height);
389 AImage_getPlaneData(pImage, 0, &yPlane, &ylen);
390 AImage_getPlaneData(pImage, 1, &uvPlane, &uvlen);
391
392 if (mWidth > mHeight) // landscape mode, copy without rotate
393 {
394 memcpy(encoderBuf, yPlane, ylen);
395 memcpy(encoderBuf + ylen, uvPlane, uvlen);
396 }
397 else
398 {
399 int32_t facing, sensorOrientation;
400 mCamera->GetSensorOrientation(mCameraId, &facing, &sensorOrientation);
401
402 switch (facing)
403 {
404 case ACAMERA_LENS_FACING_FRONT:
405 {
406 result = ImsMediaImageRotate::YUV420_SP_Rotate270(
407 encoderBuf, buffCapacity, mCodecStride, yPlane, uvPlane, width, height);
408 }
409 break;
410
411 case ACAMERA_LENS_FACING_BACK:
412 {
413 result = ImsMediaImageRotate::YUV420_SP_Rotate90(
414 encoderBuf, buffCapacity, mCodecStride, yPlane, uvPlane, width, height);
415 }
416 break;
417
418 case ACAMERA_LENS_FACING_EXTERNAL:
419 {
420 uint32_t size = width * height;
421 memcpy(encoderBuf, yPlane, size);
422 memcpy(encoderBuf + size, uvPlane, size / 2);
423 }
424 break;
425 }
426 }
427
428 IMLOGD_PACKET1(IM_PACKET_LOG_VIDEO, "[onCameraFrame] queue buffer size[%d]", ylen + uvlen);
429
430 if (result == 0)
431 {
432 AMediaCodec_queueInputBuffer(
433 mCodec, index, 0, ylen + uvlen, ImsMediaTimer::GetTimeInMicroSeconds(), 0);
434 }
435 else
436 {
437 IMLOGE5("Camera image resolution[%dx%d]. Encoder resolution[%dx%d] buffer size[%d]",
438 width, height, mWidth, mHeight, buffCapacity);
439 AMediaCodec_queueInputBuffer(mCodec, index, 0, 0, 0, 0);
440 return;
441 }
442 }
443 else
444 {
445 IMLOGE1("[onCameraFrame] dequeueInputBuffer returned index[%d]", index);
446 }
447
448 processOutputBuffer();
449 }
450
changeBitrate(const uint32_t bitrate)451 bool ImsMediaVideoSource::changeBitrate(const uint32_t bitrate)
452 {
453 IMLOGD1("[changeBitrate] bitrate[%d]", bitrate);
454 std::lock_guard<std::mutex> guard(mMutex);
455
456 if (mStopped)
457 {
458 return false;
459 }
460
461 AMediaFormat* params = AMediaFormat_new();
462 AMediaFormat_setInt32(params, AMEDIACODEC_KEY_VIDEO_BITRATE, bitrate);
463 media_status_t status = AMediaCodec_setParameters(mCodec, params);
464
465 if (status != AMEDIA_OK)
466 {
467 IMLOGE1("[changeBitrate] error[%d]", status);
468 return false;
469 }
470
471 return true;
472 }
473
requestIdrFrame()474 void ImsMediaVideoSource::requestIdrFrame()
475 {
476 IMLOGD0("[requestIdrFrame]");
477 std::lock_guard<std::mutex> guard(mMutex);
478
479 if (mStopped)
480 {
481 return;
482 }
483
484 AMediaFormat* params = AMediaFormat_new();
485 AMediaFormat_setInt32(params, AMEDIACODEC_KEY_REQUEST_SYNC_FRAME, 0);
486 media_status_t status = AMediaCodec_setParameters(mCodec, params);
487
488 if (status != AMEDIA_OK)
489 {
490 IMLOGE1("[requestIdrFrame] error[%d]", status);
491 }
492 }
493
EncodePauseImage()494 void ImsMediaVideoSource::EncodePauseImage()
495 {
496 IMLOGD0("[EncodePauseImage] start");
497
498 uint32_t nextTime = ImsMediaTimer::GetTimeInMilliSeconds();
499 uint32_t timeInterval = 66;
500
501 if (mFramerate != 0)
502 {
503 timeInterval = 1000 / mFramerate;
504 }
505
506 while (!IsStopped())
507 {
508 mMutex.lock();
509 auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO);
510
511 if (index >= 0)
512 {
513 size_t buffCapacity = 0;
514 uint8_t* encoderBuf = AMediaCodec_getInputBuffer(mCodec, index, &buffCapacity);
515 if (!encoderBuf || !buffCapacity)
516 {
517 IMLOGE1("[EncodePauseImage] returned null buffer pointer or buffCapacity[%d]",
518 buffCapacity);
519 return;
520 }
521
522 size_t len = mPauseImageSource.GetYuvImage(encoderBuf, buffCapacity);
523 AMediaCodec_queueInputBuffer(
524 mCodec, index, 0, len, ImsMediaTimer::GetTimeInMicroSeconds(), 0);
525 }
526 else
527 {
528 IMLOGE1("[EncodePauseImage] dequeueInputBuffer returned index[%d]", index);
529 }
530
531 processOutputBuffer();
532 mMutex.unlock();
533
534 if (IsStopped())
535 {
536 break;
537 }
538
539 nextTime += timeInterval;
540 uint32_t nCurrTime = ImsMediaTimer::GetTimeInMilliSeconds();
541
542 if (nextTime > nCurrTime)
543 {
544 uint32_t timeDiff = nextTime - nCurrTime;
545 IMLOGD_PACKET1(IM_PACKET_LOG_VIDEO, "[EncodePauseImage] timeDiff[%u]", timeDiff);
546 ImsMediaTimer::Sleep(timeDiff);
547 }
548 }
549
550 IMLOGD0("[EncodePauseImage] end");
551 mConditionExit.signal();
552 }
553
processOutputBuffer()554 void ImsMediaVideoSource::processOutputBuffer()
555 {
556 AMediaCodecBufferInfo info;
557 auto index = AMediaCodec_dequeueOutputBuffer(mCodec, &info, CODEC_TIMEOUT_NANO);
558
559 if (index >= 0)
560 {
561 IMLOGD_PACKET5(IM_PACKET_LOG_VIDEO,
562 "[processOutputBuffer] index[%d], size[%d], offset[%d], time[%ld], flags[%d]",
563 index, info.size, info.offset, info.presentationTimeUs, info.flags);
564
565 if (info.size > 0)
566 {
567 size_t buffCapacity;
568 uint8_t* buf = AMediaCodec_getOutputBuffer(mCodec, index, &buffCapacity);
569
570 if (buf != nullptr && buffCapacity > 0)
571 {
572 if (mListener != nullptr)
573 {
574 mListener->OnUplinkEvent(
575 buf + info.offset, info.size, info.presentationTimeUs, info.flags);
576 }
577 }
578
579 AMediaCodec_releaseOutputBuffer(mCodec, index, false);
580 }
581 }
582 else if (index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
583 {
584 IMLOGI0("[processOutputBuffer] Encoder output buffer changed");
585 }
586 else if (index == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
587 {
588 if (mFormat != nullptr)
589 {
590 AMediaFormat_delete(mFormat);
591 }
592
593 mFormat = AMediaCodec_getOutputFormat(mCodec);
594 IMLOGI1("[processOutputBuffer] Encoder format changed, format[%s]",
595 AMediaFormat_toString(mFormat));
596 }
597 else if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
598 {
599 IMLOGD_PACKET0(IM_PACKET_LOG_VIDEO, "[processOutputBuffer] no output buffer");
600 }
601 else
602 {
603 IMLOGI1("[processOutputBuffer] unexpected index[%d]", index);
604 }
605 }
606
ImageCallback(void * context,AImageReader * reader)607 static void ImageCallback(void* context, AImageReader* reader)
608 {
609 if (context == nullptr)
610 {
611 return;
612 }
613
614 ImsMediaVideoSource* pVideoSource = static_cast<ImsMediaVideoSource*>(context);
615
616 AImage* image = nullptr;
617 auto status = AImageReader_acquireNextImage(reader, &image);
618
619 if (status != AMEDIA_OK)
620 {
621 return;
622 }
623
624 pVideoSource->onCameraFrame(image);
625 AImage_delete(image);
626 }
627
CreateImageReader(int width,int height)628 ANativeWindow* ImsMediaVideoSource::CreateImageReader(int width, int height)
629 {
630 media_status_t status =
631 AImageReader_new(width, height, AIMAGE_FORMAT_YUV_420_888, 2, &mImageReader);
632
633 if (status != AMEDIA_OK)
634 {
635 return nullptr;
636 }
637
638 AImageReader_ImageListener listener{
639 .context = this,
640 .onImageAvailable = ImageCallback,
641 };
642
643 AImageReader_setImageListener(mImageReader, &listener);
644
645 ANativeWindow* nativeWindow;
646 AImageReader_getWindow(mImageReader, &nativeWindow);
647 return nativeWindow;
648 }
649