1 /**
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <stdio.h>
18 #include <sys/time.h>
19 #include <sys/types.h>
20 #include <sys/stat.h>
21 #include <fcntl.h>
22 #include <ImsMediaDefine.h>
23 #include <ImsMediaTimer.h>
24 #include <ImsMediaTrace.h>
25 #include <ImsMediaAudioUtil.h>
26 #include <ImsMediaAudioSource.h>
27 #include <utils/Errors.h>
28 #include <thread>
29 
30 #define AAUDIO_STATE_TIMEOUT_NANO (100 * 1000000L)
31 #define NUM_FRAMES_PER_SEC        (50)
32 #define DEFAULT_SAMPLING_RATE     (8000)
33 #define CODEC_TIMEOUT_NANO        (100000)
34 
35 using namespace android;
36 
ImsMediaAudioSource()37 ImsMediaAudioSource::ImsMediaAudioSource()
38 {
39     mAudioStream = nullptr;
40     mCodec = nullptr;
41     mFormat = nullptr;
42     mCallback = nullptr;
43     mCodecType = -1;
44     mMode = 0;
45     mPtime = 0;
46     mSamplingRate = DEFAULT_SAMPLING_RATE;
47     mBufferSize = 0;
48     mEvsBandwidth = kEvsBandwidthNone;
49     mEvsBitRate = 0;
50     mEvsChAwOffset = 0;
51     mIsEvsInitialized = false;
52     mMediaDirection = 0;
53     mIsDtxEnabled = false;
54     mIsOctetAligned = false;
55 }
56 
~ImsMediaAudioSource()57 ImsMediaAudioSource::~ImsMediaAudioSource() {}
58 
SetUplinkCallback(IFrameCallback * callback)59 void ImsMediaAudioSource::SetUplinkCallback(IFrameCallback* callback)
60 {
61     std::lock_guard<std::mutex> guard(mMutexUplink);
62     mCallback = callback;
63 }
64 
SetCodec(int32_t type)65 void ImsMediaAudioSource::SetCodec(int32_t type)
66 {
67     IMLOGD1("[SetCodec] type[%d]", type);
68     mCodecType = type;
69 }
70 
SetCodecMode(uint32_t mode)71 void ImsMediaAudioSource::SetCodecMode(uint32_t mode)
72 {
73     IMLOGD1("[SetCodecMode] mode[%d]", mode);
74     mMode = mode;
75 }
76 
SetEvsBitRate(uint32_t bitrate)77 void ImsMediaAudioSource::SetEvsBitRate(uint32_t bitrate)
78 {
79     IMLOGD1("[SetEvsBitRate] bitrate[%d]", bitrate);
80     mEvsBitRate = bitrate;
81 }
82 
SetSamplingRate(int32_t samplingRate)83 void ImsMediaAudioSource::SetSamplingRate(int32_t samplingRate)
84 {
85     mSamplingRate = samplingRate;
86 }
87 
SetEvsChAwOffset(int32_t offset)88 void ImsMediaAudioSource::SetEvsChAwOffset(int32_t offset)
89 {
90     mEvsChAwOffset = offset;
91 }
92 
SetPtime(uint32_t time)93 void ImsMediaAudioSource::SetPtime(uint32_t time)
94 {
95     IMLOGD1("[SetPtime] Ptime[%d]", time);
96     mPtime = time;
97 }
98 
SetEvsBandwidth(int32_t evsBandwidth)99 void ImsMediaAudioSource::SetEvsBandwidth(int32_t evsBandwidth)
100 {
101     mEvsBandwidth = (kEvsBandwidth)evsBandwidth;
102 }
103 
SetMediaDirection(int32_t direction)104 void ImsMediaAudioSource::SetMediaDirection(int32_t direction)
105 {
106     mMediaDirection = direction;
107 }
108 
SetDtxEnabled(bool isDtxEnabled)109 void ImsMediaAudioSource::SetDtxEnabled(bool isDtxEnabled)
110 {
111     mIsDtxEnabled = isDtxEnabled;
112 }
113 
SetOctetAligned(bool isOctetAligned)114 void ImsMediaAudioSource::SetOctetAligned(bool isOctetAligned)
115 {
116     mIsOctetAligned = isOctetAligned;
117 }
118 
Start()119 bool ImsMediaAudioSource::Start()
120 {
121     openAudioStream();
122 
123     if (mAudioStream == nullptr)
124     {
125         IMLOGE0("[Start] create audio stream failed");
126         return false;
127     }
128 
129     // configure and start codec
130     if (!startCodec())
131     {
132         IMLOGE0("[Start] start codec failed");
133         return false;
134     }
135 
136     // start audio
137     auto audioResult = AAudioStream_requestStart(mAudioStream);
138 
139     if (audioResult != AAUDIO_OK)
140     {
141         IMLOGE1("[Start] Error start stream[%s]", AAudio_convertResultToText(audioResult));
142 
143         stopCodec();
144         return false;
145     }
146 
147     aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STARTING;
148     aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
149     audioResult = AAudioStream_waitForStateChange(
150             mAudioStream, inputState, &nextState, 10 * AAUDIO_STATE_TIMEOUT_NANO);
151 
152     if (audioResult != AAUDIO_OK)
153     {
154         IMLOGE1("[Start] Error start stream[%s]", AAudio_convertResultToText(audioResult));
155 
156         if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
157         {
158             AMediaCodec_delete(mCodec);
159             mCodec = nullptr;
160             AMediaFormat_delete(mFormat);
161             mFormat = nullptr;
162         }
163 
164         return false;
165     }
166 
167     IMLOGI1("[Start] start stream state[%s]", AAudio_convertStreamStateToText(nextState));
168 
169     // start audio read thread
170     StartThread("ImsMediaAudioSource");
171     return true;
172 }
173 
Stop()174 void ImsMediaAudioSource::Stop()
175 {
176     IMLOGD0("[Stop]");
177     StopThread();
178 
179     if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
180     {
181         mConditionExit.reset();
182         mConditionExit.wait_timeout(AUDIO_STOP_TIMEOUT);
183     }
184 
185     std::lock_guard<std::mutex> guard(mMutexUplink);
186 
187     if (mAudioStream != nullptr)
188     {
189         aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STOPPING;
190         aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
191         aaudio_result_t result = AAudioStream_requestStop(mAudioStream);
192 
193         if (result != AAUDIO_OK)
194         {
195             IMLOGE1("[Stop] Error stop stream[%s]", AAudio_convertResultToText(result));
196         }
197 
198         // TODO: if it causes extra delay in stop, optimize later
199         result = AAudioStream_waitForStateChange(
200                 mAudioStream, inputState, &nextState, AAUDIO_STATE_TIMEOUT_NANO);
201 
202         if (result != AAUDIO_OK)
203         {
204             IMLOGE1("[Stop] Error stop stream[%s]", AAudio_convertResultToText(result));
205         }
206 
207         IMLOGI1("[Stop] stream state[%s]", AAudio_convertStreamStateToText(nextState));
208 
209         AAudioStream_close(mAudioStream);
210         mAudioStream = nullptr;
211     }
212 
213     stopCodec();
214 }
215 
ProcessCmr(const uint32_t cmr)216 void ImsMediaAudioSource::ProcessCmr(const uint32_t cmr)
217 {
218     IMLOGI1("[ProcessCmr] cmr[%d]", cmr);
219 
220     if (IsThreadStopped())
221     {
222         return;
223     }
224 
225     std::lock_guard<std::mutex> guard(mMutexUplink);
226     mMode = cmr;
227     stopCodec();
228     startCodec();
229 }
230 
audioErrorCallback(AAudioStream * stream,void * userData,aaudio_result_t error)231 void ImsMediaAudioSource::audioErrorCallback(
232         AAudioStream* stream, void* userData, aaudio_result_t error)
233 {
234     if (stream == nullptr || userData == nullptr)
235     {
236         return;
237     }
238 
239     aaudio_stream_state_t streamState = AAudioStream_getState(stream);
240     IMLOGW2("[errorCallback] error[%s], state[%d]", AAudio_convertResultToText(error), streamState);
241 
242     if (error == AAUDIO_ERROR_DISCONNECTED)
243     {
244         // Handle stream restart on a separate thread
245         std::thread streamRestartThread(&ImsMediaAudioSource::restartAudioStream,
246                 reinterpret_cast<ImsMediaAudioSource*>(userData));
247         streamRestartThread.detach();
248     }
249 }
250 
run()251 void* ImsMediaAudioSource::run()
252 {
253     IMLOGD0("[run] enter");
254     uint32_t nNextTime = ImsMediaTimer::GetTimeInMilliSeconds();
255     int16_t buffer[PCM_BUFFER_SIZE];
256     int64_t ptsUsec = 0;
257     uint32_t evsFlags = 2;
258     uint8_t outputBuf[PCM_BUFFER_SIZE];
259     int size = 0;
260 
261     outputBuf[0] = 0;
262 
263     for (;;)
264     {
265         if (IsThreadStopped())
266         {
267             IMLOGD0("[run] terminated");
268             break;
269         }
270 
271         mMutexUplink.lock();
272 
273         if (mAudioStream != nullptr &&
274                 AAudioStream_getState(mAudioStream) == AAUDIO_STREAM_STATE_STARTED)
275         {
276             aaudio_result_t readSize = AAudioStream_read(mAudioStream, buffer, mBufferSize, 0);
277 
278             if (readSize > 0)
279             {
280                 IMLOGD_PACKET1(IM_PACKET_LOG_AUDIO, "[run] nReadSize[%d]", readSize);
281                 if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
282                 {
283                     queueInputBuffer(buffer, readSize * sizeof(uint16_t));
284                     dequeueOutputBuffer();
285                 }
286                 else if (mCodecType == kAudioCodecEvs)
287                 {
288                     // TODO: Integration with libEVS is required.
289                     if (!mIsEvsInitialized)
290                     {
291                         mIsEvsInitialized = true;
292                     }
293 
294                     if (ptsUsec == 0)
295                     {
296                         ptsUsec = ImsMediaTimer::GetTimeInMicroSeconds() / 1000;
297                     }
298 
299                     if (mCallback != nullptr && outputBuf[0] != 0)
300                     {
301                         // TODO: integration with libEVS is require to encode outputBuf
302                         mCallback->onDataFrame(outputBuf, size, ptsUsec, evsFlags);
303                     }
304                     size = 0;
305                 }
306             }
307         }
308 
309         mMutexUplink.unlock();
310 
311         nNextTime += mPtime;
312         uint32_t nCurrTime = ImsMediaTimer::GetTimeInMilliSeconds();
313 
314         if (nNextTime > nCurrTime)
315         {
316             ImsMediaTimer::Sleep(nNextTime - nCurrTime);
317         }
318     }
319 
320     mConditionExit.signal();
321     return nullptr;
322 }
323 
openAudioStream()324 void ImsMediaAudioSource::openAudioStream()
325 {
326     AAudioStreamBuilder* builder = nullptr;
327     aaudio_result_t result = AAudio_createStreamBuilder(&builder);
328 
329     if (result != AAUDIO_OK)
330     {
331         IMLOGE1("[openAudioStream] Error creating stream builder[%s]",
332                 AAudio_convertResultToText(result));
333         return;
334     }
335 
336     // setup builder
337     AAudioStreamBuilder_setInputPreset(builder, AAUDIO_INPUT_PRESET_VOICE_COMMUNICATION);
338     AAudioStreamBuilder_setDirection(builder, AAUDIO_DIRECTION_INPUT);
339     AAudioStreamBuilder_setFormat(builder, AAUDIO_FORMAT_PCM_I16);
340     AAudioStreamBuilder_setChannelCount(builder, 1);
341     AAudioStreamBuilder_setSampleRate(builder, mSamplingRate);
342     AAudioStreamBuilder_setSharingMode(builder, AAUDIO_SHARING_MODE_EXCLUSIVE);
343     AAudioStreamBuilder_setPerformanceMode(builder, AAUDIO_PERFORMANCE_MODE_LOW_LATENCY);
344     AAudioStreamBuilder_setUsage(builder, AAUDIO_USAGE_VOICE_COMMUNICATION);
345     AAudioStreamBuilder_setErrorCallback(builder, audioErrorCallback, this);
346     AAudioStreamBuilder_setPrivacySensitive(builder, true);
347 
348     // open stream
349     result = AAudioStreamBuilder_openStream(builder, &mAudioStream);
350     AAudioStreamBuilder_delete(builder);
351 
352     if (result == AAUDIO_OK && mAudioStream != nullptr)
353     {
354         mBufferSize = AAudioStream_getFramesPerBurst(mAudioStream);
355         IMLOGD3("[openAudioStream] samplingRate[%d], framesPerBurst[%d], "
356                 "performanceMode[%d]",
357                 AAudioStream_getSampleRate(mAudioStream), mBufferSize,
358                 AAudioStream_getPerformanceMode(mAudioStream));
359         // Set the buffer size to the burst size - this will give us the minimum
360         // possible latency
361         AAudioStream_setBufferSizeInFrames(mAudioStream, mBufferSize);
362     }
363     else
364     {
365         IMLOGE1("[openAudioStream] Failed to openStream. Error[%s]",
366                 AAudio_convertResultToText(result));
367         mAudioStream = nullptr;
368     }
369 }
370 
restartAudioStream()371 void ImsMediaAudioSource::restartAudioStream()
372 {
373     std::lock_guard<std::mutex> guard(mMutexUplink);
374 
375     if (mAudioStream == nullptr)
376     {
377         return;
378     }
379 
380     AAudioStream_requestStop(mAudioStream);
381     AAudioStream_close(mAudioStream);
382     mAudioStream = nullptr;
383     openAudioStream();
384 
385     if (mAudioStream == nullptr)
386     {
387         return;
388     }
389 
390     aaudio_stream_state_t inputState = AAUDIO_STREAM_STATE_STARTING;
391     aaudio_stream_state_t nextState = AAUDIO_STREAM_STATE_UNINITIALIZED;
392     aaudio_result_t result = AAudioStream_requestStart(mAudioStream);
393 
394     if (result != AAUDIO_OK)
395     {
396         IMLOGE1("[restartAudioStream] Error start stream[%s]", AAudio_convertResultToText(result));
397         return;
398     }
399 
400     result = AAudioStream_waitForStateChange(
401             mAudioStream, inputState, &nextState, 10 * AAUDIO_STATE_TIMEOUT_NANO);
402 
403     if (result != AAUDIO_OK)
404     {
405         IMLOGE1("[restartAudioStream] Error start stream[%s]", AAudio_convertResultToText(result));
406         return;
407     }
408 
409     IMLOGI1("[restartAudioStream] start stream state[%s]",
410             AAudio_convertStreamStateToText(nextState));
411 }
412 
startCodec()413 bool ImsMediaAudioSource::startCodec()
414 {
415     char kMimeType[128] = {'\0'};
416     int amrBitrate = 0;
417     // TODO: Integration with libEVS is required.
418     ImsMediaAudioUtil::ConvertEvsBandwidthToStr(mEvsBandwidth, mEvsbandwidthStr, MAX_EVS_BW_STRLEN);
419 
420     switch (mCodecType)
421     {
422         case kAudioCodecAmr:
423             sprintf(kMimeType, "audio/3gpp");
424             amrBitrate = ImsMediaAudioUtil::ConvertAmrModeToBitrate(mMode);
425             break;
426         case kAudioCodecAmrWb:
427             sprintf(kMimeType, "audio/amr-wb");
428             amrBitrate = ImsMediaAudioUtil::ConvertAmrWbModeToBitrate(mMode);
429             break;
430         case kAudioCodecEvs:
431             // TODO: Integration with libEVS is required.
432             sprintf(kMimeType, "audio/evs");
433             break;
434         default:
435             return false;
436     }
437 
438     IMLOGD1("[startCodec] codec type[%s]", kMimeType);
439 
440     if (mCodecType == kAudioCodecAmr || mCodecType == kAudioCodecAmrWb)
441     {
442         mFormat = AMediaFormat_new();
443         AMediaFormat_setString(mFormat, AMEDIAFORMAT_KEY_MIME, kMimeType);
444         AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_SAMPLE_RATE, mSamplingRate);
445         AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_CHANNEL_COUNT, 1);
446         AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_BIT_RATE, amrBitrate);
447 
448         mCodec = AMediaCodec_createEncoderByType(kMimeType);
449 
450         if (mCodec == nullptr)
451         {
452             IMLOGE1("[startCodec] unable to create %s codec instance", kMimeType);
453             AMediaFormat_delete(mFormat);
454             mFormat = nullptr;
455             return false;
456         }
457 
458         IMLOGD0("[startCodec] configure codec");
459         media_status_t codecResult = AMediaCodec_configure(
460                 mCodec, mFormat, nullptr, nullptr, AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
461 
462         if (codecResult != AMEDIA_OK)
463         {
464             IMLOGE2("[startCodec] unable to configure[%s] codec - err[%d]", kMimeType, codecResult);
465             AMediaCodec_delete(mCodec);
466             mCodec = nullptr;
467             AMediaFormat_delete(mFormat);
468             mFormat = nullptr;
469             return false;
470         }
471 
472         codecResult = AMediaCodec_start(mCodec);
473 
474         if (codecResult != AMEDIA_OK)
475         {
476             IMLOGE1("[Start] unable to start codec - err[%d]", codecResult);
477             AMediaCodec_delete(mCodec);
478             mCodec = nullptr;
479             AMediaFormat_delete(mFormat);
480             mFormat = nullptr;
481             return false;
482         }
483     }
484     else if (mCodecType == kAudioCodecEvs)
485     {
486         // TODO: Integration with libEVS is required.
487         mIsEvsInitialized = true;
488     }
489 
490     return true;
491 }
492 
stopCodec()493 void ImsMediaAudioSource::stopCodec()
494 {
495     if (mCodec != nullptr)
496     {
497         AMediaCodec_stop(mCodec);
498         AMediaCodec_delete(mCodec);
499         mCodec = nullptr;
500     }
501 
502     if (mFormat != nullptr)
503     {
504         AMediaFormat_delete(mFormat);
505         mFormat = nullptr;
506     }
507 }
508 
queueInputBuffer(int16_t * buffer,uint32_t size)509 void ImsMediaAudioSource::queueInputBuffer(int16_t* buffer, uint32_t size)
510 {
511     if (mCodec == nullptr)
512     {
513         return;
514     }
515 
516     ssize_t index = AMediaCodec_dequeueInputBuffer(mCodec, 0);
517 
518     if (index >= 0)
519     {
520         size_t bufferSize = 0;
521         uint8_t* inputBuffer = AMediaCodec_getInputBuffer(mCodec, index, &bufferSize);
522 
523         if (inputBuffer != nullptr)
524         {
525             memcpy(inputBuffer, buffer, size);
526             IMLOGD_PACKET2(IM_PACKET_LOG_AUDIO,
527                     "[queueInputBuffer] queue input buffer index[%d], size[%d]", index, size);
528 
529             auto err = AMediaCodec_queueInputBuffer(
530                     mCodec, index, 0, size, ImsMediaTimer::GetTimeInMicroSeconds(), 0);
531 
532             if (err != AMEDIA_OK)
533             {
534                 IMLOGE1("[queueInputBuffer] Unable to queue input buffers - err[%d]", err);
535             }
536         }
537     }
538 }
539 
dequeueOutputBuffer()540 void ImsMediaAudioSource::dequeueOutputBuffer()
541 {
542     AMediaCodecBufferInfo info;
543     auto index = AMediaCodec_dequeueOutputBuffer(mCodec, &info, CODEC_TIMEOUT_NANO);
544 
545     if (index >= 0)
546     {
547         IMLOGD_PACKET5(IM_PACKET_LOG_AUDIO,
548                 "[dequeueOutputBuffer] index[%d], size[%d], offset[%d], time[%ld], flags[%d]",
549                 index, info.size, info.offset, info.presentationTimeUs, info.flags);
550 
551         if (info.size > 0)
552         {
553             size_t buffCapacity;
554             uint8_t* buf = AMediaCodec_getOutputBuffer(mCodec, index, &buffCapacity);
555 
556             if (mCallback != nullptr)
557             {
558                 mCallback->onDataFrame(buf, info.size, info.presentationTimeUs, info.flags);
559             }
560         }
561 
562         AMediaCodec_releaseOutputBuffer(mCodec, index, false);
563     }
564     else if (index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
565     {
566         IMLOGD0("[dequeueOutputBuffer] Encoder output buffer changed");
567     }
568     else if (index == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
569     {
570         if (mFormat != nullptr)
571         {
572             AMediaFormat_delete(mFormat);
573         }
574 
575         mFormat = AMediaCodec_getOutputFormat(mCodec);
576         IMLOGD1("[dequeueOutputBuffer] Encoder format changed, format[%s]",
577                 AMediaFormat_toString(mFormat));
578     }
579     else if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
580     {
581         IMLOGD0("[dequeueOutputBuffer] no output buffer");
582     }
583     else
584     {
585         IMLOGD1("[dequeueOutputBuffer] unexpected index[%d]", index);
586     }
587 }
588