1 /**
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <algorithm>
18 #include <VideoJitterBuffer.h>
19 #include <ImsMediaDataQueue.h>
20 #include <ImsMediaTrace.h>
21 #include <ImsMediaVideoUtil.h>
22 #include <ImsMediaTimer.h>
23 
24 #define DEFAULT_MAX_SAVE_FRAME_NUM          (5)
25 #define DEFAULT_IDR_FRAME_CHECK_INTRERVAL   (3)
26 #define DEFAULT_VIDEO_JITTER_IDR_WAIT_DELAY (200)
27 #define RTCPNACK_SEQ_INCREASE(seq)          ((seq) == 0xffff ? 0 : (seq) + 1)
28 #define RTCPNACK_SEQ_ROUND_COMPARE(a, b)    (((a) > (b)) && ((a) > 0xfff0) && ((b) < 0x000f))
29 #define DEFAULT_PACKET_LOSS_MONITORING_TIME (5)     // sec
30 #define PACKET_LOSS_RATIO                   (2)     // percentage
31 #define BITRATE_ADAPTIVE_RATIO              (0.1f)  // ratio
32 
VideoJitterBuffer()33 VideoJitterBuffer::VideoJitterBuffer() :
34         BaseJitterBuffer()
35 {
36     mCodecType = kVideoCodecAvc;
37     mNewInputData = false;
38     mFramerate = 15;
39     mFrameInterval = 1000 / mFramerate;
40     mMaxSaveFrameNum = DEFAULT_MAX_SAVE_FRAME_NUM;
41     mSavedFrameNum = 0;
42     mMarkedFrameNum = 0;
43     InitLostPktList();
44     mResponseWaitTime = 0;
45     mLastPlayedTime = 0;
46     mNumAddedPacket = 0;
47     mNumLossPacket = 0;
48     mAccumulatedPacketSize = 0;
49     mLastAddedTimestamp = 0;
50     mLastAddedSeqNum = 0;
51     mIDRCheckCnt = DEFAULT_IDR_FRAME_CHECK_INTRERVAL;
52     mFirTimeStamp = 0;
53     mMaxBitrate = 0;
54     mIncomingBitrate = 0;
55     mLossDuration = DEFAULT_PACKET_LOSS_MONITORING_TIME;
56     mLossRateThreshold = 0;
57     mCountTimerExpired = 0;
58     mTimer = nullptr;
59 }
60 
~VideoJitterBuffer()61 VideoJitterBuffer::~VideoJitterBuffer()
62 {
63     InitLostPktList();
64 
65     if (mTimer != nullptr)
66     {
67         IMLOGD0("[~VideoJitterBuffer] stop timer");
68         ImsMediaTimer::TimerStop(mTimer, nullptr);
69         mTimer = nullptr;
70     }
71 }
72 
SetJitterBufferSize(uint32_t nInit,uint32_t nMin,uint32_t nMax)73 void VideoJitterBuffer::SetJitterBufferSize(uint32_t nInit, uint32_t nMin, uint32_t nMax)
74 {
75     BaseJitterBuffer::SetJitterBufferSize(nInit, nMin, nMax);
76     mMaxSaveFrameNum = mMaxJitterBufferSize * 20 / mFrameInterval;
77     mIDRCheckCnt = DEFAULT_VIDEO_JITTER_IDR_WAIT_DELAY / mFrameInterval;
78     mFirTimeStamp = 0;
79     IMLOGD2("[SetJitterBufferSize] maxSaveFrameNum[%u], IDRCheckCnt[%d]", mMaxSaveFrameNum,
80             mIDRCheckCnt);
81 }
82 
SetCodecType(uint32_t type)83 void VideoJitterBuffer::SetCodecType(uint32_t type)
84 {
85     mCodecType = type;
86 }
87 
SetFramerate(uint32_t framerate)88 void VideoJitterBuffer::SetFramerate(uint32_t framerate)
89 {
90     mFramerate = framerate;
91     mFrameInterval = 1000 / mFramerate;
92     IMLOGD2("[SetFramerate] framerate[%u], frameInterval[%d]", mFramerate, mFrameInterval);
93 }
94 
InitLostPktList()95 void VideoJitterBuffer::InitLostPktList()
96 {
97     IMLOGD0("[InitLostPktList]");
98 
99     while (!mLostPktList.empty())
100     {
101         LostPacket* entry = mLostPktList.front();
102 
103         if (entry != nullptr)
104         {
105             delete entry;
106         }
107 
108         mLostPktList.pop_front();
109     }
110 
111     mLostPktList.clear();
112 }
113 
Reset()114 void VideoJitterBuffer::Reset()
115 {
116     BaseJitterBuffer::Reset();
117     mSavedFrameNum = 0;
118     mMarkedFrameNum = 0;
119     mLastPlayedTime = 0;
120     mNumAddedPacket = 0;
121     mNumLossPacket = 0;
122     mAccumulatedPacketSize = 0;
123     mLastPlayedTimestamp = 0;
124     mLastAddedTimestamp = 0;
125     mLastAddedSeqNum = 0;
126     InitLostPktList();
127     mResponseWaitTime = 0;
128 }
129 
StartTimer(uint32_t time,uint32_t rate)130 void VideoJitterBuffer::StartTimer(uint32_t time, uint32_t rate)
131 {
132     IMLOGD2("[StartTimer] time[%d], rate[%u]", time, rate);
133 
134     if (mTimer == nullptr)
135     {
136         IMLOGD0("[StartTimer] timer start");
137         mTimer = ImsMediaTimer::TimerStart(1000, true, OnTimer, this);
138     }
139 }
140 
StopTimer()141 void VideoJitterBuffer::StopTimer()
142 {
143     if (mTimer != nullptr)
144     {
145         IMLOGD0("[StopTimer] stop timer");
146 
147         if (ImsMediaTimer::TimerStop(mTimer, nullptr) == false)
148         {
149             IMLOGE0("[StopTimer] stop timer error");
150         }
151 
152         mTimer = nullptr;
153     }
154 }
155 
SetResponseWaitTime(const uint32_t time)156 void VideoJitterBuffer::SetResponseWaitTime(const uint32_t time)
157 {
158     IMLOGD1("[SetResponseWaitTime] time[%u]", time);
159     mResponseWaitTime = time;
160 }
161 
Add(ImsMediaSubType subtype,uint8_t * pbBuffer,uint32_t nBufferSize,uint32_t nTimestamp,bool bMark,uint32_t nSeqNum,ImsMediaSubType eDataType,uint32_t arrivalTime)162 void VideoJitterBuffer::Add(ImsMediaSubType subtype, uint8_t* pbBuffer, uint32_t nBufferSize,
163         uint32_t nTimestamp, bool bMark, uint32_t nSeqNum, ImsMediaSubType eDataType,
164         uint32_t arrivalTime)
165 {
166     if (subtype == MEDIASUBTYPE_REFRESHED)
167     {
168         // nBufferSize is ssrc value
169         mSsrc = nBufferSize;
170         IMLOGI1("[Add] ssrc[%u]", mSsrc);
171         return;
172     }
173 
174     DataEntry currEntry = DataEntry();
175     currEntry.pbBuffer = pbBuffer;
176     currEntry.nBufferSize = nBufferSize;
177     currEntry.nTimestamp = nTimestamp;
178     currEntry.bMark = bMark;
179     currEntry.nSeqNum = nSeqNum;
180     currEntry.bHeader = CheckHeader(pbBuffer);
181     currEntry.bValid = false;
182     currEntry.subtype = subtype;
183     currEntry.eDataType = eDataType;
184     currEntry.arrivalTime = arrivalTime;
185 
186     std::lock_guard<std::mutex> guard(mMutex);
187 
188     if (currEntry.bHeader == true)
189     {
190         if (eDataType == MEDIASUBTYPE_VIDEO_CONFIGSTRING)
191         {
192             currEntry.bMark = true;
193         }
194 
195         if (eDataType == MEDIASUBTYPE_VIDEO_SEI_FRAME)
196         {
197             IMLOGD1("[Add] ignore SEI frame seq[%d]", nSeqNum);
198             return;
199         }
200     }
201 
202     IMLOGD_PACKET6(IM_PACKET_LOG_VIDEO_JITTER,
203             "[Add] eDataType[%u], Seq[%u], Mark[%u], Header[%u], TS[%u], Size[%u]",
204             currEntry.eDataType, nSeqNum, bMark, currEntry.bHeader, nTimestamp, nBufferSize);
205 
206     // very old frame, don't add this frame, nothing to do
207     if ((!USHORT_SEQ_ROUND_COMPARE(nSeqNum, mLastPlayedSeqNum)) && (mLastPlayedTime != 0))
208     {
209         IMLOGE2("[Add] Receive very old frame!!! Drop Packet. Seq[%u], LastPlayedSeqNum[%u]",
210                 nSeqNum, mLastPlayedSeqNum);
211     }
212     else if (mDataQueue.GetCount() == 0)
213     {  // jitter buffer is empty
214         mDataQueue.Add(&currEntry);
215         mNumAddedPacket++;
216         mAccumulatedPacketSize += nBufferSize;
217         IMLOGD_PACKET4(IM_PACKET_LOG_VIDEO_JITTER,
218                 "[Add] queue[%u] Seq[%u], LastPlayedSeqNum[%u], LastAddedTimestamp[%u]",
219                 mDataQueue.GetCount(), nSeqNum, mLastPlayedSeqNum, mLastAddedTimestamp);
220         mLastAddedTimestamp = nTimestamp;
221         mLastAddedSeqNum = nSeqNum;
222     }
223     else
224     {
225         DataEntry* pEntry;
226         mDataQueue.GetLast(&pEntry);
227 
228         if (pEntry == nullptr)
229         {
230             return;
231         }
232 
233         if (USHORT_SEQ_ROUND_COMPARE(nSeqNum, pEntry->nSeqNum))
234         {
235             // current data is the latest data
236             if (nSeqNum == pEntry->nSeqNum && nBufferSize == pEntry->nBufferSize)
237             {
238                 IMLOGD1("[Add] drop duplicate Seq[%u]", nSeqNum);
239                 return;
240             }
241 
242             mDataQueue.Add(&currEntry);
243             mNumAddedPacket++;
244             mAccumulatedPacketSize += nBufferSize;
245             IMLOGD_PACKET4(IM_PACKET_LOG_VIDEO_JITTER,
246                     "[Add] queue[%u] Seq[%u], LastPlayedSeqNum[%u], LastAddedTimestamp[%u]",
247                     mDataQueue.GetCount(), nSeqNum, mLastPlayedSeqNum, mLastAddedTimestamp);
248         }
249         else
250         {
251             // find the position of current data and insert current data to the correct position
252             uint32_t i;
253             mDataQueue.SetReadPosFirst();
254 
255             for (i = 0; mDataQueue.GetNext(&pEntry); i++)
256             {
257                 if (nSeqNum == pEntry->nSeqNum && nBufferSize == pEntry->nBufferSize)
258                 {
259                     IMLOGD1("[Add] drop duplicate Seq[%u]", nSeqNum);
260                     return;
261                 }
262 
263                 if (!USHORT_SEQ_ROUND_COMPARE(nSeqNum, pEntry->nSeqNum))
264                 {
265                     mDataQueue.InsertAt(i, &currEntry);
266                     break;
267                 }
268             }
269         }
270 
271         // Remove mark of packets with same Timestamp and less SeqNum
272         mDataQueue.SetReadPosFirst();
273 
274         while (mDataQueue.GetNext(&pEntry))
275         {
276             if (pEntry->nSeqNum != nSeqNum && pEntry->nTimestamp == nTimestamp &&
277                     pEntry->bMark == true && pEntry->eDataType != MEDIASUBTYPE_VIDEO_CONFIGSTRING &&
278                     USHORT_SEQ_ROUND_COMPARE(nSeqNum, pEntry->nSeqNum))
279             {
280                 IMLOGD_PACKET6(IM_PACKET_LOG_VIDEO_JITTER,
281                         "[Add] Remove marker of Seq/TS/Mark[%u/%u/%u], pEntry "
282                         "Seq/TS/Mark[%u/%u/%u]",
283                         pEntry->nSeqNum, pEntry->nTimestamp, pEntry->bMark, currEntry.nSeqNum,
284                         currEntry.nTimestamp, currEntry.bMark);
285                 pEntry->bMark = false;
286                 continue;
287             }
288             else if (!USHORT_SEQ_ROUND_COMPARE(nSeqNum, pEntry->nSeqNum))
289             {
290                 break;
291             }
292         }
293 
294         mLastAddedTimestamp = nTimestamp;
295         mLastAddedSeqNum = nSeqNum;
296     }
297 
298     mNewInputData = true;
299 }
300 
Get(ImsMediaSubType * subtype,uint8_t ** ppData,uint32_t * pnDataSize,uint32_t * pnTimestamp,bool * pbMark,uint32_t * pnSeqNum,uint32_t currentTime,ImsMediaSubType *)301 bool VideoJitterBuffer::Get(ImsMediaSubType* subtype, uint8_t** ppData, uint32_t* pnDataSize,
302         uint32_t* pnTimestamp, bool* pbMark, uint32_t* pnSeqNum, uint32_t currentTime,
303         ImsMediaSubType* /*pDataType*/)
304 {
305     DataEntry* pEntry = nullptr;
306     bool bValidPacket = false;
307     std::lock_guard<std::mutex> guard(mMutex);
308 
309     // check validation
310     if (mNewInputData)
311     {
312         mSavedFrameNum = 0;
313         mMarkedFrameNum = 0;
314         bool bFoundHeader = false;
315         uint16_t nLastRecvSeq = 0;  // for NACK generation
316         mDataQueue.SetReadPosFirst();
317 
318         uint32_t nIndex = 0;
319         uint32_t nHeaderIndex = 0;
320         uint16_t nHeaderSeq = 0;
321         uint32_t nHeaderTimestamp = 0;
322         uint32_t nLastTimeStamp = 0;
323         uint32_t nSavedIdrFrame = 0;
324         for (nIndex = 0; mDataQueue.GetNext(&pEntry); nIndex++)
325         {
326             IMLOGD_PACKET8(IM_PACKET_LOG_VIDEO_JITTER,
327                     "[Get] queue[%u/%u] bValid[%u], Seq[%u], Mark[%u], Header[%u], TS[%u], "
328                     "Size[%u]",
329                     nIndex, mDataQueue.GetCount(), pEntry->bValid, pEntry->nSeqNum, pEntry->bMark,
330                     pEntry->bHeader, pEntry->nTimestamp, pEntry->nBufferSize);
331 
332             if (mResponseWaitTime > 0 && nLastTimeStamp != 0)
333             {
334                 CheckPacketLoss(pEntry->nSeqNum, nLastRecvSeq);
335             }
336 
337             nLastRecvSeq = pEntry->nSeqNum;
338 
339             if (pEntry->nTimestamp != nLastTimeStamp || nLastTimeStamp == 0)
340             {
341                 if (pEntry->eDataType != MEDIASUBTYPE_VIDEO_CONFIGSTRING)
342                 {
343                     if (pEntry->eDataType == MEDIASUBTYPE_VIDEO_IDR_FRAME)
344                     {
345                         nSavedIdrFrame++;
346                     }
347 
348                     mSavedFrameNum++;
349                     nLastTimeStamp = pEntry->nTimestamp;
350                 }
351             }
352 
353             if (pEntry->bMark)
354             {
355                 mMarkedFrameNum++;
356             }
357 
358             if (nSavedIdrFrame == mIDRCheckCnt && pEntry->eDataType == MEDIASUBTYPE_VIDEO_IDR_FRAME)
359             {
360                 /** TODO: improve this logic later */
361                 CheckValidIDR(pEntry);
362             }
363 
364             IMLOGD_PACKET3(IM_PACKET_LOG_VIDEO_JITTER,
365                     "[Get] SavedFrameNum[%u], mMarkedFrameNum[%u], nLastTimeStamp[%u]",
366                     mSavedFrameNum, mMarkedFrameNum, nLastTimeStamp);
367 
368             if (pEntry->bHeader)
369             {
370                 if (pEntry->bMark)
371                 {
372                     pEntry->bValid = true;
373 
374                     if (pEntry->nTimestamp > nHeaderTimestamp)
375                     {
376                         bFoundHeader = false;  // make Header false only for new frame
377                     }
378                 }
379                 else
380                 {
381                     // Check Header with new timestamp. Otherwise(duplicated header with
382                     // same timestamp), do not update Header Information
383                     if (bFoundHeader == false || pEntry->nTimestamp < nHeaderTimestamp ||
384                             nHeaderTimestamp == 0)
385                     {
386                         nHeaderTimestamp = pEntry->nTimestamp;
387                         nHeaderIndex = nIndex;
388                         nHeaderSeq = pEntry->nSeqNum;
389                         bFoundHeader = true;
390 
391                         IMLOGD_PACKET3(IM_PACKET_LOG_VIDEO_JITTER,
392                                 "[Get] New Header Found at [%u] - Seq[%u], Timestamp[%u]",
393                                 nHeaderIndex, nHeaderSeq, nHeaderTimestamp);
394                     }
395                 }
396             }
397 
398             if (bFoundHeader)
399             {
400                 IMLOGD_PACKET4(IM_PACKET_LOG_VIDEO_JITTER,
401                         "[Get] bFoundHeader[%u] - Check Valid of Seq[%u ~ %u], "
402                         "nHeaderTimestamp[%u]",
403                         bFoundHeader, nHeaderSeq, pEntry->nSeqNum, nHeaderTimestamp);
404 
405                 if (pEntry->bMark)
406                 {
407                     uint32_t nMarkIndex = nIndex;
408                     uint16_t nMarkSeq = pEntry->nSeqNum;
409 
410                     // make sure type of 16bit unsigned int sequence number
411                     if (nMarkIndex - nHeaderIndex == nMarkSeq - nHeaderSeq)
412                     {
413                         uint32_t i;
414 
415                         for (i = nHeaderIndex; i <= nMarkIndex; i++)
416                         {
417                             DataEntry* pValidEntry;
418                             mDataQueue.GetAt(i, &pValidEntry);
419 
420                             if (pValidEntry == nullptr)
421                             {
422                                 return false;
423                             }
424 
425                             pValidEntry->bValid = true;
426 
427                             IMLOGD_PACKET3(IM_PACKET_LOG_VIDEO_JITTER,
428                                     "[Get] Validation Check for Seq[%u] true :: nHeaderIndex[%u] "
429                                     "to nMarkIndex[%u]",
430                                     pValidEntry->nSeqNum, nHeaderIndex, nMarkIndex);
431                         }
432                     }
433 
434                     bFoundHeader = false;
435                 }
436             }
437         }
438 
439         if (mSavedFrameNum > mMaxSaveFrameNum)
440         {
441             IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO_JITTER,
442                     "[Get] Delete - SavedFrameNum[%u], nMaxFrameNum[%u]", mSavedFrameNum,
443                     mMaxSaveFrameNum);
444 
445             mDataQueue.Get(&pEntry);
446 
447             if (pEntry == nullptr)
448                 return false;
449 
450             if (pEntry->bValid == false)
451             {
452                 uint32_t nDeleteTimeStamp = pEntry->nTimestamp;
453                 uint32_t nDeleteSeqNum = pEntry->nSeqNum;
454 
455                 while (nDeleteTimeStamp == pEntry->nTimestamp)
456                 {
457                     IMLOGD_PACKET7(IM_PACKET_LOG_VIDEO_JITTER,
458                             "[Get] Delete - Seq[%u], Count[%u], bValid[%u], eDataType[%u], "
459                             "bHeader[%u], TimeStamp[%u], Size[%u]",
460                             pEntry->nSeqNum, mDataQueue.GetCount(), pEntry->bValid,
461                             pEntry->eDataType, pEntry->bHeader, pEntry->nTimestamp,
462                             pEntry->nBufferSize);
463 
464                     nDeleteSeqNum = pEntry->nSeqNum;
465                     mDataQueue.Delete();
466                     mDataQueue.Get(&pEntry);  // next packet
467 
468                     if (pEntry == nullptr)
469                     {
470                         break;
471                     }
472                 }
473 
474                 mSavedFrameNum--;
475 
476                 // remove the packets from NACK / PLI checkList
477                 if (mLostPktList.size() > 0)
478                 {
479                     RemovePacketFromLostList(nDeleteSeqNum, true);
480                 }
481             }
482         }
483 
484         if (mSavedFrameNum >= mMaxSaveFrameNum)
485         {
486             mDataQueue.Get(&pEntry);
487 
488             if (pEntry == nullptr)
489             {
490                 return false;
491             }
492 
493             mLastPlayedSeqNum = pEntry->nSeqNum - 1;
494         }
495 
496         mNewInputData = false;
497     }
498 
499     if (mSavedFrameNum >= (mMaxSaveFrameNum / 2) && mDataQueue.Get(&pEntry) == true &&
500             pEntry->bValid && (mLastPlayedSeqNum == 0 || pEntry->nSeqNum <= mLastPlayedSeqNum + 1))
501     {
502         IMLOGD_PACKET4(IM_PACKET_LOG_VIDEO_JITTER,
503                 "[Get] bValid[%u], LastPlayedTS[%u], Seq[%u], LastPlayedSeq[%u]", pEntry->bValid,
504                 mLastPlayedTimestamp, pEntry->nSeqNum, mLastPlayedSeqNum);
505 
506         uint32_t nCurrTime = currentTime;
507 
508         if (mLastPlayedTimestamp == 0 || mLastPlayedTime == 0 ||
509                 pEntry->nTimestamp == mLastPlayedTimestamp)
510         {
511             bValidPacket = true;
512         }
513         else
514         {
515             uint32_t nTimeDiff = nCurrTime - mLastPlayedTime;
516             uint32_t nThreshold;
517 
518             // this is optimized in 15fps
519             if (mSavedFrameNum <= 2 && mMarkedFrameNum <= 1)
520             {
521                 nThreshold = nTimeDiff;
522             }
523             else if (mSavedFrameNum <= 3)
524             {
525                 nThreshold = nTimeDiff / 2;
526             }
527             else
528             {
529                 nThreshold = nTimeDiff / ((mSavedFrameNum + 2) / 2);
530             }
531 
532             if (nThreshold > 66)
533             {
534                 nThreshold = 66;  // 15fps
535             }
536 
537             if (nTimeDiff >= nThreshold || (mLastPlayedTimestamp > pEntry->nTimestamp) ||
538                     (mLastPlayedTime > nCurrTime))
539             {
540                 bValidPacket = true;
541             }
542             else
543             {
544                 IMLOGD_PACKET3(IM_PACKET_LOG_VIDEO_JITTER,
545                         "[Get] bValidPacket[%u], nTimeDiff[%u], nThreshold[%u]", bValidPacket,
546                         nTimeDiff, nThreshold);
547 
548                 bValidPacket = false;
549             }
550         }
551 
552         if (bValidPacket)
553         {
554             mLastPlayedTimestamp = pEntry->nTimestamp;
555             mLastPlayedTime = nCurrTime;
556         }
557 
558         mNewInputData = true;
559     }
560     else
561     {
562         bValidPacket = false;
563     }
564 
565     if (bValidPacket && pEntry != nullptr)
566     {
567         if (subtype)
568             *subtype = pEntry->subtype;
569         if (ppData)
570             *ppData = pEntry->pbBuffer;
571         if (pnDataSize)
572             *pnDataSize = pEntry->nBufferSize;
573         if (pnTimestamp)
574             *pnTimestamp = pEntry->nTimestamp;
575         if (pbMark)
576             *pbMark = pEntry->bMark;
577         if (pnSeqNum)
578             *pnSeqNum = pEntry->nSeqNum;
579 
580         mLastPlayedSeqNum = pEntry->nSeqNum;
581 
582         IMLOGD_PACKET5(IM_PACKET_LOG_VIDEO_JITTER,
583                 "[Get] OK - seq=%u, TS=%u, size=%u, dataType=%d, delay=%d", pEntry->nSeqNum,
584                 pEntry->nTimestamp, pEntry->nBufferSize, pEntry->eDataType,
585                 currentTime - pEntry->arrivalTime);
586         return true;
587     }
588     else
589     {
590         if (subtype)
591             *subtype = MEDIASUBTYPE_UNDEFINED;
592         if (ppData)
593             *ppData = nullptr;
594         if (pnDataSize)
595             *pnDataSize = 0;
596         if (pnTimestamp)
597             *pnTimestamp = 0;
598         if (pbMark)
599             *pbMark = false;
600         if (pnSeqNum)
601             *pnSeqNum = 0;
602         IMLOGD_PACKET3(IM_PACKET_LOG_VIDEO_JITTER,
603                 "[Get] fail - saved frames=%u, marked frames=%u, queue=%u", mSavedFrameNum,
604                 mMarkedFrameNum, mDataQueue.GetCount());
605         return false;
606     }
607 }
608 
CheckValidIDR(DataEntry * pIDREntry)609 void VideoJitterBuffer::CheckValidIDR(DataEntry* pIDREntry)
610 {
611     if (pIDREntry == nullptr)
612     {
613         return;
614     }
615 
616     if (pIDREntry->bValid == false)
617     {
618         IMLOGD2("[CheckValidIDR] mFirTimeStamp[%u] -> nTimestamp[%u]", mFirTimeStamp,
619                 pIDREntry->nTimestamp);
620 
621         if (pIDREntry->nTimestamp == mFirTimeStamp)
622         {
623             return;
624         }
625 
626         RequestToSendPictureLost(kPsfbFir);
627         mFirTimeStamp = pIDREntry->nTimestamp;
628         return;
629     }
630 }
631 
Delete()632 void VideoJitterBuffer::Delete()
633 {
634     DataEntry* pEntry;
635     std::lock_guard<std::mutex> guard(mMutex);
636     mDataQueue.Get(&pEntry);
637 
638     if (pEntry == nullptr)
639     {
640         return;
641     }
642 
643     IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO_JITTER, "[Delete] Seq[%u] / BufferCount[%u]",
644             pEntry->nSeqNum, mDataQueue.GetCount());
645     mLastPlayedSeqNum = pEntry->nSeqNum;
646     mDataQueue.Delete();
647     mNewInputData = true;
648 
649     if (mLostPktList.size() > 0)
650     {
651         RemovePacketFromLostList(mLastPlayedSeqNum, true);
652     }
653 }
654 
GetCount()655 uint32_t VideoJitterBuffer::GetCount()
656 {
657     return mDataQueue.GetCount();
658 }
659 
CheckHeader(uint8_t * pbBuffer)660 bool VideoJitterBuffer::CheckHeader(uint8_t* pbBuffer)
661 {
662     if (pbBuffer == nullptr)
663     {
664         return false;
665     }
666 
667     // check start code
668     if ((pbBuffer[0] == 0x00 && pbBuffer[1] == 0x00 && pbBuffer[2] == 0x00) ||
669             (pbBuffer[0] == 0x00 && pbBuffer[1] == 0x00 && pbBuffer[2] == 0x01))
670     {
671         return true;
672     }
673     else
674     {
675         return false;
676     }
677 }
678 
RemovePacketFromLostList(uint16_t seqNum,bool bRemoveOldPacket)679 void VideoJitterBuffer::RemovePacketFromLostList(uint16_t seqNum, bool bRemoveOldPacket)
680 {
681     LostPacket* pEntry = nullptr;
682     std::list<LostPacket*>::iterator it = mLostPktList.begin();
683 
684     while (it != mLostPktList.end())
685     {
686         pEntry = *it;
687 
688         if (pEntry == nullptr)
689         {
690             mLostPktList.erase(it++);
691             continue;
692         }
693 
694         if (bRemoveOldPacket && pEntry->seqNum < seqNum)
695         {
696             IMLOGD_PACKET3(IM_PACKET_LOG_VIDEO_JITTER,
697                     "[RemovePacketFromLostList] delete lost seq[%u], target seq[%u], "
698                     "bRemoveOldPacket[%u]",
699                     pEntry->seqNum, seqNum, bRemoveOldPacket);
700 
701             it = mLostPktList.erase(it);
702             delete pEntry;
703         }
704         else if (pEntry->seqNum == seqNum)
705         {
706             IMLOGD_PACKET1(IM_PACKET_LOG_VIDEO_JITTER,
707                     "[RemovePacketFromLostList] remove lost seq[%u]", pEntry->seqNum);
708 
709             it = mLostPktList.erase(it);
710             delete pEntry;
711         }
712         else
713         {
714             it++;
715         }
716     }
717 }
718 
CheckPacketLoss(uint16_t seqNum,uint16_t nLastRecvPkt)719 void VideoJitterBuffer::CheckPacketLoss(uint16_t seqNum, uint16_t nLastRecvPkt)
720 {
721     if (mLostPktList.size() > 0)
722     {
723         RemovePacketFromLostList(seqNum);
724     }
725 
726     // normal case : no packet loss
727     if (RTCPNACK_SEQ_INCREASE(nLastRecvPkt) == seqNum)
728     {
729         return;
730     }
731 
732     if (nLastRecvPkt == seqNum)
733     {
734         return;  // same packet should be removed in STAP-A type case.
735     }
736 
737     // the first lost packet.
738     uint16_t PID = RTCPNACK_SEQ_INCREASE(nLastRecvPkt);
739 
740     // the number of lost packets
741     uint16_t nLossGap = RTCPNACK_SEQ_ROUND_COMPARE(nLastRecvPkt, seqNum)
742             ? ((0xffff - nLastRecvPkt) + seqNum)
743             : (seqNum - PID);
744 
745     if (nLossGap > 0x000f)
746     {
747         nLossGap = 0x000f;
748     }
749 
750     uint16_t countNackPacketNum = 0;
751     uint16_t countSecondNack = 0;
752     uint16_t nPLIPkt = 0;
753     bool bPLIPkt = false;
754     bool bSentPLI = false;
755 
756     for (int32_t index = 0; index < nLossGap; index++)
757     {
758         if (UpdateLostPacketList(PID + index, &countSecondNack, &nPLIPkt, &bPLIPkt))
759         {
760             countNackPacketNum++;
761         }
762 
763         // request PLI Message
764         if (bPLIPkt && !bSentPLI)
765         {
766             IMLOGD1("[CheckPacketLoss] nPLI pkt[%u]", nPLIPkt);
767             RequestToSendPictureLost(kPsfbPli);
768             bSentPLI = true;
769         }
770     }
771 
772     if (nPLIPkt > PID)
773     {
774         PID = nPLIPkt + 1;
775     }
776 
777     // return if PID is same as the current seq.
778     if (PID == seqNum)
779     {
780         return;
781     }
782 
783     // request NACK Message
784     if (countNackPacketNum > 0)
785     {
786         RequestSendNack(countNackPacketNum, PID, countSecondNack);
787     }
788 }
789 
UpdateLostPacketList(uint16_t lostSeq,uint16_t * countSecondNack,uint16_t * nPLIPkt,bool * bPLIPkt)790 bool VideoJitterBuffer::UpdateLostPacketList(
791         uint16_t lostSeq, uint16_t* countSecondNack, uint16_t* nPLIPkt, bool* bPLIPkt)
792 {
793     LostPacket* foundLostPacket = nullptr;
794     auto result = std::find_if(mLostPktList.begin(), mLostPktList.end(),
795             [lostSeq, &foundLostPacket](LostPacket* entry)
796             {
797                 foundLostPacket = entry;
798                 return (entry->seqNum == lostSeq);
799             });
800 
801     if (result != mLostPktList.end() && foundLostPacket != nullptr)
802     {
803         return UpdateNackStatus(foundLostPacket, lostSeq, countSecondNack, nPLIPkt, bPLIPkt);
804     }
805 
806     IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO_JITTER,
807             "[UpdateLostPacketList] add lost seq[%u], queue size[%d]", lostSeq,
808             mLostPktList.size());
809 
810     LostPacket* entry =
811             new LostPacket(lostSeq, ImsMediaTimer::GetTimeInMilliSeconds(), kRequestSendNackNone);
812     mLostPktList.push_back(entry);
813     mNumLossPacket++;
814     return false;
815 }
816 
UpdateNackStatus(LostPacket * pEntry,uint16_t lostSeq,uint16_t * countSecondNack,uint16_t * nPLIPkt,bool * bPLIPkt)817 bool VideoJitterBuffer::UpdateNackStatus(LostPacket* pEntry, uint16_t lostSeq,
818         uint16_t* countSecondNack, uint16_t* nPLIPkt, bool* bPLIPkt)
819 {
820     /**
821      * Send initial NACK if there is error in decoding frame due to packet loss
822      */
823     if (pEntry->option == kRequestSendNackNone)
824     {
825         if ((ImsMediaTimer::GetTimeInMilliSeconds() - pEntry->markedTime) < mFrameInterval)
826         {
827             return false;
828         }
829 
830         pEntry->markedTime = ImsMediaTimer::GetTimeInMilliSeconds();
831         pEntry->option = kRequestInitialNack;
832         IMLOGD_PACKET1(
833                 IM_PACKET_LOG_VIDEO_JITTER, "[UpdateNackStatus] initial NACK, seq[%u]", lostSeq);
834         return true;
835     }
836 
837     if ((ImsMediaTimer::GetTimeInMilliSeconds() - pEntry->markedTime) < mResponseWaitTime)
838     {
839         return false;
840     }
841 
842     /**
843      * Send Second NACK if there is first packet still not arrived within RWT duration
844      */
845     if (pEntry->option == kRequestInitialNack)
846     {
847         (*countSecondNack)++;
848         pEntry->markedTime = ImsMediaTimer::GetTimeInMilliSeconds();
849         pEntry->option = kRequestSecondNack;
850         IMLOGD_PACKET1(
851                 IM_PACKET_LOG_VIDEO_JITTER, "[UpdateNackStatus] second NACK, seq[%u]", lostSeq);
852         return true;
853     }
854     else if (pEntry->option == kRequestSecondNack)
855     {
856         /**
857          * Send PLI if the recovery picture does not arrived within two RWT duration
858          */
859         *nPLIPkt = lostSeq;
860         *bPLIPkt = true;
861         pEntry->markedTime = ImsMediaTimer::GetTimeInMilliSeconds();
862         pEntry->option = kRequestPli;
863         IMLOGD_PACKET1(
864                 IM_PACKET_LOG_VIDEO_JITTER, "[UpdateNackStatus] request PLI seq[%u]", lostSeq);
865     }
866     else if (pEntry->option == kRequestPli)
867     {
868         *nPLIPkt = lostSeq;
869         pEntry->markedTime = ImsMediaTimer::GetTimeInMilliSeconds();
870     }
871 
872     return false;
873 }
874 
RequestSendNack(uint16_t nLossGap,uint16_t PID,uint16_t countSecondNack,bool bNACK)875 void VideoJitterBuffer::RequestSendNack(
876         uint16_t nLossGap, uint16_t PID, uint16_t countSecondNack, bool bNACK)
877 {
878     uint32_t BLP = 0x0;  // bitmask of following lost packets
879 
880     if (nLossGap > 1)
881     {
882         BLP = (0x01 << (nLossGap - 1)) - 1;
883     }
884 
885     InternalRequestEventParam* pParam = new InternalRequestEventParam(
886             kRequestVideoSendNack, NackParams(PID, BLP, countSecondNack, bNACK));
887 
888     IMLOGD0("[RequestSendNack]");
889     mCallback->SendEvent(kRequestVideoSendNack, reinterpret_cast<uint64_t>(pParam));
890 }
891 
RequestToSendPictureLost(uint32_t type)892 void VideoJitterBuffer::RequestToSendPictureLost(uint32_t type)
893 {
894     IMLOGD0("[RequestToSendPictureLost]");
895     InternalRequestEventParam* pParam =
896             new InternalRequestEventParam(kRequestVideoSendPictureLost, type);
897     mCallback->SendEvent(kRequestVideoSendPictureLost, reinterpret_cast<uint64_t>(pParam));
898 }
899 
RequestToSendTmmbr(uint32_t bitrate)900 void VideoJitterBuffer::RequestToSendTmmbr(uint32_t bitrate)
901 {
902     IMLOGD2("[RequestToSendTmmbr] ssrc[%x], bitrate[%d]", mSsrc, bitrate);
903     uint32_t exp = 0;
904     uint32_t mantissa = 0;
905     ImsMediaVideoUtil::ConvertBitrateToPower(bitrate, exp, mantissa);
906 
907     InternalRequestEventParam* pParam =
908             new InternalRequestEventParam(kRtpFbTmmbr, TmmbrParams(mSsrc, exp, mantissa, 40));
909     mCallback->SendEvent(kRequestVideoSendTmmbr, reinterpret_cast<uint64_t>(pParam));
910 }
911 
OnTimer(hTimerHandler hTimer,void * pUserData)912 void VideoJitterBuffer::OnTimer(hTimerHandler hTimer, void* pUserData)
913 {
914     (void)hTimer;
915     VideoJitterBuffer* jitter = reinterpret_cast<VideoJitterBuffer*>(pUserData);
916 
917     if (jitter != nullptr)
918     {
919         jitter->ProcessTimer();
920     }
921 }
922 
ProcessTimer()923 void VideoJitterBuffer::ProcessTimer()
924 {
925     mCountTimerExpired++;
926 
927     /** calculate bitrate */
928     mIncomingBitrate = mAccumulatedPacketSize * 8;
929 
930     if (mIncomingBitrate > mMaxBitrate)
931     {
932         mMaxBitrate = mIncomingBitrate;
933     }
934 
935     IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO_JITTER, "[ProcessTimer] bitrate[%d] maxBitrate[%d]",
936             mIncomingBitrate, mMaxBitrate);
937 
938     mAccumulatedPacketSize = 0;
939 
940     /** calculate loss rate in every seconds */
941     double lossRate =
942             static_cast<double>(mNumLossPacket) * 100 / (mNumAddedPacket + mNumLossPacket);
943 
944     IMLOGD3("[ProcessTimer] rate[%lf], lossPackets[%d], addedPackets[%d]", lossRate, mNumLossPacket,
945             mNumAddedPacket);
946 
947     if (mIncomingBitrate > 0)
948     {
949         CheckBitrateAdaptation(lossRate);
950     }
951 
952     /** compare loss rate with threshold */
953     if (mLossDuration != 0 && mCountTimerExpired >= mLossDuration)
954     {
955         if (lossRate >= mLossRateThreshold && mLossRateThreshold != 0)
956         {
957             /** TODO: request send loss indication event */
958         }
959 
960         mNumLossPacket = 0;
961         mNumAddedPacket = 0;
962         mCountTimerExpired = 0;
963     }
964 }
965 
CheckBitrateAdaptation(double lossRate)966 void VideoJitterBuffer::CheckBitrateAdaptation(double lossRate)
967 {
968     if (mCountTimerExpired % DEFAULT_PACKET_LOSS_MONITORING_TIME == 0)
969     {
970         if (lossRate >= PACKET_LOSS_RATIO)
971         {
972             RequestToSendTmmbr(mIncomingBitrate);
973         }
974         else if (lossRate == 0)
975         {
976             mRequestedBitrate = mIncomingBitrate + mIncomingBitrate * BITRATE_ADAPTIVE_RATIO;
977 
978             if (mRequestedBitrate > mMaxBitrate)
979             {
980                 mRequestedBitrate = mMaxBitrate;
981             }
982 
983             RequestToSendTmmbr(mRequestedBitrate);
984         }
985     }
986 }