1 /**
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <ImsMediaDefine.h>
18 #include <VideoRtpPayloadDecoderNode.h>
19 #include <ImsMediaVideoUtil.h>
20 #include <ImsMediaTrace.h>
21 #include <VideoConfig.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <string.h>
25 
26 using namespace android::telephony::imsmedia;
27 
VideoRtpPayloadDecoderNode(BaseSessionCallback * callback)28 VideoRtpPayloadDecoderNode::VideoRtpPayloadDecoderNode(BaseSessionCallback* callback) :
29         BaseNode(callback)
30 {
31     mCodecType = 0;
32     mPayloadMode = 0;
33     mBuffer = nullptr;
34     mSbitfirstByte = 0;
35 }
36 
~VideoRtpPayloadDecoderNode()37 VideoRtpPayloadDecoderNode::~VideoRtpPayloadDecoderNode() {}
38 
GetNodeId()39 kBaseNodeId VideoRtpPayloadDecoderNode::GetNodeId()
40 {
41     return kNodeIdVideoPayloadDecoder;
42 }
43 
Start()44 ImsMediaResult VideoRtpPayloadDecoderNode::Start()
45 {
46     IMLOGD2("[Start] Codec[%d], PayloadMode[%d]", mCodecType, mPayloadMode);
47 
48     mBuffer = reinterpret_cast<uint8_t*>(malloc(MAX_RTP_PAYLOAD_BUFFER_SIZE * sizeof(uint8_t)));
49 
50     if (mBuffer == nullptr)
51     {
52         return RESULT_NO_MEMORY;
53     }
54 
55     mNodeState = kNodeStateRunning;
56     return RESULT_SUCCESS;
57 }
58 
Stop()59 void VideoRtpPayloadDecoderNode::Stop()
60 {
61     if (mBuffer != nullptr)
62     {
63         free(mBuffer);
64         mBuffer = nullptr;
65     }
66 
67     mNodeState = kNodeStateStopped;
68 }
69 
IsRunTime()70 bool VideoRtpPayloadDecoderNode::IsRunTime()
71 {
72     return true;
73 }
74 
IsSourceNode()75 bool VideoRtpPayloadDecoderNode::IsSourceNode()
76 {
77     return false;
78 }
79 
SetConfig(void * config)80 void VideoRtpPayloadDecoderNode::SetConfig(void* config)
81 {
82     if (config == nullptr)
83     {
84         return;
85     }
86 
87     VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(config);
88     mCodecType = pConfig->getCodecType();
89     mPayloadMode = pConfig->getPacketizationMode();
90 }
91 
IsSameConfig(void * config)92 bool VideoRtpPayloadDecoderNode::IsSameConfig(void* config)
93 {
94     if (config == nullptr)
95     {
96         return false;
97     }
98 
99     VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(config);
100     return (mCodecType == pConfig->getCodecType() &&
101             mPayloadMode == pConfig->getPacketizationMode());
102 }
103 
OnDataFromFrontNode(ImsMediaSubType subtype,uint8_t * pData,uint32_t nDataSize,uint32_t nTimeStamp,bool bMark,uint32_t nSeqNum,ImsMediaSubType nDataType,uint32_t arrivalTime)104 void VideoRtpPayloadDecoderNode::OnDataFromFrontNode(ImsMediaSubType subtype, uint8_t* pData,
105         uint32_t nDataSize, uint32_t nTimeStamp, bool bMark, uint32_t nSeqNum,
106         ImsMediaSubType nDataType, uint32_t arrivalTime)
107 {
108     if (subtype == MEDIASUBTYPE_REFRESHED)
109     {
110         SendDataToRearNode(
111                 subtype, nullptr, nDataSize, 0, 0, 0, MEDIASUBTYPE_UNDEFINED, arrivalTime);
112         return;
113     }
114 
115     switch (mCodecType)
116     {
117         case VideoConfig::CODEC_AVC:
118             DecodeAvc(subtype, pData, nDataSize, nTimeStamp, bMark, nSeqNum, arrivalTime);
119             break;
120         case VideoConfig::CODEC_HEVC:
121             DecodeHevc(subtype, pData, nDataSize, nTimeStamp, bMark, nSeqNum, arrivalTime);
122             break;
123         default:
124             IMLOGE1("[OnDataFromFrontNode] invalid codec type[%d]", mCodecType);
125             SendDataToRearNode(MEDIASUBTYPE_UNDEFINED, pData, nDataSize, nTimeStamp, bMark, nSeqNum,
126                     nDataType, arrivalTime);
127             break;
128     }
129 }
130 
DecodeAvc(ImsMediaSubType subtype,uint8_t * pData,uint32_t nDataSize,uint32_t nTimeStamp,bool bMark,uint32_t nSeqNum,uint32_t arrivalTime)131 void VideoRtpPayloadDecoderNode::DecodeAvc(ImsMediaSubType subtype, uint8_t* pData,
132         uint32_t nDataSize, uint32_t nTimeStamp, bool bMark, uint32_t nSeqNum, uint32_t arrivalTime)
133 {
134     if (pData == nullptr || nDataSize == 0 || mBuffer == nullptr)
135     {
136         return;
137     }
138 
139     // check packet type
140     uint8_t bPacketType = pData[0] & 0x1F;
141     ImsMediaSubType eDataType = MEDIASUBTYPE_UNDEFINED;
142 
143     // make start code prefix
144     mBuffer[0] = 0x00;
145     mBuffer[1] = 0x00;
146     mBuffer[2] = 0x00;
147     mBuffer[3] = 0x01;
148 
149     IMLOGD_PACKET2(IM_PACKET_LOG_PH, "[DecodeAvc] [%02X] nDataSize[%d]", pData[0], nDataSize);
150 
151     if (bPacketType >= 1 && bPacketType <= 23)
152     {  // Single NAL unit packet
153         memcpy(mBuffer + 4, pData, nDataSize);
154 
155         if ((bPacketType & 0x1F) == 7 || (bPacketType & 0x1F) == 8)
156         {
157             eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
158         }
159         else if ((bPacketType & 0x1F) == 5)
160         {
161             eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
162         }
163         else if ((bPacketType & 0x1F) == 6)
164         {
165             eDataType = MEDIASUBTYPE_VIDEO_SEI_FRAME;
166         }
167         else
168         {
169             eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
170         }
171 
172         SendDataToRearNode(subtype, mBuffer, nDataSize + 4, nTimeStamp, bMark, nSeqNum, eDataType,
173                 arrivalTime);
174     }
175     else if (bPacketType == 24)
176     {  // STAP-A
177         uint8_t* pCurrData = pData + 1;
178         int32_t nRemainSize = (int32_t)(nDataSize - 1);
179 
180         if (mPayloadMode == kRtpPayloadHeaderModeSingleNalUnit)
181         {
182             IMLOGD_PACKET0(IM_PACKET_LOG_PH, "[DecodeAvc] Warning - single nal unit mode");
183         }
184 
185         bPacketType = pCurrData[2] & 0x1F;
186 
187         if ((bPacketType & 0x1F) == 7 || (bPacketType & 0x1F) == 8)
188         {
189             eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
190         }
191         else if ((bPacketType & 0x1F) == 5)
192         {  // check idr frame
193             eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
194         }
195         else
196         {
197             eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
198         }
199 
200         IMLOGD2("[DecodeAvc] eDataType[%u], nRemainSize[%u]", eDataType, nRemainSize);
201 
202         while (nRemainSize > 2)
203         {
204             // read NAL unit size
205             uint32_t nNALUnitsize = pCurrData[0];
206             nNALUnitsize = (nNALUnitsize << 8) + pCurrData[1];
207             IMLOGD_PACKET1(IM_PACKET_LOG_PH, "[DecodeAvc] STAP-A nNALUnitsize[%d]", nNALUnitsize);
208             pCurrData += 2;
209             nRemainSize -= 2;
210             // Read and Send NAL
211             if (nRemainSize >= (int32_t)nNALUnitsize)
212             {
213                 IMLOGD_PACKET1(IM_PACKET_LOG_PH, "[DecodeAvc] STAP-A [%02X] nNALUnitsize[%d]",
214                         nNALUnitsize);
215                 memcpy(mBuffer + 4, pCurrData, nNALUnitsize);
216                 SendDataToRearNode(subtype, mBuffer, nNALUnitsize + 4, nTimeStamp, bMark, nSeqNum,
217                         eDataType, arrivalTime);
218             }
219             pCurrData += nNALUnitsize;
220             nRemainSize -= nNALUnitsize;
221         }
222     }
223     else if (bPacketType == 28)
224     {  // FU-A
225         uint8_t bFUIndicator;
226         uint8_t bFUHeader;
227         uint8_t bNALUnitType;
228         uint8_t bStartBit;
229         uint8_t bEndBit;
230 
231         if (mPayloadMode == kRtpPayloadHeaderModeSingleNalUnit)
232         {
233             IMLOGW0("[DecodeAvc] Warning - (FU-A, 28) for single nal unit mode");
234         }
235 
236         bFUIndicator = pData[0];
237         bFUHeader = pData[1];
238         bNALUnitType = (bFUIndicator & 0xE0) | (bFUHeader & 0x1F);
239         bStartBit = (bFUHeader >> 7) & 0x01;
240         bEndBit = (bFUHeader >> 6) & 0x01;
241 
242         if ((bNALUnitType & 0x1F) == 7 || (bNALUnitType & 0x1F) == 8)
243         {
244             eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
245         }
246         else if ((bNALUnitType & 0x1F) == 5)
247         {  // check idr frame
248             eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
249         }
250         else
251         {
252             eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
253         }
254 
255         if (bStartBit)
256         {
257             mBuffer[4] = bNALUnitType;  // for video decoder
258             memcpy(mBuffer + 5, pData + 2, nDataSize - 2);
259 
260             if (bEndBit)
261             {
262                 SendDataToRearNode(subtype, mBuffer, nDataSize + 3, nTimeStamp, bMark, nSeqNum,
263                         eDataType, arrivalTime);
264             }
265             else
266             {
267                 SendDataToRearNode(subtype, mBuffer, nDataSize + 3, nTimeStamp, bEndBit, nSeqNum,
268                         eDataType, arrivalTime);
269             }
270         }
271         else
272         {
273             SendDataToRearNode(subtype, pData + 2, nDataSize - 2, nTimeStamp, bEndBit, nSeqNum,
274                     eDataType, arrivalTime);
275         }
276     }
277     else
278     {
279         IMLOGE1("[DecodeAvc] Unsupported payload type[%d]", bPacketType);
280     }
281 }
282 
DecodeHevc(ImsMediaSubType subtype,uint8_t * pData,uint32_t nDataSize,uint32_t nTimeStamp,bool bMark,uint32_t nSeqNum,uint32_t arrivalTime)283 void VideoRtpPayloadDecoderNode::DecodeHevc(ImsMediaSubType subtype, uint8_t* pData,
284         uint32_t nDataSize, uint32_t nTimeStamp, bool bMark, uint32_t nSeqNum, uint32_t arrivalTime)
285 {
286     if (pData == nullptr || nDataSize == 0)
287     {
288         IMLOGE1("[DecodeHevc] INVALID Data, Size[%d]", nDataSize);
289         return;
290     }
291 
292     if (mBuffer == nullptr)
293     {
294         return;
295     }
296 
297     // check packet type
298     uint8_t bPacketType = (pData[0] & 0x7E) >> 1;
299     ImsMediaSubType eDataType = MEDIASUBTYPE_UNDEFINED;
300 
301     // Please check Decoder Start Code...
302     //  make start code prefix
303     mBuffer[0] = 0x00;
304     mBuffer[1] = 0x00;
305     mBuffer[2] = 0x00;
306     mBuffer[3] = 0x01;
307 
308     IMLOGD_PACKET3(IM_PACKET_LOG_PH, "[DecodeHevc] [%02X %02X] nDataSize[%d]", pData[0], pData[1],
309             nDataSize);
310 
311     if (bPacketType <= 40)
312     {  // Single NAL unit packet
313         memcpy(mBuffer + 4, pData, nDataSize);
314 
315         if (bPacketType >= 32 && bPacketType <= 34)
316         {  // 32: VPS, 33: SPS, 34: PPS
317             eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
318         }
319         else if (bPacketType == 19 || bPacketType == 20)
320         {  // IDR
321             eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
322         }
323         else
324         {
325             eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
326         }
327 
328         SendDataToRearNode(subtype, mBuffer, nDataSize + 4, nTimeStamp, bMark, nSeqNum, eDataType,
329                 arrivalTime);
330     }
331     else if (bPacketType == 48)
332     {  // Aggregation packet(AP)
333        // need to implement
334     }
335     else if (bPacketType == 49)
336     {  // FU-A
337         uint8_t bFUIndicator1;
338         uint8_t bFUIndicator2;
339         uint8_t bFUHeader;
340         uint8_t bNALUnitType;
341         uint8_t bStartBit;
342         uint8_t bEndBit;
343 
344         if (mPayloadMode == kRtpPayloadHeaderModeSingleNalUnit)
345         {
346             IMLOGW0("[DecodeHevc] Warning - invalid packet type(FU, 49) for single nal unit mode");
347         }
348 
349         bFUIndicator1 = pData[0];
350         bFUIndicator2 = pData[1];
351         bFUHeader = pData[2];
352         bNALUnitType = (bFUIndicator1 & 0x81) | ((bFUHeader & 0x3F) << 1);
353         bStartBit = (bFUHeader >> 7) & 0x01;
354         bEndBit = (bFUHeader >> 6) & 0x01;
355 
356         uint8_t frameType = (bNALUnitType & 0x7E) >> 1;
357 
358         if (frameType >= 32 && frameType <= 34)
359         {  // 32: VPS, 33: SPS, 34: PPS
360             eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
361         }
362         else if (frameType == 19 || frameType == 20)
363         {  // IDR
364             eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
365         }
366         else
367         {
368             eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
369         }
370 
371         if (bStartBit)
372         {
373             mBuffer[4] = bNALUnitType;  // for decoder
374             mBuffer[5] = bFUIndicator2;
375             memcpy(mBuffer + 6, pData + 3, nDataSize - 3);
376             // exclude FU header
377             SendDataToRearNode(subtype, mBuffer, nDataSize + 3, nTimeStamp, bEndBit, nSeqNum,
378                     eDataType, arrivalTime);
379         }
380         else
381         {  // exclude start code
382             SendDataToRearNode(subtype, pData + 3, nDataSize - 3, nTimeStamp, bEndBit, nSeqNum,
383                     eDataType, arrivalTime);
384         }
385     }
386     else
387     {
388         IMLOGE1("[DecodeHevc] Unsupported payload type[%d]", bPacketType);
389     }
390 }
391