1 /**
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <RtpEncoderNode.h>
18 #include <ImsMediaTimer.h>
19 #include <ImsMediaTrace.h>
20 #include <ImsMediaVideoUtil.h>
21 #include <AudioConfig.h>
22 #include <VideoConfig.h>
23 #include <TextConfig.h>
24 #include <string.h>
25 
RtpEncoderNode(BaseSessionCallback * callback)26 RtpEncoderNode::RtpEncoderNode(BaseSessionCallback* callback) :
27         BaseNode(callback)
28 {
29     mRtpSession = nullptr;
30     mDtmfMode = false;
31     mMark = false;
32     mPrevTimestamp = 0;
33     mSamplingRate = 0;
34     mRtpPayloadTx = 0;
35     mRtpPayloadRx = 0;
36     mRtpTxDtmfPayload = 0;
37     mRtpRxDtmfPayload = 0;
38     mDtmfSamplingRate = 0;
39     mDtmfTimestamp = 0;
40     mCvoValue = CVO_DEFINE_NONE;
41     mRedundantLevel = 0;
42     mRedundantPayload = 0;
43 }
44 
~RtpEncoderNode()45 RtpEncoderNode::~RtpEncoderNode()
46 {
47     // remove IRtpSession here to avoid shared instance in other node from unable to use
48     if (mRtpSession)
49     {
50         mRtpSession->StopRtp();
51         mRtpSession->SetRtpEncoderListener(nullptr);
52         IRtpSession::ReleaseInstance(mRtpSession);
53         mRtpSession = nullptr;
54     }
55 }
56 
GetNodeId()57 kBaseNodeId RtpEncoderNode::GetNodeId()
58 {
59     return kNodeIdRtpEncoder;
60 }
61 
Start()62 ImsMediaResult RtpEncoderNode::Start()
63 {
64     IMLOGD1("[Start] type[%d]", mMediaType);
65 
66     if (mRtpPayloadTx == 0 || mRtpPayloadRx == 0)
67     {
68         IMLOGE0("[Start] invalid payload number");
69         return RESULT_INVALID_PARAM;
70     }
71 
72     if (mRtpSession == nullptr)
73     {
74         mRtpSession = IRtpSession::GetInstance(mMediaType, mLocalAddress, mPeerAddress);
75 
76         if (mRtpSession == nullptr)
77         {
78             IMLOGE0("[Start] Can't create rtp session");
79             return RESULT_NOT_READY;
80         }
81     }
82 
83     mRtpSession->SetRtpEncoderListener(this);
84 
85     if (mMediaType == IMS_MEDIA_AUDIO)
86     {
87         mRtpSession->SetRtpPayloadParam(mRtpPayloadTx, mRtpPayloadRx, mSamplingRate * 1000,
88                 mRtpTxDtmfPayload, mRtpRxDtmfPayload, mDtmfSamplingRate * 1000);
89 
90         if (mRtpContextParams.getSequenceNumber() >= 0)
91         {
92             // Set the next sequence number to use by RTP stack.
93             mRtpSession->SetRtpContext(mRtpContextParams.getSsrc(),
94                     mRtpContextParams.getTimestamp(), mRtpContextParams.getSequenceNumber() + 1);
95         }
96     }
97     else if (mMediaType == IMS_MEDIA_VIDEO)
98     {
99         mRtpSession->SetRtpPayloadParam(mRtpPayloadTx, mRtpPayloadRx, mSamplingRate * 1000);
100     }
101     else if (mMediaType == IMS_MEDIA_TEXT)
102     {
103         if (mRedundantPayload > 0)
104         {
105             mRtpSession->SetRtpPayloadParam(mRtpPayloadTx, mRtpPayloadRx, mSamplingRate * 1000,
106                     mRedundantPayload, mSamplingRate * 1000);
107         }
108         else
109         {
110             mRtpSession->SetRtpPayloadParam(mRtpPayloadTx, mRtpPayloadRx, mSamplingRate * 1000);
111         }
112 
113         if (mRtpContextParams.getSequenceNumber() > 0)
114         {
115             // Set the next sequence number to use by RTP stack.
116             mRtpSession->SetRtpContext(mRtpContextParams.getSsrc(),
117                     mRtpContextParams.getTimestamp(), mRtpContextParams.getSequenceNumber() + 1);
118         }
119     }
120 
121     mRtpSession->StartRtp();
122     mDtmfMode = false;
123     mMark = true;
124     mPrevTimestamp = 0;
125 #ifdef DEBUG_JITTER_GEN_SIMULATION_DELAY
126     mNextTime = 0;
127 #endif
128 #ifdef DEBUG_JITTER_GEN_SIMULATION_REORDER
129     jitterData.Clear();
130     mReorderDataCount = 0;
131 #endif
132     mNodeState = kNodeStateRunning;
133     return RESULT_SUCCESS;
134 }
135 
Stop()136 void RtpEncoderNode::Stop()
137 {
138     IMLOGD1("[Stop] type[%d]", mMediaType);
139 
140     if (mRtpSession)
141     {
142         mRtpSession->StopRtp();
143     }
144 
145     mRtpContextParams.setDefaultConfig();
146 
147     ClearDataQueue();
148     mNodeState = kNodeStateStopped;
149 }
150 
OnDataFromFrontNode(ImsMediaSubType subtype,uint8_t * data,uint32_t size,uint32_t timestamp,bool mark,uint32_t,ImsMediaSubType,uint32_t arrivalTime)151 void RtpEncoderNode::OnDataFromFrontNode(ImsMediaSubType subtype, uint8_t* data, uint32_t size,
152         uint32_t timestamp, bool mark, uint32_t /*seq*/, ImsMediaSubType /*dataType*/,
153         uint32_t arrivalTime)
154 {
155     if (mNodeState != kNodeStateRunning)
156     {
157         return;
158     }
159 
160     if (mMediaType == IMS_MEDIA_AUDIO)
161     {
162         mArrivalTime = arrivalTime;
163         ProcessAudioData(subtype, data, size, timestamp);
164     }
165     else if (mMediaType == IMS_MEDIA_VIDEO)
166     {
167         ProcessVideoData(subtype, data, size, timestamp, mark);
168     }
169     else if (mMediaType == IMS_MEDIA_TEXT)
170     {
171         ProcessTextData(subtype, data, size, timestamp, mark);
172     }
173 }
174 
IsSourceNode()175 bool RtpEncoderNode::IsSourceNode()
176 {
177     return false;
178 }
179 
SetConfig(void * config)180 void RtpEncoderNode::SetConfig(void* config)
181 {
182     IMLOGD1("[SetConfig] media[%d]", mMediaType);
183 
184     if (config == nullptr)
185     {
186         return;
187     }
188 
189     if (mMediaType == IMS_MEDIA_AUDIO)
190     {
191         AudioConfig* pConfig = reinterpret_cast<AudioConfig*>(config);
192         mPeerAddress = RtpAddress(pConfig->getRemoteAddress().c_str(), pConfig->getRemotePort());
193         mSamplingRate = pConfig->getSamplingRateKHz();
194         mRtpPayloadTx = pConfig->getTxPayloadTypeNumber();
195         mRtpPayloadRx = pConfig->getRxPayloadTypeNumber();
196         mRtpTxDtmfPayload = pConfig->getTxDtmfPayloadTypeNumber();
197         mRtpRxDtmfPayload = pConfig->getRxDtmfPayloadTypeNumber();
198         mDtmfSamplingRate = pConfig->getDtmfsamplingRateKHz();
199 
200         RtpContextParams rtpContextParams = pConfig->getRtpContextParams();
201 
202         if (pConfig->getAccessNetwork() == ACCESS_NETWORK_IWLAN &&
203                           rtpContextParams.getSequenceNumber() >= 0)
204         {
205             SetRtpContext(rtpContextParams);
206         }
207     }
208     else if (mMediaType == IMS_MEDIA_VIDEO)
209     {
210         VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(config);
211         mPeerAddress = RtpAddress(pConfig->getRemoteAddress().c_str(), pConfig->getRemotePort());
212         mSamplingRate = pConfig->getSamplingRateKHz();
213         mRtpPayloadTx = pConfig->getTxPayloadTypeNumber();
214         mRtpPayloadRx = pConfig->getRxPayloadTypeNumber();
215         mCvoValue = pConfig->getCvoValue();
216     }
217     else if (mMediaType == IMS_MEDIA_TEXT)
218     {
219         TextConfig* pConfig = reinterpret_cast<TextConfig*>(config);
220         mPeerAddress = RtpAddress(pConfig->getRemoteAddress().c_str(), pConfig->getRemotePort());
221         mSamplingRate = pConfig->getSamplingRateKHz();
222         mRtpPayloadTx = pConfig->getTxPayloadTypeNumber();
223         mRtpPayloadRx = pConfig->getRxPayloadTypeNumber();
224         mRedundantPayload = pConfig->getRedundantPayload();
225         mRedundantLevel = pConfig->getRedundantLevel();
226 
227         RtpContextParams rtpContextParams = pConfig->getRtpContextParams();
228 
229         if (pConfig->getAccessNetwork() == ACCESS_NETWORK_IWLAN &&
230                             rtpContextParams.getSequenceNumber() > 0)
231         {
232             SetRtpContext(rtpContextParams);
233         }
234     }
235 
236     IMLOGD2("[SetConfig] peer Ip[%s], port[%d]", mPeerAddress.ipAddress, mPeerAddress.port);
237 }
238 
IsSameConfig(void * config)239 bool RtpEncoderNode::IsSameConfig(void* config)
240 {
241     if (config == nullptr)
242     {
243         return true;
244     }
245 
246     if (mMediaType == IMS_MEDIA_AUDIO)
247     {
248         AudioConfig* pConfig = reinterpret_cast<AudioConfig*>(config);
249         return (mPeerAddress ==
250                         RtpAddress(pConfig->getRemoteAddress().c_str(), pConfig->getRemotePort()) &&
251                 mRtpContextParams == pConfig->getRtpContextParams() &&
252                 mSamplingRate == pConfig->getSamplingRateKHz() &&
253                 mRtpPayloadTx == pConfig->getTxPayloadTypeNumber() &&
254                 mRtpPayloadRx == pConfig->getRxPayloadTypeNumber() &&
255                 mRtpTxDtmfPayload == pConfig->getTxDtmfPayloadTypeNumber() &&
256                 mRtpRxDtmfPayload == pConfig->getRxDtmfPayloadTypeNumber() &&
257                 mDtmfSamplingRate == pConfig->getDtmfsamplingRateKHz());
258     }
259     else if (mMediaType == IMS_MEDIA_VIDEO)
260     {
261         VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(config);
262         return (mPeerAddress ==
263                         RtpAddress(pConfig->getRemoteAddress().c_str(), pConfig->getRemotePort()) &&
264                 mSamplingRate == pConfig->getSamplingRateKHz() &&
265                 mRtpPayloadTx == pConfig->getTxPayloadTypeNumber() &&
266                 mRtpPayloadRx == pConfig->getRxPayloadTypeNumber() &&
267                 mCvoValue == pConfig->getCvoValue());
268     }
269     else if (mMediaType == IMS_MEDIA_TEXT)
270     {
271         TextConfig* pConfig = reinterpret_cast<TextConfig*>(config);
272         return (mPeerAddress ==
273                         RtpAddress(pConfig->getRemoteAddress().c_str(), pConfig->getRemotePort()) &&
274                 mRtpContextParams == pConfig->getRtpContextParams() &&
275                 mSamplingRate == pConfig->getSamplingRateKHz() &&
276                 mRtpPayloadTx == pConfig->getTxPayloadTypeNumber() &&
277                 mRtpPayloadRx == pConfig->getRxPayloadTypeNumber() &&
278                 mRedundantPayload == pConfig->getRedundantPayload() &&
279                 mRedundantLevel == pConfig->getRedundantLevel());
280     }
281 
282     return false;
283 }
284 
OnRtpPacket(unsigned char * data,uint32_t nSize)285 void RtpEncoderNode::OnRtpPacket(unsigned char* data, uint32_t nSize)
286 {
287     SendDataToRearNode(
288             MEDIASUBTYPE_RTPPACKET, data, nSize, 0, 0, 0, MEDIASUBTYPE_UNDEFINED, mArrivalTime);
289 }
290 
SetLocalAddress(const RtpAddress & address)291 void RtpEncoderNode::SetLocalAddress(const RtpAddress& address)
292 {
293     mLocalAddress = address;
294 }
295 
SetPeerAddress(const RtpAddress & address)296 void RtpEncoderNode::SetPeerAddress(const RtpAddress& address)
297 {
298     mPeerAddress = address;
299 }
300 
SetCvoExtension(const int64_t facing,const int64_t orientation)301 bool RtpEncoderNode::SetCvoExtension(const int64_t facing, const int64_t orientation)
302 {
303     IMLOGD3("[SetCvoExtension] cvoValue[%d], facing[%ld], orientation[%ld]", mCvoValue, facing,
304             orientation);
305 
306     if (mCvoValue > 0)
307     {
308         uint32_t rotation = 0;
309         uint32_t cameraId = 0;
310 
311         if (facing == kCameraFacingRear)
312         {
313             cameraId = 1;
314         }
315 
316         switch (orientation)
317         {
318             default:
319             case 0:
320                 rotation = 0;
321                 break;
322             case 270:
323                 rotation = 1;
324                 break;
325             case 180:
326                 rotation = 2;
327                 break;
328             case 90:
329                 rotation = 3;
330                 break;
331         }
332 
333         if (cameraId == 1)  // rear camera
334         {
335             if (rotation == 1)  // CCW90
336             {
337                 rotation = 3;
338             }
339             else if (rotation == 3)  // CCW270
340             {
341                 rotation = 1;
342             }
343         }
344 
345         int8_t extensionData[4];  // 32bit
346         IMLOGD3("[SetCvoExtension] cvoValue[%d], facing[%d], orientation[%d]", mCvoValue, cameraId,
347                 rotation);
348 
349         extensionData[0] = (mCvoValue << 4) | 1;  // local identifier and data length
350         extensionData[1] = (cameraId << 3) | rotation;
351         extensionData[2] = 0;  // padding
352         extensionData[3] = 0;  // padding
353 
354         mListRtpExtension.clear();
355         mListRtpExtension.push_back(RtpHeaderExtensionInfo(
356                 RtpHeaderExtensionInfo::kBitPatternForOneByteHeader, 1, extensionData, 4));
357         return true;
358     }
359 
360     return false;
361 }
362 
SetRtpHeaderExtension(std::list<RtpHeaderExtension> * listExtension)363 void RtpEncoderNode::SetRtpHeaderExtension(std::list<RtpHeaderExtension>* listExtension)
364 {
365     if (listExtension == nullptr || listExtension->empty())
366     {
367         return;
368     }
369 
370     /**
371      * Check number of byte of the header. Based on RFC8285 4.2, one byte header has a local
372      * identifier in range of 1 to 14. Two byte header has is a range of 1 to 255.
373      */
374     bool useTwoByteHeader = false;
375     int32_t totalPayloadLength = 0;  // accumulate payload length except the header size
376 
377     for (auto extension : *listExtension)
378     {
379         if (extension.getLocalIdentifier() > 15)
380         {
381             useTwoByteHeader = true;
382         }
383 
384         totalPayloadLength += extension.getExtensionDataSize();
385     }
386 
387     // accumulate header size
388     useTwoByteHeader ? totalPayloadLength += 2 * listExtension->size()
389                      : totalPayloadLength += listExtension->size();
390 
391     // padding size
392     int32_t paddingSize = totalPayloadLength % IMS_MEDIA_WORD_SIZE == 0
393             ? 0
394             : IMS_MEDIA_WORD_SIZE - totalPayloadLength % IMS_MEDIA_WORD_SIZE;
395     totalPayloadLength += paddingSize;
396 
397     int8_t* extensionData = new int8_t[totalPayloadLength];
398     int offset = 0;
399 
400     for (auto extension : *listExtension)
401     {
402         if (useTwoByteHeader)
403         {
404             extensionData[offset++] = extension.getLocalIdentifier();
405             extensionData[offset++] = extension.getExtensionDataSize();
406         }
407         else
408         {
409             extensionData[offset++] =
410                     extension.getLocalIdentifier() << 4 | (extension.getExtensionDataSize() - 1);
411         }
412 
413         memcpy(extensionData + offset, extension.getExtensionData(),
414                 extension.getExtensionDataSize());
415         offset += extension.getExtensionDataSize();
416     }
417 
418     // add padding
419     memset(extensionData + offset, 0, paddingSize);
420 
421     IMLOGD3("[SetRtpHeaderExtension] twoByte[%d], size[%d], list size[%d]", useTwoByteHeader,
422             totalPayloadLength, listExtension->size());
423 
424     int16_t defineByProfile = useTwoByteHeader
425             ? RtpHeaderExtensionInfo::kBitPatternForTwoByteHeader
426             : RtpHeaderExtensionInfo::kBitPatternForOneByteHeader;
427     mListRtpExtension.push_back(RtpHeaderExtensionInfo(
428             defineByProfile, totalPayloadLength / 4, extensionData, totalPayloadLength));
429 
430     delete[] extensionData;
431 }
432 
ProcessAudioData(ImsMediaSubType subtype,uint8_t * data,uint32_t size,uint32_t timestamp)433 void RtpEncoderNode::ProcessAudioData(
434         ImsMediaSubType subtype, uint8_t* data, uint32_t size, uint32_t timestamp)
435 {
436     std::lock_guard<std::mutex> guard(mMutex);
437 
438     uint32_t timeDiff;
439     uint32_t timestampDiff;
440 
441     if (subtype == MEDIASUBTYPE_DTMFSTART)
442     {
443         IMLOGD0("[ProcessAudioData] SetDTMF mode true");
444         mDtmfMode = true;
445         mMark = true;
446     }
447     else if (subtype == MEDIASUBTYPE_DTMFEND)
448     {
449         IMLOGD0("[ProcessAudioData] SetDTMF mode false");
450         mDtmfMode = false;
451         mMark = true;
452     }
453     else if (subtype == MEDIASUBTYPE_DTMF_PAYLOAD)
454     {
455         if (mDtmfMode)
456         {
457             timeDiff = timestamp - mPrevTimestamp;
458             mMark ? mDtmfTimestamp = timestamp : timeDiff = 0;
459             mPrevTimestamp = timestamp;
460             timestampDiff = timeDiff * mSamplingRate;
461 
462             IMLOGD_PACKET3(IM_PACKET_LOG_RTP,
463                     "[ProcessAudioData] dtmf payload, size[%u], TS[%u], diff[%u]", size,
464                     mDtmfTimestamp, timestampDiff);
465             mRtpSession->SendRtpPacket(
466                     mRtpTxDtmfPayload, data, size, mDtmfTimestamp, mMark, timestampDiff);
467             mMark = false;
468         }
469     }
470     else  // MEDIASUBTYPE_RTPPAYLOAD
471     {
472         if (!mDtmfMode)
473         {
474             timeDiff = mPrevTimestamp == 0 ? 0 : ((timestamp - mPrevTimestamp) + 10) / 20 * 20;
475             mPrevTimestamp = timestamp;
476 
477             RtpPacket* packet = new RtpPacket();
478             packet->rtpDataType = kRtpDataTypeNormal;
479             mCallback->SendEvent(
480                     kCollectPacketInfo, kStreamRtpTx, reinterpret_cast<uint64_t>(packet));
481 
482             timestampDiff = timeDiff * mSamplingRate;
483             IMLOGD_PACKET3(IM_PACKET_LOG_RTP, "[ProcessAudioData] size[%u], TS[%u], diff[%u]", size,
484                     timestamp, timestampDiff);
485 
486             if (!mListRtpExtension.empty())
487             {
488                 mRtpSession->SendRtpPacket(mRtpPayloadTx, data, size, timestamp, mMark,
489                         timestampDiff, &mListRtpExtension.front());
490                 mListRtpExtension.pop_front();
491             }
492             else
493             {
494                 mRtpSession->SendRtpPacket(
495                         mRtpPayloadTx, data, size, timestamp, mMark, timestampDiff);
496             }
497 
498             if (mMark)
499             {
500                 mMark = false;
501             }
502         }
503     }
504 }
505 
ProcessVideoData(ImsMediaSubType subtype,uint8_t * data,uint32_t size,uint32_t timestamp,bool mark)506 void RtpEncoderNode::ProcessVideoData(
507         ImsMediaSubType subtype, uint8_t* data, uint32_t size, uint32_t timestamp, bool mark)
508 {
509     IMLOGD_PACKET4(IM_PACKET_LOG_RTP, "[ProcessVideoData] subtype[%d], size[%d], TS[%u], mark[%d]",
510             subtype, size, timestamp, mark);
511 
512 #ifdef SIMULATE_VIDEO_CVO_UPDATE
513     const int64_t kCameraFacing = kCameraFacingFront;
514     static int64_t sDeviceOrientation = 0;
515     static int64_t sCount = 0;
516     if ((++sCount % 100) == 0)
517     {
518         SetCvoExtension(kCameraFacing, (sDeviceOrientation += 90) % 360);
519     }
520 #endif
521 
522     if (mCvoValue > 0 && mark && subtype == MEDIASUBTYPE_VIDEO_IDR_FRAME)
523     {
524         mRtpSession->SendRtpPacket(mRtpPayloadTx, data, size, timestamp, mark, 0,
525                 mListRtpExtension.empty() ? nullptr : &mListRtpExtension.front());
526     }
527     else
528     {
529         mRtpSession->SendRtpPacket(mRtpPayloadTx, data, size, timestamp, mark, 0);
530     }
531 }
532 
ProcessTextData(ImsMediaSubType subtype,uint8_t * data,uint32_t size,uint32_t timestamp,bool mark)533 void RtpEncoderNode::ProcessTextData(
534         ImsMediaSubType subtype, uint8_t* data, uint32_t size, uint32_t timestamp, bool mark)
535 {
536     IMLOGD_PACKET4(IM_PACKET_LOG_RTP,
537             "[ProcessTextData] subtype[%d], size[%d], timestamp[%d], mark[%d]", subtype, size,
538             timestamp, mark);
539 
540     uint32_t timeDiff = mMark ? 0 : timestamp - mPrevTimestamp;
541 
542     if (subtype == MEDIASUBTYPE_BITSTREAM_T140)
543     {
544         if (mRedundantLevel > 1 && mRedundantPayload > 0)
545         {
546             mRtpSession->SendRtpPacket(mRedundantPayload, data, size, timestamp, mark, timeDiff);
547         }
548         else
549         {
550             mRtpSession->SendRtpPacket(mRtpPayloadRx, data, size, timestamp, mark, timeDiff);
551         }
552     }
553     else if (subtype == MEDIASUBTYPE_BITSTREAM_T140_RED)
554     {
555         mRtpSession->SendRtpPacket(mRtpPayloadTx, data, size, timestamp, mark, timeDiff);
556     }
557 
558     mMark = false;
559     mPrevTimestamp = timestamp;
560 }
561 
SetRtpContext(RtpContextParams & rtpContextParams)562 void RtpEncoderNode::SetRtpContext(RtpContextParams& rtpContextParams)
563 {
564     mRtpContextParams = rtpContextParams;
565 }
566 
GetRtpContext(RtpContextParams & rtpContextParams)567 void RtpEncoderNode::GetRtpContext(RtpContextParams& rtpContextParams)
568 {
569     uint32_t ssrc = 0;
570     uint32_t timestamp = 0;
571     uint16_t seqNumber = 0;
572 
573     if (mRtpSession != nullptr)
574     {
575         mRtpSession->GetRtpContext(ssrc, timestamp, seqNumber);
576     }
577 
578     rtpContextParams.setSsrc(ssrc);
579     rtpContextParams.setTimestamp(timestamp);
580     rtpContextParams.setSequenceNumber(seqNumber);
581 }