1 /**
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <VideoStreamGraphRtpTx.h>
18 #include <ImsMediaTrace.h>
19 #include <ImsMediaNetworkUtil.h>
20 #include <VideoConfig.h>
21 #include <RtpEncoderNode.h>
22 #include <SocketWriterNode.h>
23 #include <VideoRtpPayloadEncoderNode.h>
24 #include <IVideoSourceNode.h>
25 
VideoStreamGraphRtpTx(BaseSessionCallback * callback,int localFd)26 VideoStreamGraphRtpTx::VideoStreamGraphRtpTx(BaseSessionCallback* callback, int localFd) :
27         VideoStreamGraph(callback, localFd)
28 {
29     mSurface = nullptr;
30     mVideoMode = -1;
31 }
32 
~VideoStreamGraphRtpTx()33 VideoStreamGraphRtpTx::~VideoStreamGraphRtpTx() {}
34 
create(RtpConfig * config)35 ImsMediaResult VideoStreamGraphRtpTx::create(RtpConfig* config)
36 {
37     IMLOGI1("[create] state[%d]", mGraphState);
38 
39     if (config == nullptr)
40     {
41         return RESULT_INVALID_PARAM;
42     }
43 
44     VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(config);
45 
46     if (pConfig->getVideoMode() == VideoConfig::VIDEO_MODE_PREVIEW)
47     {
48         return createPreviewMode(pConfig);
49     }
50 
51     if (mConfig != nullptr)
52     {
53         delete mConfig;
54         mConfig = nullptr;
55     }
56 
57     mConfig = new VideoConfig(pConfig);
58 
59     char localIp[MAX_IP_LEN];
60     uint32_t localPort = 0;
61     ImsMediaNetworkUtil::getLocalIpPortFromSocket(mLocalFd, localIp, MAX_IP_LEN, localPort);
62     RtpAddress localAddress(localIp, localPort);
63 
64     BaseNode* pNodeSource = new IVideoSourceNode(mCallback);
65     pNodeSource->SetMediaType(IMS_MEDIA_VIDEO);
66     pNodeSource->SetConfig(mConfig);
67     AddNode(pNodeSource);
68 
69     BaseNode* pNodeRtpPayloadEncoder = new VideoRtpPayloadEncoderNode(mCallback);
70     pNodeRtpPayloadEncoder->SetMediaType(IMS_MEDIA_VIDEO);
71     pNodeRtpPayloadEncoder->SetConfig(mConfig);
72     AddNode(pNodeRtpPayloadEncoder);
73     pNodeSource->ConnectRearNode(pNodeRtpPayloadEncoder);
74 
75     BaseNode* pNodeRtpEncoder = new RtpEncoderNode(mCallback);
76     pNodeRtpEncoder->SetMediaType(IMS_MEDIA_VIDEO);
77     pNodeRtpEncoder->SetConfig(mConfig);
78     (static_cast<RtpEncoderNode*>(pNodeRtpEncoder))->SetLocalAddress(localAddress);
79     AddNode(pNodeRtpEncoder);
80     pNodeRtpPayloadEncoder->ConnectRearNode(pNodeRtpEncoder);
81 
82     BaseNode* pNodeSocketWriter = new SocketWriterNode(mCallback);
83     pNodeSocketWriter->SetMediaType(IMS_MEDIA_VIDEO);
84     (static_cast<SocketWriterNode*>(pNodeSocketWriter))->SetLocalFd(mLocalFd);
85     (static_cast<SocketWriterNode*>(pNodeSocketWriter))->SetLocalAddress(localAddress);
86     (static_cast<SocketWriterNode*>(pNodeSocketWriter))->SetProtocolType(kProtocolRtp);
87     pNodeSocketWriter->SetConfig(config);
88     AddNode(pNodeSocketWriter);
89     pNodeRtpEncoder->ConnectRearNode(pNodeSocketWriter);
90 
91     setState(kStreamStateCreated);
92     mVideoMode = pConfig->getVideoMode();
93     return RESULT_SUCCESS;
94 }
95 
update(RtpConfig * config)96 ImsMediaResult VideoStreamGraphRtpTx::update(RtpConfig* config)
97 {
98     IMLOGI2("[update] current mode[%d], state[%d]", mVideoMode, mGraphState);
99 
100     if (config == nullptr)
101     {
102         return RESULT_INVALID_PARAM;
103     }
104 
105     VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(config);
106 
107     if (*reinterpret_cast<VideoConfig*>(mConfig) == *pConfig)
108     {
109         IMLOGI0("[update] no update");
110         return RESULT_SUCCESS;
111     }
112 
113     if (mGraphState == kStreamStateWaitSurface)
114     {
115         setState(kStreamStateCreated);
116     }
117 
118     if (mConfig != nullptr)
119     {
120         delete mConfig;
121         mConfig = nullptr;
122     }
123 
124     ImsMediaResult result = RESULT_NOT_READY;
125 
126     if (pConfig->getVideoMode() != mVideoMode &&
127             (mVideoMode == VideoConfig::VIDEO_MODE_PREVIEW ||
128                     pConfig->getVideoMode() == VideoConfig::VIDEO_MODE_PREVIEW))
129     {
130         result = stop();
131 
132         if (result != RESULT_SUCCESS)
133         {
134             return result;
135         }
136 
137         /** delete nodes */
138         deleteNodes();
139         mSurface = nullptr;
140 
141         /** create nodes */
142         result = create(pConfig);
143 
144         if (result != RESULT_SUCCESS)
145         {
146             return result;
147         }
148 
149         return start();
150     }
151 
152     mConfig = new VideoConfig(pConfig);
153 
154     if (mConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_NO_FLOW ||
155             mConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_RECEIVE_ONLY ||
156             mConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_INACTIVE)
157     {
158         IMLOGI0("[update] pause TX");
159         return stop();
160     }
161 
162     if (pConfig->getVideoMode() != VideoConfig::VIDEO_MODE_PAUSE_IMAGE && mSurface == nullptr)
163     {
164         IMLOGI2("[update] direction[%d], mode[%d], surface is not ready, wait",
165                 pConfig->getMediaDirection(), pConfig->getVideoMode());
166 
167         if (mGraphState == kStreamStateRunning)
168         {
169             stop();
170         }
171 
172         updateNodes(mConfig);
173         setState(kStreamStateWaitSurface);
174         return RESULT_SUCCESS;
175     }
176 
177     result = updateNodes(mConfig);
178 
179     if (result != RESULT_SUCCESS)
180     {
181         return result;
182     }
183 
184     if (mGraphState == kStreamStateCreated &&
185             (pConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_SEND_ONLY ||
186                     pConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_SEND_RECEIVE))
187     {
188         IMLOGI0("[update] resume TX");
189         return start();
190     }
191 
192     return result;
193 }
194 
start()195 ImsMediaResult VideoStreamGraphRtpTx::start()
196 {
197     IMLOGI2("[start] current mode[%d], state[%d]", mVideoMode, mGraphState);
198 
199     if (mConfig == nullptr)
200     {
201         return RESULT_INVALID_PARAM;
202     }
203 
204     VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(mConfig);
205 
206     if (pConfig->getVideoMode() != VideoConfig::VIDEO_MODE_PREVIEW &&
207             (pConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_NO_FLOW ||
208                     pConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_RECEIVE_ONLY ||
209                     pConfig->getMediaDirection() == RtpConfig::MEDIA_DIRECTION_INACTIVE))
210     {
211         IMLOGI1("[start] direction[%d] no need to start", pConfig->getMediaDirection());
212         return RESULT_SUCCESS;
213     }
214 
215     if (pConfig->getVideoMode() != VideoConfig::VIDEO_MODE_PAUSE_IMAGE && mSurface == nullptr)
216     {
217         IMLOGI2("[start] direction[%d], mode[%d], surface is not ready, wait",
218                 pConfig->getMediaDirection(), pConfig->getVideoMode());
219         setState(kStreamStateWaitSurface);
220         return RESULT_SUCCESS;
221     }
222 
223     ImsMediaResult result = startNodes();
224 
225     if (result != RESULT_SUCCESS)
226     {
227         setState(kStreamStateCreated);
228         mCallback->SendEvent(kImsMediaEventNotifyError, result, kStreamModeRtpTx);
229         return result;
230     }
231 
232     setState(kStreamStateRunning);
233     mVideoMode = mConfig->getVideoMode();
234     return RESULT_SUCCESS;
235 }
236 
setMediaQualityThreshold(MediaQualityThreshold * threshold)237 bool VideoStreamGraphRtpTx::setMediaQualityThreshold(MediaQualityThreshold* threshold)
238 {
239     if (threshold != nullptr)
240     {
241         BaseNode* node = findNode(kNodeIdVideoSource);
242 
243         if (node != nullptr)
244         {
245             IVideoSourceNode* source = reinterpret_cast<IVideoSourceNode*>(node);
246             source->SetBitrateThreshold(threshold->getVideoBitrateBps());
247             return true;
248         }
249     }
250 
251     return false;
252 }
253 
setSurface(ANativeWindow * surface)254 void VideoStreamGraphRtpTx::setSurface(ANativeWindow* surface)
255 {
256     IMLOGI1("[setSurface] state[%d]", mGraphState);
257 
258     if (surface != nullptr)
259     {
260         mSurface = surface;
261 
262         BaseNode* node = findNode(kNodeIdVideoSource);
263 
264         if (node != nullptr)
265         {
266             IVideoSourceNode* source = reinterpret_cast<IVideoSourceNode*>(node);
267             source->UpdateSurface(surface);
268 
269             if (getState() == kStreamStateWaitSurface)
270             {
271                 setState(kStreamStateCreated);
272 
273                 if (start() != RESULT_SUCCESS)
274                 {
275                     IMLOGE0("[setSurface] start fail");
276                 }
277             }
278         }
279     }
280 }
281 
createPreviewMode(RtpConfig * config)282 ImsMediaResult VideoStreamGraphRtpTx::createPreviewMode(RtpConfig* config)
283 {
284     if (config == nullptr)
285     {
286         return RESULT_INVALID_PARAM;
287     }
288 
289     if (mConfig != nullptr)
290     {
291         delete mConfig;
292         mConfig = nullptr;
293     }
294 
295     IMLOGI0("[createPreviewMode]");
296     mConfig = new VideoConfig(reinterpret_cast<VideoConfig*>(config));
297     BaseNode* pNodeSource = new IVideoSourceNode(mCallback);
298     pNodeSource->SetMediaType(IMS_MEDIA_VIDEO);
299     pNodeSource->SetConfig(mConfig);
300     AddNode(pNodeSource);
301 
302     setState(kStreamStateCreated);
303     mVideoMode = VideoConfig::VIDEO_MODE_PREVIEW;
304     return RESULT_SUCCESS;
305 }
306 
updateNodes(RtpConfig * config)307 ImsMediaResult VideoStreamGraphRtpTx::updateNodes(RtpConfig* config)
308 {
309     IMLOGD1("[updateNodes] state[%d]", mGraphState);
310 
311     ImsMediaResult result = RESULT_NOT_READY;
312 
313     if (mGraphState == kStreamStateRunning)
314     {
315         mScheduler->Stop();
316 
317         for (auto& node : mListNodeStarted)
318         {
319             if (node != nullptr)
320             {
321                 IMLOGD1("[updateNodes] update node[%s]", node->GetNodeName());
322                 result = node->UpdateConfig(config);
323 
324                 if (result != RESULT_SUCCESS)
325                 {
326                     IMLOGE2("[updateNodes] error in update node[%s], result[%d]",
327                             node->GetNodeName(), result);
328                     return result;
329                 }
330             }
331         }
332 
333         mScheduler->Start();
334     }
335     else if (mGraphState == kStreamStateCreated)
336     {
337         for (auto& node : mListNodeToStart)
338         {
339             if (node != nullptr)
340             {
341                 IMLOGD1("[updateNodes] update node[%s]", node->GetNodeName());
342                 result = node->UpdateConfig(config);
343 
344                 if (result != RESULT_SUCCESS)
345                 {
346                     IMLOGE2("[updateNodes] error in update node[%s], result[%d]",
347                             node->GetNodeName(), result);
348                     return result;
349                 }
350             }
351         }
352     }
353 
354     return result;
355 }
356 
OnEvent(int32_t type,uint64_t param1,uint64_t param2)357 bool VideoStreamGraphRtpTx::OnEvent(int32_t type, uint64_t param1, uint64_t param2)
358 {
359     IMLOGI3("[OnEvent] type[%d], param1[%d], param2[%d]", type, param1, param2);
360 
361     switch (type)
362     {
363         case kRequestVideoCvoUpdate:
364         {
365             BaseNode* node = findNode(kNodeIdRtpEncoder);
366 
367             if (node != nullptr)
368             {
369                 RtpEncoderNode* pNode = reinterpret_cast<RtpEncoderNode*>(node);
370                 return pNode->SetCvoExtension(param1, param2);
371             }
372 
373             return false;
374         }
375         break;
376         case kRequestVideoBitrateChange:
377         case kRequestVideoIdrFrame:
378         {
379             BaseNode* node = findNode(kNodeIdVideoSource);
380 
381             if (node != nullptr)
382             {
383                 IVideoSourceNode* pNode = reinterpret_cast<IVideoSourceNode*>(node);
384                 pNode->OnEvent(type, param1, param2);
385                 return true;
386             }
387         }
388         break;
389         default:
390             break;
391     }
392 
393     return false;
394 }