1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.video.cts;
18 
19 import static org.junit.Assert.assertNotNull;
20 import static org.junit.Assert.assertTrue;
21 import static org.junit.Assume.assumeTrue;
22 
23 import android.media.MediaCodec;
24 import android.media.MediaCodecInfo;
25 import android.media.MediaCodecList;
26 import android.media.MediaFormat;
27 import android.util.Log;
28 
29 import java.io.IOException;
30 import java.nio.ByteBuffer;
31 import java.util.ArrayList;
32 import java.util.HashMap;
33 import java.util.Map;
34 import java.util.concurrent.locks.Condition;
35 import java.util.concurrent.locks.Lock;
36 import java.util.concurrent.locks.ReentrantLock;
37 
38 class CodecEncoderPerformanceTestBase extends CodecPerformanceTestBase {
39     private static final String LOG_TAG = CodecEncoderPerformanceTest.class.getSimpleName();
40     private static final Map<String, Float> transcodeAVCToTargetBitrateMap = new HashMap<>();
41     private static final boolean ENABLE_LOGS = false;
42 
43     final String mEncoderMime;
44     final String mEncoderName;
45     final int mBitrate;
46     double mAchievedFps;
47     boolean mIsAsync;
48     int mMaxBFrames;
49 
50     private boolean mSawEncInputEOS = false;
51     private boolean mSawEncOutputEOS = false;
52     private int mEncOutputNum = 0;
53     private MediaCodec mEncoder;
54     private MediaFormat mEncoderFormat;
55     private boolean mIsCodecInAsyncMode;
56 
57     private final Lock mLock = new ReentrantLock();
58     private final Condition mCondition = mLock.newCondition();
59 
60     // Suggested bitrate scaling factors for transcoding avc to target format.
61     static {
transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, 1.25f)62         transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_VP8, 1.25f);
transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, 1.0f)63         transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_AVC, 1.0f);
transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, 0.7f)64         transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_VP9, 0.7f);
transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, 0.6f)65         transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_HEVC, 0.6f);
transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, 0.4f)66         transcodeAVCToTargetBitrateMap.put(MediaFormat.MIMETYPE_VIDEO_AV1, 0.4f);
67     }
68 
getBitrateScalingFactor(String mime)69     public static float getBitrateScalingFactor(String mime) {
70         return transcodeAVCToTargetBitrateMap.getOrDefault(mime, 1.5f);
71     }
72 
CodecEncoderPerformanceTestBase(String decoderName, String testFile, String encoderMime, String encoderName, int bitrate, int keyPriority, float scalingFactor, boolean isAsync, int maxBFrames)73     public CodecEncoderPerformanceTestBase(String decoderName, String testFile, String encoderMime,
74             String encoderName, int bitrate, int keyPriority, float scalingFactor,
75             boolean isAsync, int maxBFrames) {
76         super(decoderName, testFile, keyPriority, scalingFactor);
77         mEncoderMime = encoderMime;
78         mEncoderName = encoderName;
79         mBitrate = bitrate;
80         mIsAsync = isAsync;
81         mMaxBFrames = maxBFrames;
82     }
83 
getMimesOfAvailableHardwareVideoEncoders()84     static ArrayList<String> getMimesOfAvailableHardwareVideoEncoders() {
85         MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
86         MediaCodecInfo[] codecInfos = codecList.getCodecInfos();
87         ArrayList<String> listOfMimes = new ArrayList<>();
88         for (MediaCodecInfo codecInfo : codecInfos) {
89             if (!codecInfo.isEncoder() || !codecInfo.isHardwareAccelerated()) continue;
90             String[] types = codecInfo.getSupportedTypes();
91             for (String type : types) {
92                 if (type.startsWith("video/") && !listOfMimes.contains(type)) {
93                     listOfMimes.add(type);
94                 }
95             }
96         }
97         return listOfMimes;
98     }
99 
setUpEncoderFormat(MediaFormat format, String mime, int bitrate)100     public static MediaFormat setUpEncoderFormat(MediaFormat format, String mime, int bitrate) {
101         MediaFormat fmt = new MediaFormat();
102         fmt.setString(MediaFormat.KEY_MIME, mime);
103         fmt.setInteger(MediaFormat.KEY_WIDTH, format.getInteger(MediaFormat.KEY_WIDTH));
104         fmt.setInteger(MediaFormat.KEY_HEIGHT, format.getInteger(MediaFormat.KEY_HEIGHT));
105         fmt.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
106         fmt.setInteger(MediaFormat.KEY_FRAME_RATE,
107                 format.getInteger(MediaFormat.KEY_FRAME_RATE, 30));
108         fmt.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1.0f);
109         fmt.setInteger(MediaFormat.KEY_COLOR_FORMAT,
110                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
111         return fmt;
112     }
113 
dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info)114     private void dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
115         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
116             mEncOutputNum++;
117         }
118         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
119             mSawEncOutputEOS = true;
120         }
121         mEncoder.releaseOutputBuffer(bufferIndex, false);
122     }
123 
setUpFormats(MediaFormat format)124     private void setUpFormats(MediaFormat format) throws IOException {
125         mDecoderFormat = new MediaFormat(format);
126         mDecoderFormat.setInteger(MediaFormat.KEY_PRIORITY, mKeyPriority);
127         mEncoderFormat = setUpEncoderFormat(mDecoderFormat, mEncoderMime, mBitrate);
128         mEncoderFormat.setInteger(MediaFormat.KEY_PRIORITY, mKeyPriority);
129         double maxOperatingRateDecoder = getMaxOperatingRate(mDecoderName, mDecoderMime);
130         double maxOperatingRateEncoder = getMaxOperatingRate(mEncoderName, mEncoderMime);
131         mOperatingRateExpected = Math.min(maxOperatingRateDecoder, maxOperatingRateEncoder);
132         // As both decoder and encoder are running in concurrently, expected rate is halved
133         mOperatingRateExpected /= 2.0;
134         if (mMaxOpRateScalingFactor > 0.0f) {
135             int operatingRateToSet = (int) (mOperatingRateExpected * mMaxOpRateScalingFactor);
136             if (mMaxOpRateScalingFactor < 1.0f) {
137                 mOperatingRateExpected = operatingRateToSet;
138             }
139 
140             if (EXCLUDE_ENCODER_OPRATE_0_TO_30) {
141                 assumeTrue("For devices launched with Android R and below, operating rate tests "
142                                 + "are limited to operating rate <= 0 or >= 30",
143                         operatingRateToSet <= 0 || operatingRateToSet >= 30);
144             }
145 
146             mDecoderFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, operatingRateToSet);
147             mEncoderFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, operatingRateToSet);
148         } else if (mMaxOpRateScalingFactor < 0.0f) {
149             mDecoderFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, -1);
150             mEncoderFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, -1);
151         }
152         mEncoderFormat.setInteger(MediaFormat.KEY_COMPLEXITY,
153                 getEncoderMinComplexity(mEncoderName, mEncoderMime));
154         mEncoderFormat.setInteger(MediaFormat.KEY_MAX_B_FRAMES, mMaxBFrames);
155     }
156 
doWork()157     private void doWork() throws InterruptedException {
158         if (mIsCodecInAsyncMode) {
159             while (!mSawEncOutputEOS) {
160                 mLock.lock();
161                 while (!mSawDecOutputEOS) {
162                     mCondition.await();
163                 }
164                 mLock.unlock();
165                 if (!mSawEncInputEOS) {
166                     mEncoder.signalEndOfInputStream();
167                     mSawEncInputEOS = true;
168                 }
169             }
170         } else {
171             while (!mSawEncOutputEOS) {
172                 if (!mSawDecInputEOS) {
173                     int inputBufIndex = mDecoder.dequeueInputBuffer(Q_DEQ_TIMEOUT_US);
174                     if (inputBufIndex >= 0) {
175                         enqueueDecoderInput(inputBufIndex);
176                     }
177                 }
178                 if (!mSawDecOutputEOS) {
179                     MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
180                     int outputBufIndex = mDecoder.dequeueOutputBuffer(info, Q_DEQ_TIMEOUT_US);
181                     if (outputBufIndex >= 0) {
182                         dequeueDecoderOutput(outputBufIndex, info, true);
183                     }
184                 }
185                 if (mSawDecOutputEOS && !mSawEncInputEOS) {
186                     mEncoder.signalEndOfInputStream();
187                     mSawEncInputEOS = true;
188                 }
189                 MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
190                 int outputBufferId = mEncoder.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
191                 if (outputBufferId >= 0) {
192                     dequeueEncoderOutput(outputBufferId, outInfo);
193                 }
194             }
195         }
196     }
197 
encode()198     public void encode() throws IOException, InterruptedException {
199         MediaFormat format = setUpDecoderInput();
200         assertNotNull("Video track not present in " + mTestFile, format);
201 
202         if (EXCLUDE_ENCODER_MAX_RESOLUTION) {
203             int maxFrameSize = getMaxFrameSize(mEncoderName, mEncoderMime);
204             assumeTrue(mWidth + "x" + mHeight + " is skipped as it not less than half of " +
205                     "maximum frame size: " + maxFrameSize + " supported by the encoder.",
206                     mWidth * mHeight < maxFrameSize / 2);
207         }
208 
209         setUpFormats(format);
210         mDecoder = MediaCodec.createByCodecName(mDecoderName);
211         mEncoder = MediaCodec.createByCodecName(mEncoderName);
212         configureCodec(mIsAsync);
213         mDecoder.start();
214         mEncoder.start();
215         long start = System.currentTimeMillis();
216         doWork();
217         long finish = System.currentTimeMillis();
218         mEncoder.stop();
219         mSurface.release();
220         mEncoder.release();
221         mDecoder.stop();
222         mDecoder.release();
223         mEncoder = null;
224         mDecoder = null;
225         assertTrue("Encoder output count is zero", mEncOutputNum > 0);
226         mAchievedFps = mEncOutputNum / ((finish - start) / 1000.0);
227     }
228 
configureCodec(boolean isAsync)229     void configureCodec(boolean isAsync) {
230         resetContext(isAsync);
231 
232         if (isAsync) {
233             mEncoder.setCallback(new MediaCodec.Callback() {
234                 @Override
235                 public void onInputBufferAvailable(MediaCodec codec, int index) {
236                 }
237 
238                 @Override
239                 public void onOutputBufferAvailable(MediaCodec codec, int index,
240                         MediaCodec.BufferInfo info) {
241                     if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
242                         mEncOutputNum++;
243                     }
244                     codec.releaseOutputBuffer(index, false);
245                     if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
246                         mSawEncOutputEOS = true;
247                     }
248                 }
249 
250                 @Override
251                 public void onError(MediaCodec codec, MediaCodec.CodecException e) {
252                 }
253 
254                 @Override
255                 public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
256                 }
257             });
258         }
259         mEncoder.configure(mEncoderFormat, null, MediaCodec.CONFIGURE_FLAG_ENCODE, null);
260 
261         mSurface = mEncoder.createInputSurface();
262         assertTrue("Surface created is null.", mSurface != null);
263         assertTrue("Surface is not valid", mSurface.isValid());
264 
265         if (isAsync) {
266             mDecoder.setCallback(new MediaCodec.Callback() {
267                 @Override
268                 public void onInputBufferAvailable(MediaCodec codec, int index) {
269                     if (index >= 0) {
270                         if (mSampleIndex <= MIN_FRAME_COUNT) {
271                             MediaCodec.BufferInfo info = mBufferInfos.get(mSampleIndex++);
272                             if (info.size > 0
273                                     && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
274                                 ByteBuffer dstBuf = mDecoder.getInputBuffer(index);
275                                 dstBuf.put(mBuff.array(), info.offset, info.size);
276                                 mDecInputNum++;
277                             }
278                             codec.queueInputBuffer(index, 0, info.size, info.presentationTimeUs,
279                                     info.flags);
280                             if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
281                                 mSawDecInputEOS = true;
282                             }
283                         }
284                     }
285                 }
286 
287                 @Override
288                 public void onOutputBufferAvailable(MediaCodec codec, int index,
289                         MediaCodec.BufferInfo info) {
290                     if (index >= 0) {
291                         if (info.size > 0) {
292                             mDecOutputNum++;
293                         }
294                         codec.releaseOutputBuffer(index, true);
295                         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
296                             mSawDecOutputEOS = true;
297                         }
298                         if (mSawDecOutputEOS) {
299                             mLock.lock();
300                             mCondition.signal();
301                             mLock.unlock();
302                         }
303                     }
304                 }
305 
306                 @Override
307                 public void onError(MediaCodec codec, MediaCodec.CodecException e) {
308                 }
309 
310                 @Override
311                 public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
312                 }
313             });
314         }
315         mDecoder.configure(mDecoderFormat, mSurface, null, 0);
316 
317         if (ENABLE_LOGS) {
318             Log.v(LOG_TAG, "codec configured");
319         }
320     }
321 
resetContext(boolean isAsync)322     void resetContext(boolean isAsync) {
323         mIsCodecInAsyncMode = isAsync;
324         mDecInputNum = 0;
325         mDecOutputNum = 0;
326         mEncOutputNum = 0;
327     }
328 
329     @Override
finalize()330     protected void finalize() throws Throwable {
331         if (mDecoder != null) {
332             mDecoder.release();
333             mDecoder = null;
334         }
335         if (mEncoder != null) {
336             mEncoder.release();
337             mEncoder = null;
338         }
339     }
340 }
341