1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.video.cts;
18 
19 import static org.junit.Assert.assertTrue;
20 import static org.junit.Assert.fail;
21 import static org.junit.Assume.assumeTrue;
22 
23 import android.media.MediaCodec;
24 import android.media.MediaCodecInfo;
25 import android.media.MediaCodecList;
26 import android.media.MediaExtractor;
27 import android.media.MediaFormat;
28 import android.media.cts.TestArgs;
29 import android.os.Build;
30 import android.os.SystemProperties;
31 import android.util.Range;
32 import android.view.Surface;
33 
34 import org.junit.Before;
35 
36 import java.io.File;
37 import java.io.IOException;
38 import java.nio.ByteBuffer;
39 import java.util.ArrayList;
40 import java.util.List;
41 
42 class CodecPerformanceTestBase {
43     private static final String LOG_TAG = CodecPerformanceTestBase.class.getSimpleName();
44     static final long Q_DEQ_TIMEOUT_US = 5000; // block at most 5ms while looking for io buffers
45     static final int PER_TEST_TIMEOUT_LARGE_TEST_MS = 300000;
46     static final int MIN_FRAME_COUNT = 500;
47     static final int SELECT_ALL = 0; // Select all codecs
48     static final int SELECT_HARDWARE = 1; // Select Hardware codecs only
49     static final int SELECT_SOFTWARE = 2; // Select Software codecs only
50     // allowed tolerance in measured fps vs expected fps, i.e. codecs achieving fps
51     // that is greater than (FPS_TOLERANCE_FACTOR * expectedFps) will be considered as
52     // passing the test
53     static final double FPS_TOLERANCE_FACTOR;
54     static final boolean IS_AT_LEAST_VNDK_S;
55 
56     static final int DEVICE_INITIAL_SDK;
57     static final int VNDK_VERSION;
58 
59     // Some older devices can not support concurrent instances of both decoder and encoder
60     // at max resolution. To handle such cases, this test is limited to test the
61     // resolutions that are less than half of max supported frame sizes of encoder.
62     static final boolean EXCLUDE_ENCODER_MAX_RESOLUTION;
63 
64     // Some older devices can not support concurrent instances of both decoder and encoder
65     // for operating rates > 0 and < 30
66     static final boolean EXCLUDE_ENCODER_OPRATE_0_TO_30;
67 
68     static final String mInputPrefix = WorkDir.getMediaDirString();
69 
70     ArrayList<MediaCodec.BufferInfo> mBufferInfos;
71     ByteBuffer mBuff;
72 
73     final String mDecoderName;
74     final String mTestFile;
75     final int mKeyPriority;
76     final float mMaxOpRateScalingFactor;
77 
78     String mDecoderMime;
79     int mWidth;
80     int mHeight;
81     int mFrameRate;
82 
83     boolean mSawDecInputEOS = false;
84     boolean mSawDecOutputEOS = false;
85     int mDecInputNum = 0;
86     int mDecOutputNum = 0;
87     int mSampleIndex = 0;
88 
89     MediaCodec mDecoder;
90     MediaFormat mDecoderFormat;
91     Surface mSurface;
92     double mOperatingRateExpected;
93 
94     static final float[] SCALING_FACTORS_LIST = new float[]{2.5f, 1.25f, 1.0f, 0.75f, 0.0f, -1.0f};
95     static final int[] KEY_PRIORITIES_LIST = new int[]{1, 0};
96 
97     static {
98         // os.Build.VERSION.DEVICE_INITIAL_SDK_INT can be used here, but it was called
99         // os.Build.VERSION.FIRST_SDK_INT in Android R and below. Using DEVICE_INITIAL_SDK_INT
100         // will mean that the tests built in Android S can't be run on Android R and below.
101         DEVICE_INITIAL_SDK = SystemProperties.getInt("ro.product.first_api_level", 0);
102 
103         VNDK_VERSION = SystemProperties.getInt("ro.vndk.version",
104                 Build.VERSION_CODES.CUR_DEVELOPMENT);
105 
106         // fps tolerance factor is kept quite low for devices with Android R VNDK or lower
107         FPS_TOLERANCE_FACTOR = VNDK_VERSION <= Build.VERSION_CODES.R ? 0.67 : 0.95;
108 
109         IS_AT_LEAST_VNDK_S = VNDK_VERSION > Build.VERSION_CODES.R;
110 
111         // Encoders on devices launched on Android Q and lower aren't tested at maximum resolution
112         EXCLUDE_ENCODER_MAX_RESOLUTION = DEVICE_INITIAL_SDK <= Build.VERSION_CODES.Q;
113 
114         // Encoders on devices launched on Android R and lower aren't tested when operating rate
115         // that is set is > 0 and < 30.
116         // This includes devices launched on Android S with R or lower vendor partition.
117         EXCLUDE_ENCODER_OPRATE_0_TO_30 =
118             !IS_AT_LEAST_VNDK_S || (DEVICE_INITIAL_SDK <= Build.VERSION_CODES.R);
119     }
120 
121     @Before
prologue()122     public void prologue() {
123         assumeTrue("For VNDK R and below, operating rate <= 0 isn't tested",
124                 IS_AT_LEAST_VNDK_S || mMaxOpRateScalingFactor > 0.0);
125 
126         assumeTrue("For devices launched on Android P and below, operating rate tests are disabled",
127                 DEVICE_INITIAL_SDK > Build.VERSION_CODES.P);
128 
129         if (DEVICE_INITIAL_SDK <= Build.VERSION_CODES.Q) {
130             assumeTrue("For devices launched with Android Q and below, operating rate tests are " +
131                             "limited to operating rate scaling factor > 0.0 and <= 1.25",
132                     mMaxOpRateScalingFactor > 0.0 && mMaxOpRateScalingFactor <= 1.25);
133         }
134     }
135 
CodecPerformanceTestBase(String decoderName, String testFile, int keyPriority, float maxOpRateScalingFactor)136     public CodecPerformanceTestBase(String decoderName, String testFile, int keyPriority,
137             float maxOpRateScalingFactor) {
138         mDecoderName = decoderName;
139         mTestFile = testFile;
140         mKeyPriority = keyPriority;
141         mMaxOpRateScalingFactor = maxOpRateScalingFactor;
142         mBufferInfos = new ArrayList<>();
143     }
144 
getVideoFormat(String filePath)145     static MediaFormat getVideoFormat(String filePath) throws IOException {
146         final String input = mInputPrefix + filePath;
147         MediaExtractor extractor = new MediaExtractor();
148         extractor.setDataSource(input);
149         for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) {
150             MediaFormat format = extractor.getTrackFormat(trackID);
151             if (format.getString(MediaFormat.KEY_MIME).startsWith("video/")) {
152                 extractor.release();
153                 return format;
154             }
155         }
156         extractor.release();
157         return null;
158     }
159 
selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)160     static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats,
161             String[] features, boolean isEncoder) {
162         return selectCodecs(mime, formats, features, isEncoder, SELECT_ALL);
163     }
164 
selectHardwareCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)165     static ArrayList<String> selectHardwareCodecs(String mime, ArrayList<MediaFormat> formats,
166             String[] features, boolean isEncoder) {
167         return selectCodecs(mime, formats, features, isEncoder, SELECT_HARDWARE);
168     }
169 
selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder, int selectCodecOption)170     static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats,
171             String[] features, boolean isEncoder, int selectCodecOption) {
172         ArrayList<String> listOfCodecs = new ArrayList<>();
173         if (TestArgs.shouldSkipMediaType(mime)) {
174             return listOfCodecs;
175         }
176         MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
177         MediaCodecInfo[] codecInfos = codecList.getCodecInfos();
178 
179         for (MediaCodecInfo codecInfo : codecInfos) {
180             if (TestArgs.shouldSkipCodec(codecInfo.getName())) {
181                 continue;
182             }
183             if (codecInfo.isEncoder() != isEncoder) continue;
184             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue;
185             if (selectCodecOption == SELECT_HARDWARE && !codecInfo.isHardwareAccelerated())
186                 continue;
187             else if (selectCodecOption == SELECT_SOFTWARE && !codecInfo.isSoftwareOnly())
188                 continue;
189             String[] types = codecInfo.getSupportedTypes();
190             for (String type : types) {
191                 if (type.equalsIgnoreCase(mime)) {
192                     boolean isOk = true;
193                     MediaCodecInfo.CodecCapabilities codecCapabilities =
194                             codecInfo.getCapabilitiesForType(type);
195                     if (formats != null) {
196                         for (MediaFormat format : formats) {
197                             if (!codecCapabilities.isFormatSupported(format)) {
198                                 isOk = false;
199                                 break;
200                             }
201                         }
202                     }
203                     if (features != null) {
204                         for (String feature : features) {
205                             if (!codecCapabilities.isFeatureSupported(feature)) {
206                                 isOk = false;
207                                 break;
208                             }
209                         }
210                     }
211                     if (isOk) listOfCodecs.add(codecInfo.getName());
212                 }
213             }
214         }
215         return listOfCodecs;
216     }
217 
setUpDecoderInput()218     MediaFormat setUpDecoderInput() throws IOException {
219         final String input = mInputPrefix + mTestFile;
220         MediaExtractor extractor = new MediaExtractor();
221         extractor.setDataSource(input);
222         for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) {
223             MediaFormat format = extractor.getTrackFormat(trackID);
224             if (format.getString(MediaFormat.KEY_MIME).startsWith("video/")) {
225                 extractor.selectTrack(trackID);
226                 File file = new File(input);
227                 int bufferSize = (int) file.length();
228                 mBuff = ByteBuffer.allocate(bufferSize);
229                 int offset = 0;
230                 long maxPTS = 0;
231                 while (true) {
232                     MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
233                     bufferInfo.size = extractor.readSampleData(mBuff, offset);
234                     if (bufferInfo.size < 0) break;
235                     bufferInfo.offset = offset;
236                     bufferInfo.presentationTimeUs = extractor.getSampleTime();
237                     maxPTS = Math.max(maxPTS, bufferInfo.presentationTimeUs);
238                     int flags = extractor.getSampleFlags();
239                     bufferInfo.flags = 0;
240                     if ((flags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
241                         bufferInfo.flags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
242                     }
243                     mBufferInfos.add(bufferInfo);
244                     extractor.advance();
245                     offset += bufferInfo.size;
246                 }
247 
248                 // If the clip doesn't have sufficient frames, loopback by copying bufferInfos
249                 // from the start of the list and incrementing the timestamp.
250                 int actualBufferInfosCount = mBufferInfos.size();
251                 long ptsOffset;
252                 while (mBufferInfos.size() < MIN_FRAME_COUNT) {
253                     ptsOffset = maxPTS + 1000000L;
254                     for (int i = 0; i < actualBufferInfosCount; i++) {
255                         MediaCodec.BufferInfo tmpBufferInfo = mBufferInfos.get(i);
256                         MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
257                         bufferInfo.set(tmpBufferInfo.offset, tmpBufferInfo.size,
258                                 ptsOffset + tmpBufferInfo.presentationTimeUs,
259                                 tmpBufferInfo.flags);
260                         maxPTS = Math.max(maxPTS, bufferInfo.presentationTimeUs);
261                         mBufferInfos.add(bufferInfo);
262                         if (mBufferInfos.size() >= MIN_FRAME_COUNT) break;
263                     }
264                 }
265                 MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
266                 bufferInfo.set(0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
267                 mBufferInfos.add(bufferInfo);
268                 mDecoderMime = format.getString(MediaFormat.KEY_MIME);
269                 mWidth = format.getInteger(MediaFormat.KEY_WIDTH);
270                 mHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
271                 mFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE, 30);
272                 extractor.release();
273                 return format;
274             }
275         }
276         extractor.release();
277         fail("No video track found in file: " + mTestFile);
278         return null;
279     }
280 
281     // TODO (b/193458026) Limit max expected fps
getMaxExpectedFps(int width, int height)282     static int getMaxExpectedFps(int width, int height) {
283         int numSamples = width * height;
284         if (numSamples > 3840 * 2160 * 2) { // 8K
285             return 30;
286         } else if (numSamples > 1920 * 1088 * 2) { // 4K
287             return 120;
288         } else {
289             return 240;
290         }
291     }
292 
getMaxOperatingRate(String codecName, String mime)293     int getMaxOperatingRate(String codecName, String mime) throws IOException {
294         MediaCodec codec = MediaCodec.createByCodecName(codecName);
295         MediaCodecInfo mediaCodecInfo = codec.getCodecInfo();
296         List<MediaCodecInfo.VideoCapabilities.PerformancePoint> pps = mediaCodecInfo
297                 .getCapabilitiesForType(mime).getVideoCapabilities()
298                 .getSupportedPerformancePoints();
299         assertTrue(pps.size() > 0);
300         MediaCodecInfo.VideoCapabilities.PerformancePoint cpp =
301                 new MediaCodecInfo.VideoCapabilities.PerformancePoint(mWidth, mHeight, mFrameRate);
302         int macroblocks = cpp.getMaxMacroBlocks();
303         int maxOperatingRate = -1;
304         for (MediaCodecInfo.VideoCapabilities.PerformancePoint pp : pps) {
305             if (pp.covers(cpp)) {
306                 maxOperatingRate = Math.max(Math.min(pp.getMaxFrameRate(),
307                         (int) pp.getMaxMacroBlockRate() / macroblocks), maxOperatingRate);
308             }
309         }
310         codec.release();
311         assumeTrue("Codec doesn't advertise performance point for " + mWidth + "x" + mHeight,
312                 maxOperatingRate != -1);
313         return maxOperatingRate;
314     }
315 
getEncoderMinComplexity(String codecName, String mime)316     int getEncoderMinComplexity(String codecName, String mime) throws IOException {
317         MediaCodec codec = MediaCodec.createByCodecName(codecName);
318         MediaCodecInfo mediaCodecInfo = codec.getCodecInfo();
319         int minComplexity = -1;
320         if (mediaCodecInfo.isEncoder()) {
321             Range<Integer> complexityRange = mediaCodecInfo
322                     .getCapabilitiesForType(mime).getEncoderCapabilities()
323                     .getComplexityRange();
324             minComplexity = complexityRange.getLower();
325         }
326         codec.release();
327         return minComplexity;
328     }
329 
getMaxFrameSize(String codecName, String mime)330     static int getMaxFrameSize(String codecName, String mime) throws IOException {
331         MediaCodec codec = MediaCodec.createByCodecName(codecName);
332         MediaCodecInfo.CodecCapabilities codecCapabilities =
333                 codec.getCodecInfo().getCapabilitiesForType(mime);
334         MediaCodecInfo.VideoCapabilities vc = codecCapabilities.getVideoCapabilities();
335         Range<Integer> heights = vc.getSupportedHeights();
336         Range<Integer> widths = vc.getSupportedWidthsFor(heights.getUpper());
337         int maxFrameSize = heights.getUpper() * widths.getUpper();
338         codec.release();
339         return maxFrameSize;
340     }
341 
enqueueDecoderInput(int bufferIndex)342     void enqueueDecoderInput(int bufferIndex) {
343         MediaCodec.BufferInfo info = mBufferInfos.get(mSampleIndex++);
344         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
345             ByteBuffer dstBuf = mDecoder.getInputBuffer(bufferIndex);
346             dstBuf.put(mBuff.array(), info.offset, info.size);
347             mDecInputNum++;
348         }
349         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
350             mSawDecInputEOS = true;
351         }
352         mDecoder.queueInputBuffer(bufferIndex, 0, info.size, info.presentationTimeUs, info.flags);
353     }
354 
dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info, boolean render)355     void dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info, boolean render) {
356         if (info.size > 0) {
357             mDecOutputNum++;
358         }
359         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
360             mSawDecOutputEOS = true;
361         }
362         mDecoder.releaseOutputBuffer(bufferIndex, render);
363     }
364 }
365