1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.mediav2.common.cts;
18 
19 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface;
20 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible;
21 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUVP010;
22 
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertNotNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27 import static org.junit.Assume.assumeTrue;
28 
29 import android.graphics.ImageFormat;
30 import android.media.Image;
31 import android.media.MediaCodec;
32 import android.media.MediaCodecInfo;
33 import android.media.MediaExtractor;
34 import android.media.MediaFormat;
35 import android.os.PersistableBundle;
36 import android.util.Log;
37 import android.util.Pair;
38 
39 import com.android.compatibility.common.util.Preconditions;
40 
41 import org.junit.After;
42 import org.junit.Before;
43 
44 import java.io.IOException;
45 import java.nio.ByteBuffer;
46 import java.util.ArrayList;
47 
48 /**
49  * Wrapper class for trying and testing mediacodec decoder components.
50  */
51 public class CodecDecoderTestBase extends CodecTestBase {
52     private static final String LOG_TAG = CodecDecoderTestBase.class.getSimpleName();
53 
54     protected final String mTestFile;
55     protected boolean mIsInterlaced;
56     protected boolean mSkipChecksumVerification;
57 
58     protected final ArrayList<ByteBuffer> mCsdBuffers;
59     protected int mCurrCsdIdx;
60 
61     protected final ByteBuffer mFlatBuffer = ByteBuffer.allocate(4 * Integer.BYTES);
62 
63     protected MediaExtractor mExtractor;
64 
CodecDecoderTestBase(String codecName, String mediaType, String testFile, String allTestParams)65     public CodecDecoderTestBase(String codecName, String mediaType, String testFile,
66             String allTestParams) {
67         super(codecName, mediaType, allTestParams);
68         mTestFile = testFile;
69         mCsdBuffers = new ArrayList<>();
70     }
71 
72     @Before
setUpCodecDecoderTestBase()73     public void setUpCodecDecoderTestBase() {
74         assertTrue("Testing a mediaType that is neither audio nor video is not supported \n"
75                 + mTestConfig, mIsAudio || mIsVideo);
76     }
77 
78     @After
tearDownCodecDecoderTestBase()79     public void tearDownCodecDecoderTestBase() {
80         if (mExtractor != null) {
81             mExtractor.release();
82             mExtractor = null;
83         }
84     }
85 
getMaxSampleSizeForMediaType(String fileName, String mediaType)86     public static int getMaxSampleSizeForMediaType(String fileName, String mediaType)
87             throws IOException {
88         Preconditions.assertTestFileExists(fileName);
89         int maxSampleSize = 0;
90         MediaExtractor extractor = new MediaExtractor();
91         extractor.setDataSource(fileName);
92         for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) {
93             MediaFormat format = extractor.getTrackFormat(trackID);
94             if (mediaType.equalsIgnoreCase(format.getString(MediaFormat.KEY_MIME))) {
95                 extractor.selectTrack(trackID);
96                 if (format.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) {
97                     maxSampleSize = format.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE);
98                 } else {
99                     int size;
100                     while ((size = (int) extractor.getSampleSize()) != -1) {
101                         maxSampleSize = Math.max(maxSampleSize, size);
102                         extractor.advance();
103                     }
104                 }
105                 extractor.release();
106                 return maxSampleSize;
107             }
108         }
109         fail("No track with mediaType: " + mediaType + " found in file: " + fileName + "\n");
110         return maxSampleSize;
111     }
112 
setUpSource(String srcFile)113     protected MediaFormat setUpSource(String srcFile) throws IOException {
114         Preconditions.assertTestFileExists(srcFile);
115         mExtractor = new MediaExtractor();
116         mExtractor.setDataSource(srcFile);
117         for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) {
118             MediaFormat format = mExtractor.getTrackFormat(trackID);
119             if (mMediaType.equalsIgnoreCase(format.getString(MediaFormat.KEY_MIME))) {
120                 // This is required for some mlaw and alaw test vectors where access unit size is
121                 // exceeding default max input size
122                 if (mMediaType.equalsIgnoreCase(MediaFormat.MIMETYPE_AUDIO_G711_ALAW)
123                         || mMediaType.equalsIgnoreCase(MediaFormat.MIMETYPE_AUDIO_G711_MLAW)) {
124                     format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE,
125                             getMaxSampleSizeForMediaType(srcFile, mMediaType));
126                 }
127                 mExtractor.selectTrack(trackID);
128                 if (mIsVideo) {
129                     ArrayList<MediaFormat> formatList = new ArrayList<>();
130                     formatList.add(format);
131                     boolean selectHBD = doesAnyFormatHaveHDRProfile(mMediaType, formatList);
132                     if (!selectHBD && srcFile.contains("10bit")) {
133                         selectHBD = true;
134                     }
135                     format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
136                             getColorFormat(mCodecName, mMediaType, mSurface != null, selectHBD));
137                     if (selectHBD && (format.getInteger(MediaFormat.KEY_COLOR_FORMAT)
138                             != COLOR_FormatYUVP010)) {
139                         mSkipChecksumVerification = true;
140                     }
141 
142                     if ((format.getInteger(MediaFormat.KEY_COLOR_FORMAT) != COLOR_FormatYUVP010)
143                             && selectHBD && mSurface == null) {
144                         // Codecs that do not advertise P010 on devices with VNDK version < T, do
145                         // not support decoding high bit depth clips when color format is set to
146                         // COLOR_FormatYUV420Flexible in byte buffer mode. Since byte buffer mode
147                         // for high bit depth decoding wasn't tested prior to Android T, skip this
148                         // when device is older
149                         assumeTrue("Skipping High Bit Depth tests on VNDK < T", VNDK_IS_AT_LEAST_T);
150                     }
151                 }
152                 // TODO: determine this from the extractor format when it becomes exposed.
153                 mIsInterlaced = srcFile.contains("_interlaced_");
154                 return format;
155             }
156         }
157         fail("No track with mediaType: " + mMediaType + " found in file: " + srcFile + "\n"
158                 + mTestConfig + mTestEnv);
159         return null;
160     }
161 
getColorFormat(String name, String mediaType, boolean surfaceMode, boolean hbdMode)162     int getColorFormat(String name, String mediaType, boolean surfaceMode, boolean hbdMode)
163             throws IOException {
164         if (surfaceMode) return COLOR_FormatSurface;
165         if (hbdMode) {
166             MediaCodec codec = MediaCodec.createByCodecName(name);
167             MediaCodecInfo.CodecCapabilities cap =
168                     codec.getCodecInfo().getCapabilitiesForType(mediaType);
169             codec.release();
170             for (int c : cap.colorFormats) {
171                 if (c == COLOR_FormatYUVP010) {
172                     return c;
173                 }
174             }
175         }
176         return COLOR_FormatYUV420Flexible;
177     }
178 
hasCSD(MediaFormat format)179     public static boolean hasCSD(MediaFormat format) {
180         return format.containsKey("csd-0");
181     }
182 
flattenBufferInfo(MediaCodec.BufferInfo info, boolean isAudio)183     protected void flattenBufferInfo(MediaCodec.BufferInfo info, boolean isAudio) {
184         if (isAudio) {
185             mFlatBuffer.putInt(info.size);
186         }
187         mFlatBuffer.putInt(info.flags & ~MediaCodec.BUFFER_FLAG_END_OF_STREAM)
188                 .putLong(info.presentationTimeUs);
189         mFlatBuffer.flip();
190     }
191 
enqueueCodecConfig(int bufferIndex)192     void enqueueCodecConfig(int bufferIndex) {
193         ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
194         ByteBuffer csdBuffer = mCsdBuffers.get(mCurrCsdIdx);
195         inputBuffer.put((ByteBuffer) csdBuffer.rewind());
196         mCodec.queueInputBuffer(bufferIndex, 0, csdBuffer.limit(), 0,
197                 MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
198         if (ENABLE_LOGS) {
199             Log.v(LOG_TAG, "queued csd: id: " + bufferIndex + " size: " + csdBuffer.limit());
200         }
201     }
202 
enqueueInput(int bufferIndex)203     protected void enqueueInput(int bufferIndex) {
204         if (mExtractor.getSampleSize() < 0) {
205             enqueueEOS(bufferIndex);
206         } else {
207             ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
208             mExtractor.readSampleData(inputBuffer, 0);
209             int size = (int) mExtractor.getSampleSize();
210             long pts = mExtractor.getSampleTime();
211             int extractorFlags = mExtractor.getSampleFlags();
212             int codecFlags = 0;
213             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
214                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
215             }
216             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_PARTIAL_FRAME) != 0) {
217                 codecFlags |= MediaCodec.BUFFER_FLAG_PARTIAL_FRAME;
218             }
219             if (!mExtractor.advance() && mSignalEOSWithLastFrame) {
220                 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
221                 mSawInputEOS = true;
222             }
223             if (ENABLE_LOGS) {
224                 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts
225                         + " flags: " + codecFlags);
226             }
227             mCodec.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags);
228             if (size > 0 && (codecFlags & (MediaCodec.BUFFER_FLAG_CODEC_CONFIG
229                     | MediaCodec.BUFFER_FLAG_PARTIAL_FRAME)) == 0) {
230                 mOutputBuff.saveInPTS(pts);
231                 mInputCount++;
232             }
233         }
234     }
235 
enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info)236     protected void enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info) {
237         ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
238         buffer.position(info.offset);
239         for (int i = 0; i < info.size; i++) {
240             inputBuffer.put(buffer.get());
241         }
242         if (ENABLE_LOGS) {
243             Log.v(LOG_TAG, "input: id: " + bufferIndex + " flags: " + info.flags + " size: "
244                     + info.size + " timestamp: " + info.presentationTimeUs);
245         }
246         mCodec.queueInputBuffer(bufferIndex, 0, info.size, info.presentationTimeUs,
247                 info.flags);
248         if (info.size > 0 && ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0)
249                 && ((info.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) == 0)) {
250             mOutputBuff.saveInPTS(info.presentationTimeUs);
251             mInputCount++;
252         }
253         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
254             mSawInputEOS = true;
255         }
256     }
257 
dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)258     protected void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) {
259         if (info.size > 0 && mSaveToMem) {
260             ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex);
261             flattenBufferInfo(info, mIsAudio);
262             mOutputBuff.checksum(mFlatBuffer, mFlatBuffer.limit());
263             if (mIsAudio) {
264                 mOutputBuff.checksum(buf, info);
265                 mOutputBuff.saveToMemory(buf, info);
266             } else {
267                 // tests both getOutputImage and getOutputBuffer. Can do time division
268                 // multiplexing but lets allow it for now
269                 Image img = mCodec.getOutputImage(bufferIndex);
270                 assertNotNull("CPU-read via ImageReader API is not available", img);
271                 mOutputBuff.checksum(img);
272                 int imgFormat = img.getFormat();
273                 int bytesPerSample = (ImageFormat.getBitsPerPixel(imgFormat) * 2) / (8 * 3);
274 
275                 MediaFormat format = mCodec.getOutputFormat();
276                 buf = mCodec.getOutputBuffer(bufferIndex);
277                 int width = format.getInteger(MediaFormat.KEY_WIDTH);
278                 int height = format.getInteger(MediaFormat.KEY_HEIGHT);
279                 int stride = format.getInteger(MediaFormat.KEY_STRIDE);
280                 mOutputBuff.checksum(buf, info.size, width, height, stride, bytesPerSample);
281             }
282         }
283         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
284             mSawOutputEOS = true;
285         }
286         if (ENABLE_LOGS) {
287             Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: "
288                     + info.size + " timestamp: " + info.presentationTimeUs);
289         }
290         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
291             mOutputBuff.saveOutPTS(info.presentationTimeUs);
292             mOutputCount++;
293         }
294         mCodec.releaseOutputBuffer(bufferIndex, false);
295     }
296 
doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list)297     protected void doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list)
298             throws InterruptedException {
299         int frameCount = 0;
300         if (mIsCodecInAsyncMode) {
301             // output processing after queuing EOS is done in waitForAllOutputs()
302             while (!mAsyncHandle.hasSeenError() && !mSawInputEOS && frameCount < list.size()) {
303                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork();
304                 if (element != null) {
305                     int bufferID = element.first;
306                     MediaCodec.BufferInfo info = element.second;
307                     if (info != null) {
308                         dequeueOutput(bufferID, info);
309                     } else {
310                         enqueueInput(bufferID, buffer, list.get(frameCount));
311                         frameCount++;
312                     }
313                 }
314             }
315         } else {
316             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
317             // output processing after queuing EOS is done in waitForAllOutputs()
318             while (!mSawInputEOS && frameCount < list.size()) {
319                 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
320                 if (outputBufferId >= 0) {
321                     dequeueOutput(outputBufferId, outInfo);
322                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
323                     mOutFormat = mCodec.getOutputFormat();
324                     mSignalledOutFormatChanged = true;
325                 }
326                 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US);
327                 if (inputBufferId != -1) {
328                     enqueueInput(inputBufferId, buffer, list.get(frameCount));
329                     frameCount++;
330                 }
331             }
332         }
333     }
334 
queueCodecConfig()335     protected void queueCodecConfig() throws InterruptedException {
336         if (mIsCodecInAsyncMode) {
337             for (mCurrCsdIdx = 0; !mAsyncHandle.hasSeenError() && mCurrCsdIdx < mCsdBuffers.size();
338                     mCurrCsdIdx++) {
339                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getInput();
340                 if (element != null) {
341                     enqueueCodecConfig(element.first);
342                 }
343             }
344         } else {
345             for (mCurrCsdIdx = 0; mCurrCsdIdx < mCsdBuffers.size(); mCurrCsdIdx++) {
346                 enqueueCodecConfig(mCodec.dequeueInputBuffer(-1));
347             }
348         }
349     }
350 
validateTestState()351     protected void validateTestState() {
352         super.validateTestState();
353         if (!mOutputBuff.isPtsStrictlyIncreasing(mPrevOutputPts)) {
354             fail("Output timestamps are not strictly increasing \n" + mTestConfig + mTestEnv
355                     + mOutputBuff.getErrMsg());
356         }
357         if (mIsVideo) {
358             // TODO: Timestamps for deinterlaced content are under review. (E.g. can decoders
359             // produce multiple progressive frames?) For now, do not verify timestamps.
360             if (!mIsInterlaced && !mOutputBuff.isOutPtsListIdenticalToInpPtsList(false)) {
361                 fail("Input pts list and Output pts list are not identical ]\n" + mTestConfig
362                         + mTestEnv + mOutputBuff.getErrMsg());
363             }
364         }
365     }
366 
decodeToMemory(String file, String decoder, OutputManager outputBuff, long pts, int mode, int frameLimit, boolean isAsync, boolean signalledEos)367     public void decodeToMemory(String file, String decoder, OutputManager outputBuff, long pts,
368             int mode, int frameLimit, boolean isAsync, boolean signalledEos)
369             throws IOException, InterruptedException {
370         mSaveToMem = true;
371         mOutputBuff = outputBuff;
372         mCodec = MediaCodec.createByCodecName(decoder);
373         MediaFormat format = setUpSource(file);
374         configureCodec(format, isAsync, signalledEos, false);
375         mCodec.start();
376         mExtractor.seekTo(pts, mode);
377         doWork(frameLimit);
378         queueEOS();
379         waitForAllOutputs();
380         mCodec.stop();
381         mCodec.release();
382         mExtractor.release();
383         mSaveToMem = false;
384     }
385 
decodeToMemory(String file, String decoder, OutputManager outputBuff, long pts, int mode, int frameLimit)386     public void decodeToMemory(String file, String decoder, OutputManager outputBuff, long pts,
387             int mode, int frameLimit) throws IOException, InterruptedException {
388         decodeToMemory(file, decoder, outputBuff, pts, mode, frameLimit, false, true);
389     }
390 
decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit)391     public void decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit)
392             throws IOException, InterruptedException {
393         decodeToMemory(file, decoder, new OutputManager(), pts, mode, frameLimit);
394     }
395 
decodeToMemory(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list, MediaFormat format, String decoder)396     public void decodeToMemory(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list,
397             MediaFormat format, String decoder) throws IOException, InterruptedException {
398         mSaveToMem = true;
399         mOutputBuff = new OutputManager();
400         mCodec = MediaCodec.createByCodecName(decoder);
401         configureCodec(format, false, true, false);
402         mCodec.start();
403         doWork(buffer, list);
404         queueEOS();
405         waitForAllOutputs();
406         mCodec.stop();
407         mCodec.release();
408         mSaveToMem = false;
409     }
410 
411     @Override
validateMetrics(String decoder, MediaFormat format)412     protected PersistableBundle validateMetrics(String decoder, MediaFormat format) {
413         PersistableBundle metrics = super.validateMetrics(decoder, format);
414         assertEquals("error! metrics#MetricsConstants.MIME_TYPE is not as expected \n" + mTestConfig
415                 + mTestEnv, metrics.getString(MediaCodec.MetricsConstants.MIME_TYPE), mMediaType);
416         assertEquals("error! metrics#MetricsConstants.ENCODER is not as expected \n" + mTestConfig
417                 + mTestEnv, 0, metrics.getInt(MediaCodec.MetricsConstants.ENCODER));
418         return metrics;
419     }
420 
validateColorAspects(int range, int standard, int transfer, boolean ignoreColorBox)421     public void validateColorAspects(int range, int standard, int transfer, boolean ignoreColorBox)
422             throws IOException, InterruptedException {
423         Preconditions.assertTestFileExists(mTestFile);
424         mOutputBuff = new OutputManager();
425         MediaFormat format = setUpSource(mTestFile);
426         if (ignoreColorBox) {
427             format.removeKey(MediaFormat.KEY_COLOR_RANGE);
428             format.removeKey(MediaFormat.KEY_COLOR_STANDARD);
429             format.removeKey(MediaFormat.KEY_COLOR_TRANSFER);
430         }
431         mCodec = MediaCodec.createByCodecName(mCodecName);
432         configureCodec(format, true, true, false);
433         mCodec.start();
434         doWork(1);
435         queueEOS();
436         waitForAllOutputs();
437         validateColorAspects(mCodec.getOutputFormat(), range, standard, transfer);
438         mCodec.stop();
439         mCodec.release();
440         mExtractor.release();
441     }
442 }
443