1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.mediapc.cts;
18 
19 import static android.mediapc.cts.common.CodecMetrics.getMetrics;
20 import static android.mediav2.common.cts.CodecTestBase.PROFILE_HLG_MAP;
21 import static android.mediapc.cts.CodecTestBase.areFormatsSupported;
22 
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25 
26 import android.media.MediaCodec;
27 import android.media.MediaCodecInfo;
28 import android.media.MediaExtractor;
29 import android.media.MediaFormat;
30 import android.mediapc.cts.common.CodecMetrics;
31 import android.util.Log;
32 import android.util.Pair;
33 import android.view.Surface;
34 
35 import java.io.IOException;
36 import java.nio.ByteBuffer;
37 import java.util.ArrayList;
38 import java.util.Objects;
39 import java.util.concurrent.Callable;
40 
41 public class CodecTranscoderTestBase {
42     private static final String LOG_TAG = CodecTranscoderTestBase.class.getSimpleName();
43     private static final boolean ENABLE_LOGS = false;
44     static final String mInpPrefix = WorkDir.getMediaDirString();
45     String mMediaType;
46     String mTestFile;
47     int mBitrate;
48     int mFrameRate;
49     double mFrameDrops;
50     long mLastPresentationTimeUs = -1;
51     boolean mUseHighBitDepth;
52     MediaExtractor mExtractor;
53     int mMaxBFrames;
54     int mLatency;
55 
56     MediaCodec mEncoder;
57     CodecAsyncHandler mAsyncHandleEncoder;
58     MediaCodec mDecoder;
59     CodecAsyncHandler mAsyncHandleDecoder;
60     Surface mSurface;
61     InputSurface mInputSurface;
62     OutputSurface mOutputSurface;
63 
64     boolean mSawDecInputEOS;
65     boolean mSawDecOutputEOS;
66     boolean mSawEncOutputEOS;
67     boolean mIsCodecInAsyncMode;
68     boolean mSignalEOSWithLastFrame;
69     boolean mReviseLatency;
70     int mDecInputCount;
71     int mDecOutputCount;
72     int mEncOutputCount;
73 
CodecTranscoderTestBase(String mediaType, String testfile, int bitrate, int frameRate, boolean useHighBitDepth)74     CodecTranscoderTestBase(String mediaType, String testfile, int bitrate, int frameRate,
75             boolean useHighBitDepth) {
76         mMediaType = mediaType;
77         mTestFile = testfile;
78         mBitrate = bitrate;
79         mFrameRate = frameRate;
80         mUseHighBitDepth = useHighBitDepth;
81         mMaxBFrames = 0;
82         mLatency = mMaxBFrames;
83         mReviseLatency = false;
84         mAsyncHandleDecoder = new CodecAsyncHandler();
85         mAsyncHandleEncoder = new CodecAsyncHandler();
86     }
87 
hasSeenError()88     boolean hasSeenError() {
89         return mAsyncHandleDecoder.hasSeenError() || mAsyncHandleEncoder.hasSeenError();
90     }
91 
setUpSource(String srcFile)92     MediaFormat setUpSource(String srcFile) throws IOException {
93         mExtractor = new MediaExtractor();
94         mExtractor.setDataSource(mInpPrefix + srcFile);
95         for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) {
96             MediaFormat format = mExtractor.getTrackFormat(trackID);
97             String mediaType = format.getString(MediaFormat.KEY_MIME);
98             if (mediaType.startsWith("video/")) {
99                 mExtractor.selectTrack(trackID);
100                 format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
101                         MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
102                 format.setInteger(MediaFormat.KEY_PRIORITY, 1); // Best effort
103                 return format;
104             }
105         }
106         mExtractor.release();
107         fail("No video track found in file: " + srcFile);
108         return null;
109     }
110 
resetContext(boolean isAsync, boolean signalEOSWithLastFrame)111     void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) {
112         mAsyncHandleDecoder.resetContext();
113         mAsyncHandleEncoder.resetContext();
114         mIsCodecInAsyncMode = isAsync;
115         mSignalEOSWithLastFrame = signalEOSWithLastFrame;
116         mSawDecInputEOS = false;
117         mSawDecOutputEOS = false;
118         mSawEncOutputEOS = false;
119         mDecInputCount = 0;
120         mDecOutputCount = 0;
121         mEncOutputCount = 0;
122         mFrameDrops = 0;
123         mLastPresentationTimeUs = -1;
124     }
125 
configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync, boolean signalEOSWithLastFrame)126     void configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync,
127             boolean signalEOSWithLastFrame) {
128         resetContext(isAsync, signalEOSWithLastFrame);
129         mAsyncHandleEncoder.setCallBack(mEncoder, isAsync);
130         mEncoder.configure(encFormat, null, MediaCodec.CONFIGURE_FLAG_ENCODE, null);
131         if (mEncoder.getInputFormat().containsKey(MediaFormat.KEY_LATENCY)) {
132             mReviseLatency = true;
133             mLatency = mEncoder.getInputFormat().getInteger(MediaFormat.KEY_LATENCY);
134         }
135         mSurface = mEncoder.createInputSurface();
136         assertTrue("Surface is not valid", mSurface.isValid());
137         mAsyncHandleDecoder.setCallBack(mDecoder, isAsync);
138         mDecoder.configure(decFormat, mSurface, null, 0);
139         if (ENABLE_LOGS) {
140             Log.v(LOG_TAG, "codec configured");
141         }
142     }
143 
enqueueDecoderEOS(int bufferIndex)144     void enqueueDecoderEOS(int bufferIndex) {
145         if (!mSawDecInputEOS) {
146             mDecoder.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
147             mSawDecInputEOS = true;
148             if (ENABLE_LOGS) {
149                 Log.v(LOG_TAG, "Queued End of Stream");
150             }
151         }
152     }
153 
enqueueDecoderInput(int bufferIndex)154     void enqueueDecoderInput(int bufferIndex) {
155         if (mExtractor.getSampleSize() < 0) {
156             enqueueDecoderEOS(bufferIndex);
157         } else {
158             ByteBuffer inputBuffer = mDecoder.getInputBuffer(bufferIndex);
159             int size = mExtractor.readSampleData(inputBuffer, 0);
160             long pts = mExtractor.getSampleTime();
161             int extractorFlags = mExtractor.getSampleFlags();
162             int codecFlags = 0;
163             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
164                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
165             }
166             if (!mExtractor.advance() && mSignalEOSWithLastFrame) {
167                 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
168                 mSawDecInputEOS = true;
169             }
170             mDecoder.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags);
171             if (size > 0 && (codecFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
172                 mDecInputCount++;
173             }
174         }
175     }
176 
dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info)177     void dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
178         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
179             mSawDecOutputEOS = true;
180         }
181         long expectedFrameDurationUs = 1000000 / mFrameRate;
182         long presentationTimeUs = info.presentationTimeUs;
183         if (mLastPresentationTimeUs != -1) {
184             if (presentationTimeUs > mLastPresentationTimeUs + expectedFrameDurationUs) {
185                 mFrameDrops++;
186             }
187         }
188         mLastPresentationTimeUs = presentationTimeUs;
189         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
190             mDecOutputCount++;
191         }
192         mDecoder.releaseOutputBuffer(bufferIndex, mSurface != null);
193     }
194 
dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info)195     void dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
196         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
197             mSawEncOutputEOS = true;
198         }
199         long expectedFrameDurationUs = 1000000 / mFrameRate;
200         long presentationTimeUs = info.presentationTimeUs;
201         if (mLastPresentationTimeUs != -1) {
202             if (presentationTimeUs > mLastPresentationTimeUs + expectedFrameDurationUs) {
203                 mFrameDrops++;
204             }
205         }
206         mLastPresentationTimeUs = presentationTimeUs;
207         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
208             mEncOutputCount++;
209         }
210         mEncoder.releaseOutputBuffer(bufferIndex, false);
211     }
212 
tryEncoderOutput(long timeOutUs)213     void tryEncoderOutput(long timeOutUs) throws InterruptedException {
214         if (mIsCodecInAsyncMode) {
215             if (!hasSeenError() && !mSawEncOutputEOS) {
216                 int retry = 0;
217                 while (mReviseLatency) {
218                     if (mAsyncHandleEncoder.hasOutputFormatChanged()) {
219                         mReviseLatency = false;
220                         int actualLatency = mAsyncHandleEncoder.getOutputFormat()
221                                 .getInteger(MediaFormat.KEY_LATENCY, mLatency);
222                         if (mLatency < actualLatency) {
223                             mLatency = actualLatency;
224                             return;
225                         }
226                     } else {
227                         if (retry > CodecTestBase.RETRY_LIMIT) throw new InterruptedException(
228                                 "did not receive output format changed for encoder after " +
229                                         CodecTestBase.Q_DEQ_TIMEOUT_US * CodecTestBase.RETRY_LIMIT +
230                                         " us");
231                         Thread.sleep(CodecTestBase.Q_DEQ_TIMEOUT_US / 1000);
232                         retry ++;
233                     }
234                 }
235                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleEncoder.getOutput();
236                 if (element != null) {
237                     dequeueEncoderOutput(element.first, element.second);
238                 }
239             }
240         } else {
241             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
242             if (!mSawEncOutputEOS) {
243                 int outputBufferId = mEncoder.dequeueOutputBuffer(outInfo, timeOutUs);
244                 if (outputBufferId >= 0) {
245                     dequeueEncoderOutput(outputBufferId, outInfo);
246                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
247                     mLatency = mEncoder.getOutputFormat()
248                             .getInteger(MediaFormat.KEY_LATENCY, mLatency);
249                 }
250             }
251         }
252     }
253 
waitForAllEncoderOutputs()254     void waitForAllEncoderOutputs() throws InterruptedException {
255         if (mIsCodecInAsyncMode) {
256             while (!hasSeenError() && !mSawEncOutputEOS) {
257                 tryEncoderOutput(CodecTestBase.Q_DEQ_TIMEOUT_US);
258             }
259         } else {
260             while (!mSawEncOutputEOS) {
261                 tryEncoderOutput(CodecTestBase.Q_DEQ_TIMEOUT_US);
262             }
263         }
264     }
265 
queueEOS()266     void queueEOS() throws InterruptedException {
267         if (mIsCodecInAsyncMode) {
268             while (!mAsyncHandleDecoder.hasSeenError() && !mSawDecInputEOS) {
269                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork();
270                 if (element != null) {
271                     int bufferID = element.first;
272                     MediaCodec.BufferInfo info = element.second;
273                     if (info != null) {
274                         dequeueDecoderOutput(bufferID, info);
275                     } else {
276                         enqueueDecoderEOS(element.first);
277                     }
278                 }
279             }
280         } else {
281             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
282             while (!mSawDecInputEOS) {
283                 int outputBufferId =
284                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
285                 if (outputBufferId >= 0) {
286                     dequeueDecoderOutput(outputBufferId, outInfo);
287                 }
288                 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US);
289                 if (inputBufferId != -1) {
290                     enqueueDecoderEOS(inputBufferId);
291                 }
292             }
293         }
294         if (mIsCodecInAsyncMode) {
295             while (!hasSeenError() && !mSawDecOutputEOS) {
296                 Pair<Integer, MediaCodec.BufferInfo> decOp = mAsyncHandleDecoder.getOutput();
297                 if (decOp != null) dequeueDecoderOutput(decOp.first, decOp.second);
298                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
299                 if (mDecOutputCount - mEncOutputCount > mLatency) {
300                     tryEncoderOutput(-1);
301                 }
302             }
303         } else {
304             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
305             while (!mSawDecOutputEOS) {
306                 int outputBufferId =
307                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
308                 if (outputBufferId >= 0) {
309                     dequeueDecoderOutput(outputBufferId, outInfo);
310                 }
311                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
312                 if (mDecOutputCount - mEncOutputCount > mLatency) {
313                     tryEncoderOutput(-1);
314                 }
315             }
316         }
317     }
318 
doWork(int frameLimit)319     void doWork(int frameLimit) throws InterruptedException {
320         int frameCnt = 0;
321         if (mIsCodecInAsyncMode) {
322             // dequeue output after inputEOS is expected to be done in waitForAllOutputs()
323             while (!hasSeenError() && !mSawDecInputEOS && frameCnt < frameLimit) {
324                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork();
325                 if (element != null) {
326                     int bufferID = element.first;
327                     MediaCodec.BufferInfo info = element.second;
328                     if (info != null) {
329                         // <id, info> corresponds to output callback. Handle it accordingly
330                         dequeueDecoderOutput(bufferID, info);
331                     } else {
332                         // <id, null> corresponds to input callback. Handle it accordingly
333                         enqueueDecoderInput(bufferID);
334                         frameCnt++;
335                     }
336                 }
337                 // check decoder EOS
338                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
339                 // encoder output
340                 if (mDecOutputCount - mEncOutputCount > mLatency) {
341                     tryEncoderOutput(-1);
342                 }
343             }
344         } else {
345             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
346             while (!mSawDecInputEOS && frameCnt < frameLimit) {
347                 // decoder input
348                 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US);
349                 if (inputBufferId != -1) {
350                     enqueueDecoderInput(inputBufferId);
351                     frameCnt++;
352                 }
353                 // decoder output
354                 int outputBufferId =
355                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
356                 if (outputBufferId >= 0) {
357                     dequeueDecoderOutput(outputBufferId, outInfo);
358                 }
359                 // check decoder EOS
360                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
361                 // encoder output
362                 if (mDecOutputCount - mEncOutputCount > mLatency) {
363                     tryEncoderOutput(-1);
364                 }
365             }
366         }
367     }
368 
setUpEncoderFormat(MediaFormat decoderFormat)369     MediaFormat setUpEncoderFormat(MediaFormat decoderFormat) {
370         MediaFormat encoderFormat = new MediaFormat();
371         encoderFormat.setString(MediaFormat.KEY_MIME, mMediaType);
372         encoderFormat.setInteger(MediaFormat.KEY_WIDTH,
373                 decoderFormat.getInteger(MediaFormat.KEY_WIDTH));
374         encoderFormat.setInteger(MediaFormat.KEY_HEIGHT,
375                 decoderFormat.getInteger(MediaFormat.KEY_HEIGHT));
376         encoderFormat.setInteger(MediaFormat.KEY_FRAME_RATE, mFrameRate);
377         encoderFormat.setInteger(MediaFormat.KEY_BIT_RATE, mBitrate);
378         encoderFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1.0f);
379         encoderFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
380                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
381         encoderFormat.setInteger(MediaFormat.KEY_MAX_B_FRAMES, mMaxBFrames);
382         encoderFormat.setInteger(MediaFormat.KEY_PRIORITY,
383                 decoderFormat.getInteger(MediaFormat.KEY_PRIORITY));
384         if (mUseHighBitDepth) {
385             encoderFormat.setInteger(MediaFormat.KEY_PROFILE,
386                     Objects.requireNonNull(PROFILE_HLG_MAP.get(mMediaType))[0]);
387             encoderFormat.setInteger(MediaFormat.KEY_LEVEL, 1);
388         }
389         return encoderFormat;
390     }
391 }
392 
393 /**
394  * The following class transcodes the given testFile and returns the achieved fps for transcoding.
395  */
396 class Transcode extends CodecTranscoderTestBase implements Callable<CodecMetrics> {
397     private static final String LOG_TAG = Transcode.class.getSimpleName();
398 
399     final String mDecoderName;
400     final String mEncoderName;
401     final boolean mIsAsync;
402 
Transcode(String mediaType, String testFile, String decoderName, String encoderName, boolean isAsync, boolean useHighBitDepth)403     Transcode(String mediaType, String testFile, String decoderName, String encoderName,
404             boolean isAsync, boolean useHighBitDepth) {
405         super(mediaType, testFile, 3000000, 30, useHighBitDepth);
406         mDecoderName = decoderName;
407         mEncoderName = encoderName;
408         mIsAsync = isAsync;
409     }
410 
doTranscode()411     public CodecMetrics doTranscode() throws Exception {
412         MediaFormat decoderFormat = setUpSource(mTestFile);
413         ArrayList<MediaFormat> formats = new ArrayList<>();
414         formats.add(decoderFormat);
415         // If the decoder doesn't support the formats, then return 0 to indicate that decode failed
416         if (!areFormatsSupported(mDecoderName, formats)) {
417             return getMetrics(0.0, 0.0);
418         }
419 
420         mDecoder = MediaCodec.createByCodecName(mDecoderName);
421         MediaFormat encoderFormat = setUpEncoderFormat(decoderFormat);
422         mEncoder = MediaCodec.createByCodecName(mEncoderName);
423         mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
424         configureCodec(decoderFormat, encoderFormat, mIsAsync, false);
425         mEncoder.start();
426         mDecoder.start();
427         long start = System.currentTimeMillis();
428         doWork(Integer.MAX_VALUE);
429         queueEOS();
430         waitForAllEncoderOutputs();
431         long end = System.currentTimeMillis();
432         mSurface.release();
433         mDecoder.stop();
434         mDecoder.release();
435         mEncoder.stop();
436         mEncoder.release();
437         mExtractor.release();
438         double fps = mEncOutputCount / ((end - start) / 1000.0);
439         Log.d(LOG_TAG, "MediaType: " + mMediaType + " Decoder: " + mDecoderName + " Encoder: "
440                 + mEncoderName + " Achieved fps: " + fps);
441         return getMetrics(fps, mFrameDrops / 30);
442     }
443 
444     @Override
call()445     public CodecMetrics call() throws Exception {
446         try {
447             return doTranscode();
448         } catch (Exception e) {
449             Log.d(LOG_TAG, "MediaType: " + mMediaType + " Decoder: " + mDecoderName + " Encoder: "
450                     + mEncoderName + " Failed due to: " + e);
451             return getMetrics(-1.0, 0.0);
452         }
453     }
454 }
455 
456 /**
457  * The following class transcodes the given testFile until loadStatus is finished.
458  * If input reaches eos, it will rewind the input to start position.
459  */
460 class TranscodeLoad extends Transcode {
461     private static final String LOG_TAG = TranscodeLoad.class.getSimpleName();
462     private static final boolean DEBUG = false;
463     private static final int TARGET_WIDTH = 1280;
464     private static final int TARGET_HEIGHT = 720;
465     private final LoadStatus mLoadStatus;
466 
467     private long mMaxPts;
468     private long mBasePts;
469 
TranscodeLoad(String mediaType, String testFile, String decoderName, String encoderName, LoadStatus loadStatus)470     TranscodeLoad(String mediaType, String testFile, String decoderName, String encoderName,
471             LoadStatus loadStatus) {
472         super(mediaType, testFile, decoderName, encoderName, false, false);
473         mLoadStatus = loadStatus;
474         mMaxPts = 0;
475         mBasePts = 0;
476     }
477 
478     @Override
setUpEncoderFormat(MediaFormat decoderFormat)479     MediaFormat setUpEncoderFormat(MediaFormat decoderFormat) {
480         MediaFormat encoderFormat = new MediaFormat();
481         encoderFormat.setString(MediaFormat.KEY_MIME, mMediaType);
482         encoderFormat.setInteger(MediaFormat.KEY_FRAME_RATE, mFrameRate);
483         encoderFormat.setInteger(MediaFormat.KEY_BIT_RATE, mBitrate);
484         encoderFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1.0f);
485         encoderFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
486                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
487         encoderFormat.setInteger(MediaFormat.KEY_MAX_B_FRAMES, mMaxBFrames);
488         encoderFormat.setInteger(MediaFormat.KEY_PRIORITY,
489                 decoderFormat.getInteger(MediaFormat.KEY_PRIORITY));
490         if ((decoderFormat.getInteger(MediaFormat.KEY_WIDTH) != TARGET_WIDTH)
491                 || (decoderFormat.getInteger(MediaFormat.KEY_HEIGHT) != TARGET_HEIGHT)) {
492             encoderFormat.setInteger(MediaFormat.KEY_WIDTH, TARGET_WIDTH);
493             encoderFormat.setInteger(MediaFormat.KEY_HEIGHT, TARGET_HEIGHT);
494         } else {
495             encoderFormat.setInteger(MediaFormat.KEY_WIDTH,
496                     decoderFormat.getInteger(MediaFormat.KEY_WIDTH));
497             encoderFormat.setInteger(MediaFormat.KEY_HEIGHT,
498                     decoderFormat.getInteger(MediaFormat.KEY_HEIGHT));
499         }
500         return encoderFormat;
501     }
502 
503     @Override
configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync, boolean signalEOSWithLastFrame)504     void configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync,
505             boolean signalEOSWithLastFrame) {
506         decFormat.setInteger(MediaFormat.KEY_PRIORITY, 1);
507         encFormat.setInteger(MediaFormat.KEY_PRIORITY, 1);
508         resetContext(isAsync, signalEOSWithLastFrame);
509         mAsyncHandleEncoder.setCallBack(mEncoder, isAsync);
510         mEncoder.configure(encFormat, null, MediaCodec.CONFIGURE_FLAG_ENCODE, null);
511         if (mEncoder.getInputFormat().containsKey(MediaFormat.KEY_LATENCY)) {
512             mReviseLatency = true;
513             mLatency = mEncoder.getInputFormat().getInteger(MediaFormat.KEY_LATENCY);
514         }
515         mSurface = mEncoder.createInputSurface();
516         assertTrue("Surface is not valid", mSurface.isValid());
517         mInputSurface = new InputSurface(mSurface);
518         mInputSurface.updateSize(TARGET_WIDTH, TARGET_HEIGHT);
519         mInputSurface.makeCurrent();
520         mAsyncHandleDecoder.setCallBack(mDecoder, isAsync);
521         mOutputSurface = new OutputSurface();
522         mDecoder.configure(decFormat, mOutputSurface.getSurface(), null, 0);
523     }
524 
525     @Override
doTranscode()526     public CodecMetrics doTranscode() throws Exception {
527         MediaFormat decoderFormat = setUpSource(mTestFile);
528         mDecoder = MediaCodec.createByCodecName(mDecoderName);
529         MediaFormat encoderFormat = setUpEncoderFormat(decoderFormat);
530         mEncoder = MediaCodec.createByCodecName(mEncoderName);
531         mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
532         configureCodec(decoderFormat, encoderFormat, false, false);
533         mEncoder.start();
534         mDecoder.start();
535         long start = System.currentTimeMillis();
536         doWork(Integer.MAX_VALUE);
537         long end = System.currentTimeMillis();
538         mSurface.release();
539         mInputSurface.release();
540         mOutputSurface.release();
541         mDecoder.stop();
542         mDecoder.release();
543         mEncoder.stop();
544         mEncoder.release();
545         mExtractor.release();
546         double fps = mEncOutputCount / ((end - start) / 1000.0);
547         Log.d(LOG_TAG,
548                 "MediaType: " + mMediaType + " Decoder: " + mDecoderName + " Encoder: "
549                         + mEncoderName + " Achieved fps: " + fps);
550         return getMetrics(fps, mFrameDrops / 30);
551     }
552 
553     @Override
doWork(int frameLimit)554     void doWork(int frameLimit) {
555         MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
556         boolean outputDone = false;
557         boolean inputDone = false;
558         boolean decoderDone = false;
559 
560         while (!outputDone) {
561             // Feed data to the decoder
562             if (!inputDone) {
563                 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US);
564                 if (inputBufferId != -1) {
565                     enqueueDecoderInput(inputBufferId);
566                     if (mSawDecInputEOS) {
567                         inputDone = true;
568                         if (DEBUG) Log.d(LOG_TAG, "Sent input EOS");
569                     }
570                 } else {
571                     if (DEBUG) Log.e(LOG_TAG, "Input buffer not available");
572                 }
573             }
574 
575             // Assume output is available.  Loop until both assumptions are false.
576             boolean decoderOutputAvailable = !decoderDone;
577             boolean encoderOutputAvailable = true;
578             while (decoderOutputAvailable || encoderOutputAvailable) {
579                 // Start by draining any pending output from the encoder.  It's important to
580                 // do this before we try to stuff any more data in.
581                 int outputBufferId =
582                         mEncoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
583                 if (outputBufferId == MediaCodec.INFO_TRY_AGAIN_LATER) {
584                     // no output available yet
585                     if (DEBUG) Log.d(LOG_TAG, "no output from encoder available");
586                     encoderOutputAvailable = false;
587                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
588                     if (DEBUG) Log.d(LOG_TAG, "encoder output buffers changed");
589                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
590                     MediaFormat newFormat = mEncoder.getOutputFormat();
591                     if (DEBUG) Log.d(LOG_TAG, "encoder output format changed: " + newFormat);
592                 } else if (outputBufferId < 0) {
593                     fail("unexpected result from encoder.dequeueOutputBuffer: " + outputBufferId);
594                 } else { // outputBufferId >= 0
595                     dequeueEncoderOutput(outputBufferId, outInfo);
596                     if (mSawEncOutputEOS) {
597                         outputDone = true;
598                     }
599                 }
600                 if (outputBufferId != MediaCodec.INFO_TRY_AGAIN_LATER) {
601                     continue;
602                 }
603 
604                 if (!decoderDone) {
605                     outputBufferId =
606                             mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
607                     if (outputBufferId == MediaCodec.INFO_TRY_AGAIN_LATER) {
608                         // no output available yet
609                         if (DEBUG) Log.d(LOG_TAG, "no output from decoder available");
610                         decoderOutputAvailable = false;
611                     } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
612                         if (DEBUG) Log.d(LOG_TAG, "decoder output buffers changed (we don't care)");
613                     } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
614                         // expected before first buffer of data
615                         MediaFormat newFormat = mDecoder.getOutputFormat();
616                         if (DEBUG) Log.d(LOG_TAG, "decoder output format changed: " + newFormat);
617                     } else if (outputBufferId < 0) {
618                         fail("unexpected result from decoder.dequeueOutputBuffer: "
619                                 + outputBufferId);
620                     } else {  // outputBufferId >= 0
621                         // The ByteBuffers are null references, but we still get a nonzero
622                         // size for the decoded data.
623                         boolean doRender = (outInfo.size != 0);
624 
625                         // As soon as we call releaseOutputBuffer, the buffer will be forwarded
626                         // to SurfaceTexture to convert to a texture.  The API doesn't
627                         // guarantee that the texture will be available before the call
628                         // returns, so we need to wait for the onFrameAvailable callback to
629                         // fire.  If we don't wait, we risk rendering from the previous frame.
630                         mDecoder.releaseOutputBuffer(outputBufferId, doRender);
631                         if (doRender) {
632                             // This waits for the image and renders it after it arrives.
633                             if (DEBUG) Log.d(LOG_TAG, "awaiting frame");
634                             mOutputSurface.awaitNewImage();
635                             mOutputSurface.drawImage();
636 
637                             // Send it to the encoder.
638                             mInputSurface.setPresentationTime(outInfo.presentationTimeUs * 1000);
639                             if (DEBUG) Log.d(LOG_TAG, "swapBuffers");
640                             mInputSurface.swapBuffers();
641                         }
642                         if ((outInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
643                             // forward decoder EOS to encoder
644                             if (DEBUG) Log.d(LOG_TAG, "signaling input EOS");
645                             mEncoder.signalEndOfInputStream();
646                         }
647                     }
648                 }
649             }
650         }
651     }
652 
653     @Override
enqueueDecoderInput(int bufferIndex)654     void enqueueDecoderInput(int bufferIndex) {
655         if (mExtractor.getSampleSize() < 0 || mLoadStatus.isLoadFinished()) {
656             enqueueDecoderEOS(bufferIndex);
657         } else {
658             ByteBuffer inputBuffer = mDecoder.getInputBuffer(bufferIndex);
659             int size = mExtractor.readSampleData(inputBuffer, 0);
660             long pts = mExtractor.getSampleTime();
661             mMaxPts = Math.max(mMaxPts, mBasePts + pts);
662             int extractorFlags = mExtractor.getSampleFlags();
663             int codecFlags = 0;
664             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
665                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
666             }
667             mDecoder.queueInputBuffer(bufferIndex, 0, size, mBasePts + pts, codecFlags);
668             if (size > 0 && (codecFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
669                 mDecInputCount++;
670             }
671             // If eos is reached, seek to start position.
672             if (!mExtractor.advance()) {
673                 mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
674                 mBasePts = mMaxPts + 1000000L;
675             }
676         }
677     }
678 }
679 
680 /**
681  * The following class tells the status of the load whether it is finished or not.
682  */
683 class LoadStatus {
684     private boolean mLoadFinished;
685 
LoadStatus()686     public LoadStatus() { mLoadFinished = false; }
687 
setLoadFinished()688     public synchronized void setLoadFinished() { mLoadFinished = true; }
689 
isLoadFinished()690     public synchronized boolean isLoadFinished() { return mLoadFinished; }
691 }
692