1 /*
2  * Copyright (C) 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.media.codec.cts;
18 
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertSame;
24 import static org.junit.Assert.assertThrows;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27 
28 import android.content.res.AssetFileDescriptor;
29 import android.hardware.HardwareBuffer;
30 import android.media.AudioFormat;
31 import android.media.AudioPresentation;
32 import android.media.MediaCodec;
33 import android.media.MediaCodec.BufferInfo;
34 import android.media.MediaCodec.CodecException;
35 import android.media.MediaCodec.CryptoException;
36 import android.media.MediaCodec.CryptoInfo;
37 import android.media.MediaCodec.CryptoInfo.Pattern;
38 import android.media.MediaCodecInfo;
39 import android.media.MediaCodecInfo.AudioCapabilities;
40 import android.media.MediaCodecInfo.CodecCapabilities;
41 import android.media.MediaCodecInfo.CodecProfileLevel;
42 import android.media.MediaCodecInfo.EncoderCapabilities;
43 import android.media.MediaCodecInfo.VideoCapabilities;
44 import android.media.MediaCodecList;
45 import android.media.MediaExtractor;
46 import android.media.MediaFormat;
47 import android.media.cts.InputSurface;
48 import android.media.cts.OutputSurface;
49 import android.media.cts.StreamUtils;
50 import android.media.cts.TestUtils;
51 import android.opengl.GLES20;
52 import android.os.Build;
53 import android.os.ConditionVariable;
54 import android.os.Handler;
55 import android.os.HandlerThread;
56 import android.os.ParcelFileDescriptor;
57 import android.os.PersistableBundle;
58 import android.platform.test.annotations.AppModeFull;
59 import android.platform.test.annotations.Presubmit;
60 import android.platform.test.annotations.RequiresDevice;
61 import android.util.Log;
62 import android.util.Range;
63 import android.view.Surface;
64 
65 import androidx.test.ext.junit.runners.AndroidJUnit4;
66 import androidx.test.filters.MediumTest;
67 import androidx.test.filters.SmallTest;
68 
69 import com.android.compatibility.common.util.ApiLevelUtil;
70 import com.android.compatibility.common.util.ApiTest;
71 import com.android.compatibility.common.util.FrameworkSpecificTest;
72 import com.android.compatibility.common.util.MediaUtils;
73 import com.android.compatibility.common.util.NonMainlineTest;
74 import com.android.compatibility.common.util.Preconditions;
75 
76 import org.junit.Test;
77 import org.junit.runner.RunWith;
78 
79 import java.io.File;
80 import java.io.IOException;
81 import java.nio.ByteBuffer;
82 import java.util.ArrayList;
83 import java.util.Arrays;
84 import java.util.List;
85 import java.util.UUID;
86 import java.util.concurrent.CountDownLatch;
87 import java.util.concurrent.LinkedBlockingQueue;
88 import java.util.concurrent.TimeUnit;
89 import java.util.concurrent.atomic.AtomicBoolean;
90 import java.util.concurrent.atomic.AtomicInteger;
91 
92 /**
93  * General MediaCodec tests.
94  *
95  * In particular, check various API edge cases.
96  */
97 @Presubmit
98 @SmallTest
99 @RequiresDevice
100 @AppModeFull(reason = "Instant apps cannot access the SD card")
101 @RunWith(AndroidJUnit4.class)
102 public class MediaCodecTest {
103     private static final String TAG = "MediaCodecTest";
104     private static final boolean VERBOSE = false;           // lots of logging
105 
106     static final String mInpPrefix = WorkDir.getMediaDirString();
107     // parameters for the video encoder
108                                                             // H.264 Advanced Video Coding
109     private static final String MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC;
110     private static final int BIT_RATE = 2000000;            // 2Mbps
111     private static final int FRAME_RATE = 15;               // 15fps
112     private static final int IFRAME_INTERVAL = 10;          // 10 seconds between I-frames
113     private static final int WIDTH = 1280;
114     private static final int HEIGHT = 720;
115     // parameters for the audio encoder
116     private static final String MIME_TYPE_AUDIO = MediaFormat.MIMETYPE_AUDIO_AAC;
117     private static final int AUDIO_SAMPLE_RATE = 44100;
118     private static final int AUDIO_AAC_PROFILE = 2; /* OMX_AUDIO_AACObjectLC */
119     private static final int AUDIO_CHANNEL_COUNT = 2; // mono
120     private static final int AUDIO_BIT_RATE = 128000;
121 
122     private static final int TIMEOUT_USEC = 100000;
123     private static final int TIMEOUT_USEC_SHORT = 100;
124 
125     private boolean mVideoEncoderHadError = false;
126     private boolean mAudioEncoderHadError = false;
127     private volatile boolean mVideoEncodingOngoing = false;
128 
129     private static final String INPUT_RESOURCE =
130             "video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_192kbps_44100hz.mp4";
131 
132     // The test should fail if the decoder never produces output frames for the input.
133     // Time out decoding, as we have no way to query whether the decoder will produce output.
134     private static final int DECODING_TIMEOUT_MS = 10000;
135 
136     private static boolean mIsAtLeastR = ApiLevelUtil.isAtLeast(Build.VERSION_CODES.R);
137     private static boolean mIsAtLeastS = ApiLevelUtil.isAtLeast(Build.VERSION_CODES.S);
138     private static boolean mIsAtLeastU = ApiLevelUtil.isAfter(Build.VERSION_CODES.TIRAMISU);
139 
140     /**
141      * Tests:
142      * <br> Exceptions for MediaCodec factory methods
143      * <br> Exceptions for MediaCodec methods when called in the incorrect state.
144      *
145      * A selective test to ensure proper exceptions are thrown from MediaCodec
146      * methods when called in incorrect operational states.
147      */
148     @ApiTest(apis = {"MediaCodec#createByCodecName", "MediaCodec#createDecoderByType",
149             "MediaCodec#createEncoderByType", "MediaCodec#start", "MediaCodec#flush",
150             "MediaCodec#configure", "MediaCodec#dequeueInputBuffer",
151             "MediaCodec#dequeueOutputBuffer", "MediaCodec#createInputSurface",
152             "MediaCodec#getInputBuffers", "MediaCodec#getQueueRequest",
153             "MediaCodec#getOutputFrame", "MediaCodec#stop", "MediaCodec#release",
154             "MediaCodec#getCodecInfo", "MediaCodec#getSupportedVendorParameters",
155             "MediaCodec#getParameterDescriptor",
156             "MediaCodec#subscribeToVendorParameters",
157             "MediaCodec#unsubscribeFromVendorParameters",
158             "MediaCodec#getInputBuffer", "MediaCodec#getOutputBuffer",
159             "MediaCodec#setCallback", "MediaCodec#getName"})
160     @Test
testException()161     public void testException() throws Exception {
162         boolean tested = false;
163         // audio decoder (MP3 should be present on all Android devices)
164         MediaFormat format = MediaFormat.createAudioFormat(
165                 MediaFormat.MIMETYPE_AUDIO_MPEG, 44100 /* sampleRate */, 2 /* channelCount */);
166         tested = verifyException(format, false /* isEncoder */) || tested;
167 
168         // audio encoder (AMR-WB may not be present on some Android devices)
169         format = MediaFormat.createAudioFormat(
170                 MediaFormat.MIMETYPE_AUDIO_AMR_WB, 16000 /* sampleRate */, 1 /* channelCount */);
171         format.setInteger(MediaFormat.KEY_BIT_RATE, 19850);
172         tested = verifyException(format, true /* isEncoder */) || tested;
173 
174         // video decoder (H.264/AVC may not be present on some Android devices)
175         format = createMediaFormat();
176         tested = verifyException(format, false /* isEncoder */) || tested;
177 
178         // video encoder (H.264/AVC may not be present on some Android devices)
179         tested = verifyException(format, true /* isEncoder */) || tested;
180 
181         // signal test is skipped due to no device media codecs.
182         if (!tested) {
183             MediaUtils.skipTest(TAG, "cannot find any compatible device codecs");
184         }
185     }
186 
187     // wrap MediaCodec encoder and decoder creation
createCodecByType(String type, boolean isEncoder)188     private static MediaCodec createCodecByType(String type, boolean isEncoder)
189             throws IOException {
190         if (isEncoder) {
191             return MediaCodec.createEncoderByType(type);
192         }
193         return MediaCodec.createDecoderByType(type);
194     }
195 
logMediaCodecException(MediaCodec.CodecException ex)196     private static void logMediaCodecException(MediaCodec.CodecException ex) {
197         if (ex.isRecoverable()) {
198             Log.w(TAG, "CodecException Recoverable: " + ex.getErrorCode());
199         } else if (ex.isTransient()) {
200             Log.w(TAG, "CodecException Transient: " + ex.getErrorCode());
201         } else {
202             Log.w(TAG, "CodecException Fatal: " + ex.getErrorCode());
203         }
204     }
205 
verifyException(MediaFormat format, boolean isEncoder)206     private static boolean verifyException(MediaFormat format, boolean isEncoder)
207             throws IOException {
208         String mimeType = format.getString(MediaFormat.KEY_MIME);
209         if (!supportsCodec(mimeType, isEncoder)) {
210             Log.i(TAG, "No " + (isEncoder ? "encoder" : "decoder")
211                     + " found for mimeType= " + mimeType);
212             return false;
213         }
214 
215         final boolean isVideoEncoder = isEncoder && mimeType.startsWith("video/");
216 
217         if (isVideoEncoder) {
218             format = new MediaFormat(format);
219             format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
220                     MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
221         }
222 
223         // create codec (enter Initialized State)
224         MediaCodec codec;
225 
226         // create improperly
227         final String methodName = isEncoder ? "createEncoderByType" : "createDecoderByType";
228         try {
229             codec = createCodecByType(null, isEncoder);
230             fail(methodName + " should return NullPointerException on null");
231         } catch (NullPointerException e) { // expected
232         }
233         try {
234             codec = createCodecByType("foobarplan9", isEncoder); // invalid type
235             fail(methodName + " should return IllegalArgumentException on invalid type");
236         } catch (IllegalArgumentException e) { // expected
237         }
238         try {
239             codec = MediaCodec.createByCodecName("foobarplan9"); // invalid name
240             fail(methodName + " should return IllegalArgumentException on invalid name");
241         } catch (IllegalArgumentException e) { // expected
242         }
243         // correct
244         codec = createCodecByType(format.getString(MediaFormat.KEY_MIME), isEncoder);
245 
246         // test a few commands
247         try {
248             codec.start();
249             fail("start should return IllegalStateException when in Initialized state");
250         } catch (MediaCodec.CodecException e) {
251             logMediaCodecException(e);
252             fail("start should not return MediaCodec.CodecException on wrong state");
253         } catch (IllegalStateException e) { // expected
254         }
255         try {
256             codec.flush();
257             fail("flush should return IllegalStateException when in Initialized state");
258         } catch (MediaCodec.CodecException e) {
259             logMediaCodecException(e);
260             fail("flush should not return MediaCodec.CodecException on wrong state");
261         } catch (IllegalStateException e) { // expected
262         }
263 
264         MediaCodecInfo codecInfo = codec.getCodecInfo(); // obtaining the codec info now is fine.
265         try {
266             int bufIndex = codec.dequeueInputBuffer(0);
267             fail("dequeueInputBuffer should return IllegalStateException"
268                     + " when in the Initialized state");
269         } catch (MediaCodec.CodecException e) {
270             logMediaCodecException(e);
271             fail("dequeueInputBuffer should not return MediaCodec.CodecException"
272                     + " on wrong state");
273         } catch (IllegalStateException e) { // expected
274         }
275         try {
276             MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
277             int bufIndex = codec.dequeueOutputBuffer(info, 0);
278             fail("dequeueOutputBuffer should return IllegalStateException"
279                     + " when in the Initialized state");
280         } catch (MediaCodec.CodecException e) {
281             logMediaCodecException(e);
282             fail("dequeueOutputBuffer should not return MediaCodec.CodecException"
283                     + " on wrong state");
284         } catch (IllegalStateException e) { // expected
285         }
286 
287         // configure (enter Configured State)
288 
289         // configure improperly
290         try {
291             codec.configure(format, null /* surface */, null /* crypto */,
292                     isEncoder ? 0 : MediaCodec.CONFIGURE_FLAG_ENCODE /* flags */);
293             fail("configure needs MediaCodec.CONFIGURE_FLAG_ENCODE for encoders only");
294         } catch (MediaCodec.CodecException e) { // expected
295             logMediaCodecException(e);
296         } catch (IllegalStateException e) {
297             fail("configure should not return IllegalStateException when improperly configured");
298         }
299         if (mIsAtLeastU) {
300             try {
301                 int flags = MediaCodec.CONFIGURE_FLAG_USE_CRYPTO_ASYNC |
302                         (isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0);
303                 codec.configure(format, null /* surface */, null /* crypto */, flags /* flags */);
304                 fail("At the minimum, CONFIGURE_FLAG_USE_CRYPTO_ASYNC requires setting callback");
305             } catch(IllegalStateException e) { //expected
306                 // Need to set callbacks
307             }
308         }
309         // correct
310         codec.configure(format, null /* surface */, null /* crypto */,
311                 isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0 /* flags */);
312 
313         // test a few commands
314         try {
315             codec.flush();
316             fail("flush should return IllegalStateException when in Configured state");
317         } catch (MediaCodec.CodecException e) {
318             logMediaCodecException(e);
319             fail("flush should not return MediaCodec.CodecException on wrong state");
320         } catch (IllegalStateException e) { // expected
321         }
322         try {
323             Surface surface = codec.createInputSurface();
324             if (!isEncoder) {
325                 fail("createInputSurface should not work on a decoder");
326             }
327         } catch (IllegalStateException |
328                  IllegalArgumentException e) { // expected for decoder and audio encoder
329             if (isVideoEncoder) {
330                 throw e;
331             }
332         }
333 
334         // test getInputBuffers before start()
335         try {
336             ByteBuffer[] buffers = codec.getInputBuffers();
337             fail("getInputBuffers called before start() should throw exception");
338         } catch (IllegalStateException e) { // expected
339         }
340 
341         // start codec (enter Executing state)
342         codec.start();
343 
344         // test getInputBuffers after start()
345         try {
346             ByteBuffer[] buffers = codec.getInputBuffers();
347             if (buffers == null) {
348                 fail("getInputBuffers called after start() should not return null");
349             }
350             if (isVideoEncoder && buffers.length > 0) {
351                 fail("getInputBuffers returned non-zero length array with input surface");
352             }
353         } catch (IllegalStateException e) {
354             fail("getInputBuffers called after start() shouldn't throw exception");
355         }
356 
357         // test a few commands
358         try {
359             codec.configure(format, null /* surface */, null /* crypto */, 0 /* flags */);
360             fail("configure should return IllegalStateException when in Executing state");
361         } catch (MediaCodec.CodecException e) {
362             logMediaCodecException(e);
363             // TODO: consider configuring after a flush.
364             fail("configure should not return MediaCodec.CodecException on wrong state");
365         } catch (IllegalStateException e) { // expected
366         }
367         if (mIsAtLeastR) {
368             try {
369                 codec.getQueueRequest(0);
370                 fail("getQueueRequest should throw IllegalStateException when not configured with " +
371                         "CONFIGURE_FLAG_USE_BLOCK_MODEL");
372             } catch (MediaCodec.CodecException e) {
373                 logMediaCodecException(e);
374                 fail("getQueueRequest should not return " +
375                         "MediaCodec.CodecException on wrong configuration");
376             } catch (IllegalStateException e) { // expected
377             }
378             try {
379                 codec.getOutputFrame(0);
380                 fail("getOutputFrame should throw IllegalStateException when not configured with " +
381                         "CONFIGURE_FLAG_USE_BLOCK_MODEL");
382             } catch (MediaCodec.CodecException e) {
383                 logMediaCodecException(e);
384                 fail("getOutputFrame should not return MediaCodec.CodecException on wrong " +
385                         "configuration");
386             } catch (IllegalStateException e) { // expected
387             }
388         }
389 
390         // two flushes should be fine.
391         codec.flush();
392         codec.flush();
393 
394         // stop codec (enter Initialized state)
395         // two stops should be fine.
396         codec.stop();
397         codec.stop();
398 
399         // release codec (enter Uninitialized state)
400         // two releases should be fine.
401         codec.release();
402         codec.release();
403 
404         try {
405             codecInfo = codec.getCodecInfo();
406             fail("getCodecInfo should should return IllegalStateException" +
407                     " when in Uninitialized state");
408         } catch (MediaCodec.CodecException e) {
409             logMediaCodecException(e);
410             fail("getCodecInfo should not return MediaCodec.CodecException on wrong state");
411         } catch (IllegalStateException e) { // expected
412         }
413         try {
414             codec.stop();
415             fail("stop should return IllegalStateException when in Uninitialized state");
416         } catch (MediaCodec.CodecException e) {
417             logMediaCodecException(e);
418             fail("stop should not return MediaCodec.CodecException on wrong state");
419         } catch (IllegalStateException e) { // expected
420         }
421 
422         if (mIsAtLeastS) {
423             try {
424                 codec.getSupportedVendorParameters();
425                 fail("getSupportedVendorParameters should throw IllegalStateException" +
426                         " when in Uninitialized state");
427             } catch (IllegalStateException e) { // expected
428             } catch (Exception e) {
429                 fail("unexpected exception: " + e.toString());
430             }
431             try {
432                 codec.getParameterDescriptor("");
433                 fail("getParameterDescriptor should throw IllegalStateException" +
434                         " when in Uninitialized state");
435             } catch (IllegalStateException e) { // expected
436             } catch (Exception e) {
437                 fail("unexpected exception: " + e.toString());
438             }
439             try {
440                 codec.subscribeToVendorParameters(List.of(""));
441                 fail("subscribeToVendorParameters should throw IllegalStateException" +
442                         " when in Uninitialized state");
443             } catch (IllegalStateException e) { // expected
444             } catch (Exception e) {
445                 fail("unexpected exception: " + e.toString());
446             }
447             try {
448                 codec.unsubscribeFromVendorParameters(List.of(""));
449                 fail("unsubscribeFromVendorParameters should throw IllegalStateException" +
450                         " when in Uninitialized state");
451             } catch (IllegalStateException e) { // expected
452             } catch (Exception e) {
453                 fail("unexpected exception: " + e.toString());
454             }
455         }
456 
457         if (mIsAtLeastR) {
458             // recreate
459             codec = createCodecByType(format.getString(MediaFormat.KEY_MIME), isEncoder);
460 
461             if (isVideoEncoder) {
462                 format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
463                         MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
464             }
465 
466             // configure improperly
467             try {
468                 codec.configure(format, null /* surface */, null /* crypto */,
469                         MediaCodec.CONFIGURE_FLAG_USE_BLOCK_MODEL |
470                         (isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0) /* flags */);
471                 fail("configure with detached buffer mode should be done after setCallback");
472             } catch (MediaCodec.CodecException e) {
473                 logMediaCodecException(e);
474                 fail("configure should not return IllegalStateException when improperly configured");
475             } catch (IllegalStateException e) { // expected
476             }
477 
478             final LinkedBlockingQueue<Integer> inputQueue = new LinkedBlockingQueue<>();
479             codec.setCallback(new MediaCodec.Callback() {
480                 @Override
481                 public void onInputBufferAvailable(MediaCodec codec, int index) {
482                     inputQueue.offer(index);
483                 }
484                 @Override
485                 public void onOutputBufferAvailable(
486                         MediaCodec codec, int index, MediaCodec.BufferInfo info) { }
487                 @Override
488                 public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) { }
489                 @Override
490                 public void onError(MediaCodec codec, CodecException e) { }
491             });
492 
493             // configure with CONFIGURE_FLAG_USE_BLOCK_MODEL (enter Configured State)
494             codec.configure(format, null /* surface */, null /* crypto */,
495                     MediaCodec.CONFIGURE_FLAG_USE_BLOCK_MODEL |
496                     (isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0) /* flags */);
497 
498             // start codec (enter Executing state)
499             codec.start();
500 
501             // grab input index (this should happen immediately)
502             Integer index = null;
503             try {
504                 index = inputQueue.poll(2, TimeUnit.SECONDS);
505             } catch (InterruptedException e) {
506             }
507             assertNotNull(index);
508 
509             // test a few commands
510             try {
511                 codec.getInputBuffers();
512                 fail("getInputBuffers called in detached buffer mode should throw exception");
513             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
514             }
515             try {
516                 codec.getOutputBuffers();
517                 fail("getOutputBuffers called in detached buffer mode should throw exception");
518             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
519             }
520             try {
521                 codec.getInputBuffer(index);
522                 fail("getInputBuffer called in detached buffer mode should throw exception");
523             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
524             }
525             try {
526                 codec.dequeueInputBuffer(0);
527                 fail("dequeueInputBuffer called in detached buffer mode should throw exception");
528             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
529             }
530             MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
531             try {
532                 codec.dequeueOutputBuffer(info, 0);
533                 fail("dequeueOutputBuffer called in detached buffer mode should throw exception");
534             } catch (MediaCodec.IncompatibleWithBlockModelException e) { // expected
535             }
536 
537             // test getQueueRequest
538             MediaCodec.QueueRequest request = codec.getQueueRequest(index);
539             try {
540                 request.queue();
541                 fail("QueueRequest should throw IllegalStateException when no buffer is set");
542             } catch (IllegalStateException e) { // expected
543             }
544             // setting a block
545             String[] names = new String[]{ codec.getName() };
546             request.setLinearBlock(MediaCodec.LinearBlock.obtain(1, names), 0, 0);
547             // setting additional block should fail
548             try (HardwareBuffer buffer = HardwareBuffer.create(
549                     16 /* width */,
550                     16 /* height */,
551                     HardwareBuffer.YCBCR_420_888,
552                     1 /* layers */,
553                     HardwareBuffer.USAGE_CPU_READ_OFTEN | HardwareBuffer.USAGE_CPU_WRITE_OFTEN)) {
554                 request.setHardwareBuffer(buffer);
555                 fail("QueueRequest should throw IllegalStateException multiple blocks are set.");
556             } catch (IllegalStateException e) { // expected
557             }
558         }
559 
560         // release codec
561         codec.release();
562 
563         return true;
564     }
565 
566     /**
567      * Tests:
568      * <br> calling createInputSurface() before configure() throws exception
569      * <br> calling createInputSurface() after start() throws exception
570      * <br> calling createInputSurface() with a non-Surface color format is not required to throw exception
571      */
572     @ApiTest(apis = "MediaCodec#createInputSurface")
573     @Test
574     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
575     @NonMainlineTest // tests only the first MIME_TYPE codec, which is usually hardware
testCreateInputSurfaceErrors()576     public void testCreateInputSurfaceErrors() {
577         if (!supportsCodec(MIME_TYPE, true)) {
578             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
579             return;
580         }
581 
582         MediaFormat format = createMediaFormat();
583         MediaCodec encoder = null;
584         Surface surface = null;
585 
586         // Replace color format with something that isn't COLOR_FormatSurface.
587         MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
588         int colorFormat = findNonSurfaceColorFormat(codecInfo, MIME_TYPE);
589         format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
590 
591         try {
592             try {
593                 encoder = MediaCodec.createByCodecName(codecInfo.getName());
594             } catch (IOException e) {
595                 fail("failed to create codec " + codecInfo.getName());
596             }
597             try {
598                 surface = encoder.createInputSurface();
599                 fail("createInputSurface should not work pre-configure");
600             } catch (IllegalStateException ise) {
601                 // good
602             }
603             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
604             encoder.start();
605             try {
606                 surface = encoder.createInputSurface();
607                 fail("createInputSurface should not work post-start");
608             } catch (IllegalStateException ise) {
609                 // good
610             }
611         } finally {
612             if (encoder != null) {
613                 encoder.stop();
614                 encoder.release();
615             }
616         }
617         assertNull(surface);
618     }
619 
620     /**
621      * Tests:
622      * <br> signaling end-of-stream before any data is sent works
623      * <br> signaling EOS twice throws exception
624      * <br> submitting a frame after EOS throws exception [TODO]
625      */
626     @ApiTest(apis = "MediaCodec#signalEndOfInputStream")
627     @Test
628     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
629     @NonMainlineTest // tests only the first MIME_TYPE codec, which is usually hardware
testSignalSurfaceEOS()630     public void testSignalSurfaceEOS() {
631         if (!supportsCodec(MIME_TYPE, true)) {
632             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
633             return;
634         }
635 
636         MediaFormat format = createMediaFormat();
637         MediaCodec encoder = null;
638         InputSurface inputSurface = null;
639 
640         try {
641             try {
642                 encoder = MediaCodec.createEncoderByType(MIME_TYPE);
643             } catch (IOException e) {
644                 fail("failed to create " + MIME_TYPE + " encoder");
645             }
646             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
647             inputSurface = new InputSurface(encoder.createInputSurface());
648             inputSurface.makeCurrent();
649             encoder.start();
650 
651             // send an immediate EOS
652             encoder.signalEndOfInputStream();
653 
654             try {
655                 encoder.signalEndOfInputStream();
656                 fail("should not be able to signal EOS twice");
657             } catch (IllegalStateException ise) {
658                 // good
659             }
660 
661             // submit a frame post-EOS
662             GLES20.glClearColor(0.0f, 0.5f, 0.0f, 1.0f);
663             GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
664             try {
665                 inputSurface.swapBuffers();
666                 if (false) {    // TODO
667                     fail("should not be able to submit frame after EOS");
668                 }
669             } catch (Exception ex) {
670                 // good
671             }
672         } finally {
673             if (encoder != null) {
674                 encoder.stop();
675                 encoder.release();
676             }
677             if (inputSurface != null) {
678                 inputSurface.release();
679             }
680         }
681     }
682 
683     /**
684      * Tests:
685      * <br> stopping with buffers in flight doesn't crash or hang
686      */
687     @ApiTest(apis = "MediaCodec#stop")
688     @Test
689     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
690     @NonMainlineTest // tests only the first MIME_TYPE codec, which is usually hardware
testAbruptStop()691     public void testAbruptStop() {
692         if (!supportsCodec(MIME_TYPE, true)) {
693             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
694             return;
695         }
696 
697         // There appears to be a race, so run it several times with a short delay between runs
698         // to allow any previous activity to shut down.
699         for (int i = 0; i < 50; i++) {
700             Log.d(TAG, "testAbruptStop " + i);
701             doTestAbruptStop();
702             try { Thread.sleep(400); } catch (InterruptedException ignored) {}
703         }
704     }
doTestAbruptStop()705     private void doTestAbruptStop() {
706         MediaFormat format = createMediaFormat();
707         MediaCodec encoder = null;
708         InputSurface inputSurface = null;
709 
710         try {
711             try {
712                 encoder = MediaCodec.createEncoderByType(MIME_TYPE);
713             } catch (IOException e) {
714                 fail("failed to create " + MIME_TYPE + " encoder");
715             }
716             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
717             inputSurface = new InputSurface(encoder.createInputSurface());
718             inputSurface.makeCurrent();
719             encoder.start();
720 
721             int totalBuffers = encoder.getOutputBuffers().length;
722             if (VERBOSE) Log.d(TAG, "Total buffers: " + totalBuffers);
723 
724             // Submit several frames quickly, without draining the encoder output, to try to
725             // ensure that we've got some queued up when we call stop().  If we do too many
726             // we'll block in swapBuffers().
727             for (int i = 0; i < totalBuffers; i++) {
728                 GLES20.glClearColor(0.0f, (i % 8) / 8.0f, 0.0f, 1.0f);
729                 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
730                 inputSurface.swapBuffers();
731             }
732             Log.d(TAG, "stopping");
733             encoder.stop();
734             Log.d(TAG, "stopped");
735         } finally {
736             if (encoder != null) {
737                 encoder.stop();
738                 encoder.release();
739             }
740             if (inputSurface != null) {
741                 inputSurface.release();
742             }
743         }
744     }
745 
746     @ApiTest(apis = {"MediaCodec#flush", "MediaCodec#release"})
747     @Test
testReleaseAfterFlush()748     public void testReleaseAfterFlush() throws IOException, InterruptedException {
749         String mimes[] = new String[] { MIME_TYPE, MIME_TYPE_AUDIO};
750         for (String mime : mimes) {
751             if (!MediaUtils.checkEncoder(mime)) {
752                 continue;
753             }
754             testReleaseAfterFlush(mime);
755         }
756     }
757 
testReleaseAfterFlush(String mime)758     private void testReleaseAfterFlush(String mime) throws IOException, InterruptedException {
759         CountDownLatch buffersExhausted = null;
760         CountDownLatch codecFlushed = null;
761         AtomicInteger numBuffers = null;
762 
763         // sync flush from same thread
764         MediaCodec encoder = MediaCodec.createEncoderByType(mime);
765         runReleaseAfterFlush(mime, encoder, buffersExhausted, codecFlushed, numBuffers);
766 
767         // sync flush from different thread
768         encoder = MediaCodec.createEncoderByType(mime);
769         buffersExhausted = new CountDownLatch(1);
770         codecFlushed = new CountDownLatch(1);
771         numBuffers = new AtomicInteger();
772         Thread flushThread = new FlushThread(encoder, buffersExhausted, codecFlushed);
773         flushThread.start();
774         runReleaseAfterFlush(mime, encoder, buffersExhausted, codecFlushed, numBuffers);
775         flushThread.join();
776 
777         // async
778         // This value is calculated in getOutputBufferIndices by calling dequeueOutputBuffer
779         // with a fixed timeout until buffers are exhausted; it is possible that random timing
780         // in dequeueOutputBuffer can result in a smaller `nBuffs` than the max possible value.
781         int nBuffs = numBuffers.get();
782         HandlerThread callbackThread = new HandlerThread("ReleaseAfterFlushCallbackThread");
783         callbackThread.start();
784         Handler handler = new Handler(callbackThread.getLooper());
785 
786         // async flush from same thread
787         encoder = MediaCodec.createEncoderByType(mime);
788         buffersExhausted = null;
789         codecFlushed = null;
790         ReleaseAfterFlushCallback callback =
791                 new ReleaseAfterFlushCallback(mime, encoder, buffersExhausted, codecFlushed, nBuffs);
792         encoder.setCallback(callback, handler); // setCallback before configure, which is called in run
793         callback.run(); // drive input on main thread
794 
795         // async flush from different thread
796         encoder = MediaCodec.createEncoderByType(mime);
797         buffersExhausted = new CountDownLatch(1);
798         codecFlushed = new CountDownLatch(1);
799         callback = new ReleaseAfterFlushCallback(mime, encoder, buffersExhausted, codecFlushed, nBuffs);
800         encoder.setCallback(callback, handler);
801         flushThread = new FlushThread(encoder, buffersExhausted, codecFlushed);
802         flushThread.start();
803         callback.run();
804         flushThread.join();
805 
806         callbackThread.quitSafely();
807         callbackThread.join();
808     }
809 
810     @ApiTest(apis = {"MediaCodec#setCallback", "MediaCodec#flush", "MediaCodec#reset"})
811     @Test
testAsyncFlushAndReset()812     public void testAsyncFlushAndReset() throws Exception, InterruptedException {
813         testAsyncReset(false /* testStop */);
814     }
815 
816     @ApiTest(apis = {"MediaCodec#setCallback", "MediaCodec#stop", "MediaCodec#reset"})
817     @Test
testAsyncStopAndReset()818     public void testAsyncStopAndReset() throws Exception, InterruptedException {
819         testAsyncReset(true /* testStop */);
820     }
821 
testAsyncReset(boolean testStop)822     private void testAsyncReset(boolean testStop) throws Exception, InterruptedException {
823         // Test video and audio 10x each
824         for (int i = 0; i < 10; i++) {
825             testAsyncReset(false /* audio */, (i % 2) == 0 /* swap */, testStop);
826         }
827         for (int i = 0; i < 10; i++) {
828             testAsyncReset(true /* audio */, (i % 2) == 0 /* swap */, testStop);
829         }
830     }
831 
832     /*
833      * This method simulates a race between flush (or stop) and reset() called from
834      * two threads. Neither call should get stuck. This should be run multiple rounds.
835      */
testAsyncReset(boolean audio, boolean swap, final boolean testStop)836     private void testAsyncReset(boolean audio, boolean swap, final boolean testStop)
837             throws Exception, InterruptedException {
838         String mimeTypePrefix  = audio ? "audio/" : "video/";
839         final MediaExtractor mediaExtractor = getMediaExtractorForMimeType(
840                 INPUT_RESOURCE, mimeTypePrefix);
841         MediaFormat mediaFormat = mediaExtractor.getTrackFormat(
842                 mediaExtractor.getSampleTrackIndex());
843         if (!MediaUtils.checkDecoderForFormat(mediaFormat)) {
844             return; // skip
845         }
846 
847         OutputSurface outputSurface = audio ? null : new OutputSurface(1, 1);
848         final Surface surface = outputSurface == null ? null : outputSurface.getSurface();
849 
850         String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
851         final MediaCodec mediaCodec = MediaCodec.createDecoderByType(mimeType);
852 
853         try {
854             mediaCodec.configure(mediaFormat, surface, null /* crypto */, 0 /* flags */);
855 
856             mediaCodec.start();
857 
858             assertTrue(runDecodeTillFirstOutput(mediaCodec, mediaExtractor));
859 
860             Thread flushingThread = new Thread(new Runnable() {
861                 @Override
862                 public void run() {
863                     try {
864                         if (testStop) {
865                             mediaCodec.stop();
866                         } else {
867                             mediaCodec.flush();
868                         }
869                     } catch (IllegalStateException e) {
870                         // This is okay, since we're simulating a race between flush and reset.
871                         // If reset executed first, flush could fail.
872                     }
873                 }
874             });
875 
876             Thread resettingThread = new Thread(new Runnable() {
877                 @Override
878                 public void run() {
879                     mediaCodec.reset();
880                 }
881             });
882 
883             // start flushing (or stopping) and resetting in two threads
884             if (swap) {
885                 flushingThread.start();
886                 resettingThread.start();
887             } else {
888                 resettingThread.start();
889                 flushingThread.start();
890             }
891 
892             // wait for at most 5 sec, and check if the thread exits properly
893             flushingThread.join(5000);
894             assertFalse(flushingThread.isAlive());
895 
896             resettingThread.join(5000);
897             assertFalse(resettingThread.isAlive());
898         } finally {
899             if (mediaCodec != null) {
900                 mediaCodec.release();
901             }
902             if (mediaExtractor != null) {
903                 mediaExtractor.release();
904             }
905             if (outputSurface != null) {
906                 outputSurface.release();
907             }
908         }
909     }
910 
911     private static class FlushThread extends Thread {
912         final MediaCodec mEncoder;
913         final CountDownLatch mBuffersExhausted;
914         final CountDownLatch mCodecFlushed;
915 
FlushThread(MediaCodec encoder, CountDownLatch buffersExhausted, CountDownLatch codecFlushed)916         FlushThread(MediaCodec encoder, CountDownLatch buffersExhausted,
917                 CountDownLatch codecFlushed) {
918             mEncoder = encoder;
919             mBuffersExhausted = buffersExhausted;
920             mCodecFlushed = codecFlushed;
921         }
922 
923         @Override
run()924         public void run() {
925             try {
926                 mBuffersExhausted.await();
927             } catch (InterruptedException e) {
928                 Thread.currentThread().interrupt();
929                 Log.w(TAG, "buffersExhausted wait interrupted; flushing immediately.", e);
930             }
931             mEncoder.flush();
932             mCodecFlushed.countDown();
933         }
934     }
935 
936     private static class ReleaseAfterFlushCallback extends MediaCodec.Callback implements Runnable {
937         final String mMime;
938         final MediaCodec mEncoder;
939         final CountDownLatch mBuffersExhausted, mCodecFlushed;
940         final int mNumBuffersBeforeFlush;
941 
942         CountDownLatch mStopInput = new CountDownLatch(1);
943         List<Integer> mInputBufferIndices = new ArrayList<>();
944         List<Integer> mOutputBufferIndices = new ArrayList<>();
945 
ReleaseAfterFlushCallback(String mime, MediaCodec encoder, CountDownLatch buffersExhausted, CountDownLatch codecFlushed, int numBuffersBeforeFlush)946         ReleaseAfterFlushCallback(String mime,
947                 MediaCodec encoder,
948                 CountDownLatch buffersExhausted,
949                 CountDownLatch codecFlushed,
950                 int numBuffersBeforeFlush) {
951             mMime = mime;
952             mEncoder = encoder;
953             mBuffersExhausted = buffersExhausted;
954             mCodecFlushed = codecFlushed;
955             mNumBuffersBeforeFlush = numBuffersBeforeFlush;
956         }
957 
958         @Override
onInputBufferAvailable(MediaCodec codec, int index)959         public void onInputBufferAvailable(MediaCodec codec, int index) {
960             assertTrue("video onInputBufferAvailable " + index, mMime.startsWith("audio/"));
961             synchronized (mInputBufferIndices) {
962                 mInputBufferIndices.add(index);
963             };
964         }
965 
966         @Override
onOutputBufferAvailable(MediaCodec codec, int index, BufferInfo info)967         public void onOutputBufferAvailable(MediaCodec codec, int index, BufferInfo info) {
968             mOutputBufferIndices.add(index);
969             if (mOutputBufferIndices.size() == mNumBuffersBeforeFlush) {
970                 releaseAfterFlush(codec, mOutputBufferIndices, mBuffersExhausted, mCodecFlushed);
971                 mStopInput.countDown();
972             }
973         }
974 
975         @Override
onError(MediaCodec codec, CodecException e)976         public void onError(MediaCodec codec, CodecException e) {
977             Log.e(TAG, codec + " onError", e);
978             fail(codec + " onError " + e.getMessage());
979         }
980 
981         @Override
onOutputFormatChanged(MediaCodec codec, MediaFormat format)982         public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
983             Log.v(TAG, codec + " onOutputFormatChanged " + format);
984         }
985 
986         @Override
run()987         public void run() {
988             InputSurface inputSurface = null;
989             try {
990                 inputSurface = initCodecAndSurface(mMime, mEncoder);
991                 do {
992                     int inputIndex = -1;
993                     if (inputSurface == null) {
994                         // asynchronous audio codec
995                         synchronized (mInputBufferIndices) {
996                             if (mInputBufferIndices.isEmpty()) {
997                                 continue;
998                             } else {
999                                 inputIndex = mInputBufferIndices.remove(0);
1000                             }
1001                         }
1002                     }
1003                     feedEncoder(mEncoder, inputSurface, inputIndex);
1004                 } while (!mStopInput.await(TIMEOUT_USEC, TimeUnit.MICROSECONDS));
1005             } catch (InterruptedException e) {
1006                 Thread.currentThread().interrupt();
1007                 Log.w(TAG, "mEncoder input frames interrupted/stopped", e);
1008             } finally {
1009                 cleanupCodecAndSurface(mEncoder, inputSurface);
1010             }
1011         }
1012     }
1013 
runReleaseAfterFlush( String mime, MediaCodec encoder, CountDownLatch buffersExhausted, CountDownLatch codecFlushed, AtomicInteger numBuffers)1014     private static void runReleaseAfterFlush(
1015             String mime,
1016             MediaCodec encoder,
1017             CountDownLatch buffersExhausted,
1018             CountDownLatch codecFlushed,
1019             AtomicInteger numBuffers) {
1020         InputSurface inputSurface = null;
1021         try {
1022             inputSurface = initCodecAndSurface(mime, encoder);
1023             List<Integer> outputBufferIndices = getOutputBufferIndices(encoder, inputSurface);
1024             if (numBuffers != null) {
1025                 numBuffers.set(outputBufferIndices.size());
1026             }
1027             releaseAfterFlush(encoder, outputBufferIndices, buffersExhausted, codecFlushed);
1028         } finally {
1029             cleanupCodecAndSurface(encoder, inputSurface);
1030         }
1031     }
1032 
initCodecAndSurface(String mime, MediaCodec encoder)1033     private static InputSurface initCodecAndSurface(String mime, MediaCodec encoder) {
1034         MediaFormat format;
1035         InputSurface inputSurface = null;
1036         if (mime.startsWith("audio/")) {
1037             format = MediaFormat.createAudioFormat(mime, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT);
1038             format.setInteger(MediaFormat.KEY_AAC_PROFILE, AUDIO_AAC_PROFILE);
1039             format.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
1040             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1041         } else if (MIME_TYPE.equals(mime)) {
1042             CodecInfo info = getAvcSupportedFormatInfo();
1043             format = MediaFormat.createVideoFormat(mime, info.mMaxW, info.mMaxH);
1044             format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1045                     MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
1046             format.setInteger(MediaFormat.KEY_BIT_RATE, info.mBitRate);
1047             format.setInteger(MediaFormat.KEY_FRAME_RATE, info.mFps);
1048             format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
1049             OutputSurface outputSurface = new OutputSurface(1, 1);
1050             encoder.configure(format, outputSurface.getSurface(), null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1051             inputSurface = new InputSurface(encoder.createInputSurface());
1052             inputSurface.makeCurrent();
1053         } else {
1054             throw new IllegalArgumentException("unsupported mime type: " + mime);
1055         }
1056         encoder.start();
1057         return inputSurface;
1058     }
1059 
cleanupCodecAndSurface(MediaCodec encoder, InputSurface inputSurface)1060     private static void cleanupCodecAndSurface(MediaCodec encoder, InputSurface inputSurface) {
1061         if (encoder != null) {
1062             encoder.stop();
1063             encoder.release();
1064         }
1065 
1066         if (inputSurface != null) {
1067             inputSurface.release();
1068         }
1069     }
1070 
getOutputBufferIndices(MediaCodec encoder, InputSurface inputSurface)1071     private static List<Integer> getOutputBufferIndices(MediaCodec encoder, InputSurface inputSurface) {
1072         boolean feedMoreFrames;
1073         MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
1074         List<Integer> indices = new ArrayList<>();
1075         do {
1076             feedMoreFrames = indices.isEmpty();
1077             feedEncoder(encoder, inputSurface, -1);
1078             // dequeue buffers until not available
1079             int index = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
1080             while (index >= 0) {
1081                 indices.add(index);
1082                 index = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC_SHORT);
1083             }
1084         } while (feedMoreFrames);
1085         assertFalse(indices.isEmpty());
1086         return indices;
1087     }
1088 
1089     /**
1090      * @param encoder audio/video encoder
1091      * @param inputSurface null for and only for audio encoders
1092      * @param inputIndex only used for audio; if -1 the function would attempt to dequeue from encoder;
1093      * do not use -1 for asynchronous encoders
1094      */
feedEncoder(MediaCodec encoder, InputSurface inputSurface, int inputIndex)1095     private static void feedEncoder(MediaCodec encoder, InputSurface inputSurface, int inputIndex) {
1096         if (inputSurface == null) {
1097             // audio
1098             while (inputIndex == -1) {
1099                 inputIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
1100             }
1101             ByteBuffer inputBuffer = encoder.getInputBuffer(inputIndex);;
1102             for (int i = 0; i < inputBuffer.capacity() / 2; i++) {
1103                 inputBuffer.putShort((short)i);
1104             }
1105             encoder.queueInputBuffer(inputIndex, 0, inputBuffer.limit(), 0, 0);
1106         } else {
1107             // video
1108             GLES20.glClearColor(0.0f, 0.5f, 0.0f, 1.0f);
1109             GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
1110             inputSurface.swapBuffers();
1111         }
1112     }
1113 
releaseAfterFlush( MediaCodec encoder, List<Integer> outputBufferIndices, CountDownLatch buffersExhausted, CountDownLatch codecFlushed)1114     private static void releaseAfterFlush(
1115             MediaCodec encoder,
1116             List<Integer> outputBufferIndices,
1117             CountDownLatch buffersExhausted,
1118             CountDownLatch codecFlushed) {
1119         if (buffersExhausted == null) {
1120             // flush from same thread
1121             encoder.flush();
1122         } else {
1123             assertNotNull(codecFlushed);
1124             buffersExhausted.countDown();
1125             try {
1126                 codecFlushed.await();
1127             } catch (InterruptedException e) {
1128                 Thread.currentThread().interrupt();
1129                 Log.w(TAG, "codecFlushed wait interrupted; releasing buffers immediately.", e);
1130             }
1131         }
1132 
1133         for (int index : outputBufferIndices) {
1134             try {
1135                 encoder.releaseOutputBuffer(index, true);
1136                 fail("MediaCodec releaseOutputBuffer after flush() does not throw exception");
1137             } catch (MediaCodec.CodecException e) {
1138                 // Expected
1139             }
1140         }
1141     }
1142 
1143     /**
1144      * Tests:
1145      * <br> dequeueInputBuffer() fails when encoder configured with an input Surface
1146      */
1147     @ApiTest(apis = {"MediaCodec#dequeueInputBuffer", "MediaCodec#getMetrics"})
1148     @Test
1149     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
1150     @NonMainlineTest // tests only the first MIME_TYPE codec, which is usually hardware
testDequeueSurface()1151     public void testDequeueSurface() {
1152         if (!supportsCodec(MIME_TYPE, true)) {
1153             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
1154             return;
1155         }
1156 
1157         MediaFormat format = createMediaFormat();
1158         MediaCodec encoder = null;
1159         Surface surface = null;
1160 
1161         try {
1162             try {
1163                 encoder = MediaCodec.createEncoderByType(MIME_TYPE);
1164             } catch (IOException e) {
1165                 fail("failed to create " + MIME_TYPE + " encoder");
1166             }
1167             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1168             surface = encoder.createInputSurface();
1169             encoder.start();
1170 
1171             try {
1172                 encoder.dequeueInputBuffer(-1);
1173                 fail("dequeueInputBuffer should fail on encoder with input surface");
1174             } catch (IllegalStateException ise) {
1175                 // good
1176             }
1177 
1178             PersistableBundle metrics = encoder.getMetrics();
1179             if (metrics == null) {
1180                 fail("getMetrics() returns null");
1181             } else if (metrics.isEmpty()) {
1182                 fail("getMetrics() returns empty results");
1183             }
1184             int encoding = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1185             if (encoding != 1) {
1186                 fail("getMetrics() returns bad encoder value " + encoding);
1187             }
1188             String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1189             if (theCodec == null) {
1190                 fail("getMetrics() returns null codec value ");
1191             }
1192 
1193         } finally {
1194             if (encoder != null) {
1195                 encoder.stop();
1196                 encoder.release();
1197             }
1198             if (surface != null) {
1199                 surface.release();
1200             }
1201         }
1202     }
1203 
1204     /**
1205      * Tests:
1206      * <br> configure() encoder with Surface, re-configure() without Surface works
1207      * <br> sending EOS with signalEndOfInputStream on non-Surface encoder fails
1208      */
1209     @ApiTest(apis = {"MediaCodec#configure", "MediaCodec#signalEndOfInputStream",
1210             "MediaCodec#getMetrics"})
1211     @Test
1212     @FrameworkSpecificTest // tests only the first MIME_TYPE codec, which is usually hardware
1213     @NonMainlineTest // tests only the first MIME_TYPE codec, which is usually hardware
testReconfigureWithoutSurface()1214     public void testReconfigureWithoutSurface() {
1215         if (!supportsCodec(MIME_TYPE, true)) {
1216             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE);
1217             return;
1218         }
1219 
1220         MediaFormat format = createMediaFormat();
1221         MediaCodec encoder = null;
1222         Surface surface = null;
1223 
1224         try {
1225             try {
1226                 encoder = MediaCodec.createEncoderByType(MIME_TYPE);
1227             } catch (IOException e) {
1228                 fail("failed to create " + MIME_TYPE + " encoder");
1229             }
1230             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1231             surface = encoder.createInputSurface();
1232             encoder.start();
1233 
1234             encoder.getOutputBuffers();
1235 
1236             // re-configure, this time without an input surface
1237             if (VERBOSE) Log.d(TAG, "reconfiguring");
1238             encoder.stop();
1239             // Use non-opaque color format for byte buffer mode.
1240             format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1241                     MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
1242             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1243             encoder.start();
1244             if (VERBOSE) Log.d(TAG, "reconfigured");
1245 
1246             encoder.getOutputBuffers();
1247             encoder.dequeueInputBuffer(-1);
1248 
1249             try {
1250                 encoder.signalEndOfInputStream();
1251                 fail("signalEndOfInputStream only works on surface input");
1252             } catch (IllegalStateException ise) {
1253                 // good
1254             }
1255 
1256             PersistableBundle metrics = encoder.getMetrics();
1257             if (metrics == null) {
1258                 fail("getMetrics() returns null");
1259             } else if (metrics.isEmpty()) {
1260                 fail("getMetrics() returns empty results");
1261             }
1262             int encoding = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1263             if (encoding != 1) {
1264                 fail("getMetrics() returns bad encoder value " + encoding);
1265             }
1266             String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1267             if (theCodec == null) {
1268                 fail("getMetrics() returns null codec value ");
1269             }
1270 
1271         } finally {
1272             if (encoder != null) {
1273                 encoder.stop();
1274                 encoder.release();
1275             }
1276             if (surface != null) {
1277                 surface.release();
1278             }
1279         }
1280     }
1281 
1282     @ApiTest(apis = "MediaCodec#flush")
1283     @Test
testDecodeAfterFlush()1284     public void testDecodeAfterFlush() throws InterruptedException {
1285         testDecodeAfterFlush(true /* audio */);
1286         testDecodeAfterFlush(false /* audio */);
1287     }
1288 
testDecodeAfterFlush(final boolean audio)1289     private void testDecodeAfterFlush(final boolean audio) throws InterruptedException {
1290         final AtomicBoolean completed = new AtomicBoolean(false);
1291         Thread decodingThread = new Thread(new Runnable() {
1292             @Override
1293             public void run() {
1294                 OutputSurface outputSurface = null;
1295                 MediaExtractor mediaExtractor = null;
1296                 MediaCodec mediaCodec = null;
1297                 try {
1298                     String mimeTypePrefix  = audio ? "audio/" : "video/";
1299                     if (!audio) {
1300                         outputSurface = new OutputSurface(1, 1);
1301                     }
1302                     mediaExtractor = getMediaExtractorForMimeType(INPUT_RESOURCE, mimeTypePrefix);
1303                     MediaFormat mediaFormat =
1304                             mediaExtractor.getTrackFormat(mediaExtractor.getSampleTrackIndex());
1305                     if (!MediaUtils.checkDecoderForFormat(mediaFormat)) {
1306                         completed.set(true);
1307                         return; // skip
1308                     }
1309                     String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
1310                     mediaCodec = MediaCodec.createDecoderByType(mimeType);
1311                     mediaCodec.configure(mediaFormat, outputSurface == null ? null : outputSurface.getSurface(),
1312                             null /* crypto */, 0 /* flags */);
1313                     mediaCodec.start();
1314 
1315                     if (!runDecodeTillFirstOutput(mediaCodec, mediaExtractor)) {
1316                         throw new RuntimeException("decoder does not generate non-empty output.");
1317                     }
1318 
1319                     PersistableBundle metrics = mediaCodec.getMetrics();
1320                     if (metrics == null) {
1321                         fail("getMetrics() returns null");
1322                     } else if (metrics.isEmpty()) {
1323                         fail("getMetrics() returns empty results");
1324                     }
1325                     int encoder = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1326                     if (encoder != 0) {
1327                         fail("getMetrics() returns bad encoder value " + encoder);
1328                     }
1329                     String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1330                     if (theCodec == null) {
1331                         fail("getMetrics() returns null codec value ");
1332                     }
1333 
1334 
1335                     // simulate application flush.
1336                     mediaExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
1337                     mediaCodec.flush();
1338 
1339                     completed.set(runDecodeTillFirstOutput(mediaCodec, mediaExtractor));
1340                     metrics = mediaCodec.getMetrics();
1341                     if (metrics == null) {
1342                         fail("getMetrics() returns null");
1343                     } else if (metrics.isEmpty()) {
1344                         fail("getMetrics() returns empty results");
1345                     }
1346                     int encoding = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1347                     if (encoding != 0) {
1348                         fail("getMetrics() returns bad encoder value " + encoding);
1349                     }
1350                     String theCodec2 = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1351                     if (theCodec2 == null) {
1352                         fail("getMetrics() returns null codec value ");
1353                     }
1354 
1355                 } catch (IOException e) {
1356                     throw new RuntimeException("error setting up decoding", e);
1357                 } finally {
1358                     if (mediaCodec != null) {
1359                         mediaCodec.stop();
1360 
1361                         PersistableBundle metrics = mediaCodec.getMetrics();
1362                         if (metrics == null) {
1363                             fail("getMetrics() returns null");
1364                         } else if (metrics.isEmpty()) {
1365                             fail("getMetrics() returns empty results");
1366                         }
1367                         int encoder = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1368                         if (encoder != 0) {
1369                             fail("getMetrics() returns bad encoder value " + encoder);
1370                         }
1371                         String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1372                         if (theCodec == null) {
1373                             fail("getMetrics() returns null codec value ");
1374                         }
1375 
1376                         mediaCodec.release();
1377                     }
1378                     if (mediaExtractor != null) {
1379                         mediaExtractor.release();
1380                     }
1381                     if (outputSurface != null) {
1382                         outputSurface.release();
1383                     }
1384                 }
1385             }
1386         });
1387         decodingThread.start();
1388         decodingThread.join(DECODING_TIMEOUT_MS);
1389         // In case it's timed out, need to stop the thread and have all resources released.
1390         decodingThread.interrupt();
1391         if (!completed.get()) {
1392             throw new RuntimeException("timed out decoding to end-of-stream");
1393         }
1394     }
1395 
1396     // Run the decoder till it generates an output buffer.
1397     // Return true when that output buffer is not empty, false otherwise.
runDecodeTillFirstOutput( MediaCodec mediaCodec, MediaExtractor mediaExtractor)1398     private static boolean runDecodeTillFirstOutput(
1399             MediaCodec mediaCodec, MediaExtractor mediaExtractor) {
1400         final int TIME_OUT_US = 10000;
1401 
1402         assertTrue("Wrong test stream which has no data.",
1403                 mediaExtractor.getSampleTrackIndex() != -1);
1404         boolean signaledEos = false;
1405         MediaCodec.BufferInfo outputBufferInfo = new MediaCodec.BufferInfo();
1406         while (!Thread.interrupted()) {
1407             // Try to feed more data into the codec.
1408             if (!signaledEos) {
1409                 int bufferIndex = mediaCodec.dequeueInputBuffer(TIME_OUT_US /* timeoutUs */);
1410                 if (bufferIndex != -1) {
1411                     ByteBuffer buffer = mediaCodec.getInputBuffer(bufferIndex);
1412                     int size = mediaExtractor.readSampleData(buffer, 0 /* offset */);
1413                     long timestampUs = mediaExtractor.getSampleTime();
1414                     mediaExtractor.advance();
1415                     signaledEos = mediaExtractor.getSampleTrackIndex() == -1;
1416                     mediaCodec.queueInputBuffer(bufferIndex,
1417                             0 /* offset */,
1418                             size,
1419                             timestampUs,
1420                             signaledEos ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
1421                     Log.i("DEBUG", "queue with " + signaledEos);
1422                 }
1423             }
1424 
1425             int outputBufferIndex = mediaCodec.dequeueOutputBuffer(
1426                     outputBufferInfo, TIME_OUT_US /* timeoutUs */);
1427 
1428             if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
1429                     || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED
1430                     || outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
1431                 continue;
1432             }
1433             assertTrue("Wrong output buffer index", outputBufferIndex >= 0);
1434 
1435             PersistableBundle metrics = mediaCodec.getMetrics();
1436             Log.d(TAG, "getMetrics after first buffer metrics says: " + metrics);
1437 
1438             int encoder = metrics.getInt(MediaCodec.MetricsConstants.ENCODER, -1);
1439             if (encoder != 0) {
1440                 fail("getMetrics() returns bad encoder value " + encoder);
1441             }
1442             String theCodec = metrics.getString(MediaCodec.MetricsConstants.CODEC, null);
1443             if (theCodec == null) {
1444                 fail("getMetrics() returns null codec value ");
1445             }
1446 
1447             mediaCodec.releaseOutputBuffer(outputBufferIndex, false /* render */);
1448             boolean eos = (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
1449             Log.i("DEBUG", "Got a frame with eos=" + eos);
1450             if (eos && outputBufferInfo.size == 0) {
1451                 return false;
1452             } else {
1453                 return true;
1454             }
1455         }
1456 
1457         return false;
1458     }
1459 
1460     /**
1461      * Tests whether decoding a short group-of-pictures succeeds. The test queues a few video frames
1462      * then signals end-of-stream. The test fails if the decoder doesn't output the queued frames.
1463      */
1464     @ApiTest(apis = {"MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface"})
1465     @Test
testDecodeShortInput()1466     public void testDecodeShortInput() throws InterruptedException {
1467         // Input buffers from this input video are queued up to and including the video frame with
1468         // timestamp LAST_BUFFER_TIMESTAMP_US.
1469         final String INPUT_RESOURCE =
1470                 "video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_192kbps_44100hz.mp4";
1471         final long LAST_BUFFER_TIMESTAMP_US = 166666;
1472 
1473         // The test should fail if the decoder never produces output frames for the truncated input.
1474         // Time out decoding, as we have no way to query whether the decoder will produce output.
1475         final int DECODING_TIMEOUT_MS = 2000;
1476 
1477         final AtomicBoolean completed = new AtomicBoolean();
1478         Thread videoDecodingThread = new Thread(new Runnable() {
1479             @Override
1480             public void run() {
1481                 completed.set(runDecodeShortInput(INPUT_RESOURCE, LAST_BUFFER_TIMESTAMP_US));
1482             }
1483         });
1484         videoDecodingThread.start();
1485         videoDecodingThread.join(DECODING_TIMEOUT_MS);
1486         if (!completed.get()) {
1487             throw new RuntimeException("timed out decoding to end-of-stream");
1488         }
1489     }
1490 
runDecodeShortInput(final String inputResource, long lastBufferTimestampUs)1491     private boolean runDecodeShortInput(final String inputResource, long lastBufferTimestampUs) {
1492         final int NO_BUFFER_INDEX = -1;
1493 
1494         OutputSurface outputSurface = null;
1495         MediaExtractor mediaExtractor = null;
1496         MediaCodec mediaCodec = null;
1497         try {
1498             outputSurface = new OutputSurface(1, 1);
1499             mediaExtractor = getMediaExtractorForMimeType(inputResource, "video/");
1500             MediaFormat mediaFormat =
1501                     mediaExtractor.getTrackFormat(mediaExtractor.getSampleTrackIndex());
1502             String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
1503             if (!supportsCodec(mimeType, false)) {
1504                 Log.i(TAG, "No decoder found for mimeType= " + MIME_TYPE);
1505                 return true;
1506             }
1507             mediaCodec =
1508                     MediaCodec.createDecoderByType(mimeType);
1509             mediaCodec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
1510             mediaCodec.start();
1511             boolean eos = false;
1512             boolean signaledEos = false;
1513             MediaCodec.BufferInfo outputBufferInfo = new MediaCodec.BufferInfo();
1514             int outputBufferIndex = NO_BUFFER_INDEX;
1515             while (!eos && !Thread.interrupted()) {
1516                 // Try to feed more data into the codec.
1517                 if (mediaExtractor.getSampleTrackIndex() != -1 && !signaledEos) {
1518                     int bufferIndex = mediaCodec.dequeueInputBuffer(0);
1519                     if (bufferIndex != NO_BUFFER_INDEX) {
1520                         ByteBuffer buffer = mediaCodec.getInputBuffers()[bufferIndex];
1521                         int size = mediaExtractor.readSampleData(buffer, 0);
1522                         long timestampUs = mediaExtractor.getSampleTime();
1523                         mediaExtractor.advance();
1524                         signaledEos = mediaExtractor.getSampleTrackIndex() == -1
1525                                 || timestampUs == lastBufferTimestampUs;
1526                         mediaCodec.queueInputBuffer(bufferIndex,
1527                                 0,
1528                                 size,
1529                                 timestampUs,
1530                                 signaledEos ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
1531                     }
1532                 }
1533 
1534                 // If we don't have an output buffer, try to get one now.
1535                 if (outputBufferIndex == NO_BUFFER_INDEX) {
1536                     outputBufferIndex = mediaCodec.dequeueOutputBuffer(outputBufferInfo, 0);
1537                 }
1538 
1539                 if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
1540                         || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED
1541                         || outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
1542                     outputBufferIndex = NO_BUFFER_INDEX;
1543                 } else if (outputBufferIndex != NO_BUFFER_INDEX) {
1544                     eos = (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
1545 
1546                     boolean render = outputBufferInfo.size > 0;
1547                     mediaCodec.releaseOutputBuffer(outputBufferIndex, render);
1548                     if (render) {
1549                         outputSurface.awaitNewImage();
1550                     }
1551 
1552                     outputBufferIndex = NO_BUFFER_INDEX;
1553                 }
1554             }
1555 
1556             return eos;
1557         } catch (IOException e) {
1558             throw new RuntimeException("error reading input resource", e);
1559         } finally {
1560             if (mediaCodec != null) {
1561                 mediaCodec.stop();
1562                 mediaCodec.release();
1563             }
1564             if (mediaExtractor != null) {
1565                 mediaExtractor.release();
1566             }
1567             if (outputSurface != null) {
1568                 outputSurface.release();
1569             }
1570         }
1571     }
1572 
1573     /**
1574      * Tests creating two decoders for {@link #MIME_TYPE_AUDIO} at the same time.
1575      */
1576     @ApiTest(apis = {"MediaCodec#createDecoderByType",
1577             "android.media.MediaFormat#KEY_MIME",
1578             "android.media.MediaFormat#KEY_SAMPLE_RATE",
1579             "android.media.MediaFormat#KEY_CHANNEL_COUNT"})
1580     @Test
testCreateTwoAudioDecoders()1581     public void testCreateTwoAudioDecoders() {
1582         final MediaFormat format = MediaFormat.createAudioFormat(
1583                 MIME_TYPE_AUDIO, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT);
1584 
1585         MediaCodec audioDecoderA = null;
1586         MediaCodec audioDecoderB = null;
1587         try {
1588             try {
1589                 audioDecoderA = MediaCodec.createDecoderByType(MIME_TYPE_AUDIO);
1590             } catch (IOException e) {
1591                 fail("failed to create first " + MIME_TYPE_AUDIO + " decoder");
1592             }
1593             audioDecoderA.configure(format, null, null, 0);
1594             audioDecoderA.start();
1595 
1596             try {
1597                 audioDecoderB = MediaCodec.createDecoderByType(MIME_TYPE_AUDIO);
1598             } catch (IOException e) {
1599                 fail("failed to create second " + MIME_TYPE_AUDIO + " decoder");
1600             }
1601             audioDecoderB.configure(format, null, null, 0);
1602             audioDecoderB.start();
1603         } finally {
1604             if (audioDecoderB != null) {
1605                 try {
1606                     audioDecoderB.stop();
1607                     audioDecoderB.release();
1608                 } catch (RuntimeException e) {
1609                     Log.w(TAG, "exception stopping/releasing codec", e);
1610                 }
1611             }
1612 
1613             if (audioDecoderA != null) {
1614                 try {
1615                     audioDecoderA.stop();
1616                     audioDecoderA.release();
1617                 } catch (RuntimeException e) {
1618                     Log.w(TAG, "exception stopping/releasing codec", e);
1619                 }
1620             }
1621         }
1622     }
1623 
1624     /**
1625      * Tests creating an encoder and decoder for {@link #MIME_TYPE_AUDIO} at the same time.
1626      */
1627     @ApiTest(apis = {"MediaCodec#createDecoderByType", "MediaCodec#createEncoderByType",
1628             "android.media.MediaFormat#KEY_MIME",
1629             "android.media.MediaFormat#KEY_SAMPLE_RATE",
1630             "android.media.MediaFormat#KEY_CHANNEL_COUNT"})
1631     @Test
testCreateAudioDecoderAndEncoder()1632     public void testCreateAudioDecoderAndEncoder() {
1633         if (!supportsCodec(MIME_TYPE_AUDIO, true)) {
1634             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE_AUDIO);
1635             return;
1636         }
1637 
1638         if (!supportsCodec(MIME_TYPE_AUDIO, false)) {
1639             Log.i(TAG, "No decoder found for mimeType= " + MIME_TYPE_AUDIO);
1640             return;
1641         }
1642 
1643         final MediaFormat encoderFormat = MediaFormat.createAudioFormat(
1644                 MIME_TYPE_AUDIO, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT);
1645         encoderFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, AUDIO_AAC_PROFILE);
1646         encoderFormat.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
1647         final MediaFormat decoderFormat = MediaFormat.createAudioFormat(
1648                 MIME_TYPE_AUDIO, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT);
1649 
1650         MediaCodec audioEncoder = null;
1651         MediaCodec audioDecoder = null;
1652         try {
1653             try {
1654                 audioEncoder = MediaCodec.createEncoderByType(MIME_TYPE_AUDIO);
1655             } catch (IOException e) {
1656                 fail("failed to create " + MIME_TYPE_AUDIO + " encoder");
1657             }
1658             audioEncoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1659             audioEncoder.start();
1660 
1661             try {
1662                 audioDecoder = MediaCodec.createDecoderByType(MIME_TYPE_AUDIO);
1663             } catch (IOException e) {
1664                 fail("failed to create " + MIME_TYPE_AUDIO + " decoder");
1665             }
1666             audioDecoder.configure(decoderFormat, null, null, 0);
1667             audioDecoder.start();
1668         } finally {
1669             if (audioDecoder != null) {
1670                 try {
1671                     audioDecoder.stop();
1672                     audioDecoder.release();
1673                 } catch (RuntimeException e) {
1674                     Log.w(TAG, "exception stopping/releasing codec", e);
1675                 }
1676             }
1677 
1678             if (audioEncoder != null) {
1679                 try {
1680                     audioEncoder.stop();
1681                     audioEncoder.release();
1682                 } catch (RuntimeException e) {
1683                     Log.w(TAG, "exception stopping/releasing codec", e);
1684                 }
1685             }
1686         }
1687     }
1688 
1689     @ApiTest(apis = {"MediaCodec#createEncoderByType",
1690             "android.media.MediaFormat#KEY_MIME",
1691             "android.media.MediaFormat#KEY_SAMPLE_RATE",
1692             "android.media.MediaFormat#KEY_CHANNEL_COUNT",
1693             "android.media.MediaFormat#KEY_WIDTH",
1694             "android.media.MediaFormat#KEY_HEIGHT"})
1695     @Test
testConcurrentAudioVideoEncodings()1696     public void testConcurrentAudioVideoEncodings() throws InterruptedException {
1697         if (!supportsCodec(MIME_TYPE_AUDIO, true)) {
1698             Log.i(TAG, "No encoder found for mimeType= " + MIME_TYPE_AUDIO);
1699             return;
1700         }
1701 
1702         if (!supportsCodec(MIME_TYPE, true)) {
1703             Log.i(TAG, "No decoder found for mimeType= " + MIME_TYPE);
1704             return;
1705         }
1706 
1707         final int VIDEO_NUM_SWAPS = 100;
1708         // audio only checks this and stop
1709         mVideoEncodingOngoing = true;
1710         final CodecInfo info = getAvcSupportedFormatInfo();
1711         long start = System.currentTimeMillis();
1712         Thread videoEncodingThread = new Thread(new Runnable() {
1713             @Override
1714             public void run() {
1715                 runVideoEncoding(VIDEO_NUM_SWAPS, info);
1716             }
1717         });
1718         Thread audioEncodingThread = new Thread(new Runnable() {
1719             @Override
1720             public void run() {
1721                 runAudioEncoding();
1722             }
1723         });
1724         videoEncodingThread.start();
1725         audioEncodingThread.start();
1726         videoEncodingThread.join();
1727         mVideoEncodingOngoing = false;
1728         audioEncodingThread.join();
1729         assertFalse("Video encoding error. Chekc logcat", mVideoEncoderHadError);
1730         assertFalse("Audio encoding error. Chekc logcat", mAudioEncoderHadError);
1731         long end = System.currentTimeMillis();
1732         Log.w(TAG, "Concurrent AV encoding took " + (end - start) + " ms for " + VIDEO_NUM_SWAPS +
1733                 " video frames");
1734     }
1735 
1736     private static class CodecInfo {
1737         public int mMaxW;
1738         public int mMaxH;
1739         public int mFps;
1740         public int mBitRate;
1741     }
1742 
1743     @ApiTest(apis = {"MediaCodec#CryptoInfo", "MediaCodec#CryptoInfo#Pattern"})
1744     @Test
1745     @FrameworkSpecificTest // media mainline doesn't update crypto
1746     @NonMainlineTest // media mainline doesn't update crypto
testCryptoInfoPattern()1747     public void testCryptoInfoPattern() {
1748         CryptoInfo info = new CryptoInfo();
1749         Pattern pattern = new Pattern(1 /*blocksToEncrypt*/, 2 /*blocksToSkip*/);
1750         assertEquals(1, pattern.getEncryptBlocks());
1751         assertEquals(2, pattern.getSkipBlocks());
1752         pattern.set(3 /*blocksToEncrypt*/, 4 /*blocksToSkip*/);
1753         assertEquals(3, pattern.getEncryptBlocks());
1754         assertEquals(4, pattern.getSkipBlocks());
1755         info.setPattern(pattern);
1756         // Check that CryptoInfo does not leak access to the underlying pattern.
1757         if (mIsAtLeastS) {
1758             // getPattern() availability SDK>=S
1759             pattern.set(10, 10);
1760             info.getPattern().set(10, 10);
1761             assertSame(3, info.getPattern().getEncryptBlocks());
1762             assertSame(4, info.getPattern().getSkipBlocks());
1763         }
1764     }
1765 
getAvcSupportedFormatInfo()1766     private static CodecInfo getAvcSupportedFormatInfo() {
1767         MediaCodecInfo mediaCodecInfo = selectCodec(MIME_TYPE);
1768         CodecCapabilities cap = mediaCodecInfo.getCapabilitiesForType(MIME_TYPE);
1769         if (cap == null) { // not supported
1770             return null;
1771         }
1772         CodecInfo info = new CodecInfo();
1773         int highestLevel = 0;
1774         for (CodecProfileLevel lvl : cap.profileLevels) {
1775             if (lvl.level > highestLevel) {
1776                 highestLevel = lvl.level;
1777             }
1778         }
1779         int maxW = 0;
1780         int maxH = 0;
1781         int bitRate = 0;
1782         int fps = 0; // frame rate for the max resolution
1783         switch(highestLevel) {
1784             // Do not support Level 1 to 2.
1785             case CodecProfileLevel.AVCLevel1:
1786             case CodecProfileLevel.AVCLevel11:
1787             case CodecProfileLevel.AVCLevel12:
1788             case CodecProfileLevel.AVCLevel13:
1789             case CodecProfileLevel.AVCLevel1b:
1790             case CodecProfileLevel.AVCLevel2:
1791                 return null;
1792             case CodecProfileLevel.AVCLevel21:
1793                 maxW = 352;
1794                 maxH = 576;
1795                 bitRate = 4000000;
1796                 fps = 25;
1797                 break;
1798             case CodecProfileLevel.AVCLevel22:
1799                 maxW = 720;
1800                 maxH = 480;
1801                 bitRate = 4000000;
1802                 fps = 15;
1803                 break;
1804             case CodecProfileLevel.AVCLevel3:
1805                 maxW = 720;
1806                 maxH = 480;
1807                 bitRate = 10000000;
1808                 fps = 30;
1809                 break;
1810             case CodecProfileLevel.AVCLevel31:
1811                 maxW = 1280;
1812                 maxH = 720;
1813                 bitRate = 14000000;
1814                 fps = 30;
1815                 break;
1816             case CodecProfileLevel.AVCLevel32:
1817                 maxW = 1280;
1818                 maxH = 720;
1819                 bitRate = 20000000;
1820                 fps = 60;
1821                 break;
1822             case CodecProfileLevel.AVCLevel4: // only try up to 1080p
1823             default:
1824                 maxW = 1920;
1825                 maxH = 1080;
1826                 bitRate = 20000000;
1827                 fps = 30;
1828                 break;
1829         }
1830         info.mMaxW = maxW;
1831         info.mMaxH = maxH;
1832         info.mFps = fps;
1833         info.mBitRate = bitRate;
1834         Log.i(TAG, "AVC Level 0x" + Integer.toHexString(highestLevel) + " bit rate " + bitRate +
1835                 " fps " + info.mFps + " w " + maxW + " h " + maxH);
1836 
1837         return info;
1838     }
1839 
runVideoEncoding(int numSwap, CodecInfo info)1840     private void runVideoEncoding(int numSwap, CodecInfo info) {
1841         MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, info.mMaxW, info.mMaxH);
1842         format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1843                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
1844         format.setInteger(MediaFormat.KEY_BIT_RATE, info.mBitRate);
1845         format.setInteger(MediaFormat.KEY_FRAME_RATE, info.mFps);
1846         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
1847         MediaCodec encoder = null;
1848         InputSurface inputSurface = null;
1849         mVideoEncoderHadError = false;
1850         try {
1851             encoder = MediaCodec.createEncoderByType(MIME_TYPE);
1852             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1853             inputSurface = new InputSurface(encoder.createInputSurface());
1854             inputSurface.makeCurrent();
1855             MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
1856             encoder.start();
1857             for (int i = 0; i < numSwap; i++) {
1858                 GLES20.glClearColor(0.0f, 0.5f, 0.0f, 1.0f);
1859                 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
1860                 inputSurface.swapBuffers();
1861                 // dequeue buffers until not available
1862                 int index = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
1863                 while (index >= 0) {
1864                     encoder.releaseOutputBuffer(index, false);
1865                     // just throw away output
1866                     // allow shorter wait for 2nd round to move on quickly.
1867                     index = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC_SHORT);
1868                 }
1869             }
1870             encoder.signalEndOfInputStream();
1871         } catch (Throwable e) {
1872             Log.w(TAG, "runVideoEncoding got error: " + e);
1873             mVideoEncoderHadError = true;
1874         } finally {
1875             if (encoder != null) {
1876                 encoder.stop();
1877                 encoder.release();
1878             }
1879             if (inputSurface != null) {
1880                 inputSurface.release();
1881             }
1882         }
1883     }
1884 
runAudioEncoding()1885     private void runAudioEncoding() {
1886         MediaFormat format = MediaFormat.createAudioFormat(MIME_TYPE_AUDIO, AUDIO_SAMPLE_RATE,
1887                 AUDIO_CHANNEL_COUNT);
1888         format.setInteger(MediaFormat.KEY_AAC_PROFILE, AUDIO_AAC_PROFILE);
1889         format.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
1890         MediaCodec encoder = null;
1891         mAudioEncoderHadError = false;
1892         try {
1893             encoder = MediaCodec.createEncoderByType(MIME_TYPE_AUDIO);
1894             encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
1895             MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
1896             encoder.start();
1897             ByteBuffer[] inputBuffers = encoder.getInputBuffers();
1898             ByteBuffer source = ByteBuffer.allocate(inputBuffers[0].capacity());
1899             for (int i = 0; i < source.capacity()/2; i++) {
1900                 source.putShort((short)i);
1901             }
1902             source.rewind();
1903             int currentInputBufferIndex = 0;
1904             long encodingLatencySum = 0;
1905             int totalEncoded = 0;
1906             int numRepeat = 0;
1907             while (mVideoEncodingOngoing) {
1908                 numRepeat++;
1909                 int inputIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
1910                 while (inputIndex == -1) {
1911                     inputIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
1912                 }
1913                 ByteBuffer inputBuffer = inputBuffers[inputIndex];
1914                 inputBuffer.rewind();
1915                 inputBuffer.put(source);
1916                 long start = System.currentTimeMillis();
1917                 totalEncoded += inputBuffers[inputIndex].limit();
1918                 encoder.queueInputBuffer(inputIndex, 0, inputBuffer.limit(), 0, 0);
1919                 source.rewind();
1920                 int index = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
1921                 long end = System.currentTimeMillis();
1922                 encodingLatencySum += (end - start);
1923                 while (index >= 0) {
1924                     encoder.releaseOutputBuffer(index, false);
1925                     // just throw away output
1926                     // allow shorter wait for 2nd round to move on quickly.
1927                     index = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC_SHORT);
1928                 }
1929             }
1930             Log.w(TAG, "Audio encoding average latency " + encodingLatencySum / numRepeat +
1931                     " ms for average write size " + totalEncoded / numRepeat +
1932                     " total latency " + encodingLatencySum + " ms for total bytes " + totalEncoded);
1933         } catch (Throwable e) {
1934             Log.w(TAG, "runAudioEncoding got error: " + e);
1935             mAudioEncoderHadError = true;
1936         } finally {
1937             if (encoder != null) {
1938                 encoder.stop();
1939                 encoder.release();
1940             }
1941         }
1942     }
1943 
1944     /**
1945      * Creates a MediaFormat with the basic set of values.
1946      */
createMediaFormat()1947     private static MediaFormat createMediaFormat() {
1948         MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, WIDTH, HEIGHT);
1949         format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
1950                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
1951         format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
1952         format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
1953         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
1954         return format;
1955     }
1956 
1957     /**
1958      * Returns the first codec capable of encoding the specified MIME type, or null if no
1959      * match was found.
1960      */
selectCodec(String mimeType)1961     private static MediaCodecInfo selectCodec(String mimeType) {
1962         // FIXME: select codecs based on the complete use-case, not just the mime
1963         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
1964         for (MediaCodecInfo info : mcl.getCodecInfos()) {
1965             if (!info.isEncoder()) {
1966                 continue;
1967             }
1968 
1969             String[] types = info.getSupportedTypes();
1970             for (int j = 0; j < types.length; j++) {
1971                 if (types[j].equalsIgnoreCase(mimeType)) {
1972                     return info;
1973                 }
1974             }
1975         }
1976         return null;
1977     }
1978 
1979     /**
1980      * Returns a color format that is supported by the codec and isn't COLOR_FormatSurface.  Throws
1981      * an exception if none found.
1982      */
findNonSurfaceColorFormat(MediaCodecInfo codecInfo, String mimeType)1983     private static int findNonSurfaceColorFormat(MediaCodecInfo codecInfo, String mimeType) {
1984         MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType);
1985         for (int i = 0; i < capabilities.colorFormats.length; i++) {
1986             int colorFormat = capabilities.colorFormats[i];
1987             if (colorFormat != MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) {
1988                 return colorFormat;
1989             }
1990         }
1991         fail("couldn't find a good color format for " + codecInfo.getName() + " / " + MIME_TYPE);
1992         return 0;   // not reached
1993     }
1994 
getMediaExtractorForMimeType(final String resource, String mimeTypePrefix)1995     private MediaExtractor getMediaExtractorForMimeType(final String resource,
1996             String mimeTypePrefix) throws IOException {
1997         Preconditions.assertTestFileExists(mInpPrefix + resource);
1998         MediaExtractor mediaExtractor = new MediaExtractor();
1999         File inpFile = new File(mInpPrefix + resource);
2000         ParcelFileDescriptor parcelFD =
2001                 ParcelFileDescriptor.open(inpFile, ParcelFileDescriptor.MODE_READ_ONLY);
2002         AssetFileDescriptor afd = new AssetFileDescriptor(parcelFD, 0, parcelFD.getStatSize());
2003         try {
2004             mediaExtractor.setDataSource(
2005                     afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
2006         } finally {
2007             afd.close();
2008         }
2009         int trackIndex;
2010         for (trackIndex = 0; trackIndex < mediaExtractor.getTrackCount(); trackIndex++) {
2011             MediaFormat trackMediaFormat = mediaExtractor.getTrackFormat(trackIndex);
2012             if (trackMediaFormat.getString(MediaFormat.KEY_MIME).startsWith(mimeTypePrefix)) {
2013                 mediaExtractor.selectTrack(trackIndex);
2014                 break;
2015             }
2016         }
2017         if (trackIndex == mediaExtractor.getTrackCount()) {
2018             throw new IllegalStateException("couldn't get a video track");
2019         }
2020 
2021         return mediaExtractor;
2022     }
2023 
supportsCodec(String mimeType, boolean encoder)2024     private static boolean supportsCodec(String mimeType, boolean encoder) {
2025         MediaCodecList list = new MediaCodecList(MediaCodecList.ALL_CODECS);
2026         for (MediaCodecInfo info : list.getCodecInfos()) {
2027             if (encoder != info.isEncoder()) {
2028                 continue;
2029             }
2030 
2031             for (String type : info.getSupportedTypes()) {
2032                 if (type.equalsIgnoreCase(mimeType)) {
2033                     return true;
2034                 }
2035             }
2036         }
2037         return false;
2038     }
2039 
2040     private static final UUID CLEARKEY_SCHEME_UUID =
2041             new UUID(0x1077efecc0b24d02L, 0xace33c1e52e2fb4bL);
2042 
2043     /**
2044      * Tests MediaCodec.CryptoException
2045      */
2046     @ApiTest(apis = "MediaCodec#CryptoException")
2047     @Test
2048     @FrameworkSpecificTest // media mainline doesn't update crypto
2049     @NonMainlineTest // media mainline doesn't update crypto
testCryptoException()2050     public void testCryptoException() {
2051         int errorCode = CryptoException.ERROR_KEY_EXPIRED;
2052         String errorMessage = "key_expired";
2053         CryptoException exception = new CryptoException(errorCode, errorMessage);
2054 
2055         assertEquals(errorCode, exception.getErrorCode());
2056         assertEquals(errorMessage, exception.getMessage());
2057     }
2058 
2059     /**
2060      * PCM encoding configuration test.
2061      *
2062      * If not specified in configure(), PCM encoding if it exists must be 16 bit.
2063      * If specified float in configure(), PCM encoding if it exists must be 16 bit, or float.
2064      *
2065      * As of Q, any codec of type "audio/raw" must support PCM encoding float.
2066      */
2067     @ApiTest(apis = {"android.media.AudioFormat#ENCODING_PCM_16BIT",
2068             "android.media.AudioFormat#ENCODING_PCM_FLOAT"})
2069     @MediumTest
2070     @Test
testPCMEncoding()2071     public void testPCMEncoding() throws Exception {
2072         final MediaCodecList mcl = new MediaCodecList(MediaCodecList.ALL_CODECS);
2073         for (MediaCodecInfo codecInfo : mcl.getCodecInfos()) {
2074             final boolean isEncoder = codecInfo.isEncoder();
2075             final String name = codecInfo.getName();
2076 
2077             for (String type : codecInfo.getSupportedTypes()) {
2078                 final MediaCodecInfo.CodecCapabilities ccaps =
2079                         codecInfo.getCapabilitiesForType(type);
2080                 final MediaCodecInfo.AudioCapabilities acaps =
2081                         ccaps.getAudioCapabilities();
2082                 if (acaps == null) {
2083                     break; // not an audio codec
2084                 }
2085 
2086                 // Deduce the minimum channel count (though prefer stereo over mono).
2087                 final int channelCount = Math.min(acaps.getMaxInputChannelCount(), 2);
2088 
2089                 // Deduce the minimum sample rate.
2090                 final int[] sampleRates = acaps.getSupportedSampleRates();
2091                 final Range<Integer>[] sampleRateRanges = acaps.getSupportedSampleRateRanges();
2092                 assertNotNull("supported sample rate ranges must be non-null", sampleRateRanges);
2093                 final int sampleRate = sampleRateRanges[0].getLower();
2094 
2095                 // If sample rate array exists (it may not),
2096                 // the minimum value must be equal with the minimum from the sample rate range.
2097                 if (sampleRates != null) {
2098                     assertEquals("sample rate range and array should have equal minimum",
2099                             sampleRate, sampleRates[0]);
2100                     Log.d(TAG, "codec: " + name + " type: " + type
2101                             + " has both sampleRate array and ranges");
2102                 } else {
2103                     Log.d(TAG, "codec: " + name + " type: " + type
2104                             + " returns null getSupportedSampleRates()");
2105                 }
2106 
2107                 // We create one format here for both tests below.
2108                 final MediaFormat format = MediaFormat.createAudioFormat(
2109                     type, sampleRate, channelCount);
2110 
2111                 // Bitrate field is mandatory for encoders (except FLAC).
2112                 if (isEncoder) {
2113                     final int bitRate = acaps.getBitrateRange().getLower();
2114                     format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
2115                 }
2116 
2117                 // First test: An audio codec must be createable from a format
2118                 // with the minimum sample rate and channel count.
2119                 // The PCM encoding must be null (doesn't exist) or 16 bit.
2120                 {
2121                     // Check encoding of codec.
2122                     final Integer actualEncoding = encodingOfAudioCodec(name, format, isEncoder);
2123                     if (actualEncoding != null) {
2124                         assertEquals("returned audio encoding must be 16 bit for codec: "
2125                                 + name + " type: " + type + " encoding: " + actualEncoding,
2126                                 AudioFormat.ENCODING_PCM_16BIT, actualEncoding.intValue());
2127                     }
2128                 }
2129 
2130                 // Second test: An audio codec configured with PCM encoding float must return
2131                 // either an encoding of null (doesn't exist), 16 bit, or float.
2132                 {
2133                     // Reuse the original format, and add float specifier.
2134                     format.setInteger(
2135                             MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_FLOAT);
2136 
2137                     // Check encoding of codec.
2138                     // The KEY_PCM_ENCODING key is advisory, so should not cause configuration
2139                     // failure.  The actual PCM encoding is returned from
2140                     // the input format (encoder) or output format (decoder).
2141                     final Integer actualEncoding = encodingOfAudioCodec(name, format, isEncoder);
2142                     if (actualEncoding != null) {
2143                         assertTrue(
2144                                 "returned audio encoding must be 16 bit or float for codec: "
2145                                 + name + " type: " + type + " encoding: " + actualEncoding,
2146                                 actualEncoding == AudioFormat.ENCODING_PCM_16BIT
2147                                 || actualEncoding == AudioFormat.ENCODING_PCM_FLOAT);
2148                         if (actualEncoding == AudioFormat.ENCODING_PCM_FLOAT) {
2149                             Log.d(TAG, "codec: " + name + " type: " + type + " supports float");
2150                         }
2151                     }
2152 
2153                     // As of Q, all codecs of type "audio/raw" must support float.
2154                     if (type.equals("audio/raw")) {
2155                         assertTrue(type + " must support float",
2156                                 actualEncoding != null &&
2157                                 actualEncoding.intValue() == AudioFormat.ENCODING_PCM_FLOAT);
2158                     }
2159                 }
2160             }
2161         }
2162     }
2163 
2164     /**
2165      * Returns the PCM encoding of an audio codec, or null if codec doesn't exist,
2166      * or not an audio codec, or PCM encoding key doesn't exist.
2167      */
encodingOfAudioCodec(String name, MediaFormat format, boolean encode)2168     private Integer encodingOfAudioCodec(String name, MediaFormat format, boolean encode)
2169             throws IOException {
2170         final int flagEncoder = encode ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0;
2171         final MediaCodec codec = MediaCodec.createByCodecName(name);
2172         Integer actualEncoding = null;
2173 
2174         try {
2175             codec.configure(format, null /* surface */, null /* crypto */, flagEncoder);
2176 
2177             // Check input/output format - this must exist.
2178             final MediaFormat actualFormat =
2179                     encode ? codec.getInputFormat() : codec.getOutputFormat();
2180             assertNotNull("cannot get format for " + name, actualFormat);
2181 
2182             // Check actual encoding - this may or may not exist
2183             try {
2184                 actualEncoding = actualFormat.getInteger(MediaFormat.KEY_PCM_ENCODING);
2185             } catch (Exception e) {
2186                 ; // trying to get a non-existent key throws exception
2187             }
2188         } finally {
2189             codec.release();
2190         }
2191         return actualEncoding;
2192     }
2193 
2194     @ApiTest(apis = "android.media.AudioFormat#KEY_FLAC_COMPRESSION_LEVEL")
2195     @SmallTest
2196     @Test
testFlacIdentity()2197     public void testFlacIdentity() throws Exception {
2198         final int PCM_FRAMES = 1152 * 4; // FIXME: requires 4 flac frames to work with OMX codecs.
2199         final int SAMPLES = PCM_FRAMES * AUDIO_CHANNEL_COUNT;
2200         final int[] SAMPLE_RATES = {AUDIO_SAMPLE_RATE, 192000}; // ensure 192kHz supported.
2201 
2202         for (int sampleRate : SAMPLE_RATES) {
2203             final MediaFormat format = new MediaFormat();
2204             format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_FLAC);
2205             format.setInteger(MediaFormat.KEY_SAMPLE_RATE, sampleRate);
2206             format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, AUDIO_CHANNEL_COUNT);
2207 
2208             Log.d(TAG, "Trying sample rate: " + sampleRate
2209                     + " channel count: " + AUDIO_CHANNEL_COUNT);
2210             // this key is only needed for encode, ignored for decode
2211             format.setInteger(MediaFormat.KEY_FLAC_COMPRESSION_LEVEL, 5);
2212 
2213             for (int i = 0; i < 2; ++i) {
2214                 final boolean useFloat = (i == 1);
2215                 final StreamUtils.PcmAudioBufferStream audioStream =
2216                     new StreamUtils.PcmAudioBufferStream(SAMPLES, sampleRate, 1000 /* frequency */,
2217                     100 /* sweep */, useFloat);
2218 
2219                 if (useFloat) {
2220                     format.setInteger(
2221                         MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_FLOAT);
2222                 }
2223 
2224                 final StreamUtils.MediaCodecStream rawToFlac = new StreamUtils.MediaCodecStream(
2225                     new StreamUtils.ByteBufferInputStream(audioStream), format, true /* encode */);
2226                 final StreamUtils.MediaCodecStream flacToRaw = new StreamUtils.MediaCodecStream(
2227                     rawToFlac, format, false /* encode */);
2228 
2229                 if (useFloat) { // ensure float precision supported at the sample rate.
2230                     assertTrue("No float FLAC encoder at " + sampleRate,
2231                             rawToFlac.mIsFloat);
2232                     assertTrue("No float FLAC decoder at " + sampleRate,
2233                             flacToRaw.mIsFloat);
2234                 }
2235 
2236                 // Note: the existence of signed zero (as well as NAN) may make byte
2237                 // comparisons invalid for floating point output. In our case, since the
2238                 // floats come through integer to float conversion, it does not matter.
2239                 assertEquals("Audio data not identical after compression",
2240                     audioStream.sizeInBytes(),
2241                     StreamUtils.compareStreams(new StreamUtils.ByteBufferInputStream(flacToRaw),
2242                         new StreamUtils.ByteBufferInputStream(
2243                         new StreamUtils.PcmAudioBufferStream(audioStream))));
2244             }
2245         }
2246     }
2247 
2248     @ApiTest(apis = "MediaCodec#release")
2249     @Test
testAsyncRelease()2250     public void testAsyncRelease() throws Exception {
2251         OutputSurface outputSurface = new OutputSurface(1, 1);
2252         MediaExtractor mediaExtractor = getMediaExtractorForMimeType(INPUT_RESOURCE, "video/");
2253         MediaFormat mediaFormat =
2254                 mediaExtractor.getTrackFormat(mediaExtractor.getSampleTrackIndex());
2255         String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
2256         for (int i = 0; i < 100; ++i) {
2257             final MediaCodec codec = MediaCodec.createDecoderByType(mimeType);
2258 
2259             try {
2260                 final ConditionVariable cv = new ConditionVariable();
2261                 Runnable first = null;
2262                 switch (i % 5) {
2263                     case 0:  // release
2264                         codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2265                         codec.start();
2266                         first = () -> { cv.block(); codec.release(); };
2267                         break;
2268                     case 1:  // start
2269                         codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2270                         first = () -> {
2271                             cv.block();
2272                             try {
2273                                 codec.start();
2274                             } catch (Exception e) {
2275                                 Log.i(TAG, "start failed", e);
2276                             }
2277                         };
2278                         break;
2279                     case 2:  // configure
2280                         first = () -> {
2281                             cv.block();
2282                             try {
2283                                 codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2284                             } catch (Exception e) {
2285                                 Log.i(TAG, "configure failed", e);
2286                             }
2287                         };
2288                         break;
2289                     case 3:  // stop
2290                         codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2291                         codec.start();
2292                         first = () -> {
2293                             cv.block();
2294                             try {
2295                                 codec.stop();
2296                             } catch (Exception e) {
2297                                 Log.i(TAG, "stop failed", e);
2298                             }
2299                         };
2300                         break;
2301                     case 4:  // flush
2302                         codec.configure(mediaFormat, outputSurface.getSurface(), null, 0);
2303                         codec.start();
2304                         codec.dequeueInputBuffer(0);
2305                         first = () -> {
2306                             cv.block();
2307                             try {
2308                                 codec.flush();
2309                             } catch (Exception e) {
2310                                 Log.i(TAG, "flush failed", e);
2311                             }
2312                         };
2313                         break;
2314                 }
2315 
2316                 Thread[] threads = new Thread[10];
2317                 threads[0] = new Thread(first);
2318                 for (int j = 1; j < threads.length; ++j) {
2319                     threads[j] = new Thread(() -> { cv.block(); codec.release(); });
2320                 }
2321                 for (Thread thread : threads) {
2322                     thread.start();
2323                 }
2324                 // Wait a little bit so that threads may reach block() call.
2325                 Thread.sleep(50);
2326                 cv.open();
2327                 for (Thread thread : threads) {
2328                     thread.join();
2329                 }
2330             } finally {
2331                 codec.release();
2332             }
2333         }
2334     }
2335 
2336     @ApiTest(apis = "MediaCodec#setAudioPresentation")
2337     @Test
testSetAudioPresentation()2338     public void testSetAudioPresentation() throws Exception {
2339         MediaFormat format = MediaFormat.createAudioFormat(
2340                 MediaFormat.MIMETYPE_AUDIO_MPEG, 44100 /* sampleRate */, 2 /* channelCount */);
2341         String mimeType = format.getString(MediaFormat.KEY_MIME);
2342         MediaCodec codec = createCodecByType(
2343                 format.getString(MediaFormat.KEY_MIME), false /* isEncoder */);
2344         assertNotNull(codec);
2345         assertThrows(NullPointerException.class, () -> {
2346             codec.setAudioPresentation(null);
2347         });
2348         codec.setAudioPresentation(
2349                 (new AudioPresentation.Builder(42 /* presentationId */)).build());
2350     }
2351 
2352     @ApiTest(apis = "android.media.MediaFormat#KEY_PREPEND_HEADER_TO_SYNC_FRAMES")
2353     @Test
testPrependHeadersToSyncFrames()2354     public void testPrependHeadersToSyncFrames() throws IOException {
2355         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
2356         for (MediaCodecInfo info : mcl.getCodecInfos()) {
2357             boolean isEncoder = info.isEncoder();
2358             for (String mime: info.getSupportedTypes()) {
2359                 CodecCapabilities caps = info.getCapabilitiesForType(mime);
2360                 boolean isVideo = (caps.getVideoCapabilities() != null);
2361                 boolean isAudio = (caps.getAudioCapabilities() != null);
2362 
2363                 MediaCodec codec = null;
2364                 MediaFormat format = null;
2365                 try {
2366                     codec = MediaCodec.createByCodecName(info.getName());
2367                     if (isVideo) {
2368                         VideoCapabilities vcaps = caps.getVideoCapabilities();
2369                         int minWidth = vcaps.getSupportedWidths().getLower();
2370                         int minHeight = vcaps.getSupportedHeightsFor(minWidth).getLower();
2371                         int minBitrate = vcaps.getBitrateRange().getLower();
2372                         int minFrameRate = Math.max(vcaps.getSupportedFrameRatesFor(
2373                                 minWidth, minHeight) .getLower().intValue(), 1);
2374                         format = MediaFormat.createVideoFormat(mime, minWidth, minHeight);
2375                         format.setInteger(MediaFormat.KEY_COLOR_FORMAT, caps.colorFormats[0]);
2376                         format.setInteger(MediaFormat.KEY_BIT_RATE, minBitrate);
2377                         format.setInteger(MediaFormat.KEY_FRAME_RATE, minFrameRate);
2378                         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
2379                     } else if(isAudio){
2380                         AudioCapabilities acaps = caps.getAudioCapabilities();
2381                         int minSampleRate = acaps.getSupportedSampleRateRanges()[0].getLower();
2382                         int minChannelCount = 1;
2383                         if (mIsAtLeastS) {
2384                             minChannelCount = acaps.getMinInputChannelCount();
2385                         }
2386                         int minBitrate = acaps.getBitrateRange().getLower();
2387                         format = MediaFormat.createAudioFormat(mime, minSampleRate, minChannelCount);
2388                         format.setInteger(MediaFormat.KEY_BIT_RATE, minBitrate);
2389                     }
2390 
2391                     if (isVideo || isAudio) {
2392                         format.setInteger(MediaFormat.KEY_PREPEND_HEADER_TO_SYNC_FRAMES, 1);
2393 
2394                         codec.configure(format, null /* surface */, null /* crypto */,
2395                             isEncoder ? codec.CONFIGURE_FLAG_ENCODE : 0);
2396                     }
2397                     if (isVideo && isEncoder) {
2398                         Log.i(TAG, info.getName() + " supports KEY_PREPEND_HEADER_TO_SYNC_FRAMES");
2399                     } else {
2400                         Log.i(TAG, info.getName() + " is not a video encoder, so" +
2401                                 " KEY_PREPEND_HEADER_TO_SYNC_FRAMES is no-op, as expected");
2402                     }
2403                     // TODO: actually test encoders prepend the headers to sync frames.
2404                 } catch (IllegalArgumentException | CodecException e) {
2405                     if (isVideo && isEncoder) {
2406                         Log.i(TAG, info.getName() + " does not support" +
2407                                 " KEY_PREPEND_HEADER_TO_SYNC_FRAMES");
2408                     } else {
2409                         fail(info.getName() + " is not a video encoder," +
2410                                 " so it should not fail to configure.\n" + e.toString());
2411                     }
2412                 } finally {
2413                     if (codec != null) {
2414                         codec.release();
2415                     }
2416                 }
2417             }
2418         }
2419     }
2420 
2421     /**
2422      * Test if flushing early in the playback does not prevent client from getting the
2423      * latest configuration. Empirically, this happens most often when the
2424      * codec is flushed after the first buffer is queued, so this test walks
2425      * through the scenario.
2426      */
2427     @ApiTest(apis = "MediaCodec#flush")
2428     @Test
testFlushAfterFirstBuffer()2429     public void testFlushAfterFirstBuffer() throws Exception {
2430         if (MediaUtils.check(mIsAtLeastR, "test needs Android 11")) {
2431             for (int i = 0; i < 100; ++i) {
2432                 doFlushAfterFirstBuffer();
2433             }
2434         }
2435     }
2436 
doFlushAfterFirstBuffer()2437     private void doFlushAfterFirstBuffer() throws Exception {
2438         MediaExtractor extractor = null;
2439         MediaCodec codec = null;
2440 
2441         try {
2442             MediaFormat newFormat = null;
2443             extractor = getMediaExtractorForMimeType(
2444                     "noise_2ch_48khz_aot29_dr_sbr_sig2_mp4.m4a", "audio/");
2445             int trackIndex = extractor.getSampleTrackIndex();
2446             MediaFormat format = extractor.getTrackFormat(trackIndex);
2447             codec = createCodecByType(
2448                     format.getString(MediaFormat.KEY_MIME), false /* isEncoder */);
2449             codec.configure(format, null, null, 0);
2450             codec.start();
2451             int firstInputIndex = codec.dequeueInputBuffer(0);
2452             while (firstInputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
2453                 firstInputIndex = codec.dequeueInputBuffer(5000);
2454             }
2455             assertTrue(firstInputIndex >= 0);
2456             extractor.readSampleData(codec.getInputBuffer(firstInputIndex), 0);
2457             codec.queueInputBuffer(
2458                     firstInputIndex, 0, Math.toIntExact(extractor.getSampleSize()),
2459                     extractor.getSampleTime(), extractor.getSampleFlags());
2460             // Don't advance, so the first buffer will be read again after flush
2461             codec.flush();
2462             ByteBuffer csd = format.getByteBuffer("csd-0");
2463             MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
2464             // We don't need to decode many frames
2465             int numFrames = 10;
2466             boolean eos = false;
2467             while (!eos) {
2468                 if (numFrames > 0) {
2469                     int inputIndex = codec.dequeueInputBuffer(0);
2470                     if (inputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
2471                         inputIndex = codec.dequeueInputBuffer(5000);
2472                     }
2473                     if (inputIndex >= 0) {
2474                         ByteBuffer inputBuffer = codec.getInputBuffer(inputIndex);
2475                         if (csd != null) {
2476                             inputBuffer.clear();
2477                             inputBuffer.put(csd);
2478                             codec.queueInputBuffer(
2479                                     inputIndex, 0, inputBuffer.position(), 0,
2480                                     MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
2481                             csd = null;
2482                         } else {
2483                             int size = extractor.readSampleData(inputBuffer, 0);
2484                             if (size <= 0) {
2485                                 break;
2486                             }
2487                             int flags = extractor.getSampleFlags();
2488                             --numFrames;
2489                             if (numFrames <= 0) {
2490                                 flags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
2491                             }
2492                             codec.queueInputBuffer(
2493                                     inputIndex, 0, size, extractor.getSampleTime(), flags);
2494                             extractor.advance();
2495                         }
2496                     }
2497                 }
2498 
2499                 int outputIndex = codec.dequeueOutputBuffer(bufferInfo, 0);
2500                 if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
2501                     outputIndex = codec.dequeueOutputBuffer(bufferInfo, 5000);
2502                 }
2503                 if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
2504                     newFormat = codec.getOutputFormat();
2505                 } else if (outputIndex >= 0) {
2506                     if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
2507                         eos = true;
2508                     }
2509                     codec.releaseOutputBuffer(outputIndex, false);
2510                 }
2511             }
2512             assertNotNull(newFormat);
2513             assertEquals(48000, newFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE));
2514         } finally {
2515             if (extractor != null) {
2516                 extractor.release();
2517             }
2518             if (codec != null) {
2519                 codec.stop();
2520                 codec.release();
2521             }
2522         }
2523     }
2524 
2525     @ApiTest(apis = {"MediaCodec#getSupportedVendorParameters",
2526             "MediaCodec#getParameterDescriptor",
2527             "MediaCodec#subscribeToVendorParameters",
2528             "MediaCodec#unsubscribeFromVendorParameters"})
2529     @Test
testVendorParameters()2530     public void testVendorParameters() {
2531         if (!MediaUtils.check(mIsAtLeastS, "test needs Android 12")) {
2532             return;
2533         }
2534         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
2535         for (MediaCodecInfo info : mcl.getCodecInfos()) {
2536             if (info.isAlias()) {
2537                 continue;
2538             }
2539             MediaCodec codec = null;
2540             if (!TestUtils.isTestableCodecInCurrentMode(info.getName())) {
2541                 Log.d(TAG, "skip testing codec " + info.getName() + " in current mode:"
2542                                 + TestUtils.currentTestModeName());
2543                 continue;
2544             }
2545             try {
2546                 codec = MediaCodec.createByCodecName(info.getName());
2547                 List<String> vendorParams = codec.getSupportedVendorParameters();
2548                 if (VERBOSE) {
2549                     Log.d(TAG, "vendor params supported by " + info.getName() + ": " +
2550                             vendorParams.toString());
2551                 }
2552                 for (String name : vendorParams) {
2553                     MediaCodec.ParameterDescriptor desc = codec.getParameterDescriptor(name);
2554                     assertNotNull(name + " is in the list of supported parameters, so the codec" +
2555                             " should be able to describe it.", desc);
2556                     assertEquals("name differs from the name in the descriptor",
2557                             name, desc.getName());
2558                     assertTrue("type in the descriptor cannot be TYPE_NULL",
2559                             MediaFormat.TYPE_NULL != desc.getType());
2560                     if (VERBOSE) {
2561                         Log.d(TAG, name + " is of type " + desc.getType());
2562                     }
2563                 }
2564                 codec.subscribeToVendorParameters(vendorParams);
2565 
2566                 // Build a MediaFormat that makes sense to the codec.
2567                 String type = info.getSupportedTypes()[0];
2568                 MediaFormat format = null;
2569                 CodecCapabilities caps = info.getCapabilitiesForType(type);
2570                 AudioCapabilities audioCaps = caps.getAudioCapabilities();
2571                 VideoCapabilities videoCaps = caps.getVideoCapabilities();
2572                 if (audioCaps != null) {
2573                     format = MediaFormat.createAudioFormat(
2574                             type,
2575                             audioCaps.getSupportedSampleRateRanges()[0].getLower(),
2576                             audioCaps.getMaxInputChannelCount());
2577                     if (info.isEncoder()) {
2578                         format.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
2579                     }
2580                 } else if (videoCaps != null) {
2581                     int width = videoCaps.getSupportedWidths().getLower();
2582                     int height = videoCaps.getSupportedHeightsFor(width).getLower();
2583                     format = MediaFormat.createVideoFormat(type, width, height);
2584                     if (info.isEncoder()) {
2585                         EncoderCapabilities encCaps = caps.getEncoderCapabilities();
2586                         if (encCaps != null) {
2587                             int bitrateMode = -1;
2588                             List<Integer> candidates = Arrays.asList(
2589                                     EncoderCapabilities.BITRATE_MODE_VBR,
2590                                     EncoderCapabilities.BITRATE_MODE_CBR,
2591                                     EncoderCapabilities.BITRATE_MODE_CQ,
2592                                     EncoderCapabilities.BITRATE_MODE_CBR_FD);
2593                             for (int candidate : candidates) {
2594                                 if (encCaps.isBitrateModeSupported(candidate)) {
2595                                     bitrateMode = candidate;
2596                                     break;
2597                                 }
2598                             }
2599                             if (VERBOSE) {
2600                                 Log.d(TAG, "video encoder: bitrate mode = " + bitrateMode);
2601                             }
2602                             format.setInteger(MediaFormat.KEY_BITRATE_MODE, bitrateMode);
2603                             switch (bitrateMode) {
2604                             case EncoderCapabilities.BITRATE_MODE_VBR:
2605                             case EncoderCapabilities.BITRATE_MODE_CBR:
2606                             case EncoderCapabilities.BITRATE_MODE_CBR_FD:
2607                                 format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
2608                                 break;
2609                             case EncoderCapabilities.BITRATE_MODE_CQ:
2610                                 format.setInteger(
2611                                         MediaFormat.KEY_QUALITY,
2612                                         encCaps.getQualityRange().getLower());
2613                                 if (VERBOSE) {
2614                                     Log.d(TAG, "video encoder: quality = " +
2615                                             encCaps.getQualityRange().getLower());
2616                                 }
2617                                 break;
2618                             default:
2619                                 format.removeKey(MediaFormat.KEY_BITRATE_MODE);
2620                             }
2621                         }
2622                         format.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
2623                         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
2624                         format.setInteger(
2625                                 MediaFormat.KEY_COLOR_FORMAT,
2626                                 CodecCapabilities.COLOR_FormatSurface);
2627                     }
2628                 } else {
2629                     Log.i(TAG, info.getName() + " is in neither audio nor video domain; skipped");
2630                     codec.release();
2631                     continue;
2632                 }
2633                 codec.configure(
2634                         format, null, null,
2635                         info.isEncoder() ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0);
2636                 Surface inputSurface = null;
2637                 if (videoCaps != null && info.isEncoder()) {
2638                     inputSurface = codec.createInputSurface();
2639                 }
2640                 codec.start();
2641                 codec.unsubscribeFromVendorParameters(vendorParams);
2642                 codec.stop();
2643             } catch (Exception e) {
2644                 throw new RuntimeException("codec name: " + info.getName(), e);
2645             } finally {
2646                 if (codec != null) {
2647                     codec.release();
2648                 }
2649             }
2650         }
2651     }
2652 }
2653