1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.codec.cts; 18 19 import static org.junit.Assert.assertEquals; 20 import static org.junit.Assert.assertTrue; 21 import static org.junit.Assert.fail; 22 23 import android.annotation.TargetApi; 24 import android.content.res.AssetFileDescriptor; 25 import android.media.MediaCodec; 26 import android.media.MediaCodecInfo; 27 import android.media.MediaCodecList; 28 import android.media.MediaExtractor; 29 import android.media.MediaFormat; 30 import android.media.MediaMuxer; 31 import android.media.MediaPlayer; 32 import android.media.cts.InputSurface; 33 import android.media.cts.MediaStubActivity; 34 import android.media.cts.MediaTestBase; 35 import android.media.cts.OutputSurface; 36 import android.os.Environment; 37 import android.os.ParcelFileDescriptor; 38 import android.platform.test.annotations.AppModeFull; 39 import android.util.Log; 40 import android.view.Surface; 41 42 import androidx.test.ext.junit.runners.AndroidJUnit4; 43 44 import com.android.compatibility.common.util.ApiTest; 45 import com.android.compatibility.common.util.Preconditions; 46 47 import org.junit.After; 48 import org.junit.Before; 49 import org.junit.Test; 50 import org.junit.runner.RunWith; 51 52 import java.io.File; 53 import java.io.FileNotFoundException; 54 import java.io.IOException; 55 import java.nio.ByteBuffer; 56 import java.util.concurrent.CountDownLatch; 57 import java.util.concurrent.atomic.AtomicReference; 58 59 /** 60 * Test for the integration of MediaMuxer and MediaCodec's encoder. 61 * 62 * <p>It uses MediaExtractor to get frames from a test stream, decodes them to a surface, uses a 63 * shader to edit them, encodes them from the resulting surface, and then uses MediaMuxer to write 64 * them into a file. 65 * 66 * <p>It does not currently check whether the result file is correct, but makes sure that nothing 67 * fails along the way. 68 * 69 * <p>It also tests the way the codec config buffers need to be passed from the MediaCodec to the 70 * MediaMuxer. 71 */ 72 @ApiTest(apis = {"android.opengl.GLES20#GL_FRAGMENT_SHADER", 73 "android.media.MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface", 74 "android.media.MediaFormat#KEY_BIT_RATE", 75 "android.media.MediaFormat#KEY_COLOR_FORMAT", 76 "android.media.MediaFormat#KEY_FRAME_RATE", 77 "android.media.MediaFormat#KEY_I_FRAME_INTERVAL", 78 "android.media.MediaFormat#KEY_SAMPLE_RATE", 79 "android.media.MediaFormat#KEY_CHANNEL_COUNT", 80 "android.media.MediaFormat#KEY_PROFILE", 81 "android.media.MediaFormat#KEY_AAC_PROFILE", 82 "android.media.MediaExtractor#setDataSource", 83 "android.media.MediaExtractor#getTrackCount", 84 "android.media.MediaExtractor#getTrackFormat", 85 "android.media.MediaExtractor#selectTrack", 86 "android.media.MediaExtractor#readSampleData", 87 "android.media.MediaExtractor#getSampleTime", 88 "android.media.MediaExtractor#getSampleFlags", 89 "android.media.MediaExtractor#advance", 90 "android.media.MediaExtractor#release", 91 "android.media.MediaMuxer#start", 92 "android.media.MediaMuxer#stop", 93 "android.media.MediaMuxer#addTrack", 94 "android.media.MediaMuxer#writeSampleData", 95 "android.media.MediaMuxer#release"}) 96 @TargetApi(18) 97 @AppModeFull(reason = "Instant apps cannot access the SD card") 98 @RunWith(AndroidJUnit4.class) 99 public class ExtractDecodeEditEncodeMuxTest extends MediaTestBase { 100 101 private static final String TAG = ExtractDecodeEditEncodeMuxTest.class.getSimpleName(); 102 private static final boolean VERBOSE = false; // lots of logging 103 static final String mInpPrefix = WorkDir.getMediaDirString(); 104 105 /** How long to wait for the next buffer to become available. */ 106 private static final int TIMEOUT_USEC = 10000; 107 108 /** Where to output the test files. */ 109 private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory(); 110 111 // parameters for the video encoder 112 private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps 113 private static final int OUTPUT_VIDEO_FRAME_RATE = 30; // 30fps 114 private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames 115 private static final int OUTPUT_VIDEO_COLOR_FORMAT = 116 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 117 118 // parameters for the audio encoder 119 // Advanced Audio Coding 120 private static final String OUTPUT_AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC; 121 private static final int OUTPUT_AUDIO_CHANNEL_COUNT = 2; // Must match the input stream. 122 private static final int OUTPUT_AUDIO_BIT_RATE = 128 * 1024; 123 private static final int OUTPUT_AUDIO_AAC_PROFILE = 124 MediaCodecInfo.CodecProfileLevel.AACObjectHE; 125 private static final int OUTPUT_AUDIO_SAMPLE_RATE_HZ = 44100; // Must match the input stream. 126 127 /** 128 * Used for editing the frames. 129 * 130 * <p>Swaps green and blue channels by storing an RBGA color in an RGBA buffer. 131 */ 132 private static final String FRAGMENT_SHADER = 133 "#extension GL_OES_EGL_image_external : require\n" + 134 "precision mediump float;\n" + 135 "varying vec2 vTextureCoord;\n" + 136 "uniform samplerExternalOES sTexture;\n" + 137 "void main() {\n" + 138 " gl_FragColor = texture2D(sTexture, vTextureCoord).rbga;\n" + 139 "}\n"; 140 141 /** Whether to copy the video from the test video. */ 142 private boolean mCopyVideo; 143 /** Whether to copy the audio from the test video. */ 144 private boolean mCopyAudio; 145 /** Whether to verify the audio format. */ 146 private boolean mVerifyAudioFormat; 147 /** Width of the output frames. */ 148 private int mWidth = -1; 149 /** Height of the output frames. */ 150 private int mHeight = -1; 151 152 /** The raw resource used as the input file. */ 153 private String mSourceRes; 154 155 /** The destination file for the encoded output. */ 156 private String mOutputFile; 157 158 private String mOutputVideoMimeType; 159 160 @Override 161 @Before setUp()162 public void setUp() throws Throwable { 163 super.setUp(); 164 } 165 166 @Override 167 @After tearDown()168 public void tearDown() { 169 super.tearDown(); 170 } 171 172 @Test testExtractDecodeEditEncodeMuxQCIF()173 public void testExtractDecodeEditEncodeMuxQCIF() throws Throwable { 174 if(!setSize(176, 144)) return; 175 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 176 setCopyVideo(); 177 setOutputVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 178 TestWrapper.runTest(this); 179 } 180 181 @Test testExtractDecodeEditEncodeMuxQVGA()182 public void testExtractDecodeEditEncodeMuxQVGA() throws Throwable { 183 if(!setSize(320, 240)) return; 184 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 185 setCopyVideo(); 186 setOutputVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 187 TestWrapper.runTest(this); 188 } 189 190 @Test testExtractDecodeEditEncodeMux720p()191 public void testExtractDecodeEditEncodeMux720p() throws Throwable { 192 if(!setSize(1280, 720)) return; 193 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 194 setCopyVideo(); 195 setOutputVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 196 TestWrapper.runTest(this); 197 } 198 199 @Test testExtractDecodeEditEncodeMux2160pHevc()200 public void testExtractDecodeEditEncodeMux2160pHevc() throws Throwable { 201 if(!setSize(3840, 2160)) return; 202 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 203 setCopyVideo(); 204 setOutputVideoMimeType(MediaFormat.MIMETYPE_VIDEO_HEVC); 205 TestWrapper.runTest(this); 206 } 207 208 @Test testExtractDecodeEditEncodeMuxAudio()209 public void testExtractDecodeEditEncodeMuxAudio() throws Throwable { 210 if(!setSize(1280, 720)) return; 211 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 212 setCopyAudio(); 213 setVerifyAudioFormat(); 214 TestWrapper.runTest(this); 215 } 216 217 @Test testExtractDecodeEditEncodeMuxAudioVideo()218 public void testExtractDecodeEditEncodeMuxAudioVideo() throws Throwable { 219 if(!setSize(1280, 720)) return; 220 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 221 setCopyAudio(); 222 setCopyVideo(); 223 setOutputVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 224 setVerifyAudioFormat(); 225 TestWrapper.runTest(this); 226 } 227 228 /** Wraps testExtractDecodeEditEncodeMux() */ 229 private static class TestWrapper implements Runnable { 230 private Throwable mThrowable; 231 private ExtractDecodeEditEncodeMuxTest mTest; 232 TestWrapper(ExtractDecodeEditEncodeMuxTest test)233 private TestWrapper(ExtractDecodeEditEncodeMuxTest test) { 234 mTest = test; 235 } 236 237 @Override run()238 public void run() { 239 try { 240 mTest.extractDecodeEditEncodeMux(); 241 } catch (Throwable th) { 242 mThrowable = th; 243 } 244 } 245 246 /** 247 * Entry point. 248 */ runTest(ExtractDecodeEditEncodeMuxTest test)249 public static void runTest(ExtractDecodeEditEncodeMuxTest test) throws Throwable { 250 test.setOutputFile(); 251 TestWrapper wrapper = new TestWrapper(test); 252 Thread th = new Thread(wrapper, "codec test"); 253 th.start(); 254 th.join(); 255 if (wrapper.mThrowable != null) { 256 throw wrapper.mThrowable; 257 } 258 } 259 } 260 261 /** 262 * Sets the test to copy the video stream. 263 */ setCopyVideo()264 private void setCopyVideo() { 265 mCopyVideo = true; 266 } 267 268 /** 269 * Sets the test to copy the video stream. 270 */ setCopyAudio()271 private void setCopyAudio() { 272 mCopyAudio = true; 273 } 274 275 /** 276 * Sets the test to verify the output audio format. 277 */ setVerifyAudioFormat()278 private void setVerifyAudioFormat() { 279 mVerifyAudioFormat = true; 280 } 281 282 /** 283 * Sets the desired frame size and returns whether the given resolution is 284 * supported. 285 * 286 * <p>If decoding/encoding using AVC as the codec, checks that the resolution 287 * is supported. For other codecs, always return {@code true}. 288 */ setSize(int width, int height)289 private boolean setSize(int width, int height) { 290 if ((width % 16) != 0 || (height % 16) != 0) { 291 Log.w(TAG, "WARNING: width or height not multiple of 16"); 292 } 293 mWidth = width; 294 mHeight = height; 295 296 // TODO: remove this logic in setSize as it is now handled when configuring codecs 297 return true; 298 } 299 300 /** 301 * Sets the raw resource used as the source video. 302 */ setSource(String res)303 private void setSource(String res) { 304 mSourceRes = res; 305 } 306 307 /** 308 * Sets the name of the output file based on the other parameters. 309 * 310 * <p>Must be called after {@link #setSize(int, int)} and {@link #setSource(String)}. 311 */ setOutputFile()312 private void setOutputFile() { 313 StringBuilder sb = new StringBuilder(); 314 sb.append(OUTPUT_FILENAME_DIR.getAbsolutePath()); 315 sb.append("/cts-media-"); 316 sb.append(getClass().getSimpleName()); 317 assertTrue("should have called setSource() first", mSourceRes != null); 318 sb.append('-'); 319 sb.append(mSourceRes); 320 if (mCopyVideo) { 321 assertTrue("should have called setSize() first", mWidth != -1); 322 assertTrue("should have called setSize() first", mHeight != -1); 323 sb.append('-'); 324 sb.append("video"); 325 sb.append('-'); 326 sb.append(mWidth); 327 sb.append('x'); 328 sb.append(mHeight); 329 } 330 if (mCopyAudio) { 331 sb.append('-'); 332 sb.append("audio"); 333 } 334 sb.append(".mp4"); 335 mOutputFile = sb.toString(); 336 } 337 setOutputVideoMimeType(String mimeType)338 private void setOutputVideoMimeType(String mimeType) { 339 mOutputVideoMimeType = mimeType; 340 } 341 342 /** 343 * Tests encoding and subsequently decoding video from frames generated into a buffer. 344 * <p> 345 * We encode several frames of a video test pattern using MediaCodec, then decode the output 346 * with MediaCodec and do some simple checks. 347 */ extractDecodeEditEncodeMux()348 private void extractDecodeEditEncodeMux() throws Exception { 349 // Exception that may be thrown during release. 350 Exception exception = null; 351 352 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 353 354 String videoEncoderName = null; 355 MediaFormat outputVideoFormat = null; 356 if (mCopyVideo) { 357 // We avoid the device-specific limitations on width and height by using values 358 // that are multiples of 16, which all tested devices seem to be able to handle. 359 outputVideoFormat = 360 MediaFormat.createVideoFormat(mOutputVideoMimeType, mWidth, mHeight); 361 362 // Set some properties. Failing to specify some of these can cause the MediaCodec 363 // configure() call to throw an unhelpful exception. 364 outputVideoFormat.setInteger( 365 MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT); 366 outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE); 367 outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE); 368 outputVideoFormat.setInteger( 369 MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); 370 if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); 371 372 videoEncoderName = mcl.findEncoderForFormat(outputVideoFormat); 373 if (videoEncoderName == null) { 374 // Don't fail CTS if they don't have an AVC codec (not here, anyway). 375 Log.e(TAG, "Unable to find an appropriate codec for " + outputVideoFormat); 376 return; 377 } 378 if (VERBOSE) Log.d(TAG, "video found codec: " + videoEncoderName); 379 } 380 381 String audioEncoderName = null; 382 MediaFormat outputAudioFormat = null; 383 if (mCopyAudio) { 384 outputAudioFormat = 385 MediaFormat.createAudioFormat( 386 OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ, 387 OUTPUT_AUDIO_CHANNEL_COUNT); 388 outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE); 389 // TODO: Bug workaround --- uncomment once fixed. 390 // outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); 391 392 audioEncoderName = mcl.findEncoderForFormat(outputAudioFormat); 393 if (audioEncoderName == null) { 394 // Don't fail CTS if they don't have an AAC codec (not here, anyway). 395 Log.e(TAG, "Unable to find an appropriate codec for " + outputAudioFormat); 396 return; 397 } 398 if (VERBOSE) Log.d(TAG, "audio found codec: " + audioEncoderName); 399 } 400 401 MediaExtractor videoExtractor = null; 402 MediaExtractor audioExtractor = null; 403 OutputSurface outputSurface = null; 404 MediaCodec videoDecoder = null; 405 MediaCodec audioDecoder = null; 406 MediaCodec videoEncoder = null; 407 MediaCodec audioEncoder = null; 408 MediaMuxer muxer = null; 409 410 InputSurface inputSurface = null; 411 412 try { 413 if (mCopyVideo) { 414 videoExtractor = createExtractor(); 415 int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); 416 assertTrue("missing video track in test video", videoInputTrack != -1); 417 MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack); 418 419 // Create a MediaCodec for the desired codec, then configure it as an encoder with 420 // our desired properties. Request a Surface to use for input. 421 AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>(); 422 videoEncoder = createVideoEncoder( 423 videoEncoderName, outputVideoFormat, inputSurfaceReference); 424 inputSurface = new InputSurface(inputSurfaceReference.get()); 425 inputSurface.makeCurrent(); 426 // Create a MediaCodec for the decoder, based on the extractor's format. 427 outputSurface = new OutputSurface(); 428 outputSurface.changeFragmentShader(FRAGMENT_SHADER); 429 videoDecoder = createVideoDecoder(mcl, inputFormat, outputSurface.getSurface()); 430 } 431 432 if (mCopyAudio) { 433 audioExtractor = createExtractor(); 434 int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor); 435 assertTrue("missing audio track in test video", audioInputTrack != -1); 436 MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack); 437 438 // Create a MediaCodec for the desired codec, then configure it as an encoder with 439 // our desired properties. Request a Surface to use for input. 440 audioEncoder = createAudioEncoder(audioEncoderName, outputAudioFormat); 441 // Create a MediaCodec for the decoder, based on the extractor's format. 442 audioDecoder = createAudioDecoder(mcl, inputFormat); 443 } 444 445 // Creates a muxer but do not start or add tracks just yet. 446 muxer = createMuxer(); 447 448 doExtractDecodeEditEncodeMux( 449 videoExtractor, 450 audioExtractor, 451 videoDecoder, 452 videoEncoder, 453 audioDecoder, 454 audioEncoder, 455 muxer, 456 inputSurface, 457 outputSurface); 458 } finally { 459 if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer"); 460 // Try to release everything we acquired, even if one of the releases fails, in which 461 // case we save the first exception we got and re-throw at the end (unless something 462 // other exception has already been thrown). This guarantees the first exception thrown 463 // is reported as the cause of the error, everything is (attempted) to be released, and 464 // all other exceptions appear in the logs. 465 try { 466 if (videoExtractor != null) { 467 videoExtractor.release(); 468 } 469 } catch(Exception e) { 470 Log.e(TAG, "error while releasing videoExtractor", e); 471 if (exception == null) { 472 exception = e; 473 } 474 } 475 try { 476 if (audioExtractor != null) { 477 audioExtractor.release(); 478 } 479 } catch(Exception e) { 480 Log.e(TAG, "error while releasing audioExtractor", e); 481 if (exception == null) { 482 exception = e; 483 } 484 } 485 try { 486 if (videoDecoder != null) { 487 videoDecoder.stop(); 488 videoDecoder.release(); 489 } 490 } catch(Exception e) { 491 Log.e(TAG, "error while releasing videoDecoder", e); 492 if (exception == null) { 493 exception = e; 494 } 495 } 496 try { 497 if (outputSurface != null) { 498 outputSurface.release(); 499 } 500 } catch(Exception e) { 501 Log.e(TAG, "error while releasing outputSurface", e); 502 if (exception == null) { 503 exception = e; 504 } 505 } 506 try { 507 if (videoEncoder != null) { 508 videoEncoder.stop(); 509 videoEncoder.release(); 510 } 511 } catch(Exception e) { 512 Log.e(TAG, "error while releasing videoEncoder", e); 513 if (exception == null) { 514 exception = e; 515 } 516 } 517 try { 518 if (audioDecoder != null) { 519 audioDecoder.stop(); 520 audioDecoder.release(); 521 } 522 } catch(Exception e) { 523 Log.e(TAG, "error while releasing audioDecoder", e); 524 if (exception == null) { 525 exception = e; 526 } 527 } 528 try { 529 if (audioEncoder != null) { 530 audioEncoder.stop(); 531 audioEncoder.release(); 532 } 533 } catch(Exception e) { 534 Log.e(TAG, "error while releasing audioEncoder", e); 535 if (exception == null) { 536 exception = e; 537 } 538 } 539 try { 540 if (muxer != null) { 541 muxer.stop(); 542 muxer.release(); 543 } 544 } catch(Exception e) { 545 Log.e(TAG, "error while releasing muxer", e); 546 if (exception == null) { 547 exception = e; 548 } 549 } 550 try { 551 if (inputSurface != null) { 552 inputSurface.release(); 553 } 554 } catch(Exception e) { 555 Log.e(TAG, "error while releasing inputSurface", e); 556 if (exception == null) { 557 exception = e; 558 } 559 } 560 } 561 if (exception != null) { 562 throw exception; 563 } 564 565 MediaExtractor mediaExtractor = null; 566 try { 567 mediaExtractor = new MediaExtractor(); 568 mediaExtractor.setDataSource(mOutputFile); 569 570 assertEquals("incorrect number of tracks", (mCopyAudio ? 1 : 0) + (mCopyVideo ? 1 : 0), 571 mediaExtractor.getTrackCount()); 572 if (mVerifyAudioFormat) { 573 boolean foundAudio = false; 574 for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { 575 MediaFormat trackFormat = mediaExtractor.getTrackFormat(i); 576 if (isAudioFormat(trackFormat)) { 577 foundAudio = true; 578 int expectedSampleRate = OUTPUT_AUDIO_SAMPLE_RATE_HZ; 579 580 // SBR mode halves the sample rate in the format. 581 // Query output profile. KEY_PROFILE gets precedence over KEY_AAC_PROFILE 582 int aac_profile = trackFormat.getInteger(MediaFormat.KEY_AAC_PROFILE, -1); 583 int profile = trackFormat.getInteger(MediaFormat.KEY_PROFILE, aac_profile); 584 585 if (profile == MediaCodecInfo.CodecProfileLevel.AACObjectHE) { 586 expectedSampleRate /= 2; 587 } 588 assertEquals("sample rates should match", expectedSampleRate, 589 trackFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); 590 } 591 } 592 593 assertTrue("output should have an audio track", foundAudio || !mCopyAudio); 594 } 595 } catch (IOException e) { 596 throw new IllegalStateException("exception verifying output file", e); 597 } finally { 598 if (mediaExtractor != null) { 599 mediaExtractor.release(); 600 } 601 } 602 603 // TODO: Check the generated output file's video format and sample data. 604 605 MediaStubActivity activity = getActivity(); 606 final MediaPlayer mp = new MediaPlayer(); 607 final Exception[] exceptionHolder = { null }; 608 final CountDownLatch playbackEndSignal = new CountDownLatch(1); 609 mp.setOnErrorListener(new MediaPlayer.OnErrorListener() { 610 @Override 611 public boolean onError(MediaPlayer origin, int what, int extra) { 612 exceptionHolder[0] = new RuntimeException("error playing output file: what=" + what 613 + " extra=" + extra); 614 // Returning false would trigger onCompletion() so that 615 // playbackEndSignal.await() can stop waiting. 616 return false; 617 } 618 }); 619 mp.setOnCompletionListener(new MediaPlayer.OnCompletionListener() { 620 @Override 621 public void onCompletion(MediaPlayer origin) { 622 playbackEndSignal.countDown(); 623 } 624 }); 625 try { 626 mp.setDataSource(mOutputFile); 627 mp.setDisplay(activity.getSurfaceHolder()); 628 mp.prepare(); 629 mp.start(); 630 playbackEndSignal.await(); 631 } catch (Exception e) { 632 exceptionHolder[0] = e; 633 } finally { 634 mp.release(); 635 } 636 637 if (exceptionHolder[0] != null) { 638 throw exceptionHolder[0]; 639 } 640 } 641 getAssetFileDescriptorFor(final String res)642 protected AssetFileDescriptor getAssetFileDescriptorFor(final String res) 643 throws FileNotFoundException { 644 Preconditions.assertTestFileExists(mInpPrefix + res); 645 File inpFile = new File(mInpPrefix + res); 646 ParcelFileDescriptor parcelFD = 647 ParcelFileDescriptor.open(inpFile, ParcelFileDescriptor.MODE_READ_ONLY); 648 return new AssetFileDescriptor(parcelFD, 0, parcelFD.getStatSize()); 649 } 650 651 /** 652 * Creates an extractor that reads its frames from {@link #mSourceRes}. 653 */ createExtractor()654 private MediaExtractor createExtractor() throws IOException { 655 MediaExtractor extractor; 656 AssetFileDescriptor srcFd = getAssetFileDescriptorFor(mSourceRes); 657 extractor = new MediaExtractor(); 658 extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(), 659 srcFd.getLength()); 660 return extractor; 661 } 662 663 /** 664 * Creates a decoder for the given format, which outputs to the given surface. 665 * 666 * @param inputFormat the format of the stream to decode 667 * @param surface into which to decode the frames 668 */ createVideoDecoder( MediaCodecList mcl, MediaFormat inputFormat, Surface surface)669 private MediaCodec createVideoDecoder( 670 MediaCodecList mcl, MediaFormat inputFormat, Surface surface) throws IOException { 671 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 672 decoder.configure(inputFormat, surface, null, 0); 673 decoder.start(); 674 return decoder; 675 } 676 677 /** 678 * Creates an encoder for the given format using the specified codec, taking input from a 679 * surface. 680 * 681 * <p>The surface to use as input is stored in the given reference. 682 * 683 * @param codecInfo of the codec to use 684 * @param format of the stream to be produced 685 * @param surfaceReference to store the surface to use as input 686 */ createVideoEncoder( String codecName, MediaFormat format, AtomicReference<Surface> surfaceReference)687 private MediaCodec createVideoEncoder( 688 String codecName, 689 MediaFormat format, 690 AtomicReference<Surface> surfaceReference) 691 throws IOException { 692 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 693 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 694 // Must be called before start() is. 695 surfaceReference.set(encoder.createInputSurface()); 696 encoder.start(); 697 return encoder; 698 } 699 700 /** 701 * Creates a decoder for the given format. 702 * 703 * @param inputFormat the format of the stream to decode 704 */ createAudioDecoder( MediaCodecList mcl, MediaFormat inputFormat)705 private MediaCodec createAudioDecoder( 706 MediaCodecList mcl, MediaFormat inputFormat) throws IOException { 707 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 708 decoder.configure(inputFormat, null, null, 0); 709 decoder.start(); 710 return decoder; 711 } 712 713 /** 714 * Creates an encoder for the given format using the specified codec. 715 * 716 * @param codecInfo of the codec to use 717 * @param format of the stream to be produced 718 */ createAudioEncoder(String codecName, MediaFormat format)719 private MediaCodec createAudioEncoder(String codecName, MediaFormat format) 720 throws IOException { 721 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 722 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 723 encoder.start(); 724 return encoder; 725 } 726 727 /** 728 * Creates a muxer to write the encoded frames. 729 * 730 * <p>The muxer is not started as it needs to be started only after all streams have been added. 731 */ createMuxer()732 private MediaMuxer createMuxer() throws IOException { 733 return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); 734 } 735 getAndSelectVideoTrackIndex(MediaExtractor extractor)736 private int getAndSelectVideoTrackIndex(MediaExtractor extractor) { 737 for (int index = 0; index < extractor.getTrackCount(); ++index) { 738 if (VERBOSE) { 739 Log.d(TAG, "format for track " + index + " is " 740 + getMimeTypeFor(extractor.getTrackFormat(index))); 741 } 742 if (isVideoFormat(extractor.getTrackFormat(index))) { 743 extractor.selectTrack(index); 744 return index; 745 } 746 } 747 return -1; 748 } 749 getAndSelectAudioTrackIndex(MediaExtractor extractor)750 private int getAndSelectAudioTrackIndex(MediaExtractor extractor) { 751 for (int index = 0; index < extractor.getTrackCount(); ++index) { 752 if (VERBOSE) { 753 Log.d(TAG, "format for track " + index + " is " 754 + getMimeTypeFor(extractor.getTrackFormat(index))); 755 } 756 if (isAudioFormat(extractor.getTrackFormat(index))) { 757 extractor.selectTrack(index); 758 return index; 759 } 760 } 761 return -1; 762 } 763 764 /** 765 * Does the actual work for extracting, decoding, encoding and muxing. 766 */ doExtractDecodeEditEncodeMux( MediaExtractor videoExtractor, MediaExtractor audioExtractor, MediaCodec videoDecoder, MediaCodec videoEncoder, MediaCodec audioDecoder, MediaCodec audioEncoder, MediaMuxer muxer, InputSurface inputSurface, OutputSurface outputSurface)767 private void doExtractDecodeEditEncodeMux( 768 MediaExtractor videoExtractor, 769 MediaExtractor audioExtractor, 770 MediaCodec videoDecoder, 771 MediaCodec videoEncoder, 772 MediaCodec audioDecoder, 773 MediaCodec audioEncoder, 774 MediaMuxer muxer, 775 InputSurface inputSurface, 776 OutputSurface outputSurface) { 777 ByteBuffer[] videoDecoderInputBuffers = null; 778 ByteBuffer[] videoDecoderOutputBuffers = null; 779 ByteBuffer[] videoEncoderOutputBuffers = null; 780 MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null; 781 MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null; 782 if (mCopyVideo) { 783 videoDecoderInputBuffers = videoDecoder.getInputBuffers(); 784 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 785 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 786 videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 787 videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 788 } 789 ByteBuffer[] audioDecoderInputBuffers = null; 790 ByteBuffer[] audioDecoderOutputBuffers = null; 791 ByteBuffer[] audioEncoderInputBuffers = null; 792 ByteBuffer[] audioEncoderOutputBuffers = null; 793 MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null; 794 MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null; 795 if (mCopyAudio) { 796 audioDecoderInputBuffers = audioDecoder.getInputBuffers(); 797 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 798 audioEncoderInputBuffers = audioEncoder.getInputBuffers(); 799 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 800 audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 801 audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 802 } 803 // We will get these from the decoders when notified of a format change. 804 MediaFormat decoderOutputVideoFormat = null; 805 MediaFormat decoderOutputAudioFormat = null; 806 // We will get these from the encoders when notified of a format change. 807 MediaFormat encoderOutputVideoFormat = null; 808 MediaFormat encoderOutputAudioFormat = null; 809 // We will determine these once we have the output format. 810 int outputVideoTrack = -1; 811 int outputAudioTrack = -1; 812 // Whether things are done on the video side. 813 boolean videoExtractorDone = false; 814 boolean videoDecoderDone = false; 815 boolean videoEncoderDone = false; 816 // Whether things are done on the audio side. 817 boolean audioExtractorDone = false; 818 boolean audioDecoderDone = false; 819 boolean audioEncoderDone = false; 820 // The audio decoder output buffer to process, -1 if none. 821 int pendingAudioDecoderOutputBufferIndex = -1; 822 823 boolean muxing = false; 824 825 int videoExtractedFrameCount = 0; 826 int videoDecodedFrameCount = 0; 827 int videoEncodedFrameCount = 0; 828 829 int audioExtractedFrameCount = 0; 830 int audioDecodedFrameCount = 0; 831 int audioEncodedFrameCount = 0; 832 833 while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) { 834 if (VERBOSE) { 835 Log.d(TAG, String.format( 836 "loop: " 837 838 + "V(%b){" 839 + "extracted:%d(done:%b) " 840 + "decoded:%d(done:%b) " 841 + "encoded:%d(done:%b)} " 842 843 + "A(%b){" 844 + "extracted:%d(done:%b) " 845 + "decoded:%d(done:%b) " 846 + "encoded:%d(done:%b) " 847 + "pending:%d} " 848 849 + "muxing:%b(V:%d,A:%d)", 850 851 mCopyVideo, 852 videoExtractedFrameCount, videoExtractorDone, 853 videoDecodedFrameCount, videoDecoderDone, 854 videoEncodedFrameCount, videoEncoderDone, 855 856 mCopyAudio, 857 audioExtractedFrameCount, audioExtractorDone, 858 audioDecodedFrameCount, audioDecoderDone, 859 audioEncodedFrameCount, audioEncoderDone, 860 pendingAudioDecoderOutputBufferIndex, 861 862 muxing, outputVideoTrack, outputAudioTrack)); 863 } 864 865 // Extract video from file and feed to decoder. 866 // Do not extract video if we have determined the output format but we are not yet 867 // ready to mux the frames. 868 while (mCopyVideo && !videoExtractorDone 869 && (encoderOutputVideoFormat == null || muxing)) { 870 int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC); 871 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 872 if (VERBOSE) Log.d(TAG, "no video decoder input buffer"); 873 break; 874 } 875 if (VERBOSE) { 876 Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex); 877 } 878 ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex]; 879 int size = videoExtractor.readSampleData(decoderInputBuffer, 0); 880 long presentationTime = videoExtractor.getSampleTime(); 881 int flags = videoExtractor.getSampleFlags(); 882 if (VERBOSE) { 883 Log.d(TAG, "video extractor: returned buffer of size " + size); 884 Log.d(TAG, "video extractor: returned buffer for time " + presentationTime); 885 } 886 videoExtractorDone = !videoExtractor.advance(); 887 if (videoExtractorDone) { 888 if (VERBOSE) Log.d(TAG, "video extractor: EOS"); 889 flags = flags | MediaCodec.BUFFER_FLAG_END_OF_STREAM; 890 } 891 if (size >= 0) { 892 videoDecoder.queueInputBuffer( 893 decoderInputBufferIndex, 894 0, 895 size, 896 presentationTime, 897 flags); 898 videoExtractedFrameCount++; 899 } 900 // We extracted a frame, let's try something else next. 901 break; 902 } 903 904 // Extract audio from file and feed to decoder. 905 // Do not extract audio if we have determined the output format but we are not yet 906 // ready to mux the frames. 907 while (mCopyAudio && !audioExtractorDone 908 && (encoderOutputAudioFormat == null || muxing)) { 909 int decoderInputBufferIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_USEC); 910 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 911 if (VERBOSE) Log.d(TAG, "no audio decoder input buffer"); 912 break; 913 } 914 if (VERBOSE) { 915 Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex); 916 } 917 ByteBuffer decoderInputBuffer = audioDecoderInputBuffers[decoderInputBufferIndex]; 918 int size = audioExtractor.readSampleData(decoderInputBuffer, 0); 919 long presentationTime = audioExtractor.getSampleTime(); 920 int flags = audioExtractor.getSampleFlags(); 921 if (VERBOSE) { 922 Log.d(TAG, "audio extractor: returned buffer of size " + size); 923 Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); 924 } 925 audioExtractorDone = !audioExtractor.advance(); 926 if (audioExtractorDone) { 927 if (VERBOSE) Log.d(TAG, "audio extractor: EOS"); 928 flags = flags | MediaCodec.BUFFER_FLAG_END_OF_STREAM; 929 } 930 if (size >= 0) { 931 audioDecoder.queueInputBuffer( 932 decoderInputBufferIndex, 933 0, 934 size, 935 presentationTime, 936 flags); 937 audioExtractedFrameCount++; 938 } 939 // We extracted a frame, let's try something else next. 940 break; 941 } 942 943 // Poll output frames from the video decoder and feed the encoder. 944 while (mCopyVideo && !videoDecoderDone 945 && (encoderOutputVideoFormat == null || muxing)) { 946 int decoderOutputBufferIndex = 947 videoDecoder.dequeueOutputBuffer( 948 videoDecoderOutputBufferInfo, TIMEOUT_USEC); 949 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 950 if (VERBOSE) Log.d(TAG, "no video decoder output buffer"); 951 break; 952 } 953 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 954 if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed"); 955 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 956 break; 957 } 958 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 959 decoderOutputVideoFormat = videoDecoder.getOutputFormat(); 960 if (VERBOSE) { 961 Log.d(TAG, "video decoder: output format changed: " 962 + decoderOutputVideoFormat); 963 } 964 break; 965 } 966 if (VERBOSE) { 967 Log.d(TAG, "video decoder: returned output buffer: " 968 + decoderOutputBufferIndex); 969 Log.d(TAG, "video decoder: returned buffer of size " 970 + videoDecoderOutputBufferInfo.size); 971 } 972 ByteBuffer decoderOutputBuffer = 973 videoDecoderOutputBuffers[decoderOutputBufferIndex]; 974 if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 975 != 0) { 976 if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer"); 977 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 978 break; 979 } 980 if (VERBOSE) { 981 Log.d(TAG, "video decoder: returned buffer for time " 982 + videoDecoderOutputBufferInfo.presentationTimeUs); 983 } 984 boolean render = videoDecoderOutputBufferInfo.size != 0; 985 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render); 986 if (render) { 987 if (VERBOSE) Log.d(TAG, "output surface: await new image"); 988 outputSurface.awaitNewImage(); 989 // Edit the frame and send it to the encoder. 990 if (VERBOSE) Log.d(TAG, "output surface: draw image"); 991 outputSurface.drawImage(); 992 inputSurface.setPresentationTime( 993 videoDecoderOutputBufferInfo.presentationTimeUs * 1000); 994 if (VERBOSE) Log.d(TAG, "input surface: swap buffers"); 995 inputSurface.swapBuffers(); 996 if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame"); 997 videoDecodedFrameCount++; 998 } 999 if ((videoDecoderOutputBufferInfo.flags 1000 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1001 if (VERBOSE) Log.d(TAG, "video decoder: EOS"); 1002 videoDecoderDone = true; 1003 videoEncoder.signalEndOfInputStream(); 1004 } 1005 // We extracted a pending frame, let's try something else next. 1006 break; 1007 } 1008 1009 // Poll output frames from the audio decoder. 1010 // Do not poll if we already have a pending buffer to feed to the encoder. 1011 while (mCopyAudio && !audioDecoderDone && pendingAudioDecoderOutputBufferIndex == -1 1012 && (encoderOutputAudioFormat == null || muxing)) { 1013 int decoderOutputBufferIndex = 1014 audioDecoder.dequeueOutputBuffer( 1015 audioDecoderOutputBufferInfo, TIMEOUT_USEC); 1016 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1017 if (VERBOSE) Log.d(TAG, "no audio decoder output buffer"); 1018 break; 1019 } 1020 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1021 if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed"); 1022 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 1023 break; 1024 } 1025 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1026 decoderOutputAudioFormat = audioDecoder.getOutputFormat(); 1027 if (VERBOSE) { 1028 Log.d(TAG, "audio decoder: output format changed: " 1029 + decoderOutputAudioFormat); 1030 } 1031 break; 1032 } 1033 if (VERBOSE) { 1034 Log.d(TAG, "audio decoder: returned output buffer: " 1035 + decoderOutputBufferIndex); 1036 } 1037 if (VERBOSE) { 1038 Log.d(TAG, "audio decoder: returned buffer of size " 1039 + audioDecoderOutputBufferInfo.size); 1040 } 1041 ByteBuffer decoderOutputBuffer = 1042 audioDecoderOutputBuffers[decoderOutputBufferIndex]; 1043 if ((audioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1044 != 0) { 1045 if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer"); 1046 audioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 1047 break; 1048 } 1049 if (VERBOSE) { 1050 Log.d(TAG, "audio decoder: returned buffer for time " 1051 + audioDecoderOutputBufferInfo.presentationTimeUs); 1052 } 1053 if (VERBOSE) { 1054 Log.d(TAG, "audio decoder: output buffer is now pending: " 1055 + pendingAudioDecoderOutputBufferIndex); 1056 } 1057 pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex; 1058 audioDecodedFrameCount++; 1059 // We extracted a pending frame, let's try something else next. 1060 break; 1061 } 1062 1063 // Feed the pending decoded audio buffer to the audio encoder. 1064 while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) { 1065 if (VERBOSE) { 1066 Log.d(TAG, "audio decoder: attempting to process pending buffer: " 1067 + pendingAudioDecoderOutputBufferIndex); 1068 } 1069 int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC); 1070 if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1071 if (VERBOSE) Log.d(TAG, "no audio encoder input buffer"); 1072 break; 1073 } 1074 if (VERBOSE) { 1075 Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex); 1076 } 1077 ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex]; 1078 int size = audioDecoderOutputBufferInfo.size; 1079 long presentationTime = audioDecoderOutputBufferInfo.presentationTimeUs; 1080 if (VERBOSE) { 1081 Log.d(TAG, "audio decoder: processing pending buffer: " 1082 + pendingAudioDecoderOutputBufferIndex); 1083 } 1084 if (VERBOSE) { 1085 Log.d(TAG, "audio decoder: pending buffer of size " + size); 1086 Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime); 1087 } 1088 if (size >= 0) { 1089 ByteBuffer decoderOutputBuffer = 1090 audioDecoderOutputBuffers[pendingAudioDecoderOutputBufferIndex] 1091 .duplicate(); 1092 decoderOutputBuffer.position(audioDecoderOutputBufferInfo.offset); 1093 decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size); 1094 encoderInputBuffer.position(0); 1095 encoderInputBuffer.put(decoderOutputBuffer); 1096 1097 audioEncoder.queueInputBuffer( 1098 encoderInputBufferIndex, 1099 0, 1100 size, 1101 presentationTime, 1102 audioDecoderOutputBufferInfo.flags); 1103 } 1104 audioDecoder.releaseOutputBuffer(pendingAudioDecoderOutputBufferIndex, false); 1105 pendingAudioDecoderOutputBufferIndex = -1; 1106 if ((audioDecoderOutputBufferInfo.flags 1107 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1108 if (VERBOSE) Log.d(TAG, "audio decoder: EOS"); 1109 audioDecoderDone = true; 1110 } 1111 // We enqueued a pending frame, let's try something else next. 1112 break; 1113 } 1114 1115 // Poll frames from the video encoder and send them to the muxer. 1116 while (mCopyVideo && !videoEncoderDone 1117 && (encoderOutputVideoFormat == null || muxing)) { 1118 int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer( 1119 videoEncoderOutputBufferInfo, TIMEOUT_USEC); 1120 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1121 if (VERBOSE) Log.d(TAG, "no video encoder output buffer"); 1122 break; 1123 } 1124 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1125 if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed"); 1126 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 1127 break; 1128 } 1129 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1130 if (VERBOSE) Log.d(TAG, "video encoder: output format changed"); 1131 if (outputVideoTrack >= 0) { 1132 fail("video encoder changed its output format again?"); 1133 } 1134 encoderOutputVideoFormat = videoEncoder.getOutputFormat(); 1135 break; 1136 } 1137 assertTrue("should have added track before processing output", muxing); 1138 if (VERBOSE) { 1139 Log.d(TAG, "video encoder: returned output buffer: " 1140 + encoderOutputBufferIndex); 1141 Log.d(TAG, "video encoder: returned buffer of size " 1142 + videoEncoderOutputBufferInfo.size); 1143 } 1144 ByteBuffer encoderOutputBuffer = 1145 videoEncoderOutputBuffers[encoderOutputBufferIndex]; 1146 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1147 != 0) { 1148 if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer"); 1149 // Simply ignore codec config buffers. 1150 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1151 break; 1152 } 1153 if (VERBOSE) { 1154 Log.d(TAG, "video encoder: returned buffer for time " 1155 + videoEncoderOutputBufferInfo.presentationTimeUs); 1156 } 1157 if (videoEncoderOutputBufferInfo.size != 0) { 1158 muxer.writeSampleData( 1159 outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo); 1160 videoEncodedFrameCount++; 1161 } 1162 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1163 != 0) { 1164 if (VERBOSE) Log.d(TAG, "video encoder: EOS"); 1165 videoEncoderDone = true; 1166 } 1167 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1168 // We enqueued an encoded frame, let's try something else next. 1169 break; 1170 } 1171 1172 // Poll frames from the audio encoder and send them to the muxer. 1173 while (mCopyAudio && !audioEncoderDone 1174 && (encoderOutputAudioFormat == null || muxing)) { 1175 int encoderOutputBufferIndex = audioEncoder.dequeueOutputBuffer( 1176 audioEncoderOutputBufferInfo, TIMEOUT_USEC); 1177 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1178 if (VERBOSE) Log.d(TAG, "no audio encoder output buffer"); 1179 break; 1180 } 1181 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1182 if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed"); 1183 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 1184 break; 1185 } 1186 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1187 if (VERBOSE) Log.d(TAG, "audio encoder: output format changed"); 1188 if (outputAudioTrack >= 0) { 1189 fail("audio encoder changed its output format again?"); 1190 } 1191 1192 encoderOutputAudioFormat = audioEncoder.getOutputFormat(); 1193 break; 1194 } 1195 assertTrue("should have added track before processing output", muxing); 1196 if (VERBOSE) { 1197 Log.d(TAG, "audio encoder: returned output buffer: " 1198 + encoderOutputBufferIndex); 1199 Log.d(TAG, "audio encoder: returned buffer of size " 1200 + audioEncoderOutputBufferInfo.size); 1201 } 1202 ByteBuffer encoderOutputBuffer = 1203 audioEncoderOutputBuffers[encoderOutputBufferIndex]; 1204 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1205 != 0) { 1206 if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer"); 1207 // Simply ignore codec config buffers. 1208 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1209 break; 1210 } 1211 if (VERBOSE) { 1212 Log.d(TAG, "audio encoder: returned buffer for time " 1213 + audioEncoderOutputBufferInfo.presentationTimeUs); 1214 } 1215 if (audioEncoderOutputBufferInfo.size != 0) { 1216 muxer.writeSampleData( 1217 outputAudioTrack, encoderOutputBuffer, audioEncoderOutputBufferInfo); 1218 } 1219 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1220 != 0) { 1221 if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); 1222 audioEncoderDone = true; 1223 } 1224 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1225 audioEncodedFrameCount++; 1226 // We enqueued an encoded frame, let's try something else next. 1227 break; 1228 } 1229 1230 if (!muxing 1231 && (!mCopyAudio || encoderOutputAudioFormat != null) 1232 && (!mCopyVideo || encoderOutputVideoFormat != null)) { 1233 if (mCopyVideo) { 1234 Log.d(TAG, "muxer: adding video track."); 1235 outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat); 1236 } 1237 if (mCopyAudio) { 1238 Log.d(TAG, "muxer: adding audio track."); 1239 outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat); 1240 } 1241 Log.d(TAG, "muxer: starting"); 1242 muxer.start(); 1243 muxing = true; 1244 } 1245 } 1246 1247 // Basic validation checks. 1248 if (mCopyVideo) { 1249 assertEquals("encoded and decoded video frame counts should match", 1250 videoDecodedFrameCount, videoEncodedFrameCount); 1251 assertTrue("decoded frame count should be less than extracted frame count", 1252 videoDecodedFrameCount <= videoExtractedFrameCount); 1253 } 1254 if (mCopyAudio) { 1255 assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex); 1256 } 1257 } 1258 isVideoFormat(MediaFormat format)1259 private static boolean isVideoFormat(MediaFormat format) { 1260 return getMimeTypeFor(format).startsWith("video/"); 1261 } 1262 isAudioFormat(MediaFormat format)1263 private static boolean isAudioFormat(MediaFormat format) { 1264 return getMimeTypeFor(format).startsWith("audio/"); 1265 } 1266 getMimeTypeFor(MediaFormat format)1267 private static String getMimeTypeFor(MediaFormat format) { 1268 return format.getString(MediaFormat.KEY_MIME); 1269 } 1270 1271 /** 1272 * Returns the first codec capable of encoding the specified MIME type, or null if no match was 1273 * found. 1274 */ selectCodec(String mimeType)1275 private static MediaCodecInfo selectCodec(String mimeType) { 1276 int numCodecs = MediaCodecList.getCodecCount(); 1277 for (int i = 0; i < numCodecs; i++) { 1278 MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); 1279 1280 if (codecInfo.isAlias()) { 1281 continue; 1282 } 1283 if (!codecInfo.isEncoder()) { 1284 continue; 1285 } 1286 1287 String[] types = codecInfo.getSupportedTypes(); 1288 for (int j = 0; j < types.length; j++) { 1289 if (types[j].equalsIgnoreCase(mimeType)) { 1290 return codecInfo; 1291 } 1292 } 1293 } 1294 return null; 1295 } 1296 } 1297