1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.mediav2.cts; 18 19 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 20 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible; 21 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUVP010; 22 import static android.mediav2.common.cts.CodecEncoderTestBase.ACCEPTABLE_WIRELESS_TX_QUALITY; 23 import static android.mediav2.common.cts.CodecEncoderTestBase.colorFormatToString; 24 import static android.mediav2.common.cts.CodecEncoderTestBase.getTempFilePath; 25 import static android.mediav2.common.cts.CodecTestBase.BOARD_SDK_IS_BEFORE_U; 26 import static android.mediav2.common.cts.CodecTestBase.PROFILE_HLG_MAP; 27 import static android.mediav2.common.cts.CodecTestBase.VNDK_IS_AT_LEAST_T; 28 import static android.mediav2.common.cts.CodecTestBase.VNDK_IS_BEFORE_U; 29 import static android.mediav2.common.cts.CodecTestBase.isDefaultCodec; 30 import static android.mediav2.common.cts.CodecTestBase.isVendorCodec; 31 32 import static org.junit.Assert.assertTrue; 33 import static org.junit.Assert.fail; 34 import static org.junit.Assume.assumeFalse; 35 36 import android.media.MediaFormat; 37 import android.mediav2.common.cts.CodecEncoderSurfaceTestBase; 38 import android.mediav2.common.cts.CodecEncoderTestBase; 39 import android.mediav2.common.cts.CodecTestBase; 40 import android.mediav2.common.cts.EncoderConfigParams; 41 import android.mediav2.common.cts.OutputManager; 42 43 import androidx.test.filters.LargeTest; 44 import androidx.test.platform.app.InstrumentationRegistry; 45 46 import com.android.compatibility.common.util.ApiTest; 47 import com.android.compatibility.common.util.CddTest; 48 49 import org.junit.After; 50 import org.junit.Test; 51 import org.junit.runner.RunWith; 52 import org.junit.runners.Parameterized; 53 54 import java.io.File; 55 import java.io.IOException; 56 import java.util.ArrayList; 57 import java.util.Arrays; 58 import java.util.Collection; 59 import java.util.List; 60 import java.util.Objects; 61 62 /** 63 * Test mediacodec api, video encoders and their interactions in surface mode. 64 * <p> 65 * The test decodes an input clip to surface. This decoded output is fed as input to encoder. 66 * Assuming no frame drops, the test expects, 67 * <ul> 68 * <li>The number of encoded frames to be identical to number of frames present in input clip 69 * .</li> 70 * <li>As encoders are expected to give consistent output for a given input and configuration 71 * parameters, the test checks for consistency across runs. For now, this attribute is not 72 * strictly enforced in this test.</li> 73 * <li>The encoder output timestamps list should be identical to decoder input timestamp list 74 * .</li> 75 * </ul> 76 * <p> 77 * The output of encoder is further verified by computing PSNR to check for obvious visual 78 * artifacts. 79 * <p> 80 * The test runs mediacodec in synchronous and asynchronous mode. 81 */ 82 @RunWith(Parameterized.class) 83 public class CodecEncoderSurfaceTest extends CodecEncoderSurfaceTestBase { 84 private static final String LOG_TAG = CodecEncoderSurfaceTest.class.getSimpleName(); 85 private static final String MEDIA_DIR = WorkDir.getMediaDirString(); 86 private final int mFrameLimit; 87 88 private final ArrayList<String> mTmpFiles = new ArrayList<>(); 89 90 static { 91 System.loadLibrary("ctsmediav2codecencsurface_jni"); 92 93 android.os.Bundle args = InstrumentationRegistry.getArguments(); 94 CodecTestBase.mediaTypeSelKeys = args.getString(CodecTestBase.MEDIA_TYPE_SEL_KEY); 95 } 96 CodecEncoderSurfaceTest(String encoder, String mediaType, String decoder, String testFileMediaType, String testFile, EncoderConfigParams encCfgParams, int colorFormat, boolean isOutputToneMapped, boolean usePersistentSurface, @SuppressWarnings("unused") String testLabel, String allTestParams)97 public CodecEncoderSurfaceTest(String encoder, String mediaType, String decoder, 98 String testFileMediaType, String testFile, EncoderConfigParams encCfgParams, 99 int colorFormat, boolean isOutputToneMapped, boolean usePersistentSurface, 100 @SuppressWarnings("unused") String testLabel, String allTestParams) { 101 super(encoder, mediaType, decoder, testFileMediaType, MEDIA_DIR + testFile, encCfgParams, 102 colorFormat, isOutputToneMapped, usePersistentSurface, allTestParams); 103 mFrameLimit = Math.max(encCfgParams.mFrameRate, 30); 104 } 105 getVideoEncoderCfgParams(String mediaType, int bitRate, int frameRate, int bitDepth, int maxBFrames)106 private static EncoderConfigParams getVideoEncoderCfgParams(String mediaType, int bitRate, 107 int frameRate, int bitDepth, int maxBFrames) { 108 EncoderConfigParams.Builder foreman = new EncoderConfigParams.Builder(mediaType) 109 .setBitRate(bitRate) 110 .setFrameRate(frameRate) 111 .setColorFormat(COLOR_FormatSurface) 112 .setInputBitDepth(bitDepth) 113 .setMaxBFrames(maxBFrames); 114 if (bitDepth == 10) { 115 foreman.setProfile(Objects.requireNonNull(PROFILE_HLG_MAP.get(mediaType))[0]); 116 } 117 return foreman.build(); 118 } 119 120 @After tearDown()121 public void tearDown() { 122 for (String tmpFile : mTmpFiles) { 123 File tmp = new File(tmpFile); 124 if (tmp.exists()) assertTrue("unable to delete file " + tmpFile, tmp.delete()); 125 } 126 mTmpFiles.clear(); 127 } 128 129 @Parameterized.Parameters(name = "{index}_{0}_{1}_{2}_{3}_{9}") input()130 public static Collection<Object[]> input() throws IOException { 131 final boolean isEncoder = true; 132 final boolean needAudio = false; 133 final boolean needVideo = true; 134 final List<Object[]> exhaustiveArgsList = new ArrayList<>(); 135 final List<Object[]> args = new ArrayList<>(Arrays.asList(new Object[][]{ 136 // mediaType, testFileMediaType, testFile, bitRate, frameRate, toneMap 137 {MediaFormat.MIMETYPE_VIDEO_H263, MediaFormat.MIMETYPE_VIDEO_H263, 138 "bbb_176x144_128kbps_15fps_h263.3gp", 128000, 15, false}, 139 {MediaFormat.MIMETYPE_VIDEO_MPEG4, MediaFormat.MIMETYPE_VIDEO_MPEG4, 140 "bbb_128x96_64kbps_12fps_mpeg4.mp4", 64000, 12, false}, 141 {MediaFormat.MIMETYPE_VIDEO_AVC, MediaFormat.MIMETYPE_VIDEO_AVC, 142 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 143 {MediaFormat.MIMETYPE_VIDEO_HEVC, MediaFormat.MIMETYPE_VIDEO_AVC, 144 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 145 {MediaFormat.MIMETYPE_VIDEO_VP8, MediaFormat.MIMETYPE_VIDEO_AVC, 146 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 147 {MediaFormat.MIMETYPE_VIDEO_VP9, MediaFormat.MIMETYPE_VIDEO_AVC, 148 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 149 {MediaFormat.MIMETYPE_VIDEO_AV1, MediaFormat.MIMETYPE_VIDEO_AVC, 150 "bbb_cif_768kbps_30fps_avc.mp4", 512000, 30, false}, 151 })); 152 153 final List<Object[]> argsHighBitDepth = new ArrayList<>(Arrays.asList(new Object[][]{ 154 {MediaFormat.MIMETYPE_VIDEO_AVC, MediaFormat.MIMETYPE_VIDEO_AVC, 155 "cosmat_520x390_24fps_crf22_avc_10bit.mkv", 512000, 30, false}, 156 {MediaFormat.MIMETYPE_VIDEO_AVC, MediaFormat.MIMETYPE_VIDEO_AVC, 157 "cosmat_520x390_24fps_crf22_avc_10bit.mkv", 512000, 30, true}, 158 {MediaFormat.MIMETYPE_VIDEO_HEVC, MediaFormat.MIMETYPE_VIDEO_HEVC, 159 "cosmat_520x390_24fps_crf22_hevc_10bit.mkv", 512000, 30, false}, 160 {MediaFormat.MIMETYPE_VIDEO_HEVC, MediaFormat.MIMETYPE_VIDEO_HEVC, 161 "cosmat_520x390_24fps_crf22_hevc_10bit.mkv", 512000, 30, true}, 162 {MediaFormat.MIMETYPE_VIDEO_VP9, MediaFormat.MIMETYPE_VIDEO_VP9, 163 "cosmat_520x390_24fps_crf22_vp9_10bit.mkv", 512000, 30, false}, 164 {MediaFormat.MIMETYPE_VIDEO_VP9, MediaFormat.MIMETYPE_VIDEO_VP9, 165 "cosmat_520x390_24fps_crf22_vp9_10bit.mkv", 512000, 30, true}, 166 {MediaFormat.MIMETYPE_VIDEO_AV1, MediaFormat.MIMETYPE_VIDEO_AV1, 167 "cosmat_520x390_24fps_768kbps_av1_10bit.mkv", 512000, 30, false}, 168 {MediaFormat.MIMETYPE_VIDEO_AV1, MediaFormat.MIMETYPE_VIDEO_AV1, 169 "cosmat_520x390_24fps_768kbps_av1_10bit.mkv", 512000, 30, true}, 170 })); 171 172 int[] colorFormats = {COLOR_FormatSurface, COLOR_FormatYUV420Flexible}; 173 int[] maxBFrames = {0, 2}; 174 boolean[] boolStates = {true, false}; 175 for (Object[] arg : args) { 176 final String mediaType = (String) arg[0]; 177 final int br = (int) arg[3]; 178 final int fps = (int) arg[4]; 179 for (int colorFormat : colorFormats) { 180 for (boolean usePersistentSurface : boolStates) { 181 for (int maxBFrame : maxBFrames) { 182 if (!mediaType.equals(MediaFormat.MIMETYPE_VIDEO_AVC) 183 && !mediaType.equals(MediaFormat.MIMETYPE_VIDEO_HEVC) 184 && maxBFrame != 0) { 185 continue; 186 } 187 Object[] testArgs = new Object[8]; 188 testArgs[0] = arg[0]; // encoder mediaType 189 testArgs[1] = arg[1]; // test file mediaType 190 testArgs[2] = arg[2]; // test file 191 testArgs[3] = getVideoEncoderCfgParams(mediaType, br, fps, 8, maxBFrame); 192 testArgs[4] = colorFormat; // color format 193 testArgs[5] = arg[5]; // tone map 194 testArgs[6] = usePersistentSurface; 195 testArgs[7] = String.format("%dkbps_%dfps_%s_%s", br / 1000, fps, 196 colorFormatToString(colorFormat, 8), 197 usePersistentSurface ? "persistentsurface" : "surface"); 198 exhaustiveArgsList.add(testArgs); 199 } 200 } 201 } 202 } 203 // P010 support was added in Android T, hence limit the following tests to Android T and 204 // above 205 if (CodecTestBase.IS_AT_LEAST_T) { 206 int[] colorFormatsHbd = {COLOR_FormatSurface, COLOR_FormatYUVP010}; 207 for (Object[] arg : argsHighBitDepth) { 208 final String mediaType = (String) arg[0]; 209 final int br = (int) arg[3]; 210 final int fps = (int) arg[4]; 211 final boolean toneMap = (boolean) arg[5]; 212 for (int colorFormat : colorFormatsHbd) { 213 for (boolean usePersistentSurface : boolStates) { 214 for (int maxBFrame : maxBFrames) { 215 if (!mediaType.equals(MediaFormat.MIMETYPE_VIDEO_AVC) 216 && !mediaType.equals(MediaFormat.MIMETYPE_VIDEO_HEVC) 217 && maxBFrame != 0) { 218 continue; 219 } 220 Object[] testArgs = new Object[8]; 221 testArgs[0] = arg[0]; // encoder mediaType 222 testArgs[1] = arg[1]; // test file mediaType 223 testArgs[2] = arg[2]; // test file 224 testArgs[3] = 225 getVideoEncoderCfgParams(mediaType, br, fps, toneMap ? 8 : 10, 226 maxBFrame); 227 if (toneMap && (colorFormat == COLOR_FormatYUVP010)) { 228 colorFormat = COLOR_FormatYUV420Flexible; 229 } 230 testArgs[4] = colorFormat; // color format 231 testArgs[5] = arg[5]; // tone map 232 testArgs[6] = usePersistentSurface; 233 testArgs[7] = String.format("%dkbps_%dfps_%s_%s_%s", br / 1000, fps, 234 colorFormatToString(colorFormat, toneMap ? 8 : 10), 235 toneMap ? "tonemapyes" : "tonemapno", 236 usePersistentSurface ? "persistentsurface" : "surface"); 237 exhaustiveArgsList.add(testArgs); 238 } 239 } 240 } 241 } 242 } 243 final List<Object[]> argsList = new ArrayList<>(); 244 for (Object[] arg : exhaustiveArgsList) { 245 ArrayList<String> decoderList = 246 CodecTestBase.selectCodecs((String) arg[1], null, null, false); 247 if (decoderList.size() == 0) { 248 decoderList.add(CodecTestBase.INVALID_CODEC + arg[1]); 249 } 250 for (String decoderName : decoderList) { 251 int argLength = exhaustiveArgsList.get(0).length; 252 Object[] testArg = new Object[argLength + 1]; 253 testArg[0] = arg[0]; // encoder mediaType 254 testArg[1] = decoderName; // decoder name 255 System.arraycopy(arg, 1, testArg, 2, argLength - 1); 256 argsList.add(testArg); 257 } 258 } 259 260 final List<Object[]> expandedArgsList = 261 CodecTestBase.prepareParamList(argsList, isEncoder, needAudio, needVideo, true); 262 263 // Prior to Android U, this test was not testing persistent surface. While this has 264 // been expected behavior for a long time, we only started testing it in Android U, so 265 // some older devices might not pass this test in persistent surface mode for some 266 // combination of codecs. These may show up as failures when running MTS tests for s/w 267 // encoders with h/w decoders in such cases. 268 269 // Prior to Android U, this test was using the first decoder for a given mediaType. 270 // In Android U, this was updated to test the encoders with all decoders for the 271 // given mediaType. There are some vendor encoders in older versions of Android 272 // and few OMX encoders which do not work as expected with the surface from s/w decoder. 273 // If the device is has vendor partition older than Android U or if the encoder is 274 // an OMX encoder, then limit the tests to first decoder like it was being done prior 275 // to Androd U 276 final List<Object[]> finalArgsList = new ArrayList<>(); 277 for (Object[] arg : expandedArgsList) { 278 String encoderName = (String) arg[0]; 279 String decoderName = (String) arg[2]; 280 String decoderMediaType = (String) arg[3]; 281 if ((BOARD_SDK_IS_BEFORE_U || VNDK_IS_BEFORE_U || encoderName.toUpperCase().startsWith("OMX")) 282 && isVendorCodec(encoderName)) { 283 if (!isDefaultCodec(decoderName, decoderMediaType, /* isEncoder */false)) { 284 continue; 285 } 286 } 287 finalArgsList.add(arg); 288 } 289 return finalArgsList; 290 } 291 292 /** 293 * Checks if the component under test can encode from surface properly. The test runs 294 * mediacodec in both synchronous and asynchronous mode. The test feeds the encoder input 295 * surface with output of decoder. Assuming no frame drops, the number of output frames from 296 * encoder should be identical to number of input frames to decoder. Also the timestamps 297 * should be identical. As encoder output is deterministic, the test expects consistent 298 * output in all runs. The output is written to a file using muxer. This file is validated 299 * for PSNR to check if the encoding happened successfully with out any obvious artifacts. 300 */ 301 @CddTest(requirements = {"2.2.2", "2.3.2", "2.5.2"}) 302 @ApiTest(apis = {"android.media.MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface", 303 "android.media.MediaCodecInfo.CodecCapabilities#COLOR_FormatYUV420Flexible", 304 "android.media.MediaCodecInfo.CodecCapabilities#COLOR_FormatYUVP010", 305 "android.media.MediaFormat#KEY_COLOR_TRANSFER_REQUEST"}) 306 @LargeTest 307 @Test(timeout = CodecTestBase.PER_TEST_TIMEOUT_LARGE_TEST_MS) testSimpleEncodeFromSurface()308 public void testSimpleEncodeFromSurface() throws IOException, InterruptedException { 309 boolean muxOutput = true; 310 if (mEncMediaType.equals(MediaFormat.MIMETYPE_VIDEO_AV1) && CodecTestBase.IS_BEFORE_U) { 311 muxOutput = false; 312 } 313 OutputManager ref = new OutputManager(); 314 OutputManager test = new OutputManager(ref.getSharedErrorLogs()); 315 boolean[] boolStates = {true, false}; 316 int count = 0; 317 String tmpPath = null; 318 boolean saveToMem = false; /* TODO(b/149027258) */ 319 for (boolean isAsync : boolStates) { 320 if (count == 0) { 321 tmpPath = getTempFilePath(mEncCfgParams.mInputBitDepth > 8 ? "10bit" : ""); 322 mTmpFiles.add(tmpPath); 323 } 324 encodeToMemory(isAsync, false, saveToMem, (count == 0 ? ref : test), muxOutput, 325 tmpPath, mFrameLimit); 326 // TODO:(b/149027258) Remove false once output is validated across runs 327 if (false) { 328 if (count != 0 && !ref.equals(test)) { 329 fail("Encoder output is not consistent across runs \n" + mTestConfig + mTestEnv 330 + test.getErrMsg()); 331 } 332 } 333 count++; 334 } 335 // Skip stream validation as there is no reference for tone mapped input 336 if (muxOutput && !mIsOutputToneMapped) { 337 if (mEncCfgParams.mInputBitDepth > 8 && !VNDK_IS_AT_LEAST_T) return; 338 CodecEncoderTestBase.validateEncodedPSNR(mTestFileMediaType, mTestFile, mEncMediaType, 339 tmpPath, false, false, ACCEPTABLE_WIRELESS_TX_QUALITY); 340 } 341 } 342 nativeTestSimpleEncode(String encoder, String decoder, String mediaType, String testFile, String testFileMediaType, String muxFile, int colorFormat, boolean usePersistentSurface, String cfgParams, String separator, StringBuilder retMsg, int frameLimit)343 private native boolean nativeTestSimpleEncode(String encoder, String decoder, String mediaType, 344 String testFile, String testFileMediaType, String muxFile, int colorFormat, 345 boolean usePersistentSurface, String cfgParams, String separator, StringBuilder retMsg, 346 int frameLimit); 347 348 /** 349 * Test is similar to {@link #testSimpleEncodeFromSurface()} but uses ndk api 350 */ 351 @CddTest(requirements = {"2.2.2", "2.3.2", "2.5.2"}) 352 @ApiTest(apis = {"android.media.MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface", 353 "android.media.MediaCodecInfo.CodecCapabilities#COLOR_FormatYUV420Flexible", 354 "android.media.MediaCodecInfo.CodecCapabilities#COLOR_FormatYUVP010"}) 355 @LargeTest 356 @Test(timeout = CodecTestBase.PER_TEST_TIMEOUT_LARGE_TEST_MS) testSimpleEncodeFromSurfaceNative()357 public void testSimpleEncodeFromSurfaceNative() throws IOException, InterruptedException { 358 // TODO(b/281661171) Update native tests to encode for tone mapped output 359 assumeFalse("tone mapping tests are skipped in native mode", mIsOutputToneMapped); 360 String tmpPath = null; 361 if (!mEncMediaType.equals(MediaFormat.MIMETYPE_VIDEO_AV1) || CodecTestBase.IS_AT_LEAST_U) { 362 tmpPath = getTempFilePath(mEncCfgParams.mInputBitDepth > 8 ? "10bit" : ""); 363 mTmpFiles.add(tmpPath); 364 } 365 int colorFormat = mDecoderFormat.getInteger(MediaFormat.KEY_COLOR_FORMAT, -1); 366 boolean isPass = nativeTestSimpleEncode(mEncoderName, mDecoderName, mEncMediaType, 367 mTestFile, mTestFileMediaType, tmpPath, colorFormat, mUsePersistentSurface, 368 EncoderConfigParams.serializeMediaFormat(mEncoderFormat), 369 EncoderConfigParams.TOKEN_SEPARATOR, mTestConfig, mFrameLimit); 370 assertTrue(mTestConfig.toString(), isPass); 371 if (tmpPath != null) { 372 if (mEncCfgParams.mInputBitDepth > 8 && !VNDK_IS_AT_LEAST_T) return; 373 CodecEncoderTestBase.validateEncodedPSNR(mTestFileMediaType, mTestFile, mEncMediaType, 374 tmpPath, false, false, ACCEPTABLE_WIRELESS_TX_QUALITY); 375 } 376 } 377 } 378