1 /* 2 * Copyright 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.util; 17 18 import android.media.MediaFormat; 19 import android.util.Log; 20 import android.util.Range; 21 22 import com.android.compatibility.common.util.DeviceReportLog; 23 import com.android.compatibility.common.util.ResultType; 24 import com.android.compatibility.common.util.ResultUnit; 25 26 import java.util.Arrays; 27 28 public class MediaPerfUtils { 29 private static final String TAG = "MediaPerfUtils"; 30 31 private static final int MOVING_AVERAGE_NUM_FRAMES = 10; 32 private static final int MOVING_AVERAGE_WINDOW_MS = 1000; 33 34 // allow a variance of 2x for measured frame rates (e.g. half of lower-limit to double of 35 // upper-limit of the published values). Also allow an extra 10% margin. This also acts as 36 // a limit for the size of the published rates (e.g. upper-limit / lower-limit <= tolerance). 37 private static final double FRAMERATE_TOLERANCE = 2.0 * 1.1; 38 39 // Allow extra tolerance when B frames are enabled 40 private static final double EXTRA_TOLERANCE_BFRAMES = 1.25; 41 /* 42 * ------------------ HELPER METHODS FOR ACHIEVABLE FRAME RATES ------------------ 43 */ 44 45 /** removes brackets from format to be included in JSON. */ formatForReport(MediaFormat format)46 private static String formatForReport(MediaFormat format) { 47 String asString = "" + format; 48 return asString.substring(1, asString.length() - 1); 49 } 50 51 /** 52 * Adds performance header info to |log| for |codecName|, |round|, |configFormat|, |inputFormat| 53 * and |outputFormat|. Also appends same to |message| and returns the resulting base message 54 * for logging purposes. 55 */ addPerformanceHeadersToLog( DeviceReportLog log, String message, int round, String codecName, MediaFormat configFormat, MediaFormat inputFormat, MediaFormat outputFormat)56 public static String addPerformanceHeadersToLog( 57 DeviceReportLog log, String message, int round, String codecName, 58 MediaFormat configFormat, MediaFormat inputFormat, MediaFormat outputFormat) { 59 String mime = configFormat.getString(MediaFormat.KEY_MIME); 60 int width = configFormat.getInteger(MediaFormat.KEY_WIDTH); 61 int height = configFormat.getInteger(MediaFormat.KEY_HEIGHT); 62 63 log.addValue("round", round, ResultType.NEUTRAL, ResultUnit.NONE); 64 log.addValue("codec_name", codecName, ResultType.NEUTRAL, ResultUnit.NONE); 65 log.addValue("mime_type", mime, ResultType.NEUTRAL, ResultUnit.NONE); 66 log.addValue("width", width, ResultType.NEUTRAL, ResultUnit.NONE); 67 log.addValue("height", height, ResultType.NEUTRAL, ResultUnit.NONE); 68 log.addValue("config_format", formatForReport(configFormat), 69 ResultType.NEUTRAL, ResultUnit.NONE); 70 log.addValue("input_format", formatForReport(inputFormat), 71 ResultType.NEUTRAL, ResultUnit.NONE); 72 log.addValue("output_format", formatForReport(outputFormat), 73 ResultType.NEUTRAL, ResultUnit.NONE); 74 75 message += " codec=" + codecName + " round=" + round + " configFormat=" + configFormat 76 + " inputFormat=" + inputFormat + " outputFormat=" + outputFormat; 77 78 Range<Double> reported = 79 MediaUtils.getVideoCapabilities(codecName, mime) 80 .getAchievableFrameRatesFor(width, height); 81 if (reported != null) { 82 log.addValue("reported_low", reported.getLower(), ResultType.NEUTRAL, ResultUnit.FPS); 83 log.addValue("reported_high", reported.getUpper(), ResultType.NEUTRAL, ResultUnit.FPS); 84 message += " reported=" + reported.getLower() + "-" + reported.getUpper(); 85 } 86 87 return message; 88 } 89 90 /** 91 * Adds performance statistics based on the raw |stats| to |log|. Also prints the same into 92 * logcat. Returns the "final fps" value. 93 */ addPerformanceStatsToLog( DeviceReportLog log, MediaUtils.Stats durationsUsStats, String message)94 public static double addPerformanceStatsToLog( 95 DeviceReportLog log, MediaUtils.Stats durationsUsStats, String message) { 96 97 MediaUtils.Stats frameAvgUsStats = 98 durationsUsStats.movingAverage(MOVING_AVERAGE_NUM_FRAMES); 99 log.addValue( 100 "window_frames", MOVING_AVERAGE_NUM_FRAMES, ResultType.NEUTRAL, ResultUnit.COUNT); 101 logPerformanceStats(log, frameAvgUsStats, "frame_avg_stats", 102 message + " window=" + MOVING_AVERAGE_NUM_FRAMES); 103 104 MediaUtils.Stats timeAvgUsStats = 105 durationsUsStats.movingAverageOverSum(MOVING_AVERAGE_WINDOW_MS * 1000); 106 log.addValue("window_time", MOVING_AVERAGE_WINDOW_MS, ResultType.NEUTRAL, ResultUnit.MS); 107 double fps = logPerformanceStats(log, timeAvgUsStats, "time_avg_stats", 108 message + " windowMs=" + MOVING_AVERAGE_WINDOW_MS); 109 110 log.setSummary("fps", fps, ResultType.HIGHER_BETTER, ResultUnit.FPS); 111 return fps; 112 } 113 114 /** 115 * Adds performance statistics based on the processed |stats| to |log| using |prefix|. 116 * Also prints the same into logcat using |message| as the base message. Returns the fps value 117 * for |stats|. |prefix| must be lowercase alphanumeric underscored format. 118 */ logPerformanceStats( DeviceReportLog log, MediaUtils.Stats statsUs, String prefix, String message)119 private static double logPerformanceStats( 120 DeviceReportLog log, MediaUtils.Stats statsUs, String prefix, String message) { 121 final String[] labels = { 122 "min", "p5", "p10", "p20", "p30", "p40", "p50", "p60", "p70", "p80", "p90", "p95", "max" 123 }; 124 final double[] points = { 125 0, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 100 126 }; 127 128 int num = statsUs.getNum(); 129 long avg = Math.round(statsUs.getAverage()); 130 long stdev = Math.round(statsUs.getStdev()); 131 log.addValue(prefix + "_num", num, ResultType.NEUTRAL, ResultUnit.COUNT); 132 log.addValue(prefix + "_avg", avg / 1000., ResultType.LOWER_BETTER, ResultUnit.MS); 133 log.addValue(prefix + "_stdev", stdev / 1000., ResultType.LOWER_BETTER, ResultUnit.MS); 134 message += " num=" + num + " avg=" + avg + " stdev=" + stdev; 135 final double[] percentiles = statsUs.getPercentiles(points); 136 for (int i = 0; i < labels.length; ++i) { 137 long p = Math.round(percentiles[i]); 138 message += " " + labels[i] + "=" + p; 139 log.addValue(prefix + "_" + labels[i], p / 1000., ResultType.NEUTRAL, ResultUnit.MS); 140 } 141 142 // print result to logcat in case test aborts before logs are written 143 Log.i(TAG, message); 144 145 return 1e6 / percentiles[points.length - 2]; 146 } 147 148 /** Verifies |measuredFps| against reported achievable rates. Returns null if at least 149 * one measurement falls within the margins of the reported range. Otherwise, returns 150 * an error message to display.*/ verifyAchievableFrameRates(String name, String mime, int w, int h, boolean fasterIsOk, double... measuredFps)151 public static String verifyAchievableFrameRates(String name, String mime, int w, 152 int h, boolean fasterIsOk, double... measuredFps) { 153 return verifyAchievableFrameRates( 154 name, mime, w, h, fasterIsOk, /* bFramesEnabled */ false, measuredFps); 155 } 156 157 /** Verifies |measuredFps| against reported achievable rates allowing extra tolerance when 158 * B frames are enabled. Returns null if at least one measurement falls within the margins 159 * of the reported range. Otherwise, returns an error message to display.*/ verifyAchievableFrameRates(String name, String mime, int w, int h, boolean fasterIsOk, boolean bFramesEnabled, double... measuredFps)160 public static String verifyAchievableFrameRates(String name, String mime, int w, int h, 161 boolean fasterIsOk, boolean bFramesEnabled, double... measuredFps) { 162 Range<Double> reported = 163 MediaUtils.getVideoCapabilities(name, mime).getAchievableFrameRatesFor(w, h); 164 String kind = "achievable frame rates for " + name + " " + mime + " " + w + "x" + h; 165 if (reported == null) { 166 return "Failed to get " + kind; 167 } 168 double tolerance = FRAMERATE_TOLERANCE; 169 if (bFramesEnabled) { 170 tolerance *= EXTRA_TOLERANCE_BFRAMES; 171 } 172 double lowerBoundary1 = reported.getLower() / tolerance; 173 double upperBoundary1 = reported.getUpper() * tolerance; 174 double lowerBoundary2 = reported.getUpper() / Math.pow(tolerance, 2); 175 double upperBoundary2 = reported.getLower() * Math.pow(tolerance, 2); 176 Log.d(TAG, name + " " + mime + " " + w + "x" + h + 177 " lowerBoundary1 " + lowerBoundary1 + " upperBoundary1 " + upperBoundary1 + 178 " lowerBoundary2 " + lowerBoundary2 + " upperBoundary2 " + upperBoundary2 + 179 " measured " + Arrays.toString(measuredFps)); 180 181 if (fasterIsOk) { 182 double lower = Math.max(lowerBoundary1, lowerBoundary2); 183 for (double measured : measuredFps) { 184 if (measured >= lower) { 185 return null; 186 } 187 } 188 } else { 189 double lower = Math.max(lowerBoundary1, lowerBoundary2); 190 double upper = Math.min(upperBoundary1, upperBoundary2); 191 for (double measured : measuredFps) { 192 if (measured >= lower && measured <= upper) { 193 return null; 194 } 195 } 196 } 197 198 return "Expected " + kind + ": " + reported + ".\n" 199 + "Measured frame rate: " + Arrays.toString(measuredFps) + ".\n"; 200 } 201 202 /** Verifies |requestedFps| does not exceed reported achievable rates. 203 * Returns null if *ALL* requested rates are claimed to be achievable. 204 * Otherwise, returns a diagnostic explaining why it's not achievable. 205 * (one of the rates was too fast, we don't have achievability information, etc). 206 * 207 * we're looking for 90% confidence, which is documented as being: 208 * "higher than half of the lower limit at least 90% of the time in tested configurations" 209 * 210 * NB: we only invoke this for the SW codecs; we use performance point info for the 211 * hardware codecs. 212 * */ areAchievableFrameRates( String name, String mime, int w, int h, double... requestedFps)213 public static String areAchievableFrameRates( 214 String name, String mime, int w, int h, double... requestedFps) { 215 Range<Double> reported = 216 MediaUtils.getVideoCapabilities(name, mime).getAchievableFrameRatesFor(w, h); 217 String kind = "achievable frame rates for " + name + " " + mime + " " + w + "x" + h; 218 if (reported == null) { 219 return "Failed to get " + kind; 220 } 221 222 double confidence90 = reported.getLower() / 2.0; 223 224 Log.d(TAG, name + " " + mime + " " + w + "x" + h + 225 " lower " + reported.getLower() + " 90% confidence " + confidence90 + 226 " requested " + Arrays.toString(requestedFps)); 227 228 // if *any* of them are too fast, we say no. 229 for (double requested : requestedFps) { 230 if (requested > confidence90) { 231 return "Expected " + kind + ": " + reported + ", 90% confidence: " + confidence90 232 + ".\n" 233 + "Requested frame rate: " + Arrays.toString(requestedFps) + ".\n"; 234 } 235 } 236 return null; 237 } 238 } 239