1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.media.videoquality.bdrate;
18 
19 import com.google.common.annotations.VisibleForTesting;
20 import com.google.common.base.Preconditions;
21 import com.google.gson.Gson;
22 import com.google.gson.GsonBuilder;
23 import com.google.gson.JsonParseException;
24 import com.google.gson.reflect.TypeToken;
25 
26 import org.kohsuke.args4j.CmdLineException;
27 import org.kohsuke.args4j.CmdLineParser;
28 import org.kohsuke.args4j.Option;
29 
30 import java.io.BufferedReader;
31 import java.io.IOException;
32 import java.nio.charset.StandardCharsets;
33 import java.nio.file.Files;
34 import java.nio.file.Path;
35 import java.text.DecimalFormat;
36 import java.text.NumberFormat;
37 import java.util.ArrayList;
38 import java.util.Arrays;
39 import java.util.Iterator;
40 import java.util.List;
41 import java.util.logging.ConsoleHandler;
42 import java.util.logging.Level;
43 import java.util.logging.Logger;
44 
45 /**
46  * Binary for calculating BD-RATE as part of the Performance Class - Video Encoding Quality CTS
47  * test.
48  *
49  * <p>Usage:
50  *
51  * <pre>
52  *  cts-media-videoquality-bdrate --REF_JSON_FILE reference_file.json
53  *      --TEST_VMAF_FILE test_result.txt
54  * </pre>
55  *
56  * Returns one of the following exit-codes:
57  *
58  * <ul>
59  *   <li>0 - The VEQ test has passed and the BD-RATE was within the threshold defined by the
60  *       reference configuration.
61  *   <li>1 - The VEQ test has failed due to the calculated BD-RATE being greater than the allowed
62  *       threshold defined by the reference configuration.
63  *   <li>2 - BD-RATE could not be calculated because one of the required conditions for calculation
64  *       was not met.
65  *   <li>3 - The configuration files could not be loaded and thus, BD-RATE could not be calculated.
66  *   <li>4 - An unknown error occurred and BD-RATE could not be calculated.
67  * </ul>
68  */
69 public class BdRateMain {
70     private static final Logger LOGGER = Logger.getLogger(BdRateMain.class.getName());
71     private static final double VERSION = 1.05;
72 
73     private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("0.00");
74     private final Gson mGson;
75 
76     private final BdRateCalculator mBdRateCalculator;
77 
78     private final BdQualityCalculator mBdQualityCalculator;
79 
BdRateMain( Gson gson, BdRateCalculator bdRateCalculator, BdQualityCalculator bdQualityCalculator)80     public BdRateMain(
81             Gson gson, BdRateCalculator bdRateCalculator, BdQualityCalculator bdQualityCalculator) {
82         mGson = gson;
83         mBdRateCalculator = bdRateCalculator;
84         mBdQualityCalculator = bdQualityCalculator;
85     }
86 
87     @Option(
88             name = "--REF_JSON_FILE",
89             usage = "The file containing the reference data.",
90             required = true)
91     private Path mRefJsonFile;
92 
93     @Option(
94             name = "--TEST_VMAF_FILE",
95             usage = "The file containing the test-generated VMAF data.",
96             required = true)
97     private Path mTestVmafFile;
98 
99     @Option(name = "-v", usage = "If set, prints the version and then exits.", required = false)
100     private boolean mVersion;
101 
run(String[] args)102     public void run(String[] args) {
103         CmdLineParser parser = new CmdLineParser(this);
104         LOGGER.info(String.format("cts-media-videoquality-bdrate v%.02f", VERSION));
105 
106         try {
107             parser.parseArgument(args);
108         } catch (CmdLineException e) {
109             throw new IllegalArgumentException("Unable to parse command-line flags!", e);
110         }
111 
112         if (mVersion) {
113             System.exit(0);
114         }
115 
116         LOGGER.info(String.format("Reading reference configuration JSON file: %s", mRefJsonFile));
117         ReferenceConfig refConfig = null;
118         try {
119             refConfig = loadReferenceConfig(mRefJsonFile, mGson);
120         } catch (IOException | JsonParseException e) {
121             throw new IllegalArgumentException("Failed to load reference configuration file!", e);
122         }
123 
124         LOGGER.info(String.format("Reading test result text file: %s", mTestVmafFile));
125         VeqTestResult veqTestResult = null;
126         try {
127             veqTestResult = loadTestResult(mTestVmafFile);
128         } catch (IOException | IllegalArgumentException e) {
129             throw new IllegalArgumentException("Failed to load VEQ Test Result file!", e);
130         }
131 
132         if (!veqTestResult.referenceFile().equals(refConfig.referenceFile())) {
133             throw new IllegalArgumentException(
134                     "Test Result file and Reference JSON file are not for the same reference file"
135                             + ".");
136         }
137 
138         logCurves(
139                 "Successfully loaded rate-distortion data: ",
140                 refConfig.referenceCurve(),
141                 veqTestResult.curve());
142         LOGGER.info(
143                 String.format(
144                         "Checking Video Encoding Quality (VEQ) for %s", refConfig.referenceFile()));
145 
146         checkVeq(
147                 mBdRateCalculator,
148                 mBdQualityCalculator,
149                 refConfig.referenceCurve(),
150                 veqTestResult.curve(),
151                 refConfig.referenceThreshold());
152     }
153 
154     /**
155      * Checks the video encoding quality of the target curve against the reference curve using
156      * Bjontegaard-Delta (BD) values, throwing a {@link VeqResultCheckFailureException} if the
157      * result is greater than the allowed threshold.
158      *
159      * @throws IllegalArgumentException if neither BD-RATE nor BD-QUALITY can be calculated for the
160      *     provided curves, which occurs when the curves do not overlap in any dimension.
161      * @throws BdPreconditionFailedException if the provided data is insufficient for BD
162      *     calculations.
163      */
164     @VisibleForTesting
checkVeq( BdRateCalculator bdRateCalculator, BdQualityCalculator bdQualityCalculator, RateDistortionCurve baseline, RateDistortionCurve target, double threshold)165     static void checkVeq(
166             BdRateCalculator bdRateCalculator,
167             BdQualityCalculator bdQualityCalculator,
168             RateDistortionCurve baseline,
169             RateDistortionCurve target,
170             double threshold) {
171         RateDistortionCurvePair curvePair =
172                 RateDistortionCurvePair.createClusteredPair(baseline, target);
173 
174         if (curvePair.canCalculateBdRate()) {
175             LOGGER.info("Calculating BD-RATE...");
176 
177             double bdRateResult = bdRateCalculator.calculate(curvePair);
178             LOGGER.info(
179                     String.format("BD-RATE: %.04f (%.02f%%)", bdRateResult, bdRateResult * 100));
180 
181             if (bdRateResult > threshold) {
182                 throw new VeqResultCheckFailureException(
183                         "BD-RATE is higher than threshold.", threshold, bdRateResult);
184             }
185         } else if (curvePair.canCalculateBdQuality()) {
186             LOGGER.warning("Unable to calculate BD-RATE, falling back to checking BD-QUALITY...");
187 
188             double bdQualityResult = bdQualityCalculator.calculate(curvePair);
189             LOGGER.info(String.format("BD-QUALITY: %.02f", bdQualityResult));
190 
191             double percentageQualityChange =
192                     bdQualityResult
193                             / Arrays.stream(curvePair.baseline().getDistortionsArray())
194                                     .average()
195                                     .getAsDouble();
196 
197             // Since distortion is measured as a higher == better value, invert
198             // the percentage so that it can be compared equivalently with the threshold.
199             if (percentageQualityChange * -1 > threshold) {
200                 throw new VeqResultCheckFailureException(
201                         "BD-QUALITY is higher than threshold.", threshold, bdQualityResult);
202             }
203         } else {
204             throw new IllegalArgumentException(
205                     "Cannot calculate BD-RATE or BD-QUALITY. Reference configuration likely does "
206                             + "not match the test result data.");
207         }
208     }
209 
logCurves( String message, RateDistortionCurve referenceCurve, RateDistortionCurve targetCurve)210     private static void logCurves(
211             String message, RateDistortionCurve referenceCurve, RateDistortionCurve targetCurve) {
212         ArrayList<String> rows = new ArrayList<>();
213         rows.add(message);
214         rows.add(
215                 String.format(
216                         "|%15s|%15s|%15s|%15s|",
217                         "Reference Rate", "Reference Dist", "Target Rate", "Target Dist"));
218         rows.add("=".repeat(rows.get(1).length()));
219 
220         Iterator<RateDistortionPoint> referencePoints = referenceCurve.points().iterator();
221         Iterator<RateDistortionPoint> targetPoints = targetCurve.points().iterator();
222 
223         while (referencePoints.hasNext() || targetPoints.hasNext()) {
224             String refRate = "";
225             String refDist = "";
226             if (referencePoints.hasNext()) {
227                 RateDistortionPoint refPoint = referencePoints.next();
228                 refRate = NUMBER_FORMAT.format(refPoint.rate());
229                 refDist = NUMBER_FORMAT.format(refPoint.distortion());
230             }
231 
232             String targetRate = "";
233             String targetDist = "";
234             if (targetPoints.hasNext()) {
235                 RateDistortionPoint targetPoint = targetPoints.next();
236                 targetRate = NUMBER_FORMAT.format(targetPoint.rate());
237                 targetDist = NUMBER_FORMAT.format(targetPoint.distortion());
238             }
239 
240             rows.add(
241                     String.format(
242                             "|%15s|%15s|%15s|%15s|", refRate, refDist, targetRate, targetDist));
243         }
244 
245         LOGGER.info(String.join("\n", rows));
246     }
247 
loadReferenceConfig(Path path, Gson gson)248     private static ReferenceConfig loadReferenceConfig(Path path, Gson gson) throws IOException {
249         Preconditions.checkArgument(Files.exists(path));
250 
251         // Each config file contains a single ReferenceConfig in a list,
252         // the first one is returned here.
253         try (BufferedReader reader = Files.newBufferedReader(path)) {
254             TypeToken<List<ReferenceConfig>> configsType = new TypeToken<>() {};
255             ArrayList<ReferenceConfig> configs = gson.fromJson(reader, configsType.getType());
256             return configs.get(0);
257         }
258     }
259 
loadTestResult(Path path)260     private static VeqTestResult loadTestResult(Path path) throws IOException {
261         Preconditions.checkState(Files.exists(path));
262 
263         String testResult = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
264         return VeqTestResult.parseFromTestResult(testResult);
265     }
266 
main(String[] args)267     public static void main(String[] args) {
268 
269         // Setup the logger.
270         Logger rootLogger = Logger.getLogger("");
271         rootLogger.setLevel(Level.FINEST);
272         rootLogger.addHandler(
273                 new ConsoleHandler() {
274                     {
275                         setOutputStream(System.out);
276                         setLevel(Level.FINEST);
277                     }
278                 });
279 
280         try {
281             new BdRateMain(
282                             new GsonBuilder()
283                                     .registerTypeAdapter(
284                                             ReferenceConfig.class,
285                                             new ReferenceConfig.Deserializer())
286                                     .create(),
287                             BdRateCalculator.create(),
288                             BdQualityCalculator.create())
289                     .run(args);
290         } catch (VeqResultCheckFailureException bdgtte) {
291             LOGGER.log(
292                     Level.SEVERE,
293                     String.format(
294                             "Failed Video Encoding Quality (VEQ) test, calculated BD-RATE was"
295                                     + " (%.04f) which was greater than the test-defined threshold"
296                                     + " of"
297                                     + " (%.04f)",
298                             bdgtte.getBdResult(), bdgtte.getThreshold()));
299             System.exit(1);
300         } catch (BdPreconditionFailedException bdfpe) {
301             LOGGER.log(
302                     Level.SEVERE,
303                     String.format("Unable to calculate BD-RATE because: %s", bdfpe.getMessage()));
304             System.exit(2);
305         } catch (IllegalArgumentException iae) {
306             LOGGER.log(Level.SEVERE, "Invalid Arguments: ", iae);
307             System.exit(3);
308         } catch (Exception e) {
309             LOGGER.log(Level.SEVERE, "Unknown error occurred!", e);
310             System.exit(4);
311         }
312 
313         LOGGER.info("Passed Video Encoding Quality (VEQ) test.");
314         System.exit(0);
315     }
316 }
317