1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.catbox.util;
17 
18 import com.android.annotations.VisibleForTesting;
19 
20 import com.android.ddmlib.Log.LogLevel;
21 
22 import com.android.tradefed.log.LogUtil.CLog;
23 import com.android.tradefed.metrics.proto.MetricMeasurement.DataType;
24 import com.android.tradefed.metrics.proto.MetricMeasurement.Measurements;
25 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
26 import com.android.tradefed.result.TestDescription;
27 import com.android.tradefed.util.proto.TfMetricProtoUtil;
28 
29 import com.google.common.base.Joiner;
30 import com.google.common.collect.ArrayListMultimap;
31 import com.google.common.math.Quantiles;
32 
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.Collections;
36 import java.util.HashMap;
37 import java.util.HashSet;
38 import java.util.LinkedHashMap;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.Map.Entry;
42 import java.util.Set;
43 import java.util.stream.Collectors;
44 
45 /**
46  * Contains common utility methods for storing the test metrics and aggregating the metrics in
47  * similar tests.
48  */
49 public class TestMetricsUtil {
50 
51     private static final String TEST_HEADER_SEPARATOR = "\n\n";
52     private static final String METRIC_SEPARATOR = "\n";
53     private static final String METRIC_KEY_VALUE_SEPARATOR = ":";
54     private static final String STATS_KEY_MIN = "min";
55     private static final String STATS_KEY_MAX = "max";
56     private static final String STATS_KEY_MEAN = "mean";
57     private static final String STATS_KEY_VAR = "var";
58     private static final String STATS_KEY_STDEV = "stdev";
59     private static final String STATS_KEY_MEDIAN = "median";
60     private static final String STATS_KEY_TOTAL = "total";
61     private static final String STATS_KEY_COUNT = "metric-count";
62     private static final String STATS_KEY_PERCENTILE_PREFIX = "p";
63     private static final String STATS_KEY_SEPARATOR = "-";
64     private static final Joiner CLASS_METHOD_JOINER = Joiner.on("#").skipNulls();
65 
66     // Used to separate the package name from the iteration number. Default is set to "$".
67     private String mTestIterationSeparator = "$";
68 
69     // Percentiles to include when calculating the aggregates.
70     private Set<Integer> mActualPercentiles = new HashSet<>();
71 
72     // Store the test metrics for aggregation at the end of test run.
73     // Outer map key is the test id and inner map key is the metric key name.
74     private Map<String, ArrayListMultimap<String, Metric>> mStoredTestMetrics =
75             new HashMap<String, ArrayListMultimap<String, Metric>>();
76 
77     /**
78      * Used for storing the individual test metrics and use it for aggregation.
79      */
storeTestMetrics(TestDescription testDescription, Map<String, Metric> testMetrics)80     public void storeTestMetrics(TestDescription testDescription, Map<String, Metric> testMetrics) {
81         if (testMetrics == null) {
82             return;
83         }
84 
85         // Group test cases which differs only by the iteration separator or test the same name.
86         String className = testDescription.getClassName();
87         int iterationSeparatorIndex = testDescription.getClassName()
88                 .indexOf(mTestIterationSeparator);
89         if (iterationSeparatorIndex != -1) {
90             className = testDescription.getClassName().substring(0, iterationSeparatorIndex);
91         }
92         String newTestId = CLASS_METHOD_JOINER.join(className, testDescription.getTestName());
93 
94         if (!mStoredTestMetrics.containsKey(newTestId)) {
95             mStoredTestMetrics.put(newTestId, ArrayListMultimap.create());
96         }
97         ArrayListMultimap<String, Metric> storedMetricsForThisTest = mStoredTestMetrics
98                 .get(newTestId);
99 
100         // Store only raw metrics
101         HashMap<String, Metric> rawMetrics = getRawMetricsOnly(testMetrics);
102 
103         for (Map.Entry<String, Metric> entry : rawMetrics.entrySet()) {
104             String key = entry.getKey();
105             // In case of Multi User test, the metric conatins className with iteration separator
106             if (key.indexOf(mTestIterationSeparator) != -1 &&
107                         key.contains(testDescription.getClassName())) {
108                 key = key.substring(0, key.indexOf(mTestIterationSeparator));
109                 key = CLASS_METHOD_JOINER.join(key, testDescription.getTestName());
110             }
111             storedMetricsForThisTest.put(key, entry.getValue());
112         }
113     }
114 
115     /**
116      * Aggregate the metrics collected from multiple iterations of the test and
117      * return aggregated metrics.
118      */
getAggregatedStoredTestMetrics()119     public Map<String, Map<String, String>> getAggregatedStoredTestMetrics() {
120         Map<String, Map<String, String>> aggregatedStoredMetrics =
121                 new HashMap<String, Map<String, String>>();
122         for (String testName : mStoredTestMetrics.keySet()) {
123             ArrayListMultimap<String, Metric> currentTest = mStoredTestMetrics.get(testName);
124 
125             Map<String, Metric> aggregateMetrics = new LinkedHashMap<String, Metric>();
126             for (String metricKey : currentTest.keySet()) {
127                 List<Metric> metrics = currentTest.get(metricKey);
128                 List<Measurements> measures = metrics.stream().map(Metric::getMeasurements)
129                         .collect(Collectors.toList());
130                 // Parse metrics into a list of SingleString values, concating lists in the process
131                 List<String> rawValues = measures.stream()
132                         .map(Measurements::getSingleString)
133                         .map(
134                                 m -> {
135                                     // Split results; also deals with the case of empty results
136                                     // in a certain run
137                                     List<String> splitVals = Arrays.asList(m.split(",", 0));
138                                     if (splitVals.size() == 1 && splitVals.get(0).isEmpty()) {
139                                         return Collections.<String> emptyList();
140                                     }
141                                     return splitVals;
142                                 })
143                         .flatMap(Collection::stream)
144                         .map(String::trim)
145                         .collect(Collectors.toList());
146                 // Do not report empty metrics
147                 if (rawValues.isEmpty()) {
148                     continue;
149                 }
150                 if (isAllDoubleValues(rawValues)) {
151                     buildStats(metricKey, rawValues, aggregateMetrics);
152                 }
153             }
154             Map<String, String> compatibleTestMetrics = TfMetricProtoUtil
155                     .compatibleConvert(aggregateMetrics);
156             aggregatedStoredMetrics.put(testName, compatibleTestMetrics);
157         }
158         return aggregatedStoredMetrics;
159     }
160 
161     /** Set percentiles */
setPercentiles(Set<Integer> percentiles)162     public void setPercentiles(Set<Integer> percentiles) {
163         mActualPercentiles = percentiles;
164     }
165 
166     /** Set iteration separator */
setIterationSeparator(String separator)167     public void setIterationSeparator(String separator) {
168         mTestIterationSeparator = separator;
169     }
170 
171     @VisibleForTesting
getStoredTestMetric()172     public Map<String, ArrayListMultimap<String, Metric>> getStoredTestMetric() {
173         return mStoredTestMetrics;
174     }
175 
176     /**
177      * Return true is all the values can be parsed to double value.
178      * Otherwise return false.
179      */
isAllDoubleValues(List<String> rawValues)180     public static boolean isAllDoubleValues(List<String> rawValues) {
181         return rawValues
182                 .stream()
183                 .allMatch(
184                         val -> {
185                             try {
186                                 Double.parseDouble(val);
187                                 return true;
188                             } catch (NumberFormatException e) {
189                                 return false;
190                             }
191                         });
192     }
193 
194     /**
195      * Compute the stats from the give list of values.
196      */
197     public static Map<String, Double> getStats(Collection<Double> values,
198             Set<Integer> percentiles) {
199         Map<String, Double> stats = new LinkedHashMap<>();
200         double sum = values.stream().mapToDouble(Double::doubleValue).sum();
201         double count = values.size();
202         // The orElse situation should never happen.
203         double mean = values.stream()
204                 .mapToDouble(Double::doubleValue)
205                 .average()
206                 .orElseThrow(IllegalStateException::new);
207         double variance = values.stream().reduce(0.0, (a, b) -> a + Math.pow(b - mean, 2) / count);
208         // Calculate percentiles. 50 th percentile will be used as medain.
209         Set<Integer> updatedPercentile = new HashSet<>(percentiles);
210         updatedPercentile.add(50);
211         Map<Integer, Double> percentileStat = Quantiles.percentiles().indexes(updatedPercentile)
212                 .compute(values);
213         double median = percentileStat.get(50);
214 
215         stats.put(STATS_KEY_MIN, Collections.min(values));
216         stats.put(STATS_KEY_MAX, Collections.max(values));
217         stats.put(STATS_KEY_MEAN, mean);
218         stats.put(STATS_KEY_VAR, variance);
219         stats.put(STATS_KEY_STDEV, Math.sqrt(variance));
220         stats.put(STATS_KEY_MEDIAN, median);
221         stats.put(STATS_KEY_TOTAL, sum);
222         stats.put(STATS_KEY_COUNT, count);
223         percentileStat
224                 .entrySet()
225                 .stream()
226                 .forEach(
227                         e -> {
228                             // If the percentile is 50, only include it if the user asks for it
229                             // explicitly.
230                             if (e.getKey() != 50 || percentiles.contains(50)) {
231                                 stats.put(
232                                         STATS_KEY_PERCENTILE_PREFIX + e.getKey().toString(),
233                                         e.getValue());
234                             }
235                         });
236         return stats;
237     }
238 
239     /**
240      * Build stats for the given set of values and build the metrics using the metric key
241      * and stats name and update the results in aggregated metrics.
242      */
243     private void buildStats(String metricKey, List<String> values,
244             Map<String, Metric> aggregateMetrics) {
245         List<Double> doubleValues = values.stream().map(Double::parseDouble)
246                 .collect(Collectors.toList());
247         Map<String, Double> stats = getStats(doubleValues, mActualPercentiles);
248         for (String statKey : stats.keySet()) {
249             Metric.Builder metricBuilder = Metric.newBuilder();
250             metricBuilder
251                     .getMeasurementsBuilder()
252                     .setSingleString(String.format("%2.2f", stats.get(statKey)));
253             aggregateMetrics.put(
254                     String.join(STATS_KEY_SEPARATOR, metricKey, statKey),
255                     metricBuilder.build());
256         }
257     }
258 
259     /**
260      * Get only raw values for processing.
261      */
262     private HashMap<String, Metric> getRawMetricsOnly(Map<String, Metric> metrics) {
263         HashMap<String, Metric> rawMetrics = new HashMap<>();
264         for (Entry<String, Metric> entry : metrics.entrySet()) {
265             if (DataType.RAW.equals(entry.getValue().getType())) {
266                 rawMetrics.put(entry.getKey(), entry.getValue());
267             }
268         }
269         return rawMetrics;
270     }
271 }