1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.postprocessor;
17 
18 import com.android.tradefed.config.Option;
19 import com.android.tradefed.config.OptionClass;
20 import com.android.tradefed.metrics.proto.MetricMeasurement.Measurements;
21 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
22 import com.android.tradefed.result.LogFile;
23 import com.android.tradefed.result.TestDescription;
24 import com.android.tradefed.util.MetricUtility;
25 
26 import com.google.common.collect.ArrayListMultimap;
27 
28 import java.util.Arrays;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.HashMap;
32 import java.util.HashSet;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.Set;
36 import java.util.stream.Collectors;
37 
38 /**
39  * A metric aggregator that gives the min, max, mean, variance, standard deviation, total, count and
40  * optionally percentiles for numeric metrics collected during multiple-iteration test runs,
41  * treating them as doubles. Non-numeric metrics are ignored. Metrics that have a mix of numeric and
42  * non-numeric values will also be ignored.
43  *
44  * <p>Note that count will only be present if a metric has all-numeric values; otherwise, it will be
45  * absent along with all the other stats.
46  *
47  * <p>It parses metrics from single string as currently metrics are passed this way.
48  */
49 @OptionClass(alias = "aggregate-post-processor")
50 public class AggregatePostProcessor extends BasePostProcessor {
51     @Option(
52             name = "report-percentiles",
53             description =
54                     "Additional percentiles of each metric to report, in integers in the 0 - 100 "
55                             + "range. Can be repeated.")
56     private Set<Integer> mPercentiles = new HashSet<>();
57 
58     // Separator for final upload
59     private static final String STATS_KEY_SEPARATOR = "-";
60 
61     // Stores the test metrics for aggregation by test description.
62     // TODO(b/118708851): Remove this workaround once AnTS is ready.
63     private HashMap<String, ArrayListMultimap<String, Metric>> mStoredTestMetrics =
64             new HashMap<String, ArrayListMultimap<String, Metric>>();
65 
66     @Override
processTestMetricsAndLogs( TestDescription testDescription, HashMap<String, Metric> testMetrics, Map<String, LogFile> testLogs)67     public Map<String, Metric.Builder> processTestMetricsAndLogs(
68             TestDescription testDescription,
69             HashMap<String, Metric> testMetrics,
70             Map<String, LogFile> testLogs) {
71         // TODO(b/118708851): Move this processing elsewhere once AnTS is ready.
72         // Use the string representation of the test description to key the tests.
73         String fullTestName = testDescription.toString();
74         // Store result from the current test.
75         if (!mStoredTestMetrics.containsKey(fullTestName)) {
76             mStoredTestMetrics.put(fullTestName, ArrayListMultimap.create());
77         }
78         ArrayListMultimap<String, Metric> storedMetricsForThisTest =
79                 mStoredTestMetrics.get(fullTestName);
80         for (Map.Entry<String, Metric> entry : testMetrics.entrySet()) {
81             storedMetricsForThisTest.put(entry.getKey(), entry.getValue());
82         }
83         // Aggregate all data in iterations of this test.
84         Map<String, Metric.Builder> aggregateMetrics = new HashMap<String, Metric.Builder>();
85         for (String metricKey : storedMetricsForThisTest.keySet()) {
86             List<Metric> metrics = storedMetricsForThisTest.get(metricKey);
87             List<Measurements> measures =
88                     metrics.stream().map(Metric::getMeasurements).collect(Collectors.toList());
89             // Parse metrics into a list of SingleString values, concating lists in the process
90             List<String> rawValues =
91                     measures.stream()
92                             .map(Measurements::getSingleString)
93                             .map(
94                                     m -> {
95                                         // Split results; also deals with the case of empty results
96                                         // in a certain run
97                                         List<String> splitVals = Arrays.asList(m.split(",", 0));
98                                         if (splitVals.size() == 1 && splitVals.get(0).isEmpty()) {
99                                             return Collections.<String>emptyList();
100                                         }
101                                         return splitVals;
102                                     })
103                             .flatMap(Collection::stream)
104                             .map(String::trim)
105                             .collect(Collectors.toList());
106             // Do not report empty metrics
107             if (rawValues.isEmpty()) {
108                 continue;
109             }
110             if (MetricUtility.isAllDoubleValues(rawValues)) {
111                 buildStats(metricKey, rawValues, aggregateMetrics);
112             }
113         }
114         return aggregateMetrics;
115     }
116 
117     @Override
processRunMetricsAndLogs( HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs)118     public Map<String, Metric.Builder> processRunMetricsAndLogs(
119             HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs) {
120         // Aggregate the test run metrics which has comma separated values which can be
121         // parsed to double values.
122         Map<String, Metric.Builder> aggregateMetrics = new HashMap<String, Metric.Builder>();
123         for (Map.Entry<String, Metric> entry : rawMetrics.entrySet()) {
124             String values = entry.getValue().getMeasurements().getSingleString();
125             List<String> splitVals = Arrays.asList(values.split(",", 0));
126             // Build stats for keys with any values, even only one.
127             if (MetricUtility.isAllDoubleValues(splitVals)) {
128                 buildStats(entry.getKey(), splitVals, aggregateMetrics);
129             }
130         }
131         return aggregateMetrics;
132     }
133 
134     /**
135      * Build stats for the given set of values and build the metrics using the metric key
136      * and stats name and update the results in aggregated metrics.
137      *
138      * @param metricKey key to which the values correspond to.
139      * @param values list of raw values.
140      * @param aggregateMetrics where final metrics will be stored.
141      */
buildStats(String metricKey, List<String> values, Map<String, Metric.Builder> aggregateMetrics)142     private void buildStats(String metricKey, List<String> values,
143             Map<String, Metric.Builder> aggregateMetrics) {
144         List<Double> doubleValues = values.stream().map(Double::parseDouble)
145                 .collect(Collectors.toList());
146         Map<String, Double> stats = MetricUtility.getStats(doubleValues, mPercentiles);
147         for (String statKey : stats.keySet()) {
148             Metric.Builder metricBuilder = Metric.newBuilder();
149             metricBuilder
150                     .getMeasurementsBuilder()
151                     .setSingleString(String.format("%2.2f", stats.get(statKey)));
152             aggregateMetrics.put(
153                     String.join(STATS_KEY_SEPARATOR, metricKey, statKey),
154                     metricBuilder);
155         }
156     }
157 }
158