1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.postprocessor;
17 
18 import com.android.annotations.VisibleForTesting;
19 import com.android.tradefed.config.Option;
20 import com.android.tradefed.config.OptionClass;
21 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
22 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric.Builder;
23 import com.android.tradefed.result.FileInputStreamSource;
24 import com.android.tradefed.result.InputStreamSource;
25 import com.android.tradefed.result.LogDataType;
26 import com.android.tradefed.result.LogFile;
27 import com.android.tradefed.result.TestDescription;
28 import com.android.tradefed.util.MetricUtility;
29 import com.android.tradefed.util.proto.TfMetricProtoUtil;
30 
31 import java.io.File;
32 import java.util.HashMap;
33 import java.util.HashSet;
34 import java.util.Map;
35 import java.util.Set;
36 
37 /**
38  * Used for uploading the metrics log file collected during the test and run level.
39  *
40  * Use "aggregate-similar-tests" option to aggregate similar tests metrics at the test run level
41  * and write it to a log file. Tests differ only by the iteration number or with the same name
42  * are considered similar tests.
43  *
44  * This will have access to only raw metrics.
45  */
46 @OptionClass(alias = "metric-file-post-processor")
47 public class MetricFilePostProcessor extends BasePostProcessor {
48 
49     private static final String AGGREGATE_TEST_SUFFIX = "_aggregate_test_metrics";
50     private static final String AGGREGATE_RUN_SUFFIX = "_aggregate_run_metrics";
51 
52     @Option(name = "enable-per-test-log", description = "Set this flag to false to disable"
53             + " writing the per test metrics to a file.")
54     private boolean mIsPerTestLogEnabled= true;
55 
56     @Option(name = "enable-run-log", description = "Set this flag to false to disable"
57             + " writing the run metrics to a file.")
58     private boolean mIsRunLogEnabled= true;
59 
60     @Option(name = "aggregate-similar-tests", description = "To aggregate the metrics from test"
61             + " cases which differ only by iteration number or having the same test name."
62             + " Used only in context with the microbenchmark test runner. Set this flag to false"
63             + " to disable aggregating the metrics.")
64     private boolean mAggregateSimilarTests= false;
65 
66     @Option(name = "aggregate-run-metrics", description = "Aggregate run metrics which has more"
67             + " than one value.")
68     private boolean mAggregateRunMetrics= false;
69 
70     @Option(name = "test-iteration-separator", description = "Separator used in between the test"
71             + " class name and the iteration number.")
72     private String mTestIterationSeparator = "$";
73 
74     @Option(name = "report-percentiles", description = "Additional percentiles of each metric to"
75             + " report in integers in the 0 - 100 range. Can be repeated.")
76     private Set<Integer> mPercentiles = new HashSet<>();
77 
78     @Option(
79             name = "strict-include-metric-filter",
80             description =
81                     "Regular expression that will be used for filtering the metrics from individual"
82                             + " test metrics and aggregated metrics.")
83     private Set<String> mStrictIncludeRegEx = new HashSet<>();
84 
85     private MetricUtility mMetricUtil = new MetricUtility();
86 
MetricFilePostProcessor()87     public MetricFilePostProcessor() {
88     }
89 
90     @VisibleForTesting
MetricFilePostProcessor(MetricUtility metricUtil)91     public MetricFilePostProcessor(MetricUtility metricUtil) {
92         mMetricUtil = metricUtil;
93     }
94 
95     @Override
processTestMetricsAndLogs( TestDescription testDescription, HashMap<String, Metric> testMetrics, Map<String, LogFile> testLogs)96     public Map<String, Metric.Builder> processTestMetricsAndLogs(
97             TestDescription testDescription,
98             HashMap<String, Metric> testMetrics,
99             Map<String, LogFile> testLogs) {
100 
101         // Store the test metric and use it for aggregation later at the end of
102         // test run.
103         if (mAggregateSimilarTests) {
104             mMetricUtil.storeTestMetrics(testDescription, testMetrics);
105         }
106 
107         // Write test metric to a file and log it.
108         if (mIsPerTestLogEnabled) {
109             Map<String, Metric> filteredTestMetrics =
110                     mStrictIncludeRegEx.size() > 0
111                             ? mMetricUtil.filterMetrics(testMetrics)
112                             : testMetrics;
113             writeMetricFile(filteredTestMetrics, testDescription.toString());
114         }
115 
116         return new HashMap<String, Builder>();
117     }
118 
119     @Override
processRunMetricsAndLogs( HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs)120     public Map<String, Builder> processRunMetricsAndLogs(
121             HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs) {
122         if (mIsRunLogEnabled) {
123             // Log the raw run metrics.
124             Map<String, Metric> filteredRawRunMetrics =
125                     mStrictIncludeRegEx.size() > 0
126                             ? mMetricUtil.filterMetrics(rawMetrics)
127                             : rawMetrics;
128             writeMetricFile(filteredRawRunMetrics, getRunName());
129 
130             // Log the aggregate run metrics.
131             if (mAggregateRunMetrics) {
132                 Map<String, Metric> aggregatedRunMetrics = mMetricUtil.aggregateMetrics(rawMetrics);
133                 Map<String, Metric> filteredAggregateRunMetrics =
134                         mStrictIncludeRegEx.size() > 0
135                                 ? mMetricUtil.filterMetrics(aggregatedRunMetrics)
136                                 : aggregatedRunMetrics;
137                 writeMetricFile(filteredAggregateRunMetrics, getRunName() + AGGREGATE_RUN_SUFFIX);
138             }
139         }
140 
141         // Aggregate similar tests metric at the run level, write it to results file and upload it.
142         if (mAggregateSimilarTests) {
143             File aggregateTestResultsFile = mMetricUtil
144                     .aggregateStoredTestMetricsAndWriteToFile(getRunName() + AGGREGATE_TEST_SUFFIX);
145             if (aggregateTestResultsFile != null) {
146                 try (InputStreamSource source = new FileInputStreamSource(aggregateTestResultsFile,
147                         true)) {
148                     testLog(aggregateTestResultsFile.getName(), LogDataType.CB_METRICS_FILE,
149                             source);
150                 }
151             }
152         }
153         return new HashMap<String, Builder>();
154     }
155 
156     /**
157      * Write the metrics to the results file and upload it.
158      *
159      * @param metrics
160      * @param testId
161      */
writeMetricFile(Map<String, Metric> metrics, String testId)162     public void writeMetricFile(Map<String, Metric> metrics, String testId) {
163         Map<String, String> compatibleMetrics = TfMetricProtoUtil
164                 .compatibleConvert(metrics);
165         File metricFile = mMetricUtil.writeResultsToFile(testId, testId,
166                 compatibleMetrics,
167                 null);
168         if (metricFile != null) {
169             try (InputStreamSource source = new FileInputStreamSource(metricFile,
170                     true)) {
171                 testLog(metricFile.getName(), LogDataType.CB_METRICS_FILE, source);
172             }
173         }
174     }
175 
176     @Override
setUp()177     public void setUp() {
178         mMetricUtil.setPercentiles(mPercentiles);
179         mMetricUtil.setIterationSeparator(mTestIterationSeparator);
180         mMetricUtil.buildMetricFilterPatterns(mStrictIncludeRegEx);
181     }
182 }
183