1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.compatibility.common.tradefed.result;
18 
19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
20 import com.android.json.stream.JsonWriter;
21 import com.android.tradefed.build.IBuildInfo;
22 import com.android.tradefed.config.Option;
23 import com.android.tradefed.config.OptionCopier;
24 import com.android.tradefed.invoker.IInvocationContext;
25 import com.android.tradefed.log.LogUtil.CLog;
26 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
27 import com.android.tradefed.result.IShardableListener;
28 import com.android.tradefed.result.TestDescription;
29 import com.android.tradefed.util.proto.TfMetricProtoUtil;
30 
31 import java.io.File;
32 import java.io.FileNotFoundException;
33 import java.io.IOException;
34 import java.io.PrintWriter;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.HashMap;
38 import java.util.LinkedList;
39 import java.util.Map;
40 
41 /**
42  * Write test metadata to the result/metadata folder.
43  */
44 public class MetadataReporter implements IShardableListener {
45 
46     @Option(name = "include-failure-time", description = "Include timing about tests that failed.")
47     private boolean mIncludeFailures = false;
48 
49     @Option(name = "min-test-duration", description = "Ignore test durations less than this.",
50             isTimeVal = true)
51     private long mMinTestDuration = 2 * 1000;
52 
53     private static final String METADATA_DIR = "metadata";
54     private CompatibilityBuildHelper mBuildHelper;
55     private File mMetadataDir;
56     private long mStartTime;
57     private String mCurrentModule;
58     private boolean mTestFailed;
59     private Collection<TestMetadata> mTestMetadata = new LinkedList<>();
60 
61     /**
62      * {@inheritDoc}
63      */
64     @Override
clone()65     public IShardableListener clone() {
66         MetadataReporter clone = new MetadataReporter();
67         OptionCopier.copyOptionsNoThrow(this, clone);
68         return clone;
69     }
70 
71     /**
72      * {@inheritDoc}
73      */
74     @Override
invocationStarted(IInvocationContext context)75     public void invocationStarted(IInvocationContext context) {
76         IBuildInfo buildInfo = context.getBuildInfos().get(0);
77         synchronized(this) {
78             if (mBuildHelper == null) {
79                 mBuildHelper = new CompatibilityBuildHelper(buildInfo);
80                 try {
81                     mMetadataDir = new File(mBuildHelper.getResultDir(), METADATA_DIR);
82                 } catch (FileNotFoundException e) {
83                     throw new RuntimeException("Metadata Directory was not created: " +
84                             mMetadataDir.getAbsolutePath());
85                 }
86             }
87         }
88     }
89 
90     /**
91      * {@inheritDoc}
92      */
93     @Override
testRunStarted(String id, int numTests)94     public void testRunStarted(String id, int numTests) {
95         this.mCurrentModule = id;
96     }
97 
98     @Override
testStarted(TestDescription test, long startTime)99     public void testStarted(TestDescription test, long startTime) {
100         mStartTime = startTime;
101         mTestFailed = false;
102     }
103 
104     /**
105      * {@inheritDoc}
106      */
107     @Override
testFailed(TestDescription test, String trace)108     public void testFailed(TestDescription test, String trace) {
109         mTestFailed = true;
110     }
111 
112     /**
113      * {@inheritDoc}
114      */
115     @Override
testIgnored(TestDescription test)116     public void testIgnored(TestDescription test) {
117         mTestFailed = true;
118     }
119 
120     /**
121      * {@inheritDoc}
122      */
123     @Override
testAssumptionFailure(TestDescription test, String trace)124     public void testAssumptionFailure(TestDescription test, String trace) {
125         mTestFailed = true;
126     }
127 
128     @Override
testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics)129     public void testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics) {
130         long duration = endTime - mStartTime;
131         if (mTestFailed && !mIncludeFailures) {
132             return;
133         }
134         if (duration < mMinTestDuration) {
135             return;
136         }
137 
138         TestMetadata metadata = new TestMetadata();
139         metadata.testId = buildTestId(test);
140         metadata.seconds = duration / 1000; // convert to second for reporting
141         mTestMetadata.add(metadata);
142     }
143 
144     /**
145      * {@inheritDoc}
146      */
147     @Override
testRunEnded(long elapsedTime, Map<String, String> metrics)148     public void testRunEnded(long elapsedTime, Map<String, String> metrics) {
149         testRunEnded(elapsedTime, TfMetricProtoUtil.upgradeConvert(metrics));
150     }
151 
152     /**
153      * {@inheritDoc}
154      */
155     @Override
testRunEnded(long elapsedTime, HashMap<String, Metric> metrics)156     public void testRunEnded(long elapsedTime, HashMap<String, Metric> metrics) {
157         if (!mTestMetadata.isEmpty()) {
158             tryWriteToFile(mBuildHelper, mCurrentModule, mMetadataDir, mTestMetadata);
159         }
160         mTestMetadata.clear();
161     }
162 
163     /** Information about a test's execution. */
164     public static class TestMetadata {
165         // The id of the test
166         String testId;
167         // The duration of the test.
168         long seconds;
169     }
170 
buildTestId(TestDescription test)171     private static String buildTestId(TestDescription test) {
172         return String.format("%s.%s", test.getClassName(), test.getTestName());
173     }
174 
tryWriteToFile( CompatibilityBuildHelper compatibilityBuildHelper, String moduleName, File metadataDir, Collection<TestMetadata> metadatas)175     private static void tryWriteToFile(
176             CompatibilityBuildHelper compatibilityBuildHelper,
177             String moduleName,
178             File metadataDir,
179             Collection<TestMetadata> metadatas) {
180 
181         metadataDir.mkdirs();
182 
183         String moduleFileName = moduleName + "." + System.currentTimeMillis() + ".json";
184         File metadataFile = new File(metadataDir, moduleFileName);
185         Map<String, String> buildAttributes =
186                 compatibilityBuildHelper.getBuildInfo().getBuildAttributes();
187         try (JsonWriter writer = new JsonWriter(new PrintWriter(metadataFile))) {
188             writer.beginObject();
189 
190             writer.name("fingerprint");
191             writer.value(buildAttributes.get("cts:build_fingerprint"));
192 
193             writer.name("product");
194             writer.value(buildAttributes.get("cts:build_product"));
195 
196             writer.name("build_id");
197             writer.value(buildAttributes.get("cts:build_id"));
198 
199             writer.name("suite_version");
200             writer.value(compatibilityBuildHelper.getSuiteVersion());
201 
202             writer.name("suite_name");
203             writer.value(compatibilityBuildHelper.getSuiteName());
204 
205             writer.name("suite_build");
206             writer.value(compatibilityBuildHelper.getSuiteBuild());
207 
208             writer.name("module_id");
209             writer.value(moduleName);
210 
211             writer.name("test");
212             writer.beginArray();
213             for (TestMetadata metadata : metadatas) {
214                 writer.beginObject();
215                 writer.name("id");
216                 writer.value(metadata.testId);
217                 writer.name("sec");
218                 writer.value(metadata.seconds);
219                 writer.endObject();
220             }
221             writer.endArray();
222 
223             writer.endObject();
224         } catch (IOException e) {
225             CLog.e("[%s] While saving metadata.", metadataFile.getAbsolutePath());
226             CLog.e(e);
227         }
228     }
229 
getTestMetadata()230     protected Collection<TestMetadata> getTestMetadata() {
231         return Collections.unmodifiableCollection(mTestMetadata);
232     }
233 }
234