1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.compatibility.common.tradefed.result.suite;
17 
18 import com.android.annotations.VisibleForTesting;
19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
20 import com.android.compatibility.common.util.DeviceInfo;
21 import com.android.tradefed.build.IBuildInfo;
22 import com.android.tradefed.cluster.SubprocessConfigBuilder;
23 import com.android.tradefed.config.IConfiguration;
24 import com.android.tradefed.config.Option;
25 import com.android.tradefed.config.OptionClass;
26 import com.android.tradefed.invoker.IInvocationContext;
27 import com.android.tradefed.invoker.ShardListener;
28 import com.android.tradefed.log.LogUtil.CLog;
29 import com.android.tradefed.result.ILogSaver;
30 import com.android.tradefed.result.ITestInvocationListener;
31 import com.android.tradefed.result.ITestSummaryListener;
32 import com.android.tradefed.result.InputStreamSource;
33 import com.android.tradefed.result.LogDataType;
34 import com.android.tradefed.result.LogFile;
35 import com.android.tradefed.result.LogFileSaver;
36 import com.android.tradefed.result.SnapshotInputStreamSource;
37 import com.android.tradefed.result.TestRunResult;
38 import com.android.tradefed.result.TestSummary;
39 import com.android.tradefed.result.suite.IFormatterGenerator;
40 import com.android.tradefed.result.suite.SuiteResultReporter;
41 import com.android.tradefed.result.suite.XmlFormattedGeneratorReporter;
42 import com.android.tradefed.util.FileUtil;
43 
44 import java.io.File;
45 import java.io.FileNotFoundException;
46 import java.io.IOException;
47 import java.io.InputStream;
48 import java.nio.file.Files;
49 import java.nio.file.Path;
50 import java.util.Collection;
51 import java.util.HashMap;
52 import java.util.HashSet;
53 import java.util.LinkedHashMap;
54 import java.util.List;
55 import java.util.Map;
56 import java.util.Set;
57 
58 /**
59  * Extension of {@link XmlFormattedGeneratorReporter} and {@link SuiteResultReporter} to handle
60  * Compatibility specific format and operations.
61  */
62 @OptionClass(alias = "result-reporter")
63 public class CertificationSuiteResultReporter extends XmlFormattedGeneratorReporter
64         implements ITestSummaryListener {
65 
66     // The known existing variant of suites.
67     // Adding a new variant requires approval from Android Partner team and Test Harness team.
68     private enum SuiteVariant {
69         CTS_ON_GSI("CTS_ON_GSI", "cts-on-gsi");
70 
71         private final String mReportDisplayName;
72         private final String mConfigName;
73 
SuiteVariant(String reportName, String configName)74         private SuiteVariant(String reportName, String configName) {
75             mReportDisplayName = reportName;
76             mConfigName = configName;
77         }
78 
getReportDisplayName()79         public String getReportDisplayName() {
80             return mReportDisplayName;
81         }
82 
getConfigName()83         public String getConfigName() {
84             return mConfigName;
85         }
86     }
87 
88     public static final String LATEST_LINK_NAME = "latest";
89     public static final String SUMMARY_FILE = "invocation_summary.txt";
90 
91     public static final String BUILD_FINGERPRINT = "cts:build_fingerprint";
92 
93     @Option(name = "result-server", description = "Server to publish test results.")
94     @Deprecated
95     private String mResultServer;
96 
97     @Option(
98             name = "disable-result-posting",
99             description = "Disable result posting into report server.")
100     @Deprecated
101     private boolean mDisableResultPosting = false;
102 
103     @Option(name = "include-test-log-tags", description = "Include test log tags in report.")
104     private boolean mIncludeTestLogTags = false;
105 
106     @Option(name = "use-log-saver", description = "Also saves generated result with log saver")
107     private boolean mUseLogSaver = false;
108 
109     @Option(name = "compress-logs", description = "Whether logs will be saved with compression")
110     private boolean mCompressLogs = true;
111 
112     public static final String INCLUDE_HTML_IN_ZIP = "html-in-zip";
113 
114     @Option(
115             name = INCLUDE_HTML_IN_ZIP,
116             description = "Whether failure summary report is included in the zip fie.")
117     @Deprecated
118     private boolean mIncludeHtml = false;
119 
120     @Option(
121             name = "result-attribute",
122             description =
123                     "Extra key-value pairs to be added as attributes and corresponding values "
124                             + "of the \"Result\" tag in the result XML.")
125     private Map<String, String> mResultAttributes = new HashMap<String, String>();
126 
127     // Should be removed for the S release.
128     @Option(
129             name = "cts-on-gsi-variant",
130             description =
131                     "Workaround for the R release to ensure the CTS-on-GSI report can be parsed "
132                             + "by the APFE.")
133     private boolean mCtsOnGsiVariant = false;
134 
135     private CompatibilityBuildHelper mBuildHelper;
136 
137     /** The directory containing the results */
138     private File mResultDir = null;
139     /** The directory containing the logs */
140     private File mLogDir = null;
141 
142     /** LogFileSaver to copy the file to the CTS results folder */
143     private LogFileSaver mTestLogSaver;
144 
145     private Map<LogFile, InputStreamSource> mPreInvocationLogs = new HashMap<>();
146     /** Invocation level Log saver to receive when files are logged */
147     private ILogSaver mLogSaver;
148 
149     private String mReferenceUrl;
150 
151     private Map<String, String> mLoggedFiles;
152 
153     private static final String[] RESULT_RESOURCES = {
154         "compatibility_result.css",
155         "compatibility_result.xsl",
156         "logo.png"
157     };
158 
CertificationSuiteResultReporter()159     public CertificationSuiteResultReporter() {
160         super();
161         mLoggedFiles = new LinkedHashMap<>();
162     }
163 
164     /**
165      * {@inheritDoc}
166      */
167     @Override
invocationStarted(IInvocationContext context)168     public final void invocationStarted(IInvocationContext context) {
169         super.invocationStarted(context);
170 
171         if (mBuildHelper == null) {
172             mBuildHelper = createBuildHelper();
173         }
174         if (mResultDir == null) {
175             initializeResultDirectories();
176         }
177     }
178 
179     @VisibleForTesting
createBuildHelper()180     CompatibilityBuildHelper createBuildHelper() {
181         return new CompatibilityBuildHelper(getPrimaryBuildInfo());
182     }
183 
184     /**
185      * {@inheritDoc}
186      */
187     @Override
testLog(String name, LogDataType type, InputStreamSource stream)188     public void testLog(String name, LogDataType type, InputStreamSource stream) {
189         if (name.endsWith(DeviceInfo.FILE_SUFFIX)) {
190             // Handle device info file case
191             testLogDeviceInfo(name, stream);
192             return;
193         }
194         if (mTestLogSaver == null) {
195             LogFile info = new LogFile(name, null, type);
196             mPreInvocationLogs.put(
197                     info, new SnapshotInputStreamSource(name, stream.createInputStream()));
198             return;
199         }
200         try {
201             File logFile = null;
202             if (mCompressLogs) {
203                 try (InputStream inputStream = stream.createInputStream()) {
204                     logFile = mTestLogSaver.saveAndGZipLogData(name, type, inputStream);
205                 }
206             } else {
207                 try (InputStream inputStream = stream.createInputStream()) {
208                     logFile = mTestLogSaver.saveLogData(name, type, inputStream);
209                 }
210             }
211             CLog.d("Saved logs for %s in %s", name, logFile.getAbsolutePath());
212         } catch (IOException e) {
213             CLog.e("Failed to write log for %s", name);
214             CLog.e(e);
215         }
216     }
217 
218     /** Write device-info files to the result */
testLogDeviceInfo(String name, InputStreamSource stream)219     private void testLogDeviceInfo(String name, InputStreamSource stream) {
220         try {
221             File ediDir = new File(mResultDir, DeviceInfo.RESULT_DIR_NAME);
222             ediDir.mkdirs();
223             File ediFile = new File(ediDir, name);
224             if (!ediFile.exists()) {
225                 // only write this file to the results if not already present
226                 FileUtil.writeToFile(stream.createInputStream(), ediFile);
227             }
228         } catch (IOException e) {
229             CLog.w("Failed to write device info %s to result", name);
230             CLog.e(e);
231         }
232     }
233 
234     /**
235      * {@inheritDoc}
236      */
237     @Override
testLogSaved(String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile)238     public void testLogSaved(String dataName, LogDataType dataType, InputStreamSource dataStream,
239             LogFile logFile) {
240         if (mIncludeTestLogTags) {
241             switch (dataType) {
242                 case BUGREPORT:
243                 case LOGCAT:
244                 case PNG:
245                     mLoggedFiles.put(dataName, logFile.getUrl());
246                     break;
247                 default:
248                     // Do nothing
249                     break;
250             }
251         }
252     }
253 
254     /**
255      * {@inheritDoc}
256      */
257     @Override
putSummary(List<TestSummary> summaries)258     public void putSummary(List<TestSummary> summaries) {
259         for (TestSummary summary : summaries) {
260             if (mReferenceUrl == null && summary.getSummary().getString() != null) {
261                 mReferenceUrl = summary.getSummary().getString();
262             }
263         }
264     }
265 
266     /**
267      * {@inheritDoc}
268      */
269     @Override
setLogSaver(ILogSaver saver)270     public void setLogSaver(ILogSaver saver) {
271         mLogSaver = saver;
272     }
273 
274     /**
275      * Create directory structure where results and logs will be written.
276      */
initializeResultDirectories()277     private void initializeResultDirectories() {
278         CLog.d("Initializing result directory");
279         try {
280             mResultDir = mBuildHelper.getResultDir();
281             if (mResultDir != null) {
282                 mResultDir.mkdirs();
283             }
284         } catch (FileNotFoundException e) {
285             throw new RuntimeException(e);
286         }
287 
288         if (mResultDir == null) {
289             throw new RuntimeException("Result Directory was not created");
290         }
291         if (!mResultDir.exists()) {
292             throw new RuntimeException("Result Directory was not created: " +
293                     mResultDir.getAbsolutePath());
294         }
295 
296         CLog.d("Results Directory: %s", mResultDir.getAbsolutePath());
297 
298         try {
299             mLogDir = mBuildHelper.getInvocationLogDir();
300         } catch (FileNotFoundException e) {
301             CLog.e(e);
302         }
303         if (mLogDir != null && mLogDir.mkdirs()) {
304             CLog.d("Created log dir %s", mLogDir.getAbsolutePath());
305         }
306         if (mLogDir == null || !mLogDir.exists()) {
307             throw new IllegalArgumentException(String.format("Could not create log dir %s",
308                     mLogDir.getAbsolutePath()));
309         }
310         // During sharding, we reach here before invocationStarted is called so the log_saver will
311         // be null at that point.
312         if (mTestLogSaver == null) {
313             mTestLogSaver = new LogFileSaver(mLogDir);
314             // Log all the early logs from before init.
315             for (LogFile earlyLog : mPreInvocationLogs.keySet()) {
316                 try (InputStreamSource source = mPreInvocationLogs.get(earlyLog)) {
317                     testLog(earlyLog.getPath(), earlyLog.getType(), source);
318                 }
319             }
320             mPreInvocationLogs.clear();
321         }
322     }
323 
324     @Override
createFormatter()325     public IFormatterGenerator createFormatter() {
326         return new CertificationResultXml(
327                 createSuiteName(mBuildHelper.getSuiteName()),
328                 mBuildHelper.getSuiteVersion(),
329                 createSuiteVariant(),
330                 mBuildHelper.getSuitePlan(),
331                 mBuildHelper.getSuiteBuild(),
332                 mReferenceUrl,
333                 getLogUrl(),
334                 mResultAttributes);
335     }
336 
337     @Override
preFormattingSetup(IFormatterGenerator formater)338     public void preFormattingSetup(IFormatterGenerator formater) {
339         super.preFormattingSetup(formater);
340         // Log the summary
341         TestSummary summary = getSummary();
342         try {
343             File summaryFile = new File(mResultDir, SUMMARY_FILE);
344             FileUtil.writeToFile(summary.getSummary().toString(), summaryFile);
345         } catch (IOException e) {
346             CLog.e("Failed to save the summary.");
347             CLog.e(e);
348         }
349 
350         copyDynamicConfigFiles();
351         copyFormattingFiles(mResultDir, mBuildHelper.getSuiteName());
352     }
353 
354     @Override
createResultDir()355     public File createResultDir() throws IOException {
356         return mResultDir;
357     }
358 
359     @Override
postFormattingStep(File resultDir, File reportFile)360     public void postFormattingStep(File resultDir, File reportFile) {
361         super.postFormattingStep(resultDir,reportFile);
362 
363         createChecksum(
364                 resultDir,
365                 getMergedTestRunResults(),
366                 getPrimaryBuildInfo().getBuildAttributes().get(BUILD_FINGERPRINT));
367 
368         Path latestLink = createLatestLinkDirectory(mResultDir.toPath());
369         if (latestLink != null) {
370             CLog.i("Latest results link: " + latestLink.toAbsolutePath());
371         }
372 
373         latestLink = createLatestLinkDirectory(mLogDir.toPath());
374         if (latestLink != null) {
375             CLog.i("Latest logs link: " + latestLink.toAbsolutePath());
376         }
377 
378         for (ITestInvocationListener resultReporter :
379                 getConfiguration().getTestInvocationListeners()) {
380             if (resultReporter instanceof CertificationReportCreator) {
381                 ((CertificationReportCreator) resultReporter).setReportFile(reportFile);
382             }
383             if (resultReporter instanceof ShardListener) {
384                 for (ITestInvocationListener subListener : ((ShardListener) resultReporter).getUnderlyingResultReporter()) {
385                     if (subListener instanceof CertificationReportCreator) {
386                         ((CertificationReportCreator) subListener).setReportFile(reportFile);
387                     }
388                 }
389             }
390         }
391     }
392 
393     /**
394      * Return the path in which log saver persists log files or null if
395      * logSaver is not enabled.
396      */
getLogUrl()397     private String getLogUrl() {
398         if (!mUseLogSaver || mLogSaver == null) {
399             return null;
400         }
401 
402         return mLogSaver.getLogReportDir().getUrl();
403     }
404 
405     /**
406      * Update the "latest" symlink to the newest result directory. CTS specific.
407      */
createLatestLinkDirectory(Path directory)408     private Path createLatestLinkDirectory(Path directory) {
409         Path link = null;
410 
411         Path parent = directory.getParent();
412 
413         if (parent != null) {
414             link = parent.resolve(LATEST_LINK_NAME);
415             try {
416                 // if latest already exists, we have to remove it before creating
417                 Files.deleteIfExists(link);
418                 Files.createSymbolicLink(link, directory);
419             } catch (IOException ioe) {
420                 CLog.e("Exception while attempting to create 'latest' link to: [%s]",
421                     directory);
422                 CLog.e(ioe);
423                 return null;
424             } catch (UnsupportedOperationException uoe) {
425                 CLog.e("Failed to create 'latest' symbolic link - unsupported operation");
426                 return null;
427             }
428         }
429         return link;
430     }
431 
432     /**
433      * move the dynamic config files to the results directory
434      */
copyDynamicConfigFiles()435     private void copyDynamicConfigFiles() {
436         File configDir = new File(mResultDir, "config");
437         if (!configDir.exists() && !configDir.mkdir()) {
438             CLog.w(
439                     "Failed to make dynamic config directory \"%s\" in the result.",
440                     configDir.getAbsolutePath());
441         }
442 
443         Set<String> uniqueModules = new HashSet<>();
444         // Check each build of the invocation, in case of multi-device invocation.
445         for (IBuildInfo buildInfo : getInvocationContext().getBuildInfos()) {
446             CompatibilityBuildHelper helper = new CompatibilityBuildHelper(buildInfo);
447             Map<String, File> dcFiles = helper.getDynamicConfigFiles();
448             for (String moduleName : dcFiles.keySet()) {
449                 File srcFile = dcFiles.get(moduleName);
450                 if (!uniqueModules.contains(moduleName)) {
451                     // have not seen config for this module yet, copy into result
452                     File destFile = new File(configDir, moduleName + ".dynamic");
453                     if (destFile.exists()) {
454                         continue;
455                     }
456                     try {
457                         FileUtil.copyFile(srcFile, destFile);
458                         uniqueModules.add(moduleName); // Add to uniqueModules if copy succeeds
459                     } catch (IOException e) {
460                         CLog.w("Failure when copying config file \"%s\" to \"%s\" for module %s",
461                                 srcFile.getAbsolutePath(), destFile.getAbsolutePath(), moduleName);
462                         CLog.e(e);
463                     }
464                 }
465                 FileUtil.deleteFile(srcFile);
466             }
467         }
468     }
469 
470     /**
471      * Copy the xml formatting files stored in this jar to the results directory. CTS specific.
472      *
473      * @param resultsDir
474      */
copyFormattingFiles(File resultsDir, String suiteName)475     private void copyFormattingFiles(File resultsDir, String suiteName) {
476         for (String resultFileName : RESULT_RESOURCES) {
477             InputStream configStream = CertificationResultXml.class.getResourceAsStream(
478                     String.format("/report/%s-%s", suiteName, resultFileName));
479             if (configStream == null) {
480                 // If suite specific files are not available, fallback to common.
481                 configStream = CertificationResultXml.class.getResourceAsStream(
482                     String.format("/report/%s", resultFileName));
483             }
484             if (configStream != null) {
485                 File resultFile = new File(resultsDir, resultFileName);
486                 try {
487                     FileUtil.writeToFile(configStream, resultFile);
488                 } catch (IOException e) {
489                     CLog.w("Failed to write %s to file", resultFileName);
490                 }
491             } else {
492                 CLog.w("Failed to load %s from jar", resultFileName);
493             }
494         }
495     }
496 
497     /**
498      * Generates a checksum files based on the results.
499      */
createChecksum(File resultDir, Collection<TestRunResult> results, String buildFingerprint)500     private void createChecksum(File resultDir, Collection<TestRunResult> results,
501             String buildFingerprint) {
502         CertificationChecksumHelper.tryCreateChecksum(resultDir, results, buildFingerprint);
503     }
504 
createSuiteName(String originalSuiteName)505     private String createSuiteName(String originalSuiteName) {
506         if (mCtsOnGsiVariant) {
507             String commandLine = getConfiguration().getCommandLine();
508             // SubprocessConfigBuilder is added to support ATS current way of running things.
509             // It won't be needed after the R release.
510             if (commandLine.startsWith("cts-on-gsi")
511                     || commandLine.startsWith(
512                             SubprocessConfigBuilder.createConfigName("cts-on-gsi"))) {
513                 return "VTS";
514             }
515         }
516         return originalSuiteName;
517     }
518 
createSuiteVariant()519     private String createSuiteVariant() {
520         IConfiguration currentConfig = getConfiguration();
521         String commandLine = currentConfig.getCommandLine();
522         for (SuiteVariant var : SuiteVariant.values()) {
523             if (commandLine.startsWith(var.getConfigName() + " ")
524                     || commandLine.equals(var.getConfigName())) {
525                 return var.getReportDisplayName();
526             }
527         }
528         return null;
529     }
530 }
531