1 /*
2  * Copyright (C) 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.result;
17 
18 import com.android.tradefed.build.IBuildInfo;
19 import com.android.tradefed.command.FatalHostError;
20 import com.android.tradefed.config.Option;
21 import com.android.tradefed.config.OptionClass;
22 import com.android.tradefed.invoker.IInvocationContext;
23 import com.android.tradefed.log.LogUtil.CLog;
24 import com.android.tradefed.result.error.InfraErrorIdentifier;
25 import com.android.tradefed.util.FileUtil;
26 
27 import java.io.File;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.util.ArrayList;
31 import java.util.List;
32 
33 /**
34  * Save logs to a file system.
35  */
36 @OptionClass(alias = "file-system-log-saver")
37 public class FileSystemLogSaver implements ILogSaver {
38 
39     @Option(name = "log-file-path", description = "root file system path to store log files.")
40     private File mRootReportDir = new File(System.getProperty("java.io.tmpdir"));
41 
42     @Option(name = "log-file-url", description =
43             "root http url of log files. Assumes files placed in log-file-path are visible via " +
44             "this url.")
45     private String mReportUrl = null;
46 
47     @Option(name = "log-retention-days", description =
48             "the number of days to keep saved log files.")
49     private Integer mLogRetentionDays = null;
50 
51     @Option(name = "compress-files", description =
52             "whether to compress files which are not already compressed")
53     private boolean mCompressFiles = true;
54 
55     private File mLogReportDir = null;
56     private LogFileSaver mFileSaver = null;
57 
58     /**
59      * A counter to control access to methods which modify this class's directories. Acting as a
60      * non-blocking reentrant lock, this int blocks access to sharded child invocations from
61      * attempting to create or delete directories.
62      */
63     private int mShardingLock = 0;
64 
65     /**
66      * {@inheritDoc}
67      *
68      * <p>Also, create a unique file system directory under {@code
69      * report-dir/[branch/]build-id/test-tag/unique_dir} for saving logs. If the creation of the
70      * directory fails, will write logs to a temporary directory on the local file system.
71      */
72     @Override
invocationStarted(IInvocationContext context)73     public void invocationStarted(IInvocationContext context) {
74         // Create log directory on first build info
75         IBuildInfo info = context.getBuildInfos().get(0);
76         synchronized (this) {
77             if (mShardingLock == 0) {
78                 mLogReportDir = createLogReportDir(info, mRootReportDir, mLogRetentionDays);
79                 mFileSaver = new LogFileSaver(mLogReportDir);
80             }
81             mShardingLock++;
82         }
83     }
84 
85     /**
86      * {@inheritDoc}
87      */
88     @Override
invocationEnded(long elapsedTime)89     public void invocationEnded(long elapsedTime) {
90         // no clean up needed.
91         synchronized (this) {
92             --mShardingLock;
93             if (mShardingLock < 0) {
94                 CLog.w(
95                         "Sharding lock exited more times than entered, possible "
96                                 + "unbalanced invocationStarted/Ended calls");
97             }
98         }
99     }
100 
101     /**
102      * {@inheritDoc}
103      * <p>
104      * Will zip and save the log file if {@link LogDataType#isCompressed()} returns false for
105      * {@code dataType} and {@code compressed-files} is set, otherwise, the stream will be saved
106      * uncompressed.
107      * </p>
108      */
109     @Override
saveLogData(String dataName, LogDataType dataType, InputStream dataStream)110     public LogFile saveLogData(String dataName, LogDataType dataType, InputStream dataStream)
111             throws IOException {
112         if (!mCompressFiles) {
113             File log = mFileSaver.saveLogData(dataName, dataType, dataStream);
114             return new LogFile(log.getAbsolutePath(), getUrl(log), dataType);
115         }
116         // saveAndGZip already handles dataType that do not need compression.
117         File log = mFileSaver.saveAndGZipLogData(dataName, dataType, dataStream);
118         return new LogFile(log.getAbsolutePath(), getUrl(log), true, dataType, log.length());
119     }
120 
121     @Override
saveLogFile(String dataName, LogDataType dataType, File fileToLog)122     public LogFile saveLogFile(String dataName, LogDataType dataType, File fileToLog)
123             throws IOException {
124         if (!mCompressFiles) {
125             File log = mFileSaver.saveLogFile(dataName, dataType, fileToLog);
126             return new LogFile(log.getAbsolutePath(), getUrl(log), dataType);
127         }
128         // saveAndGZip already handles dataType that do not need compression.
129         File log = mFileSaver.saveAndGZipLogFile(dataName, dataType, fileToLog);
130         return new LogFile(log.getAbsolutePath(), getUrl(log), true, dataType, log.length());
131     }
132 
133     /**
134      * {@inheritDoc}
135      */
136     @Override
getLogReportDir()137     public LogFile getLogReportDir() {
138         return new LogFile(mLogReportDir.getAbsolutePath(), getUrl(mLogReportDir), LogDataType.DIR);
139     }
140 
141     /**
142      * A helper method to create an invocation directory unique for saving logs.
143      * <p>
144      * Create a unique file system directory with the structure
145      * {@code report-dir/[branch/]build-id/test-tag/unique_dir} for saving logs.  If the creation
146      * of the directory fails, will write logs to a temporary directory on the local file system.
147      * </p>
148      *
149      * @param buildInfo the {@link IBuildInfo}
150      * @param reportDir the {@link File} for the report directory.
151      * @param logRetentionDays how many days logs should be kept for. If {@code null}, then no log
152      * retention file is writen.
153      * @return The directory created.
154      */
createLogReportDir(IBuildInfo buildInfo, File reportDir, Integer logRetentionDays)155     private File createLogReportDir(IBuildInfo buildInfo, File reportDir,
156             Integer logRetentionDays) {
157         File logReportDir;
158         // now create unique directory within the buildDir
159         try {
160             logReportDir = generateLogReportDir(buildInfo, reportDir);
161         } catch (IOException e) {
162             CLog.e("Unable to create unique directory in %s. Attempting to use tmp dir instead",
163                     reportDir.getAbsolutePath());
164             CLog.e(e);
165             // try to create one in a tmp location instead
166             logReportDir = createTempDir();
167         }
168 
169         boolean setPerms = FileUtil.chmodGroupRWX(logReportDir);
170         if (!setPerms) {
171             CLog.w(String.format("Failed to set dir %s to be group accessible.", logReportDir));
172         }
173 
174         if (logRetentionDays != null && logRetentionDays > 0) {
175             new RetentionFileSaver().writeRetentionFile(logReportDir, logRetentionDays);
176         }
177         CLog.d("Using log file directory %s", logReportDir.getAbsolutePath());
178         return logReportDir;
179     }
180 
181     /**
182      * An exposed method that allow subclass to customize generating path logic.
183      *
184      * @param buildInfo the {@link IBuildInfo}
185      * @param reportDir the {@link File} for the report directory.
186      * @return The directory created.
187      */
generateLogReportDir(IBuildInfo buildInfo, File reportDir)188     protected File generateLogReportDir(IBuildInfo buildInfo, File reportDir) throws IOException {
189         File buildDir = createBuildDir(buildInfo, reportDir);
190         return FileUtil.createTempDir("inv_", buildDir);
191     }
192 
193     /**
194      * A helper method to get or create a build directory based on the build info of the invocation.
195      * <p>
196      * Create a unique file system directory with the structure
197      * {@code report-dir/[branch/]build-id/test-tag} for saving logs.
198      * </p>
199      *
200      * @param buildInfo the {@link IBuildInfo}
201      * @param reportDir the {@link File} for the report directory.
202      * @return The directory where invocations for the same build should be saved.
203      * @throws IOException if the directory could not be created because a file with the same name
204      * exists or there are no permissions to write to it.
205      */
createBuildDir(IBuildInfo buildInfo, File reportDir)206     private File createBuildDir(IBuildInfo buildInfo, File reportDir) throws IOException {
207         List<String> pathSegments = new ArrayList<String>();
208         if (buildInfo.getBuildBranch() != null) {
209             pathSegments.add(buildInfo.getBuildBranch());
210         }
211         pathSegments.add(buildInfo.getBuildId());
212         pathSegments.add(buildInfo.getTestTag());
213         File buildReportDir = FileUtil.getFileForPath(reportDir,
214                 pathSegments.toArray(new String[] {}));
215 
216         // if buildReportDir already exists and is a directory - use it.
217         if (buildReportDir.exists()) {
218             if (buildReportDir.isDirectory()) {
219                 return buildReportDir;
220             } else {
221                 final String msg = String.format("Cannot create build-specific output dir %s. " +
222                         "File already exists.", buildReportDir.getAbsolutePath());
223                 CLog.w(msg);
224                 throw new IOException(msg);
225             }
226         } else {
227             if (FileUtil.mkdirsRWX(buildReportDir)) {
228                 return buildReportDir;
229             } else {
230                 final String msg = String.format("Cannot create build-specific output dir %s. " +
231                         "Failed to create directory.", buildReportDir.getAbsolutePath());
232                 CLog.w(msg);
233                 throw new IOException(msg);
234             }
235         }
236     }
237 
238     /**
239      * A helper method to create a temp directory for an invocation.
240      */
createTempDir()241     private File createTempDir() {
242         try {
243             return FileUtil.createTempDir("inv_");
244         } catch (IOException e) {
245             // Abort tradefed if a temp directory cannot be created
246             throw new FatalHostError(
247                     "Cannot create tmp directory.",
248                     e,
249                     InfraErrorIdentifier.LAB_HOST_FILESYSTEM_ERROR);
250         }
251     }
252 
253     /**
254      * A helper method that returns a URL for a given {@link File}.
255      *
256      * @param file the {@link File} of the log.
257      * @return The report directory path replaced with the report-url and path separators normalized
258      * (for Windows), or {@code null} if the report-url is not set, report-url ends with /,
259      * report-dir ends with {@link File#separator}, or the file is not in the report directory.
260      */
getUrl(File file)261     private String getUrl(File file) {
262         if (mReportUrl == null) {
263             return null;
264         }
265 
266         final String filePath = file.getAbsolutePath();
267         final String reportPath = mRootReportDir.getAbsolutePath();
268 
269         if (reportPath.endsWith(File.separator)) {
270             CLog.w("Cannot create URL. getAbsolutePath() returned %s which ends with %s",
271                     reportPath, File.separator);
272             return null;
273         }
274 
275         // Log file starts with the mReportDir path, so do a simple replacement.
276         if (filePath.startsWith(reportPath)) {
277             String relativePath = filePath.substring(reportPath.length());
278             // relativePath should start with /, drop the / from the url if it exists.
279             String url = mReportUrl;
280             if (url.endsWith("/")) {
281                 url =  url.substring(0, url.length() - 1);
282             }
283             // FIXME: Sanitize the URL.
284             return String.format("%s%s", url, relativePath.replace(File.separator, "/"));
285         }
286 
287         return null;
288     }
289 
290     /**
291      * Set the report directory. Exposed for unit testing.
292      */
setReportDir(File reportDir)293     void setReportDir(File reportDir) {
294         mRootReportDir = reportDir;
295     }
296 
297     /**
298      * Set the log retentionDays. Exposed for unit testing.
299      */
setLogRetentionDays(int logRetentionDays)300     void setLogRetentionDays(int logRetentionDays) {
301         mLogRetentionDays = logRetentionDays;
302     }
303 
setCompressFiles(boolean compress)304     public void setCompressFiles(boolean compress) {
305         mCompressFiles = compress;
306     }
307 }
308