1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.cluster;
17 
18 import com.android.annotations.VisibleForTesting;
19 import com.android.tradefed.build.BuildRetrievalError;
20 import com.android.tradefed.build.IBuildInfo;
21 import com.android.tradefed.build.IBuildProvider;
22 import com.android.tradefed.config.Option;
23 import com.android.tradefed.config.OptionClass;
24 import com.android.tradefed.invoker.logger.InvocationLocal;
25 import com.android.tradefed.log.LogUtil.CLog;
26 import com.android.tradefed.util.FileUtil;
27 import com.android.tradefed.util.FuseUtil;
28 import com.android.tradefed.util.TarUtil;
29 import com.android.tradefed.util.ZipUtil2;
30 import org.apache.commons.compress.archivers.zip.ZipFile;
31 import org.json.JSONException;
32 import org.json.JSONObject;
33 
34 import java.io.File;
35 import java.io.IOException;
36 import java.io.UncheckedIOException;
37 import java.util.ArrayList;
38 import java.util.List;
39 import java.util.concurrent.ConcurrentHashMap;
40 
41 /** A {@link IBuildProvider} to download TFC test resources. */
42 @OptionClass(alias = "cluster", global_namespace = false)
43 public class ClusterBuildProvider implements IBuildProvider {
44 
45     private static final String DEFAULT_FILE_VERSION = "0";
46 
47     @Option(name = "root-dir", description = "A root directory", mandatory = true)
48     private File mRootDir;
49 
50     @Option(
51             name = "test-resource",
52             description = "A list of JSON-serialized test resource objects",
53             mandatory = true)
54     private List<String> mTestResources = new ArrayList<>();
55 
56     @Option(name = "build-id", description = "Build ID")
57     private String mBuildId = IBuildInfo.UNKNOWN_BUILD_ID;
58 
59     @Option(name = "build-target", description = "Build target name")
60     private String mBuildTarget = "stub";
61 
62     // The keys are the URLs; the values are the downloaded files shared among all build providers
63     // in the invocation.
64     // TODO(b/139876060): Use dynamic download when it supports caching HTTPS and GCS files.
65     @VisibleForTesting
66     static final InvocationLocal<ConcurrentHashMap<String, File>> sDownloadCache =
67             new InvocationLocal<ConcurrentHashMap<String, File>>() {
68                 @Override
69                 protected ConcurrentHashMap<String, File> initialValue() {
70                     return new ConcurrentHashMap<String, File>();
71                 }
72             };
73 
74     // The keys are the resource names; the values are the files and directories.
75     @VisibleForTesting
76     static final InvocationLocal<ConcurrentHashMap<String, File>> sCreatedResources =
77             new InvocationLocal<ConcurrentHashMap<String, File>>() {
78                 @Override
79                 protected ConcurrentHashMap<String, File> initialValue() {
80                     return new ConcurrentHashMap<String, File>();
81                 }
82             };
83 
parseTestResources()84     private List<TestResource> parseTestResources() {
85         final List<TestResource> objs = new ArrayList<>();
86         for (final String s : mTestResources) {
87             try {
88                 final JSONObject json = new JSONObject(s);
89                 final TestResource obj = TestResource.fromJson(json);
90                 objs.add(obj);
91             } catch (JSONException e) {
92                 throw new RuntimeException("Failed to parse a test resource option: " + s, e);
93             }
94         }
95         return objs;
96     }
97 
98     @Override
getBuild()99     public IBuildInfo getBuild() throws BuildRetrievalError {
100         mRootDir.mkdirs();
101         final ClusterBuildInfo buildInfo = new ClusterBuildInfo(mRootDir, mBuildId, mBuildTarget);
102         final TestResourceDownloader downloader = createTestResourceDownloader();
103         final ConcurrentHashMap<String, File> cache = sDownloadCache.get();
104         final ConcurrentHashMap<String, File> createdResources = sCreatedResources.get();
105 
106         final List<TestResource> testResources = parseTestResources();
107         for (TestResource resource : testResources) {
108             // For backward compatibility.
109             if (resource.getName().endsWith(".zip") && !resource.getDecompress()) {
110                 resource =
111                         new TestResource(
112                                 resource.getName(),
113                                 resource.getUrl(),
114                                 true,
115                                 new File(resource.getName()).getParent(),
116                                 resource.mountZip(),
117                                 resource.getDecompressFiles());
118             }
119             // Validate the paths before the file operations.
120             final File resourceFile = resource.getFile(mRootDir);
121             validateTestResourceFile(mRootDir, resourceFile);
122             if (resource.getDecompress()) {
123                 File dir = resource.getDecompressDir(mRootDir);
124                 validateTestResourceFile(mRootDir, dir);
125                 for (String name : resource.getDecompressFiles()) {
126                     validateTestResourceFile(dir, new File(dir, name));
127                 }
128             }
129             // Download and decompress.
130             File file;
131             try {
132                 File cachedFile = retrieveFile(resource.getUrl(), cache, downloader, resourceFile);
133                 file = prepareTestResource(resource, createdResources, cachedFile, buildInfo);
134             } catch (UncheckedIOException e) {
135                 throw new BuildRetrievalError("failed to get test resources", e);
136             }
137             buildInfo.setFile(resource.getName(), file, DEFAULT_FILE_VERSION);
138         }
139         return buildInfo;
140     }
141 
142     /** Check if a resource file is under the working directory. */
validateTestResourceFile(File workDir, File file)143     private static void validateTestResourceFile(File workDir, File file)
144             throws BuildRetrievalError {
145         if (!file.toPath().normalize().startsWith(workDir.toPath().normalize())) {
146             throw new BuildRetrievalError(file + " is outside of working directory.");
147         }
148     }
149 
150     /**
151      * Retrieve a file from cache or URL.
152      *
153      * <p>If the URL is in the cache, this method returns the cached file. Otherwise, it downloads
154      * and adds the file to the cache. If any file operation fails, this method throws {@link
155      * UncheckedIOException}.
156      *
157      * @param downloadUrl the file to be retrieved.
158      * @param cache the cache that maps URLs to files.
159      * @param downloader the downloader that gets the file.
160      * @param downloadDest the file to be created if the URL isn't in the cache.
161      * @return the cached or downloaded file.
162      */
retrieveFile( String downloadUrl, ConcurrentHashMap<String, File> cache, TestResourceDownloader downloader, File downloadDest)163     private File retrieveFile(
164             String downloadUrl,
165             ConcurrentHashMap<String, File> cache,
166             TestResourceDownloader downloader,
167             File downloadDest) {
168         return cache.computeIfAbsent(
169                 downloadUrl,
170                 url -> {
171                     CLog.i("Download %s from %s.", downloadDest, url);
172                     try {
173                         downloader.download(url, downloadDest);
174                     } catch (IOException e) {
175                         throw new UncheckedIOException(e);
176                     }
177                     return downloadDest;
178                 });
179     }
180 
181     /**
182      * Create a resource file from cache and decompress it if needed.
183      *
184      * <p>If any file operation fails, this method throws {@link UncheckedIOException}.
185      *
186      * @param resource the resource to be created.
187      * @param createdResources the map from created resource names to paths.
188      * @param source the local cache of the file.
189      * @param buildInfo the current build info.
190      * @return the file or directory to be added to build info.
191      */
prepareTestResource( TestResource resource, ConcurrentHashMap<String, File> createdResources, File source, ClusterBuildInfo buildInfo)192     private File prepareTestResource(
193             TestResource resource,
194             ConcurrentHashMap<String, File> createdResources,
195             File source,
196             ClusterBuildInfo buildInfo) {
197         return createdResources.computeIfAbsent(
198                 resource.getName(),
199                 name -> {
200                     // Create the file regardless of the decompress flag.
201                     final File file = resource.getFile(mRootDir);
202                     if (!source.equals(file)) {
203                         if (file.exists()) {
204                             CLog.w("Overwrite %s.", name);
205                             file.delete();
206                         } else {
207                             CLog.i("Create %s.", name);
208                             file.getParentFile().mkdirs();
209                         }
210                         try {
211                             FileUtil.hardlinkFile(source, file);
212                         } catch (IOException e) {
213                             throw new UncheckedIOException(e);
214                         }
215                     }
216                     // Decompress if needed.
217                     if (resource.getDecompress()) {
218                         final File dir = resource.getDecompressDir(mRootDir);
219                         try {
220                             decompressArchive(
221                                     file,
222                                     dir,
223                                     resource.mountZip(),
224                                     resource.getDecompressFiles(),
225                                     buildInfo);
226                         } catch (IOException e) {
227                             throw new UncheckedIOException(e);
228                         }
229                         return dir;
230                     }
231                     return file;
232                 });
233     }
234 
235     @VisibleForTesting
236     FuseUtil getFuseUtil() {
237         return new FuseUtil();
238     }
239 
240     /**
241      * Extracts a zip or a gzip to a directory.
242      *
243      * @param archive the archive to be extracted.
244      * @param destDir the directory where the archive is extracted.
245      * @param mountZip whether to mount the zip or extract it.
246      * @param fileNames the files to be extracted from the archive. If the list is empty, all files
247      *     are extracted.
248      * @param buildInfo the {@link ClusterBuildInfo} that records mounted zip files.
249      * @throws IOException if any file operation fails or any file name is not found in the archive.
250      */
251     private void decompressArchive(
252             File archive,
253             File destDir,
254             boolean mountZip,
255             List<String> fileNames,
256             ClusterBuildInfo buildInfo)
257             throws IOException {
258         if (!destDir.exists()) {
259             if (!destDir.mkdirs()) {
260                 CLog.e("Cannot create %s.", destDir);
261             }
262         }
263 
264         if (TarUtil.isGzip(archive)) {
265             decompressTarGzip(archive, destDir, fileNames);
266             return;
267         }
268 
269         if (mountZip) {
270             FuseUtil fuseUtil = getFuseUtil();
271             if (fuseUtil.canMountZip()) {
272                 File mountDir = mountZip(fuseUtil, archive, buildInfo);
273                 // Build a shadow directory structure with symlinks to allow a test to create files
274                 // within it. This allows xTS to write result files under its own directory
275                 // structure (e.g. android-cts/results).
276                 symlinkFiles(mountDir, destDir, fileNames);
277                 return;
278             }
279             CLog.w("Mounting zip requested but not supported; falling back to extracting...");
280         }
281 
282         decompressZip(archive, destDir, fileNames);
283     }
284 
285     private void decompressTarGzip(File archive, File destDir, List<String> fileNames)
286             throws IOException {
287         File unGzipDir = FileUtil.createTempDir("ClusterBuildProviderUnGzip");
288         try {
289             File tar = TarUtil.unGzip(archive, unGzipDir);
290             if (fileNames.isEmpty()) {
291                 TarUtil.unTar(tar, destDir);
292             } else {
293                 TarUtil.unTar(tar, destDir, fileNames);
294             }
295         } finally {
296             FileUtil.recursiveDelete(unGzipDir);
297         }
298     }
299 
300     /** Mount a zip to a temporary directory if zip mounting is supported. */
301     private File mountZip(FuseUtil fuseUtil, File archive, ClusterBuildInfo buildInfo)
302             throws IOException {
303         File mountDir = FileUtil.createTempDir("ClusterBuildProviderZipMount");
304         buildInfo.addZipMount(mountDir);
305         CLog.i("Mounting %s to %s...", archive, mountDir);
306         fuseUtil.mountZip(archive, mountDir);
307         return mountDir;
308     }
309 
310     private void symlinkFiles(File origDir, File destDir, List<String> fileNames)
311             throws IOException {
312         if (fileNames.isEmpty()) {
313             CLog.i("Recursive symlink %s to %s...", origDir, destDir);
314             FileUtil.recursiveSymlink(origDir, destDir);
315         } else {
316             for (String name : fileNames) {
317                 File origFile = new File(origDir, name);
318                 if (!origFile.exists()) {
319                     throw new IOException(String.format("%s does not exist.", origFile));
320                 }
321                 File destFile = new File(destDir, name);
322                 CLog.i("Symlink %s to %s", origFile, destFile);
323                 destFile.getParentFile().mkdirs();
324                 FileUtil.symlinkFile(origFile, destFile);
325             }
326         }
327     }
328 
329     private void decompressZip(File archive, File destDir, List<String> fileNames)
330             throws IOException {
331         try (ZipFile zip = new ZipFile(archive)) {
332             if (fileNames.isEmpty()) {
333                 CLog.i("Extracting %s to %s...", archive, destDir);
334                 ZipUtil2.extractZip(zip, destDir);
335             } else {
336                 for (String name : fileNames) {
337                     File destFile = new File(destDir, name);
338                     CLog.i("Extracting %s from %s to %s", name, archive, destFile);
339                     destFile.getParentFile().mkdirs();
340                     if (!ZipUtil2.extractFileFromZip(zip, name, destFile)) {
341                         throw new IOException(
342                                 String.format("%s is not found in %s", name, archive));
343                     }
344                 }
345             }
346         }
347     }
348 
349     @Override
350     public void buildNotTested(IBuildInfo info) {}
351 
352     @Override
353     public void cleanUp(IBuildInfo info) {
354         if (!(info instanceof ClusterBuildInfo)) {
355             throw new IllegalArgumentException("info is not an instance of ClusterBuildInfo");
356         }
357         FuseUtil fuseUtil = getFuseUtil();
358         for (File dir : ((ClusterBuildInfo) info).getZipMounts()) {
359             fuseUtil.unmountZip(dir);
360             FileUtil.recursiveDelete(dir);
361         }
362     }
363 
364     @VisibleForTesting
365     TestResourceDownloader createTestResourceDownloader() {
366         return new TestResourceDownloader();
367     }
368 
369     @VisibleForTesting
370     void setRootDir(File rootDir) {
371         mRootDir = rootDir;
372     }
373 
374     @VisibleForTesting
375     void addTestResource(TestResource resource) throws JSONException {
376         mTestResources.add(resource.toJson().toString());
377     }
378 
379     @VisibleForTesting
380     List<TestResource> getTestResources() {
381         return parseTestResources();
382     }
383 }
384