1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.util; 17 18 import com.android.tradefed.log.ITestLogger; 19 import com.android.tradefed.log.LogUtil.CLog; 20 import com.android.tradefed.result.FileInputStreamSource; 21 import com.android.tradefed.result.InputStreamSource; 22 import com.android.tradefed.result.LogDataType; 23 24 import org.apache.commons.compress.archivers.ArchiveException; 25 import org.apache.commons.compress.archivers.ArchiveStreamFactory; 26 import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 27 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 28 import org.apache.commons.compress.utils.IOUtils; 29 30 import java.io.File; 31 import java.io.FileInputStream; 32 import java.io.FileNotFoundException; 33 import java.io.FileOutputStream; 34 import java.io.IOException; 35 import java.io.InputStream; 36 import java.io.OutputStream; 37 import java.util.Arrays; 38 import java.util.Collection; 39 import java.util.HashSet; 40 import java.util.LinkedList; 41 import java.util.List; 42 import java.util.function.Predicate; 43 import java.util.stream.Collectors; 44 import java.util.zip.GZIPInputStream; 45 import java.util.zip.GZIPOutputStream; 46 47 /** 48 * Utility to manipulate a tar file. It wraps the commons-compress in order to provide tar support. 49 */ 50 public class TarUtil { 51 52 private static final byte[] GZIP_SIGNATURE = {0x1f, (byte) 0x8b}; 53 54 /** 55 * Determine whether a file is a gzip. 56 * 57 * @param file the file to check. 58 * @return whether the file is a gzip. 59 * @throws IOException if the file could not be read. 60 */ isGzip(File file)61 public static boolean isGzip(File file) throws IOException { 62 byte[] signature = new byte[GZIP_SIGNATURE.length]; 63 try (InputStream stream = new FileInputStream(file)) { 64 if (stream.read(signature) != signature.length) { 65 return false; 66 } 67 } 68 return Arrays.equals(GZIP_SIGNATURE, signature); 69 } 70 71 /** 72 * Untar a tar file into a directory. tar.gz file needs to be {@link #unGzip(File, File)} first. 73 * 74 * @param inputFile The tar file to extract 75 * @param outputDir the directory where to put the extracted files. 76 * @return The list of {@link File} untarred. 77 * @throws FileNotFoundException 78 * @throws IOException 79 */ unTar(final File inputFile, final File outputDir)80 public static List<File> unTar(final File inputFile, final File outputDir) 81 throws FileNotFoundException, IOException { 82 return unTar(inputFile, outputDir, entryName -> true); 83 } 84 85 /** 86 * Untar a tar file into a directory. tar.gz file needs to be {@link #unGzip(File, File)} first. 87 * 88 * @param inputFile The tar file to extract 89 * @param outputDir the directory where to put the extracted files. 90 * @param fileNames the files to be extracted from the tar. 91 * @return The list of {@link File} untarred. 92 * @throws FileNotFoundException 93 * @throws IOException 94 */ unTar( final File inputFile, final File outputDir, Collection<String> fileNames)95 public static List<File> unTar( 96 final File inputFile, final File outputDir, Collection<String> fileNames) 97 throws FileNotFoundException, IOException { 98 HashSet<String> fileNameSet = new HashSet<>(fileNames); 99 List<File> untarredFiles = 100 unTar(inputFile, outputDir, (entryName) -> fileNameSet.contains(entryName)); 101 if (untarredFiles.size() != fileNameSet.size()) { 102 HashSet<File> untarredFileSet = new HashSet<File>(untarredFiles); 103 String missingFileNames = 104 fileNameSet.stream() 105 .filter(name -> !untarredFileSet.contains(new File(outputDir, name))) 106 .collect(Collectors.joining(" ")); 107 throw new IOException( 108 String.format("Couldn't find files in %s: %s", inputFile, missingFileNames)); 109 } 110 return untarredFiles; 111 } 112 113 /** 114 * Untar a tar file into a directory. tar.gz file needs to be {@link #unGzip(File, File)} first. 115 * 116 * @param inputFile The tar file to extract 117 * @param outputDir the directory where to put the extracted files. 118 * @param shouldExtract a function that takes an entry name as input and returns whether the 119 * entry should be extracted. 120 * @return The list of {@link File} untarred. 121 * @throws FileNotFoundException 122 * @throws IOException 123 */ unTar( final File inputFile, final File outputDir, Predicate<String> shouldExtract)124 private static List<File> unTar( 125 final File inputFile, final File outputDir, Predicate<String> shouldExtract) 126 throws FileNotFoundException, IOException { 127 CLog.i(String.format("Untaring %s to dir %s.", inputFile.getAbsolutePath(), 128 outputDir.getAbsolutePath())); 129 final List<File> untaredFiles = new LinkedList<File>(); 130 final InputStream is = new FileInputStream(inputFile); 131 TarArchiveInputStream debInputStream = null; 132 try { 133 debInputStream = (TarArchiveInputStream) 134 new ArchiveStreamFactory().createArchiveInputStream("tar", is); 135 TarArchiveEntry entry = null; 136 while ((entry = (TarArchiveEntry)debInputStream.getNextEntry()) != null) { 137 if (!shouldExtract.test(entry.getName())) { 138 continue; 139 } 140 final File outputFile = new File(outputDir, entry.getName()); 141 if (entry.isDirectory()) { 142 CLog.i(String.format("Attempting to write output directory %s.", 143 outputFile.getAbsolutePath())); 144 if (!outputFile.exists()) { 145 CLog.i(String.format("Attempting to create output directory %s.", 146 outputFile.getAbsolutePath())); 147 if (!outputFile.mkdirs()) { 148 throw new IllegalStateException( 149 String.format("Couldn't create directory %s.", 150 outputFile.getAbsolutePath())); 151 } 152 } 153 } else { 154 final File parent = outputFile.getParentFile(); 155 if (parent != null && !parent.exists()) { 156 if (!parent.mkdirs()) { 157 throw new IOException( 158 String.format( 159 "Couldn't create directory %s.", 160 parent.getAbsolutePath())); 161 } 162 } 163 if (entry.isSymbolicLink()) { 164 CLog.i( 165 String.format( 166 "Creating symbolic link %s -> %s.", 167 outputFile.getAbsolutePath(), entry.getLinkName())); 168 FileUtil.symlinkFile(new File(entry.getLinkName()), outputFile); 169 } else { 170 CLog.i(String.format("Creating file %s.", outputFile.getAbsolutePath())); 171 try (OutputStream outputFileStream = new FileOutputStream(outputFile)) { 172 IOUtils.copy(debInputStream, outputFileStream); 173 } 174 } 175 } 176 untaredFiles.add(outputFile); 177 } 178 } catch (ArchiveException ae) { 179 // We rethrow the ArchiveException through a more generic one. 180 throw new IOException(ae); 181 } finally { 182 StreamUtil.close(debInputStream); 183 StreamUtil.close(is); 184 } 185 return untaredFiles; 186 } 187 188 /** 189 * UnGZip a file: a tar.gz or tgz file will become a tar file. 190 * 191 * @param inputFile The {@link File} to ungzip 192 * @param outputDir The directory where to put the ungzipped file. 193 * @return a {@link File} pointing to the ungzipped file. 194 * @throws FileNotFoundException 195 * @throws IOException 196 */ unGzip(final File inputFile, final File outputDir)197 public static File unGzip(final File inputFile, final File outputDir) 198 throws FileNotFoundException, IOException { 199 CLog.i(String.format("Ungzipping %s to dir %s.", inputFile.getAbsolutePath(), 200 outputDir.getAbsolutePath())); 201 // Updates the file format from ".tar.gz" or ".tgz" to ".tar". 202 String outputFileName = 203 inputFile.getName().endsWith(".tgz") 204 ? inputFile.getName().replace(".tgz", ".tar") 205 : inputFile.getName().substring(0, inputFile.getName().length() - 3); 206 final File outputFile = new File(outputDir, outputFileName); 207 GZIPInputStream in = null; 208 FileOutputStream out = null; 209 try { 210 in = new GZIPInputStream(new FileInputStream(inputFile)); 211 out = new FileOutputStream(outputFile); 212 IOUtils.copy(in, out); 213 } finally { 214 StreamUtil.close(in); 215 StreamUtil.close(out); 216 } 217 return outputFile; 218 } 219 220 /** 221 * Utility function to gzip (.gz) a file. the .gz extension will be added to base file name. 222 * 223 * @param inputFile the {@link File} to be gzipped. 224 * @return the gzipped file. 225 * @throws IOException 226 */ gzip(final File inputFile)227 public static File gzip(final File inputFile) throws IOException { 228 File outputFile = FileUtil.createTempFile(inputFile.getName(), ".gz"); 229 GZIPOutputStream out = null; 230 FileInputStream in = null; 231 try { 232 out = new GZIPOutputStream(new FileOutputStream(outputFile)); 233 in = new FileInputStream(inputFile); 234 IOUtils.copy(in, out); 235 } catch (IOException e) { 236 // delete the tmp file if we failed to gzip. 237 FileUtil.deleteFile(outputFile); 238 throw e; 239 } finally { 240 StreamUtil.close(in); 241 StreamUtil.close(out); 242 } 243 return outputFile; 244 } 245 246 /** 247 * Untar and ungzip a tar.gz file to a temp directory. 248 * 249 * @param targzFile the tar.gz file to extract. 250 * @param nameHint the prefix for the temp directory. 251 * @return the temp directory. 252 * @throws FileNotFoundException 253 * @throws IOException 254 */ extractTarGzipToTemp(File targzFile, String nameHint)255 public static File extractTarGzipToTemp(File targzFile, String nameHint) 256 throws FileNotFoundException, IOException { 257 File unGzipDir = null; 258 File unTarDir = null; 259 try { 260 unGzipDir = FileUtil.createTempDir("extractTarGzip"); 261 File tarFile = TarUtil.unGzip(targzFile, unGzipDir); 262 unTarDir = FileUtil.createTempDir(nameHint); 263 TarUtil.unTar(tarFile, unTarDir); 264 return unTarDir; 265 } catch (IOException e) { 266 FileUtil.recursiveDelete(unTarDir); 267 throw e; 268 } finally { 269 FileUtil.recursiveDelete(unGzipDir); 270 } 271 } 272 273 /** 274 * Helper to extract and log to the reporters a tar gz file and its content 275 * 276 * @param listener the {@link ITestLogger} where to log the files. 277 * @param targzFile the tar.gz {@link File} that needs its content log. 278 * @param baseName the base name under which the files will be found. 279 */ extractAndLog(ITestLogger listener, File targzFile, String baseName)280 public static void extractAndLog(ITestLogger listener, File targzFile, String baseName) 281 throws FileNotFoundException, IOException { 282 // First upload the tar.gz file 283 InputStreamSource inputStream = null; 284 try { 285 inputStream = new FileInputStreamSource(targzFile); 286 listener.testLog(baseName, LogDataType.TAR_GZ, inputStream); 287 } finally { 288 StreamUtil.cancel(inputStream); 289 } 290 291 // extract and upload internal files. 292 File dir = FileUtil.createTempDir("tmp_tar_dir"); 293 File ungzipLog = null; 294 try { 295 ungzipLog = TarUtil.unGzip(targzFile, dir); 296 List<File> logs = TarUtil.unTar(ungzipLog, dir); 297 for (File f : logs) { 298 InputStreamSource s = null; 299 try { 300 s = new FileInputStreamSource(f); 301 listener.testLog(String.format("%s_%s", baseName, f.getName()), 302 LogDataType.TEXT, s); 303 } finally { 304 StreamUtil.cancel(s); 305 } 306 } 307 } finally { 308 FileUtil.deleteFile(ungzipLog); 309 FileUtil.recursiveDelete(dir); 310 } 311 } 312 } 313