1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.util; 18 19 import com.android.tradefed.build.BuildRetrievalError; 20 21 import com.google.api.client.googleapis.batch.BatchCallback; 22 import com.google.api.client.googleapis.batch.BatchRequest; 23 import com.google.api.client.http.HttpHeaders; 24 import com.google.api.client.http.InputStreamContent; 25 import com.google.api.services.storage.Storage; 26 import com.google.api.services.storage.Storage.Objects.List; 27 import com.google.api.services.storage.model.Objects; 28 import com.google.api.services.storage.model.StorageObject; 29 30 import org.junit.After; 31 import org.junit.Assert; 32 import org.junit.Before; 33 import org.junit.Test; 34 import org.junit.runner.RunWith; 35 import org.junit.runners.JUnit4; 36 37 import java.io.ByteArrayInputStream; 38 import java.io.File; 39 import java.io.IOException; 40 import java.io.InputStream; 41 import java.nio.file.Paths; 42 import java.util.Collections; 43 44 /** {@link GCSFileDownloader} functional test. */ 45 @RunWith(JUnit4.class) 46 public class GCSFileDownloaderFuncTest { 47 48 private static final String BUCKET_NAME = "tradefed_function_test"; 49 private static final String FILE_NAME1 = "a_host_config.xml"; 50 private static final String FILE_NAME2 = "file2.txt"; 51 private static final String FILE_NAME3 = "file3.txt"; 52 private static final String FILE_NAME4 = "file4.txt"; 53 private static final String FILE_NAME5 = "file5.txt"; 54 private static final String FILE_NAME6 = "file6.txt"; 55 private static final String EMPTY_FILE = "empty.txt"; 56 private static final String FOLDER_NAME1 = "folder1"; 57 private static final String FOLDER_NAME2 = "folder2"; 58 private static final String FOLDER_NAME3 = "folder3"; 59 private static final String FOLDER_NAME4 = "folder4"; 60 private static final String FILE_CONTENT = "Hello World!"; 61 62 private GCSFileDownloader mDownloader; 63 private String mRemoteRoot; 64 private File mLocalRoot; 65 private Storage mStorage; 66 createFile( Storage storage, String content, String bucketName, String... pathSegs)67 private static void createFile( 68 Storage storage, String content, String bucketName, String... pathSegs) 69 throws IOException { 70 String path = String.join("/", pathSegs); 71 StorageObject object = new StorageObject(); 72 object.setName(path); 73 storage.objects() 74 .insert( 75 bucketName, 76 object, 77 new InputStreamContent(null, new ByteArrayInputStream(content.getBytes()))) 78 .execute(); 79 } 80 81 @Before setUp()82 public void setUp() throws IOException { 83 File tempFile = 84 FileUtil.createTempFile(GCSFileDownloaderFuncTest.class.getSimpleName(), ""); 85 mRemoteRoot = tempFile.getName(); 86 FileUtil.deleteFile(tempFile); 87 mDownloader = 88 new GCSFileDownloader() { 89 90 @Override 91 File createTempFile(String remoteFilePath, File rootDir) 92 throws BuildRetrievalError { 93 try { 94 File tmpFile = 95 FileUtil.createTempFileForRemote(remoteFilePath, mLocalRoot); 96 tmpFile.delete(); 97 return tmpFile; 98 } catch (IOException e) { 99 throw new BuildRetrievalError(e.getMessage(), e); 100 } 101 } 102 }; 103 mStorage = 104 mDownloader.getStorage( 105 Collections.singleton( 106 "https://www.googleapis.com/auth/devstorage.read_write")); 107 createFile(mStorage, FILE_CONTENT, BUCKET_NAME, mRemoteRoot, FILE_NAME1); 108 createFile(mStorage, FILE_NAME2, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME2); 109 createFile(mStorage, FILE_NAME3, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME3); 110 // Create a special case condition where folder name is also a file name. 111 createFile(mStorage, FILE_NAME3, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FOLDER_NAME2); 112 createFile( 113 mStorage, 114 FILE_NAME4, 115 BUCKET_NAME, 116 mRemoteRoot, 117 FOLDER_NAME1, 118 FOLDER_NAME2, 119 FILE_NAME4); 120 // folder3 is a special file, it's a text file with 0 size. 121 // it is the same as a folder created from UI. 122 createFile(mStorage, "", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FOLDER_NAME3 + '/'); 123 createFile( 124 mStorage, 125 FILE_NAME5, 126 BUCKET_NAME, 127 mRemoteRoot, 128 FOLDER_NAME1, 129 FOLDER_NAME3, 130 FILE_NAME5); 131 // folder4 is a special file, it's a text file with 0 size. 132 // it is the same as a folder created from UI. 133 createFile( 134 mStorage, 135 "", 136 BUCKET_NAME, 137 mRemoteRoot, 138 FOLDER_NAME1, 139 FOLDER_NAME3, 140 FOLDER_NAME4 + "/"); 141 createFile( 142 mStorage, 143 FILE_NAME6, 144 BUCKET_NAME, 145 mRemoteRoot, 146 FOLDER_NAME1, 147 FOLDER_NAME3, 148 FOLDER_NAME4, 149 FILE_NAME6); 150 151 createFile(mStorage, "", BUCKET_NAME, mRemoteRoot, EMPTY_FILE); 152 mLocalRoot = FileUtil.createTempDir(GCSFileDownloaderFuncTest.class.getSimpleName()); 153 } 154 155 @After tearDown()156 public void tearDown() throws IOException { 157 FileUtil.recursiveDelete(mLocalRoot); 158 String pageToken = null; 159 BatchRequest batchRequest = mStorage.batch(); 160 161 while (true) { 162 List listOperation = mStorage.objects().list(BUCKET_NAME).setPrefix(mRemoteRoot); 163 if (pageToken == null) { 164 listOperation.setPageToken(pageToken); 165 } 166 Objects objects = listOperation.execute(); 167 for (StorageObject object : objects.getItems()) { 168 batchRequest.queue( 169 mStorage.objects().delete(BUCKET_NAME, object.getName()).buildHttpRequest(), 170 Void.class, 171 IOException.class, 172 new BatchCallback<Void, IOException>() { 173 @Override 174 public void onSuccess(Void arg0, HttpHeaders arg1) throws IOException {} 175 176 @Override 177 public void onFailure(IOException e, HttpHeaders arg1) 178 throws IOException { 179 throw e; 180 } 181 }); 182 } 183 pageToken = objects.getNextPageToken(); 184 if (pageToken == null) { 185 batchRequest.execute(); 186 return; 187 } 188 } 189 } 190 191 @Test testDownloadFile_streamOutput()192 public void testDownloadFile_streamOutput() throws Exception { 193 InputStream inputStream = 194 mDownloader.downloadFile(BUCKET_NAME, mRemoteRoot + "/" + FILE_NAME1); 195 String content = StreamUtil.getStringFromStream(inputStream); 196 Assert.assertEquals(FILE_CONTENT, content); 197 inputStream.reset(); 198 } 199 200 @Test testDownloadFile_streamOutput_notExist()201 public void testDownloadFile_streamOutput_notExist() throws Exception { 202 try { 203 mDownloader.downloadFile(BUCKET_NAME, mRemoteRoot + "/" + "non_exist_file"); 204 Assert.fail("Should throw IOException."); 205 } catch (IOException e) { 206 // Expect IOException 207 } 208 } 209 210 @Test testGetRemoteFileMetaData()211 public void testGetRemoteFileMetaData() throws Exception { 212 String filename = mRemoteRoot + "/" + FILE_NAME1; 213 StorageObject object = mDownloader.getRemoteFileMetaData(BUCKET_NAME, filename); 214 Assert.assertEquals(filename, object.getName()); 215 } 216 217 @Test testGetRemoteFileMetaData_notExist()218 public void testGetRemoteFileMetaData_notExist() throws Exception { 219 String filename = mRemoteRoot + "/" + "not_exist"; 220 StorageObject object = mDownloader.getRemoteFileMetaData(BUCKET_NAME, filename); 221 Assert.assertNull(object); 222 } 223 224 @Test testIsRemoteFolder()225 public void testIsRemoteFolder() throws Exception { 226 Assert.assertFalse( 227 mDownloader.isRemoteFolder( 228 BUCKET_NAME, Paths.get(mRemoteRoot, FILE_NAME1).toString())); 229 Assert.assertTrue( 230 mDownloader.isRemoteFolder( 231 BUCKET_NAME, Paths.get(mRemoteRoot, FOLDER_NAME1).toString())); 232 } 233 234 @Test testDownloadFile()235 public void testDownloadFile() throws Exception { 236 File localFile = 237 mDownloader.downloadFile( 238 String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1)); 239 String content = FileUtil.readStringFromFile(localFile); 240 Assert.assertEquals(FILE_CONTENT, content); 241 } 242 243 @Test testDownloadFile_nonExist()244 public void testDownloadFile_nonExist() throws Exception { 245 try { 246 mDownloader.downloadFile( 247 String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, "non_exist_file")); 248 Assert.fail("Should throw BuildRetrievalError."); 249 } catch (BuildRetrievalError e) { 250 // Expect BuildRetrievalError 251 } 252 } 253 254 @Test testDownloadFile_folder()255 public void testDownloadFile_folder() throws Exception { 256 File localFile = 257 mDownloader.downloadFile( 258 String.format("gs://%s/%s/%s/", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1)); 259 checkDownloadedFolder(localFile); 260 } 261 262 @Test testDownloadFile_folderNotsanitize()263 public void testDownloadFile_folderNotsanitize() throws Exception { 264 File localFile = 265 mDownloader.downloadFile( 266 String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1)); 267 checkDownloadedFolder(localFile); 268 } 269 checkDownloadedFolder(File localFile)270 private void checkDownloadedFolder(File localFile) throws Exception { 271 Assert.assertTrue(localFile.isDirectory()); 272 Assert.assertEquals(5, localFile.list().length); 273 for (String filename : localFile.list()) { 274 if (filename.equals(FILE_NAME2)) { 275 Assert.assertEquals( 276 FILE_NAME2, 277 FileUtil.readStringFromFile( 278 new File(localFile.getAbsolutePath(), filename))); 279 } else if (filename.equals(FILE_NAME3)) { 280 Assert.assertEquals( 281 FILE_NAME3, 282 FileUtil.readStringFromFile( 283 new File(localFile.getAbsolutePath(), filename))); 284 } else if (filename.equals(FOLDER_NAME2 + "_folder")) { 285 File subFolder = new File(localFile.getAbsolutePath(), filename); 286 Assert.assertTrue(subFolder.isDirectory()); 287 Assert.assertEquals(1, subFolder.list().length); 288 Assert.assertEquals( 289 FILE_NAME4, 290 FileUtil.readStringFromFile( 291 new File(subFolder.getAbsolutePath(), subFolder.list()[0]))); 292 } else if (filename.equals(FOLDER_NAME2)) { 293 File fileWithFolderName = new File(localFile.getAbsolutePath(), filename); 294 Assert.assertTrue(fileWithFolderName.isFile()); 295 } else if (filename.equals(FOLDER_NAME3)) { 296 File subFolder = new File(localFile.getAbsolutePath(), filename); 297 Assert.assertTrue(subFolder.isDirectory()); 298 Assert.assertEquals(2, subFolder.list().length); 299 Assert.assertEquals( 300 FILE_NAME5, 301 FileUtil.readStringFromFile( 302 new File(subFolder.getAbsolutePath(), FILE_NAME5))); 303 File subSubFolder = new File(subFolder, FOLDER_NAME4); 304 Assert.assertTrue(subSubFolder.isDirectory()); 305 Assert.assertEquals(1, subSubFolder.list().length); 306 Assert.assertEquals( 307 FILE_NAME6, 308 FileUtil.readStringFromFile( 309 new File(subSubFolder.getAbsolutePath(), FILE_NAME6))); 310 } else { 311 Assert.assertTrue(String.format("Unknonwn file %s", filename), false); 312 } 313 } 314 } 315 316 @Test testDownloadFile_folder_nonExist()317 public void testDownloadFile_folder_nonExist() throws Exception { 318 try { 319 mDownloader.downloadFile( 320 String.format("gs://%s/%s/%s/", BUCKET_NAME, "mRemoteRoot", "nonExistFolder")); 321 Assert.fail("Should throw BuildRetrievalError."); 322 } catch (BuildRetrievalError e) { 323 // Expect BuildRetrievalError 324 } 325 } 326 327 @Test testDownloadFile_size0Folder()328 public void testDownloadFile_size0Folder() throws Exception { 329 // Test the downloader work with size0 folder correctly. 330 File localFile = 331 mDownloader.downloadFile( 332 String.format( 333 "gs://%s/%s/%s/%s/", 334 BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FOLDER_NAME3)); 335 Assert.assertTrue(localFile.isDirectory()); 336 Assert.assertEquals(2, localFile.list().length); 337 Assert.assertEquals( 338 FILE_NAME5, 339 FileUtil.readStringFromFile(new File(localFile.getAbsolutePath(), FILE_NAME5))); 340 341 File subFolder = new File(localFile.getAbsolutePath(), FOLDER_NAME4); 342 Assert.assertTrue(subFolder.isDirectory()); 343 Assert.assertEquals( 344 FILE_NAME6, 345 FileUtil.readStringFromFile(new File(subFolder.getAbsolutePath(), FILE_NAME6))); 346 } 347 348 @Test testDownloadFile_folderWithOnlyOneFile()349 public void testDownloadFile_folderWithOnlyOneFile() throws Exception { 350 File localFile = 351 mDownloader.downloadFile( 352 String.format( 353 "gs://%s/%s/%s/%s/%s/", 354 BUCKET_NAME, 355 mRemoteRoot, 356 FOLDER_NAME1, 357 FOLDER_NAME3, 358 FOLDER_NAME4)); 359 Assert.assertTrue(localFile.isDirectory()); 360 Assert.assertEquals(1, localFile.list().length); 361 Assert.assertEquals( 362 FILE_NAME6, 363 FileUtil.readStringFromFile(new File(localFile.getAbsolutePath(), FILE_NAME6))); 364 } 365 366 @Test testDownloadFile_emptyFile()367 public void testDownloadFile_emptyFile() throws Exception { 368 try { 369 mDownloader.downloadFile( 370 String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, EMPTY_FILE)); 371 Assert.fail("Should throw BuildRetrievalError."); 372 } catch (BuildRetrievalError e) { 373 // Expect BuildRetrievalError 374 } 375 } 376 377 @Test testCheckFreshness()378 public void testCheckFreshness() throws Exception { 379 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 380 File localFile = mDownloader.downloadFile(remotePath); 381 Assert.assertTrue(mDownloader.isFresh(localFile, remotePath)); 382 } 383 384 @Test testCheckFreshness_notExist()385 public void testCheckFreshness_notExist() throws Exception { 386 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 387 Assert.assertFalse(mDownloader.isFresh(new File("/not/exist"), remotePath)); 388 } 389 390 @Test testCheckFreshness_folderNotExist()391 public void testCheckFreshness_folderNotExist() throws Exception { 392 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 393 Assert.assertFalse(mDownloader.isFresh(new File("/not/exist"), remotePath)); 394 } 395 396 @Test testCheckFreshness_remoteNotExist()397 public void testCheckFreshness_remoteNotExist() throws Exception { 398 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 399 String remoteNotExistPath = String.format("gs://%s/%s/no_exist", BUCKET_NAME, mRemoteRoot); 400 File localFile = mDownloader.downloadFile(remotePath); 401 Assert.assertFalse(mDownloader.isFresh(localFile, remoteNotExistPath)); 402 } 403 404 @Test testCheckFreshness_remoteFolderNotExist()405 public void testCheckFreshness_remoteFolderNotExist() throws Exception { 406 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 407 String remoteNotExistPath = String.format("gs://%s/%s/no_exist/", BUCKET_NAME, mRemoteRoot); 408 File localFolder = mDownloader.downloadFile(remotePath); 409 Assert.assertFalse(mDownloader.isFresh(localFolder, remoteNotExistPath)); 410 } 411 412 @Test testCheckFreshness_notFresh()413 public void testCheckFreshness_notFresh() throws Exception { 414 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 415 File localFile = mDownloader.downloadFile(remotePath); 416 mDownloader.clearCache(); 417 // Change the remote file. 418 createFile(mStorage, "New content.", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 419 Assert.assertFalse(mDownloader.isFresh(localFile, remotePath)); 420 } 421 422 @Test testCheckFreshness_folder()423 public void testCheckFreshness_folder() throws Exception { 424 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 425 File localFolder = mDownloader.downloadFile(remotePath); 426 Assert.assertTrue(mDownloader.isFresh(localFolder, remotePath)); 427 } 428 429 @Test testCheckFreshness_folder_addFile()430 public void testCheckFreshness_folder_addFile() throws Exception { 431 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 432 File localFolder = mDownloader.downloadFile(remotePath); 433 mDownloader.clearCache(); 434 createFile( 435 mStorage, 436 "A new file", 437 BUCKET_NAME, 438 mRemoteRoot, 439 FOLDER_NAME1, 440 FOLDER_NAME2, 441 "new_file.txt"); 442 Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath)); 443 } 444 445 @Test testCheckFreshness_folder_removeFile()446 public void testCheckFreshness_folder_removeFile() throws Exception { 447 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 448 File localFolder = mDownloader.downloadFile(remotePath); 449 mDownloader.clearCache(); 450 mStorage.objects() 451 .delete(BUCKET_NAME, Paths.get(mRemoteRoot, FOLDER_NAME1, FILE_NAME3).toString()) 452 .execute(); 453 Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath)); 454 } 455 456 @Test testCheckFreshness_folder_changeFile()457 public void testCheckFreshness_folder_changeFile() throws Exception { 458 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 459 File localFolder = mDownloader.downloadFile(remotePath); 460 mDownloader.clearCache(); 461 createFile(mStorage, "New content", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME3); 462 Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath)); 463 } 464 } 465