1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.build;
17 
18 import static org.junit.Assert.assertEquals;
19 import static org.junit.Assert.assertFalse;
20 import static org.junit.Assert.assertNotNull;
21 import static org.junit.Assert.assertTrue;
22 import static org.mockito.Mockito.doAnswer;
23 import static org.mockito.Mockito.times;
24 import static org.mockito.Mockito.when;
25 
26 import com.android.tradefed.log.LogUtil.CLog;
27 import com.android.tradefed.result.error.InfraErrorIdentifier;
28 import com.android.tradefed.util.FileUtil;
29 import com.android.tradefed.util.RunUtil;
30 import com.android.tradefed.util.StreamUtil;
31 
32 import org.junit.After;
33 import org.junit.Before;
34 import org.junit.Test;
35 import org.junit.runner.RunWith;
36 import org.junit.runners.JUnit4;
37 import org.mockito.InOrder;
38 import org.mockito.Mock;
39 import org.mockito.Mockito;
40 import org.mockito.MockitoAnnotations;
41 import org.mockito.invocation.InvocationOnMock;
42 import org.mockito.stubbing.Answer;
43 
44 import java.io.File;
45 import java.io.FileInputStream;
46 import java.util.ArrayList;
47 import java.util.List;
48 import java.util.concurrent.atomic.AtomicBoolean;
49 
50 /** Longer running, concurrency based tests for {@link FileDownloadCache}. */
51 @RunWith(JUnit4.class)
52 public class FileDownloadCacheFuncTest {
53 
54     private static final String REMOTE_PATH = "path";
55     private static final String DOWNLOADED_CONTENTS = "downloaded contents";
56 
57     @Mock IFileDownloader mMockDownloader;
58 
59     private FileDownloadCache mCache;
60     private File mTmpDir;
61     private List<File> mReturnedFiles;
62 
63     @Before
setUp()64     public void setUp() throws Exception {
65         MockitoAnnotations.initMocks(this);
66 
67         mTmpDir = FileUtil.createTempDir("functest");
68         mCache = new FileDownloadCache(mTmpDir);
69         mReturnedFiles = new ArrayList<File>(2);
70     }
71 
72     @After
tearDown()73     public void tearDown() throws Exception {
74         for (File file : mReturnedFiles) {
75             file.delete();
76         }
77         FileUtil.recursiveDelete(mTmpDir);
78     }
79 
80     /**
81      * Test {@link FileDownloadCache#fetchRemoteFile(IFileDownloader, String)} being called
82      * concurrently by two separate threads.
83      */
84     @Test
testFetchRemoteFile_concurrent()85     public void testFetchRemoteFile_concurrent() throws Exception {
86         // Simulate a relatively slow file download
87         Answer<Object> slowDownloadAnswer =
88                 invocation -> {
89                     Thread.sleep(500);
90                     File fileArg = (File) invocation.getArguments()[1];
91                     FileUtil.writeToFile(DOWNLOADED_CONTENTS, fileArg);
92                     return null;
93                 };
94         // Download is only called once, second thread will wait on synchronized until the download
95         // is done, then link the downloaded file.
96         doAnswer(slowDownloadAnswer)
97                 .when(mMockDownloader)
98                 .downloadFile(Mockito.eq(REMOTE_PATH), Mockito.<File>any());
99         when(mMockDownloader.isFresh(Mockito.any(), Mockito.eq(REMOTE_PATH))).thenReturn(true);
100 
101         Thread downloadThread1 = createDownloadThread(mMockDownloader, REMOTE_PATH);
102         downloadThread1.setName("FileDownloadCacheFuncTest#testFetchRemoteFile_concurrent-1");
103         Thread downloadThread2 = createDownloadThread(mMockDownloader, REMOTE_PATH);
104         downloadThread2.setName("FileDownloadCacheFuncTest#testFetchRemoteFile_concurrent-2");
105         downloadThread1.start();
106         downloadThread2.start();
107         downloadThread1.join();
108         downloadThread2.join();
109         assertNotNull(mCache.getCachedFile(REMOTE_PATH));
110         assertEquals(2, mReturnedFiles.size());
111         // returned files should be identical in content, but be different files
112         assertTrue(!mReturnedFiles.get(0).equals(mReturnedFiles.get(1)));
113         assertEquals(
114                 DOWNLOADED_CONTENTS,
115                 StreamUtil.getStringFromStream(new FileInputStream(mReturnedFiles.get(0))));
116         assertEquals(
117                 DOWNLOADED_CONTENTS,
118                 StreamUtil.getStringFromStream(new FileInputStream(mReturnedFiles.get(1))));
119         InOrder inOrder = Mockito.inOrder(mMockDownloader);
120         inOrder.verify(mMockDownloader).downloadFile(Mockito.eq(REMOTE_PATH), Mockito.<File>any());
121     }
122 
123     /**
124      * Test {@link FileDownloadCache#fetchRemoteFile(IFileDownloader, String)} being called
125      * concurrently by multiple threads trying to download different files.
126      */
127     @Test
testFetchRemoteFile_multiConcurrent()128     public void testFetchRemoteFile_multiConcurrent() throws Exception {
129         IFileDownloader mockDownloader1 = Mockito.mock(IFileDownloader.class);
130         IFileDownloader mockDownloader2 = Mockito.mock(IFileDownloader.class);
131         IFileDownloader mockDownloader3 = Mockito.mock(IFileDownloader.class);
132         String remotePath1 = "path1";
133         String remotePath2 = "path2";
134         String remotePath3 = "path3";
135 
136         // Block first download, but allow other downloads to pass.
137         final AtomicBoolean startedDownload = new AtomicBoolean(false);
138         final AtomicBoolean blockDownload = new AtomicBoolean(true);
139         mCache.setMaxCacheSize(DOWNLOADED_CONTENTS.length() + 1);
140         Answer<Void> blockedAnswer =
141                 new Answer<Void>() {
142                     @Override
143                     public Void answer(InvocationOnMock invocation) throws Throwable {
144                         if (!startedDownload.get()) {
145                             startedDownload.set(true);
146                             while (blockDownload.get()) {
147                                 RunUtil.getDefault().sleep(10);
148                             }
149                         }
150                         File fileArg = (File) invocation.getArguments()[1];
151                         FileUtil.writeToFile(DOWNLOADED_CONTENTS, fileArg);
152                         return null;
153                     }
154                 };
155 
156         // Download is called once per files since they are different files.
157         Mockito.doAnswer(blockedAnswer)
158                 .when(mockDownloader1)
159                 .downloadFile(Mockito.eq(remotePath1), Mockito.any());
160         Mockito.doAnswer(blockedAnswer)
161                 .when(mockDownloader2)
162                 .downloadFile(Mockito.eq(remotePath2), Mockito.any());
163         Mockito.doAnswer(blockedAnswer)
164                 .when(mockDownloader3)
165                 .downloadFile(Mockito.eq(remotePath3), Mockito.any());
166 
167         Thread downloadThread1 = createDownloadThread(mockDownloader1, remotePath1);
168         downloadThread1.setName("FileDownloadCacheFuncTest#testFetchRemoteFile_multiConcurrent-1");
169         Thread downloadThread2 = createDownloadThread(mockDownloader2, remotePath2);
170         downloadThread2.setName("FileDownloadCacheFuncTest#testFetchRemoteFile_multiConcurrent-2");
171         Thread downloadThread3 = createDownloadThread(mockDownloader3, remotePath3);
172         downloadThread3.setName("FileDownloadCacheFuncTest#testFetchRemoteFile_multiConcurrent-3");
173 
174         // Start first thread, and wait for download to begin
175         downloadThread1.start();
176         while (!startedDownload.get()) {
177             RunUtil.getDefault().sleep(10);
178         }
179 
180         // Start the other threads, which should run to completion. The cache should be adjusted,
181         // but the file in the first thread should not be deleted since it is still being
182         // downloaded.
183         downloadThread2.start();
184         downloadThread3.start();
185         downloadThread2.join(2000);
186         downloadThread3.join(2000);
187         assertFalse(downloadThread2.isAlive());
188         assertFalse(downloadThread3.isAlive());
189         assertNotNull(mCache.getCachedFile(remotePath1));
190 
191         // Complete download of first thread, and let the thread run to completion. What files are
192         // left in the cache can depend on implementation, so we're not testing for it.
193         blockDownload.set(false);
194         downloadThread1.join(2000);
195         assertFalse(downloadThread1.isAlive());
196 
197         Mockito.verify(mockDownloader1).downloadFile(Mockito.eq(remotePath1), Mockito.any());
198         Mockito.verify(mockDownloader2).downloadFile(Mockito.eq(remotePath2), Mockito.any());
199         Mockito.verify(mockDownloader3).downloadFile(Mockito.eq(remotePath3), Mockito.any());
200     }
201 
202     /**
203      * Test {@link FileDownloadCache#fetchRemoteFile(IFileDownloader, String)} being called
204      * concurrently by multiple threads trying to download the same file, with one thread failing.
205      */
206     @Test
testFetchRemoteFile_concurrentFail()207     public void testFetchRemoteFile_concurrentFail() throws Exception {
208         // Block first download, and later raise an error, but allow other downloads to pass.
209         final AtomicBoolean startedDownload = new AtomicBoolean(false);
210         final AtomicBoolean throwException = new AtomicBoolean(false);
211         Answer<Object> blockedDownloadAnswer =
212                 invocation -> {
213                     if (!startedDownload.get()) {
214                         startedDownload.set(true);
215                         while (!throwException.get()) {
216                             Thread.sleep(10);
217                         }
218                         throw new BuildRetrievalError(
219                                 "download error", InfraErrorIdentifier.ARTIFACT_DOWNLOAD_ERROR);
220                     }
221                     File fileArg = (File) invocation.getArguments()[1];
222                     FileUtil.writeToFile(DOWNLOADED_CONTENTS, fileArg);
223                     return null;
224                 };
225 
226         // Download should be called twice. The first call will result in an error, and the second
227         // will run to completion.
228         doAnswer(blockedDownloadAnswer)
229                 .when(mMockDownloader)
230                 .downloadFile(Mockito.eq(REMOTE_PATH), Mockito.<File>any());
231         when(mMockDownloader.isFresh(Mockito.any(), Mockito.eq(REMOTE_PATH))).thenReturn(true);
232 
233         Thread downloadThread1 = createDownloadThread(mMockDownloader, REMOTE_PATH);
234         downloadThread1.setName("FileDownloadCacheFuncTest#testFetchRemoteFile_concurrentFail-1");
235         Thread downloadThread2 = createDownloadThread(mMockDownloader, REMOTE_PATH);
236         downloadThread2.setName("FileDownloadCacheFuncTest#testFetchRemoteFile_concurrentFail-2");
237         Thread downloadThread3 = createDownloadThread(mMockDownloader, REMOTE_PATH);
238         downloadThread3.setName("FileDownloadCacheFuncTest#testFetchRemoteFile_concurrentFail-3");
239 
240         // Start first thread, and wait for download to begin
241         downloadThread1.start();
242         while (!startedDownload.get()) {
243             Thread.sleep(10);
244         }
245 
246         // Start second thread, and allow it to attempt to obtain the file lock.
247         downloadThread2.start();
248         Thread.sleep(100);
249 
250         // Throw a BuildRetrievalError, and allow both threads to run to completion.
251         throwException.set(true);
252         downloadThread1.join(2000);
253         downloadThread2.join(2000);
254         assertFalse(downloadThread1.isAlive());
255         assertFalse(downloadThread2.isAlive());
256         assertNotNull(mCache.getCachedFile(REMOTE_PATH));
257 
258         // A third attempt to retrive the file should not result in another download call.
259         downloadThread3.start();
260         downloadThread3.join(2000);
261 
262         InOrder inOrder = Mockito.inOrder(mMockDownloader);
263         inOrder.verify(mMockDownloader, times(2))
264                 .downloadFile(Mockito.eq(REMOTE_PATH), Mockito.<File>any());
265     }
266 
267     /** Verify the cache is built from disk contents on creation */
268     @Test
testConstructor_createCache()269     public void testConstructor_createCache() throws Exception {
270         // create cache contents on disk
271         File cacheRoot = FileUtil.createTempDir("constructorTest");
272         try {
273             final String filecontents = "these are the file contents";
274             File file1 = new File(cacheRoot, REMOTE_PATH);
275             FileUtil.writeToFile(filecontents, file1);
276             // this is lame, but sleep for a small amount to ensure nestedFile has later timestamp
277             // TODO: use mock File instead
278             Thread.sleep(1000);
279             File nestedDir = new File(cacheRoot, "aa");
280             nestedDir.mkdir();
281             File nestedFile = new File(nestedDir, "anotherpath");
282             FileUtil.writeToFile(filecontents, nestedFile);
283 
284             FileDownloadCache cache = new FileDownloadCache(cacheRoot);
285             assertNotNull(cache.getCachedFile(REMOTE_PATH));
286             assertNotNull(cache.getCachedFile("aa/anotherpath"));
287             assertEquals(REMOTE_PATH, cache.getOldestEntry());
288         } finally {
289             FileUtil.recursiveDelete(cacheRoot);
290         }
291     }
292 
293     /** Test scenario where an already too large cache is built from disk contents. */
294     @Test
testConstructor_cacheExceeded()295     public void testConstructor_cacheExceeded() throws Exception {
296         File cacheRoot = FileUtil.createTempDir("testConstructor_cacheExceeded");
297         try {
298             // create a couple existing files in cache
299             final String filecontents = "these are the file contents";
300             final File file1 = new File(cacheRoot, REMOTE_PATH);
301             FileUtil.writeToFile(filecontents, file1);
302             // sleep for a small amount to ensure file2 has later timestamp
303             // TODO: use mock File instead
304             Thread.sleep(1000);
305             final File file2 = new File(cacheRoot, "anotherpath");
306             FileUtil.writeToFile(filecontents, file2);
307 
308             new FileDownloadCache(cacheRoot) {
309                 @Override
310                 long getMaxFileCacheSize() {
311                     return file2.length() + 1;
312                 }
313             };
314             // expect cache to be cleaned on startup, with oldest file1 deleted, but newest file
315             // retained
316             assertFalse(file1.exists());
317             assertTrue(file2.exists());
318         } finally {
319             FileUtil.recursiveDelete(cacheRoot);
320         }
321     }
322 
323     /** Utility method to create thread that calls fetchRemoteFile. */
createDownloadThread(IFileDownloader downloader, String remotePath)324     private Thread createDownloadThread(IFileDownloader downloader, String remotePath) {
325         return new Thread() {
326             @Override
327             public void run() {
328                 try {
329                     mReturnedFiles.add(mCache.fetchRemoteFile(downloader, remotePath));
330                 } catch (BuildRetrievalError e) {
331                     CLog.e(e);
332                 }
333             }
334         };
335     }
336 }
337