1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.util;
18 
19 import com.android.annotations.Nullable;
20 import com.android.tradefed.cache.ExecutableAction;
21 import com.android.tradefed.cache.ExecutableActionResult;
22 import com.android.tradefed.cache.ICacheClient;
23 import com.android.tradefed.command.CommandInterrupter;
24 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
25 import com.android.tradefed.invoker.tracing.CloseableTraceScope;
26 import com.android.tradefed.log.LogUtil.CLog;
27 import com.android.tradefed.result.error.ErrorIdentifier;
28 import com.android.tradefed.result.error.InfraErrorIdentifier;
29 
30 import com.google.common.annotations.VisibleForTesting;
31 import com.google.common.base.Strings;
32 
33 import java.io.BufferedOutputStream;
34 import java.io.File;
35 import java.io.FileInputStream;
36 import java.io.FileOutputStream;
37 import java.io.IOException;
38 import java.io.InputStream;
39 import java.io.OutputStream;
40 import java.lang.ProcessBuilder.Redirect;
41 import java.util.ArrayList;
42 import java.util.Arrays;
43 import java.util.HashMap;
44 import java.util.HashSet;
45 import java.util.List;
46 import java.util.Map;
47 import java.util.Set;
48 import java.util.concurrent.CountDownLatch;
49 import java.util.concurrent.TimeUnit;
50 
51 import javax.annotation.Nonnull;
52 
53 /**
54  * A collection of helper methods for executing operations.
55  */
56 public class RunUtil implements IRunUtil {
57 
58     public static final String RUNNABLE_NOTIFIER_NAME = "RunnableNotifier";
59     public static final String INHERITIO_PREFIX = "inheritio-";
60 
61     private static final int POLL_TIME_INCREASE_FACTOR = 4;
62     private static final long THREAD_JOIN_POLL_INTERVAL = 30 * 1000;
63     private static final long PROCESS_DESTROY_TIMEOUT_SEC = 2;
64     private long mPollingInterval = THREAD_JOIN_POLL_INTERVAL;
65     private static IRunUtil sDefaultInstance = null;
66     private File mWorkingDir = null;
67     private Map<String, String> mEnvVariables = new HashMap<String, String>();
68     private Set<String> mUnsetEnvVariables = new HashSet<String>();
69     private EnvPriority mEnvVariablePriority = EnvPriority.UNSET;
70     private boolean mRedirectStderr = false;
71     private boolean mLinuxInterruptProcess = false;
72     private static final String PROGRESS_MONITOR_ENV = "RUN_PROGRESS_MONITOR";
73     private static final String PROGRESS_MONITOR_TIMEOUT_ENV = "RUN_PROGRESS_MONITOR_TIMEOUT";
74 
75     private final CommandInterrupter mInterrupter;
76 
77     /**
78      * Create a new {@link RunUtil} object to use.
79      */
RunUtil()80     public RunUtil() {
81         this(CommandInterrupter.INSTANCE);
82     }
83 
84     @VisibleForTesting
RunUtil(@onnull CommandInterrupter interrupter)85     RunUtil(@Nonnull CommandInterrupter interrupter) {
86         mInterrupter = interrupter;
87     }
88 
89     /**
90      * Get a reference to the default {@link RunUtil} object.
91      * <p/>
92      * This is useful for callers who want to use IRunUtil without customization.
93      * Its recommended that callers who do need a custom IRunUtil instance
94      * (ie need to call either {@link #setEnvVariable(String, String)} or
95      * {@link #setWorkingDir(File)} create their own copy.
96      */
getDefault()97     public static IRunUtil getDefault() {
98         if (sDefaultInstance == null) {
99             sDefaultInstance = new RunUtil();
100         }
101         return sDefaultInstance;
102     }
103 
104     /**
105      * Sets a new value for the internal polling interval. In most cases, you should never have a
106      * need to change the polling interval except for specific cases such as unit tests.
107      *
108      * @param pollInterval in ms for polling interval. Must be larger than 100ms.
109      */
110     @VisibleForTesting
setPollingInterval(long pollInterval)111     void setPollingInterval(long pollInterval) {
112         if (pollInterval >= 100) {
113             mPollingInterval = pollInterval;
114             return;
115         }
116         throw new IllegalArgumentException("Polling interval set too low. Try 100ms.");
117     }
118 
119     /**
120      * {@inheritDoc}
121      */
122     @Override
setWorkingDir(File dir)123     public synchronized void setWorkingDir(File dir) {
124         if (this.equals(sDefaultInstance)) {
125             throw new UnsupportedOperationException("Cannot setWorkingDir on default RunUtil");
126         }
127         mWorkingDir = dir;
128     }
129 
130     /**
131      * {@inheritDoc}
132      */
133     @Override
setEnvVariable(String name, String value)134     public synchronized void setEnvVariable(String name, String value) {
135         if (this.equals(sDefaultInstance)) {
136             throw new UnsupportedOperationException("Cannot setEnvVariable on default RunUtil");
137         }
138         mEnvVariables.put(name, value);
139     }
140 
141     /**
142      * {@inheritDoc}
143      * Environment variables may inherit from the parent process, so we need to delete
144      * the environment variable from {@link ProcessBuilder#environment()}
145      *
146      * @param key the variable name
147      * @see ProcessBuilder#environment()
148      */
149     @Override
unsetEnvVariable(String key)150     public synchronized void unsetEnvVariable(String key) {
151         if (this.equals(sDefaultInstance)) {
152             throw new UnsupportedOperationException("Cannot unsetEnvVariable on default RunUtil");
153         }
154         mUnsetEnvVariables.add(key);
155     }
156 
157     /** {@inheritDoc} */
158     @Override
setRedirectStderrToStdout(boolean redirect)159     public void setRedirectStderrToStdout(boolean redirect) {
160         if (this.equals(sDefaultInstance)) {
161             throw new UnsupportedOperationException(
162                     "Cannot setRedirectStderrToStdout on default RunUtil");
163         }
164         mRedirectStderr = redirect;
165     }
166 
167     /**
168      * {@inheritDoc}
169      */
170     @Override
runTimedCmd(final long timeout, final String... command)171     public CommandResult runTimedCmd(final long timeout, final String... command) {
172         return runTimedCmd(timeout, (OutputStream) null, (OutputStream) null, command);
173     }
174 
175     /** {@inheritDoc} */
176     @Override
runTimedCmdWithOutputMonitor( final long timeout, final long idleOutputTimeout, final String... command)177     public CommandResult runTimedCmdWithOutputMonitor(
178             final long timeout, final long idleOutputTimeout, final String... command) {
179         return runTimedCmdWithOutputMonitor(
180                 timeout, idleOutputTimeout, (OutputStream) null, (OutputStream) null, command);
181     }
182 
183     /** {@inheritDoc} */
184     @Override
runTimedCmd( final long timeout, final OutputStream stdout, OutputStream stderr, final String... command)185     public CommandResult runTimedCmd(
186             final long timeout,
187             final OutputStream stdout,
188             OutputStream stderr,
189             final String... command) {
190         return runTimedCmdWithOutputMonitor(timeout, 0, stdout, stderr, command);
191     }
192 
193     /** {@inheritDoc} */
194     @Override
runTimedCmdWithOutputMonitor( final long timeout, final long idleOutputTimeout, final OutputStream stdout, OutputStream stderr, final String... command)195     public CommandResult runTimedCmdWithOutputMonitor(
196             final long timeout,
197             final long idleOutputTimeout,
198             final OutputStream stdout,
199             OutputStream stderr,
200             final String... command) {
201         return runTimedCmdWithOutputMonitor(
202                 timeout, idleOutputTimeout, stdout, stderr, null, command);
203     }
204 
205     /** {@inheritDoc} */
206     @Override
runTimedCmdWithOutputMonitor( final long timeout, final long idleOutputTimeout, OutputStream stdout, OutputStream stderr, ICacheClient cacheClient, final String... command)207     public CommandResult runTimedCmdWithOutputMonitor(
208             final long timeout,
209             final long idleOutputTimeout,
210             OutputStream stdout,
211             OutputStream stderr,
212             ICacheClient cacheClient,
213             final String... command) {
214         ProcessBuilder processBuilder = createProcessBuilder(command);
215         ExecutableAction action = null;
216         if (cacheClient != null) {
217             try {
218                 action =
219                         ExecutableAction.create(
220                                 processBuilder.directory(),
221                                 processBuilder.command(),
222                                 processBuilder.environment(),
223                                 timeout);
224             } catch (IOException e) {
225                 CLog.e("Exception occurred when building executable action! Disabling cache...");
226                 CLog.e(e);
227                 // Disable caching.
228                 cacheClient = null;
229             }
230         }
231 
232         ExecutableActionResult cachedResult = null;
233         try {
234             cachedResult =
235                     action != null && cacheClient != null ? cacheClient.lookupCache(action) : null;
236         } catch (IOException e) {
237             CLog.e("Failed to lookup cache!");
238             CLog.e(e);
239         } catch (InterruptedException e) {
240             throw new RunInterruptedException(e.getMessage(), e, InfraErrorIdentifier.UNDETERMINED);
241         }
242         if (cachedResult != null) {
243             try {
244                 return handleCachedResult(cachedResult, stdout, stderr);
245             } catch (IOException e) {
246                 CLog.e("Exception occurred when handling cached result!");
247                 CLog.e(e);
248             }
249         }
250 
251         File stdoutBuffer = null;
252         File stderrBuffer = null;
253         if (cacheClient != null) {
254             try {
255                 stdoutBuffer = FileUtil.createTempFile("stdout-to-upload", ".txt");
256                 stdoutBuffer.deleteOnExit();
257                 stdout = new ForkedOutputStream(stdout, new FileOutputStream(stdoutBuffer));
258                 if (stderr != null) {
259                     stderrBuffer = FileUtil.createTempFile("stderr-to-upload", ".txt");
260                     stderrBuffer.deleteOnExit();
261                     stderr = new ForkedOutputStream(stderr, new FileOutputStream(stderrBuffer));
262                 }
263             } catch (IOException e) {
264                 CLog.e("Failed to catch command execution output! Skipping the cache upload...");
265                 CLog.e(e);
266                 // Disable cache upload.
267                 cacheClient = null;
268             }
269         }
270         RunnableResult osRunnable = createRunnableResult(stdout, stderr, processBuilder);
271 
272         CommandStatus status =
273                 runTimedWithOutputMonitor(timeout, idleOutputTimeout, osRunnable, true);
274         CommandResult result = osRunnable.getResult();
275         result.setStatus(status);
276 
277         try {
278             if (CommandStatus.SUCCESS.equals(status) && action != null && cacheClient != null) {
279                 ForkedOutputStream stdoutForkedStream = (ForkedOutputStream) stdout;
280                 ForkedOutputStream stderrForkedStream =
281                         stderr != null ? (ForkedOutputStream) stderr : null;
282                 if (stdoutForkedStream.isSuccess()
283                         && (stderr == null || stderrForkedStream.isSuccess())) {
284                     cacheClient.uploadCache(
285                             action,
286                             ExecutableActionResult.create(
287                                     result.getExitCode(), stdoutBuffer, stderrBuffer));
288                 }
289             }
290         } catch (IOException e) {
291             CLog.e("Failed to upload cache!");
292             CLog.e(e);
293         } catch (InterruptedException e) {
294             throw new RunInterruptedException(e.getMessage(), e, InfraErrorIdentifier.UNDETERMINED);
295         } finally {
296             FileUtil.deleteFile(stdoutBuffer);
297             FileUtil.deleteFile(stderrBuffer);
298         }
299         return result;
300     }
301 
302     /**
303      * Create a {@link com.android.tradefed.util.IRunUtil.IRunnableResult} that will run the
304      * command.
305      */
306     @VisibleForTesting
createRunnableResult( OutputStream stdout, OutputStream stderr, ProcessBuilder processBuilder)307     RunnableResult createRunnableResult(
308             OutputStream stdout, OutputStream stderr, ProcessBuilder processBuilder) {
309         return new RunnableResult(
310                 /* input= */ null,
311                 processBuilder,
312                 stdout,
313                 stderr,
314                 /* inputRedirect= */ null,
315                 false);
316     }
317 
318     /** {@inheritDoc} */
319     @Override
runTimedCmdRetry( long timeout, long retryInterval, int attempts, String... command)320     public CommandResult runTimedCmdRetry(
321             long timeout, long retryInterval, int attempts, String... command) {
322         return runTimedCmdRetryWithOutputMonitor(timeout, 0, retryInterval, attempts, command);
323     }
324 
325     /** {@inheritDoc} */
326     @Override
runTimedCmdRetryWithOutputMonitor( long timeout, long idleOutputTimeout, long retryInterval, int attempts, String... command)327     public CommandResult runTimedCmdRetryWithOutputMonitor(
328             long timeout,
329             long idleOutputTimeout,
330             long retryInterval,
331             int attempts,
332             String... command) {
333         CommandResult result = null;
334         int counter = 0;
335         while (counter < attempts) {
336             result = runTimedCmdWithOutputMonitor(timeout, idleOutputTimeout, command);
337             if (CommandStatus.SUCCESS.equals(result.getStatus())) {
338                 return result;
339             }
340             sleep(retryInterval);
341             counter++;
342         }
343         return result;
344     }
345 
createProcessBuilder(String... command)346     private synchronized ProcessBuilder createProcessBuilder(String... command) {
347         return createProcessBuilder(Arrays.asList(command));
348     }
349 
createProcessBuilder(Redirect redirect, String... command)350     private synchronized ProcessBuilder createProcessBuilder(Redirect redirect, String... command) {
351         return createProcessBuilder(redirect, Arrays.asList(command));
352     }
353 
createProcessBuilder(List<String> commandList)354     private synchronized ProcessBuilder createProcessBuilder(List<String> commandList) {
355         return createProcessBuilder(null, commandList);
356     }
357 
createProcessBuilder( Redirect redirect, List<String> commandList)358     private synchronized ProcessBuilder createProcessBuilder(
359             Redirect redirect, List<String> commandList) {
360         ProcessBuilder processBuilder = new ProcessBuilder();
361         if (mWorkingDir != null) {
362             processBuilder.directory(mWorkingDir);
363         }
364         // By default unset an env. for process has higher priority, but in some case we might want
365         // the 'set' to have priority.
366         if (EnvPriority.UNSET.equals(mEnvVariablePriority)) {
367             if (!mEnvVariables.isEmpty()) {
368                 processBuilder.environment().putAll(mEnvVariables);
369             }
370             if (!mUnsetEnvVariables.isEmpty()) {
371                 // in this implementation, the unsetEnv's priority is higher than set.
372                 processBuilder.environment().keySet().removeAll(mUnsetEnvVariables);
373             }
374         } else {
375             if (!mUnsetEnvVariables.isEmpty()) {
376                 processBuilder.environment().keySet().removeAll(mUnsetEnvVariables);
377             }
378             if (!mEnvVariables.isEmpty()) {
379                 // in this implementation, the setEnv's priority is higher than set.
380                 processBuilder.environment().putAll(mEnvVariables);
381             }
382         }
383         processBuilder.redirectErrorStream(mRedirectStderr);
384         if (redirect != null) {
385             processBuilder.redirectOutput(redirect);
386             processBuilder.redirectError(redirect);
387         }
388         return processBuilder.command(commandList);
389     }
390 
391     /**
392      * {@inheritDoc}
393      */
394     @Override
runTimedCmdWithInput(final long timeout, String input, final String... command)395     public CommandResult runTimedCmdWithInput(final long timeout, String input,
396             final String... command) {
397         return runTimedCmdWithInput(timeout, input, ArrayUtil.list(command));
398     }
399 
400     /**
401      * {@inheritDoc}
402      */
403     @Override
runTimedCmdWithInput(final long timeout, String input, final List<String> command)404     public CommandResult runTimedCmdWithInput(final long timeout, String input,
405             final List<String> command) {
406         RunnableResult osRunnable = new RunnableResult(input, createProcessBuilder(command));
407         CommandStatus status = runTimed(timeout, osRunnable, true);
408         CommandResult result = osRunnable.getResult();
409         result.setStatus(status);
410         return result;
411     }
412 
413     /** {@inheritDoc} */
414     @Override
runTimedCmdWithInput( long timeout, String input, File stdoutFile, File stderrFile, String... command)415     public CommandResult runTimedCmdWithInput(
416             long timeout, String input, File stdoutFile, File stderrFile, String... command) {
417         ProcessBuilder pb = createProcessBuilder(command);
418         pb.redirectOutput(ProcessBuilder.Redirect.to(stdoutFile));
419         pb.redirectError(ProcessBuilder.Redirect.to(stderrFile));
420         RunnableResult osRunnable = new RunnableResult(input, pb);
421         CommandStatus status = runTimed(timeout, osRunnable, true);
422         CommandResult result = osRunnable.getResult();
423         result.setStatus(status);
424         // In case of error backfill, copy stderr to its file
425         if (result.getExitCode() == 88) {
426             try {
427                 FileUtil.writeToFile(result.getStderr(), stderrFile, true);
428             } catch (IOException e) {
429                 // Ignore
430             }
431         }
432         return result;
433     }
434 
435     /** {@inheritDoc} */
436     @Override
runTimedCmdWithInputRedirect( final long timeout, @Nullable File inputRedirect, final String... command)437     public CommandResult runTimedCmdWithInputRedirect(
438             final long timeout, @Nullable File inputRedirect, final String... command) {
439         RunnableResult osRunnable =
440                 new RunnableResult(
441                         /* input= */ null,
442                         createProcessBuilder(command),
443                         /* stdoutStream= */ null,
444                         /* stderrStream= */ null,
445                         inputRedirect,
446                         true);
447         CommandStatus status = runTimed(timeout, osRunnable, true);
448         CommandResult result = osRunnable.getResult();
449         result.setStatus(status);
450         return result;
451     }
452 
453     /**
454      * {@inheritDoc}
455      */
456     @Override
runTimedCmdSilently(final long timeout, final String... command)457     public CommandResult runTimedCmdSilently(final long timeout, final String... command) {
458         RunnableResult osRunnable = new RunnableResult(null, createProcessBuilder(command), false);
459         CommandStatus status = runTimed(timeout, osRunnable, false);
460         CommandResult result = osRunnable.getResult();
461         result.setStatus(status);
462         return result;
463     }
464 
465     /**
466      * {@inheritDoc}
467      */
468     @Override
runTimedCmdSilentlyRetry(long timeout, long retryInterval, int attempts, String... command)469     public CommandResult runTimedCmdSilentlyRetry(long timeout, long retryInterval, int attempts,
470             String... command) {
471         CommandResult result = null;
472         int counter = 0;
473         while (counter < attempts) {
474             result = runTimedCmdSilently(timeout, command);
475             if (CommandStatus.SUCCESS.equals(result.getStatus())) {
476                 return result;
477             }
478             sleep(retryInterval);
479             counter++;
480         }
481         return result;
482     }
483 
484     /**
485      * {@inheritDoc}
486      */
487     @Override
runCmdInBackground(final String... command)488     public Process runCmdInBackground(final String... command) throws IOException  {
489         return runCmdInBackground(null, command);
490     }
491 
492     /** {@inheritDoc} */
493     @Override
runCmdInBackground(Redirect redirect, final String... command)494     public Process runCmdInBackground(Redirect redirect, final String... command)
495             throws IOException {
496         final String fullCmd = Arrays.toString(command);
497         CLog.v("Running in background: %s", fullCmd);
498         return createProcessBuilder(redirect, command).start();
499     }
500 
501     /**
502      * {@inheritDoc}
503      */
504     @Override
runCmdInBackground(final List<String> command)505     public Process runCmdInBackground(final List<String> command) throws IOException  {
506         return runCmdInBackground(null, command);
507     }
508 
509     /** {@inheritDoc} */
510     @Override
runCmdInBackground(Redirect redirect, final List<String> command)511     public Process runCmdInBackground(Redirect redirect, final List<String> command)
512             throws IOException {
513         CLog.v("Running in background: %s", command);
514         return createProcessBuilder(redirect, command).start();
515     }
516 
517     /**
518      * {@inheritDoc}
519      */
520     @Override
runCmdInBackground(List<String> command, OutputStream output)521     public Process runCmdInBackground(List<String> command, OutputStream output)
522             throws IOException {
523         CLog.v("Running in background: %s", command);
524         Process process = createProcessBuilder(command).start();
525         inheritIO(
526                 process.getInputStream(),
527                 output,
528                 String.format(INHERITIO_PREFIX + "stdout-%s", command));
529         inheritIO(
530                 process.getErrorStream(),
531                 output,
532                 String.format(INHERITIO_PREFIX + "stderr-%s", command));
533         return process;
534     }
535 
536     /**
537      * {@inheritDoc}
538      */
539     @Override
runTimed(long timeout, IRunUtil.IRunnableResult runnable, boolean logErrors)540     public CommandStatus runTimed(long timeout, IRunUtil.IRunnableResult runnable,
541             boolean logErrors) {
542         return runTimedWithOutputMonitor(timeout, 0, runnable, logErrors);
543     }
544 
545     /** {@inheritDoc} */
546     @Override
runTimedWithOutputMonitor( long timeout, long idleOutputTimeout, IRunUtil.IRunnableResult runnable, boolean logErrors)547     public CommandStatus runTimedWithOutputMonitor(
548             long timeout,
549             long idleOutputTimeout,
550             IRunUtil.IRunnableResult runnable,
551             boolean logErrors) {
552         mInterrupter.checkInterrupted();
553         RunnableNotifier runThread = new RunnableNotifier(runnable, logErrors);
554         if (logErrors) {
555             if (timeout > 0L) {
556                 CLog.d(
557                         "Running command %s with timeout: %s",
558                         runnable.getCommand(), TimeUtil.formatElapsedTime(timeout));
559             } else {
560                 CLog.d("Running command %s without timeout.", runnable.getCommand());
561             }
562         }
563         CommandStatus status = CommandStatus.TIMED_OUT;
564         try {
565             runThread.start();
566             long startTime = System.currentTimeMillis();
567             long pollInterval = 0;
568             if (timeout > 0L && timeout < mPollingInterval) {
569                 // only set the pollInterval if we have a timeout
570                 pollInterval = timeout;
571             } else {
572                 pollInterval = mPollingInterval;
573             }
574             do {
575                 try {
576                     // Check if the command is still making progress.
577                     if (idleOutputTimeout != 0
578                             && runThread.isAlive()
579                             && !runnable.checkOutputMonitor(idleOutputTimeout)) {
580                         // Set to Failed.
581                         runThread.cancel();
582                     } else {
583                         runThread.join(pollInterval);
584                     }
585                 } catch (InterruptedException e) {
586                     if (isInterruptAllowed()) {
587                         CLog.i("runTimed: interrupted while joining the runnable");
588                         break;
589                     } else {
590                         CLog.i("runTimed: currently uninterruptible, ignoring interrupt");
591                     }
592                 }
593                 mInterrupter.checkInterrupted();
594             } while ((timeout == 0L || (System.currentTimeMillis() - startTime) < timeout)
595                     && runThread.isAlive());
596         } catch (RunInterruptedException e) {
597             runThread.cancel();
598             throw e;
599         } finally {
600             // Snapshot the status when out of the run loop because thread may terminate and return
601             // a false FAILED instead of TIMED_OUT.
602             status = runThread.getStatus();
603             if (CommandStatus.TIMED_OUT.equals(status) || CommandStatus.EXCEPTION.equals(status)) {
604                 CLog.i("runTimed: Calling interrupt, status is %s", status);
605                 runThread.cancel();
606             }
607         }
608         mInterrupter.checkInterrupted();
609         return status;
610     }
611 
612     /**
613      * {@inheritDoc}
614      */
615     @Override
runTimedRetry(long opTimeout, long pollInterval, int attempts, IRunUtil.IRunnableResult runnable)616     public boolean runTimedRetry(long opTimeout, long pollInterval, int attempts,
617             IRunUtil.IRunnableResult runnable) {
618         return runTimedRetryWithOutputMonitor(opTimeout, 0, pollInterval, attempts, runnable);
619     }
620 
621     /** {@inheritDoc} */
622     @Override
runTimedRetryWithOutputMonitor( long opTimeout, long idleOutputTimeout, long pollInterval, int attempts, IRunUtil.IRunnableResult runnable)623     public boolean runTimedRetryWithOutputMonitor(
624             long opTimeout,
625             long idleOutputTimeout,
626             long pollInterval,
627             int attempts,
628             IRunUtil.IRunnableResult runnable) {
629         for (int i = 0; i < attempts; i++) {
630             if (runTimedWithOutputMonitor(opTimeout, idleOutputTimeout, runnable, true)
631                     == CommandStatus.SUCCESS) {
632                 return true;
633             }
634             CLog.d("operation failed, waiting for %d ms", pollInterval);
635             sleep(pollInterval);
636         }
637         return false;
638     }
639 
640     /**
641      * {@inheritDoc}
642      */
643     @Override
runFixedTimedRetry(final long opTimeout, final long pollInterval, final long maxTime, final IRunUtil.IRunnableResult runnable)644     public boolean runFixedTimedRetry(final long opTimeout, final long pollInterval,
645             final long maxTime, final IRunUtil.IRunnableResult runnable) {
646         return runFixedTimedRetryWithOutputMonitor(opTimeout, 0, pollInterval, maxTime, runnable);
647     }
648 
649     /** {@inheritDoc} */
650     @Override
runFixedTimedRetryWithOutputMonitor( final long opTimeout, long idleOutputTimeout, final long pollInterval, final long maxTime, final IRunUtil.IRunnableResult runnable)651     public boolean runFixedTimedRetryWithOutputMonitor(
652             final long opTimeout,
653             long idleOutputTimeout,
654             final long pollInterval,
655             final long maxTime,
656             final IRunUtil.IRunnableResult runnable) {
657         final long initialTime = getCurrentTime();
658         while (getCurrentTime() < (initialTime + maxTime)) {
659             if (runTimedWithOutputMonitor(opTimeout, idleOutputTimeout, runnable, true)
660                     == CommandStatus.SUCCESS) {
661                 return true;
662             }
663             CLog.d("operation failed, waiting for %d ms", pollInterval);
664             sleep(pollInterval);
665         }
666         return false;
667     }
668 
669     /** {@inheritDoc} */
670     @Override
runEscalatingTimedRetry( final long opTimeout, final long initialPollInterval, final long maxPollInterval, final long maxTime, final IRunUtil.IRunnableResult runnable)671     public boolean runEscalatingTimedRetry(
672             final long opTimeout,
673             final long initialPollInterval,
674             final long maxPollInterval,
675             final long maxTime,
676             final IRunUtil.IRunnableResult runnable) {
677         // wait an initial time provided
678         long pollInterval = initialPollInterval;
679         final long initialTime = getCurrentTime();
680         while (true) {
681             if (runTimedWithOutputMonitor(opTimeout, 0, runnable, true) == CommandStatus.SUCCESS) {
682                 return true;
683             }
684             long remainingTime = maxTime - (getCurrentTime() - initialTime);
685             if (remainingTime <= 0) {
686                 CLog.d("operation is still failing after retrying for %d ms", maxTime);
687                 return false;
688             } else if (remainingTime < pollInterval) {
689                 // cap pollInterval to a max of remainingTime
690                 pollInterval = remainingTime;
691             }
692             CLog.d("operation failed, waiting for %d ms", pollInterval);
693             sleep(pollInterval);
694             // somewhat arbitrarily, increase the poll time by a factor of 4 for each attempt,
695             // up to the previously decided maximum
696             pollInterval *= POLL_TIME_INCREASE_FACTOR;
697             if (pollInterval > maxPollInterval) {
698                 pollInterval = maxPollInterval;
699             }
700         }
701     }
702 
703     /**
704      * Retrieves the current system clock time.
705      * <p/>
706      * Exposed so it can be mocked for unit testing
707      */
getCurrentTime()708     long getCurrentTime() {
709         return System.currentTimeMillis();
710     }
711 
712     /**
713      * {@inheritDoc}
714      */
715     @Override
sleep(long time)716     public void sleep(long time) {
717         mInterrupter.checkInterrupted();
718         if (time <= 0) {
719             return;
720         }
721         try (CloseableTraceScope sleep =
722                 new CloseableTraceScope(InvocationMetricKey.host_sleep.toString())) {
723             Thread.sleep(time);
724         } catch (InterruptedException e) {
725             // ignore
726             CLog.d("sleep interrupted");
727         }
728         mInterrupter.checkInterrupted();
729     }
730 
731     /** {@inheritDoc} */
732     @Override
allowInterrupt(boolean allow)733     public void allowInterrupt(boolean allow) {
734         if (allow) {
735             mInterrupter.allowInterrupt();
736         } else {
737             mInterrupter.blockInterrupt();
738         }
739     }
740 
741     /** {@inheritDoc} */
742     @Override
isInterruptAllowed()743     public boolean isInterruptAllowed() {
744         return mInterrupter.isInterruptible();
745     }
746 
747     /** {@inheritDoc} */
748     @Override
setInterruptibleInFuture(Thread thread, final long timeMs)749     public void setInterruptibleInFuture(Thread thread, final long timeMs) {
750         mInterrupter.allowInterruptAsync(thread, timeMs, TimeUnit.MILLISECONDS);
751     }
752 
753     /** {@inheritDoc} */
754     @Override
interrupt(Thread thread, String message)755     public synchronized void interrupt(Thread thread, String message) {
756         interrupt(thread, message, null);
757     }
758 
759     /** {@inheritDoc} */
760     @Override
interrupt(Thread thread, String message, ErrorIdentifier errorId)761     public synchronized void interrupt(Thread thread, String message, ErrorIdentifier errorId) {
762         mInterrupter.interrupt(thread, message, errorId);
763         mInterrupter.checkInterrupted();
764     }
765 
766     /**
767      * Helper thread that wraps a runnable, and notifies when done.
768      */
769     private static class RunnableNotifier extends Thread {
770 
771         private final IRunUtil.IRunnableResult mRunnable;
772         private CommandStatus mStatus = CommandStatus.TIMED_OUT;
773         private boolean mLogErrors = true;
774 
RunnableNotifier(IRunUtil.IRunnableResult runnable, boolean logErrors)775         RunnableNotifier(IRunUtil.IRunnableResult runnable, boolean logErrors) {
776             // Set this thread to be a daemon so that it does not prevent
777             // TF from shutting down.
778             setName(RUNNABLE_NOTIFIER_NAME);
779             setDaemon(true);
780             mRunnable = runnable;
781             mLogErrors = logErrors;
782         }
783 
784         @Override
run()785         public void run() {
786             CommandStatus status;
787             try {
788                 status = mRunnable.run() ? CommandStatus.SUCCESS : CommandStatus.FAILED;
789             } catch (InterruptedException e) {
790                 CLog.i("runutil interrupted");
791                 status = CommandStatus.EXCEPTION;
792                 backFillException(mRunnable.getResult(), e);
793             } catch (Exception e) {
794                 if (mLogErrors) {
795                     CLog.e("Exception occurred when executing runnable");
796                     CLog.e(e);
797                 }
798                 status = CommandStatus.EXCEPTION;
799                 backFillException(mRunnable.getResult(), e);
800             }
801             synchronized (this) {
802                 mStatus = status;
803             }
804         }
805 
cancel()806         public void cancel() {
807             mRunnable.cancel();
808         }
809 
getStatus()810         synchronized CommandStatus getStatus() {
811             return mStatus;
812         }
813 
backFillException(CommandResult result, Exception e)814         private void backFillException(CommandResult result, Exception e) {
815             if (result == null) {
816                 return;
817             }
818             if (Strings.isNullOrEmpty(result.getStderr())) {
819                 result.setStderr(StreamUtil.getStackTrace(e));
820             }
821             if (result.getExitCode() == null) {
822                 // Set non-zero exit code
823                 result.setExitCode(88);
824             }
825         }
826     }
827 
828     class RunnableResult implements IRunUtil.IRunnableResult {
829         private final ProcessBuilder mProcessBuilder;
830         private final CommandResult mCommandResult;
831         private final String mInput;
832         private Process mProcess = null;
833         private CountDownLatch mCountDown = null;
834         private Thread mExecutionThread;
835         private OutputStream mStdOut = null;
836         private OutputStream mStdErr = null;
837         private final File mInputRedirect;
838         private final Object mLock = new Object();
839         private boolean mCancelled = false;
840         private boolean mLogErrors = true;
841         private File mOutputMonitorStdoutFile;
842         private File mOutputMonitorStderrFile;
843         private long mOutputMonitorFileLastSize;
844         private long mOutputMonitorLastChangeTime;
845 
RunnableResult(final String input, final ProcessBuilder processBuilder)846         RunnableResult(final String input, final ProcessBuilder processBuilder) {
847             this(input, processBuilder, null, null, null, true);
848         }
849 
RunnableResult(final String input, final ProcessBuilder processBuilder, boolean logErrors)850         RunnableResult(final String input, final ProcessBuilder processBuilder, boolean logErrors) {
851             this(input, processBuilder, null, null, null, logErrors);
852         }
853 
854         /**
855          * Alternative constructor that allows redirecting the output to any Outputstream. Stdout
856          * and stderr can be independently redirected to different Outputstream implementations. If
857          * streams are null, default behavior of using a buffer will be used.
858          *
859          * <p>Additionally, Stdin can be redirected from a File.
860          */
RunnableResult( final String input, final ProcessBuilder processBuilder, final OutputStream stdoutStream, final OutputStream stderrStream, final File inputRedirect, final boolean logErrors)861         RunnableResult(
862                 final String input,
863                 final ProcessBuilder processBuilder,
864                 final OutputStream stdoutStream,
865                 final OutputStream stderrStream,
866                 final File inputRedirect,
867                 final boolean logErrors) {
868             mProcessBuilder = processBuilder;
869             mInput = input;
870             mLogErrors = logErrors;
871 
872             mInputRedirect = inputRedirect;
873             if (mInputRedirect != null) {
874                 // Set Stdin to mInputRedirect file.
875                 mProcessBuilder.redirectInput(mInputRedirect);
876             }
877 
878             mCommandResult = newCommandResult();
879             mCountDown = new CountDownLatch(1);
880 
881             // Redirect IO, so that the outputstream for the spawn process does not fill up
882             // and cause deadlock.
883             mStdOut = stdoutStream;
884             mStdErr = stderrStream;
885         }
886 
887         @Override
getCommand()888         public List<String> getCommand() {
889             return new ArrayList<>(mProcessBuilder.command());
890         }
891 
892         @Override
getResult()893         public CommandResult getResult() {
894             return mCommandResult;
895         }
896 
897         /** Start a {@link Process} based on the {@link ProcessBuilder}. */
898         @VisibleForTesting
startProcess()899         Process startProcess() throws IOException {
900             return mProcessBuilder.start();
901         }
902 
903         @Override
run()904         public boolean run() throws Exception {
905             File stdoutFile = mProcessBuilder.redirectOutput().file();
906             File stderrFile = mProcessBuilder.redirectError().file();
907             boolean temporaryStdout = false;
908             boolean temporaryErrOut = false;
909             Thread stdoutThread = null;
910             Thread stderrThread = null;
911             synchronized (mLock) {
912                 if (mCancelled) {
913                     // if cancel() was called before run() took the lock, we do not even attempt
914                     // to run.
915                     return false;
916                 }
917                 mExecutionThread = Thread.currentThread();
918                 if (stdoutFile == null && mStdOut == null) {
919                     temporaryStdout = true;
920                     stdoutFile =
921                             FileUtil.createTempFile(
922                                     String.format(
923                                             "temporary-stdout-%s",
924                                             mProcessBuilder.command().get(0)),
925                                     ".txt");
926                     stdoutFile.deleteOnExit();
927                     mProcessBuilder.redirectOutput(Redirect.appendTo(stdoutFile));
928                 }
929                 if (stderrFile == null && mStdErr == null) {
930                     temporaryErrOut = true;
931                     stderrFile =
932                             FileUtil.createTempFile(
933                                     String.format(
934                                             "temporary-errout-%s",
935                                             mProcessBuilder.command().get(0)),
936                                     ".txt");
937                     stderrFile.deleteOnExit();
938                     mProcessBuilder.redirectError(Redirect.appendTo(stderrFile));
939                 }
940                 // Obtain a reference to the output stream redirect file for progress monitoring.
941                 mOutputMonitorStdoutFile = stdoutFile;
942                 mOutputMonitorStderrFile = stderrFile;
943                 mOutputMonitorFileLastSize = 0;
944                 mOutputMonitorLastChangeTime = 0;
945                 try {
946                     mProcess = startProcess();
947                 } catch (IOException | RuntimeException e) {
948                     if (temporaryStdout) {
949                         FileUtil.deleteFile(stdoutFile);
950                     }
951                     if (temporaryErrOut) {
952                         FileUtil.deleteFile(stderrFile);
953                     }
954                     throw e;
955                 }
956                 if (mInput != null) {
957                     BufferedOutputStream processStdin =
958                             new BufferedOutputStream(mProcess.getOutputStream());
959                     processStdin.write(mInput.getBytes("UTF-8"));
960                     processStdin.flush();
961                     processStdin.close();
962                 }
963                 if (mStdOut != null) {
964                     stdoutThread =
965                             inheritIO(
966                                     mProcess.getInputStream(),
967                                     mStdOut,
968                                     String.format(
969                                             "inheritio-stdout-%s", mProcessBuilder.command()));
970                 }
971                 if (mStdErr != null) {
972                     stderrThread =
973                             inheritIO(
974                                     mProcess.getErrorStream(),
975                                     mStdErr,
976                                     String.format(
977                                             "inheritio-stderr-%s", mProcessBuilder.command()));
978                 }
979             }
980             // Wait for process to complete.
981             Integer rc = null;
982             try {
983                 try {
984                     rc = mProcess.waitFor();
985                     // wait for stdout and stderr to be read
986                     if (stdoutThread != null) {
987                         stdoutThread.join();
988                     }
989                     if (stderrThread != null) {
990                         stderrThread.join();
991                     }
992                 } finally {
993                     rc = (rc != null) ? rc : 1; // In case of interruption ReturnCode is null
994                     mCommandResult.setExitCode(rc);
995 
996                     // Write out the streams to the result.
997                     if (temporaryStdout) {
998                         mCommandResult.setStdout(FileUtil.readStringFromFile(stdoutFile));
999                     } else {
1000                         final String stdoutDest =
1001                                 stdoutFile != null
1002                                         ? stdoutFile.getAbsolutePath()
1003                                         : mStdOut.getClass().getSimpleName();
1004                         mCommandResult.setStdout("redirected to " + stdoutDest);
1005                     }
1006                     if (temporaryErrOut) {
1007                         mCommandResult.setStderr(FileUtil.readStringFromFile(stderrFile));
1008                     } else {
1009                         final String stderrDest =
1010                                 stderrFile != null
1011                                         ? stderrFile.getAbsolutePath()
1012                                         : mStdErr.getClass().getSimpleName();
1013                         mCommandResult.setStderr("redirected to " + stderrDest);
1014                     }
1015                 }
1016             } finally {
1017                 if (temporaryStdout) {
1018                     FileUtil.deleteFile(stdoutFile);
1019                 }
1020                 if (temporaryErrOut) {
1021                     FileUtil.deleteFile(stderrFile);
1022                 }
1023                 mCountDown.countDown();
1024             }
1025 
1026             if (rc != null && rc == 0) {
1027                 return true;
1028             } else if (mLogErrors) {
1029                 CLog.d("%s command failed. return code %d", mProcessBuilder.command(), rc);
1030             }
1031             return false;
1032         }
1033 
1034         @Override
cancel()1035         public void cancel() {
1036             if (mCancelled) {
1037                 return;
1038             }
1039             mCancelled = true;
1040             synchronized (mLock) {
1041                 if (mProcess == null || !mProcess.isAlive()) {
1042                     return;
1043                 }
1044                 CLog.d("Cancelling the process execution.");
1045                 if (mLinuxInterruptProcess) {
1046                     long pid = mProcess.pid();
1047                     CommandResult killRes = RunUtil.getDefault().runTimedCmd(
1048                             60000L, "kill", "-2", "" + pid);
1049                     CLog.d("status=%s. stdout=%s . stderr=%s",
1050                             killRes.getStatus(), killRes.getStdout(), killRes.getStderr());
1051                     // Just give a little bit of time to terminate.
1052                     if (mProcess.isAlive()) {
1053                         RunUtil.getDefault().sleep(1000L);
1054                     }
1055                 }
1056                 // Always destroy to ensure it terminates.
1057                 mProcess.destroy();
1058                 try {
1059                     // Only allow to continue if the Stdout has been read
1060                     // RunnableNotifier#Interrupt is the next call and will terminate the thread
1061                     if (!mCountDown.await(PROCESS_DESTROY_TIMEOUT_SEC, TimeUnit.SECONDS)) {
1062                         CLog.i("Process still not terminated, interrupting the execution thread");
1063                         mExecutionThread.interrupt();
1064                         mCountDown.await();
1065                     }
1066                 } catch (InterruptedException e) {
1067                     CLog.i("interrupted while waiting for process output to be saved");
1068                 }
1069             }
1070         }
1071 
1072         @Override
toString()1073         public String toString() {
1074             return "RunnableResult [command="
1075                     + ((mProcessBuilder != null) ? mProcessBuilder.command() : null)
1076                     + "]";
1077         }
1078 
1079         /**
1080          * Checks if the currently running operation has made progress since the last check.
1081          *
1082          * @param idleOutputTimeout ms idle with no observed progress before beginning to assume no
1083          *     progress is being made.
1084          * @return true if progress has been detected otherwise false.
1085          */
1086         @Override
checkOutputMonitor(Long idleOutputTimeout)1087         public boolean checkOutputMonitor(Long idleOutputTimeout) {
1088             synchronized (mLock) {
1089                 // If we don't have what we need to check for progress, abort the check.
1090                 if ((mOutputMonitorStdoutFile == null || !mOutputMonitorStdoutFile.exists())
1091                         && (mOutputMonitorStderrFile == null
1092                                 || !mOutputMonitorStderrFile.exists())) {
1093                     // Let the operation timeout on its own.
1094                     return true;
1095                 }
1096 
1097                 if (mOutputMonitorLastChangeTime == 0) {
1098                     mOutputMonitorLastChangeTime = System.currentTimeMillis();
1099                     // If this is the start of a new command invocation, log only once.
1100                     CLog.d(
1101                             "checkOutputMonitor activated with idle timeout set for %.2f seconds",
1102                             idleOutputTimeout / 1000f);
1103                 }
1104 
1105                 // Observing progress by monitoring the size of the output changing.
1106                 long currentFileSize = getMonitoredStdoutSize() + getMonitoredStderrSize();
1107                 long idleTime = System.currentTimeMillis() - mOutputMonitorLastChangeTime;
1108                 if (currentFileSize == mOutputMonitorFileLastSize && idleTime > idleOutputTimeout) {
1109                     CLog.d(
1110                             "checkOutputMonitor: No new progress detected for over %.2f seconds",
1111                             idleTime / 1000f);
1112                     return false;
1113                 }
1114 
1115                 // Update change time only when new data appears on the streams.
1116                 if (currentFileSize != mOutputMonitorFileLastSize) {
1117                     mOutputMonitorLastChangeTime = System.currentTimeMillis();
1118                     idleTime = 0;
1119                 }
1120                 mOutputMonitorFileLastSize = currentFileSize;
1121             }
1122             // Always default to progress being made.
1123             return true;
1124         }
1125 
getMonitoredStdoutSize()1126         private long getMonitoredStdoutSize() {
1127             if (mOutputMonitorStdoutFile != null && mOutputMonitorStdoutFile.exists()) {
1128                 return mOutputMonitorStdoutFile.length();
1129             }
1130             return 0;
1131         }
1132 
getMonitoredStderrSize()1133         private long getMonitoredStderrSize() {
1134             if (mOutputMonitorStderrFile != null && mOutputMonitorStderrFile.exists()) {
1135                 return mOutputMonitorStderrFile.length();
1136             }
1137             return 0;
1138         }
1139     }
1140 
1141     /**
1142      * Helper method to redirect input stream.
1143      *
1144      * @param src {@link InputStream} to inherit/redirect from
1145      * @param dest {@link BufferedOutputStream} to inherit/redirect to
1146      * @param name the name of the thread returned.
1147      * @return a {@link Thread} started that receives the IO.
1148      */
inheritIO(final InputStream src, final OutputStream dest, String name)1149     private static Thread inheritIO(final InputStream src, final OutputStream dest, String name) {
1150         // In case of some Process redirect, source stream can be null.
1151         if (src == null) {
1152             return null;
1153         }
1154         Thread t =
1155                 new Thread(
1156                         new Runnable() {
1157                             @Override
1158                             public void run() {
1159                                 try {
1160                                     StreamUtil.copyStreams(src, dest);
1161                                 } catch (IOException e) {
1162                                     CLog.e("Failed to read input stream %s.", name);
1163                                 }
1164                             }
1165                         });
1166         t.setName(name);
1167         t.start();
1168         return t;
1169     }
1170 
1171     /** {@inheritDoc} */
1172     @Override
setEnvVariablePriority(EnvPriority priority)1173     public void setEnvVariablePriority(EnvPriority priority) {
1174         if (this.equals(sDefaultInstance)) {
1175             throw new UnsupportedOperationException(
1176                     "Cannot setEnvVariablePriority on default RunUtil");
1177         }
1178         mEnvVariablePriority = priority;
1179     }
1180 
1181     /** {@inheritDoc} */
1182     @Override
setLinuxInterruptProcess(boolean interrupt)1183     public void setLinuxInterruptProcess(boolean interrupt) {
1184         if (this.equals(sDefaultInstance)) {
1185             throw new UnsupportedOperationException(
1186                     "Cannot setLinuxInterruptProcess on default RunUtil");
1187         }
1188         mLinuxInterruptProcess = interrupt;
1189     }
1190 
newCommandResult()1191     private static CommandResult newCommandResult() {
1192         CommandResult commandResult = new CommandResult();
1193         // Ensure the outputs are never null
1194         commandResult.setStdout("");
1195         commandResult.setStderr("");
1196         return commandResult;
1197     }
1198 
handleCachedResult( ExecutableActionResult result, OutputStream stdout, OutputStream stderr)1199     private static CommandResult handleCachedResult(
1200             ExecutableActionResult result, OutputStream stdout, OutputStream stderr)
1201             throws IOException {
1202 
1203         CommandResult commandResult = newCommandResult();
1204         commandResult.setExitCode(result.exitCode());
1205         // Only success run will be cached.
1206         commandResult.setStatus(CommandStatus.SUCCESS);
1207         commandResult.setCached(true);
1208         if (result.stdOut() != null && stdout != null) {
1209             FileInputStream stdoutStream = new FileInputStream(result.stdOut());
1210             try {
1211                 StreamUtil.copyStreams(stdoutStream, stdout);
1212             } finally {
1213                 stdoutStream.close();
1214                 FileUtil.deleteFile(result.stdOut());
1215             }
1216         }
1217         if (result.stdErr() != null && stderr != null) {
1218             FileInputStream stderrStream = new FileInputStream(result.stdErr());
1219             try {
1220                 StreamUtil.copyStreams(stderrStream, stderr);
1221             } finally {
1222                 stderrStream.close();
1223                 FileUtil.deleteFile(result.stdErr());
1224             }
1225         }
1226         return commandResult;
1227     }
1228 
1229     /**
1230      * Utility subclass of OutputStream that forwards the data to both underlying {@link
1231      * OutputStream} and {@link FileOutputStream}.
1232      */
1233     private static class ForkedOutputStream extends OutputStream {
1234         private final FileOutputStream mFileOutputStream;
1235         private final OutputStream mOut;
1236         private boolean mSuccess = true;
1237 
ForkedOutputStream(OutputStream out, FileOutputStream fileOutputStream)1238         public ForkedOutputStream(OutputStream out, FileOutputStream fileOutputStream) {
1239             mOut = out;
1240             mFileOutputStream = fileOutputStream;
1241         }
1242 
1243         @Override
write(int b)1244         public void write(int b) throws IOException {
1245             mOut.write(b);
1246             try {
1247                 mFileOutputStream.write(b);
1248             } catch (IOException e) {
1249                 CLog.e("Failed to write to the file output stream!");
1250                 CLog.e(e);
1251                 mSuccess = false;
1252             }
1253         }
1254 
1255         @Override
write(byte[] b)1256         public void write(byte[] b) throws IOException {
1257             mOut.write(b);
1258             try {
1259                 mFileOutputStream.write(b);
1260             } catch (IOException e) {
1261                 CLog.e("Failed to write to the file output stream!");
1262                 CLog.e(e);
1263                 mSuccess = false;
1264             }
1265         }
1266 
1267         @Override
write(byte[] b, int off, int len)1268         public void write(byte[] b, int off, int len) throws IOException {
1269             mOut.write(b, off, len);
1270             try {
1271                 mFileOutputStream.write(b, off, len);
1272             } catch (IOException e) {
1273                 CLog.e("Failed to write to the file output stream!");
1274                 CLog.e(e);
1275                 mSuccess = false;
1276             }
1277         }
1278 
1279         @Override
flush()1280         public void flush() throws IOException {
1281             mOut.flush();
1282             try {
1283                 mFileOutputStream.flush();
1284             } catch (IOException e) {
1285                 CLog.e("Failed to flush the file output stream!");
1286                 CLog.e(e);
1287                 mSuccess = false;
1288             }
1289         }
1290 
1291         @Override
close()1292         public void close() throws IOException {
1293             mOut.close();
1294             try {
1295                 mFileOutputStream.close();
1296             } catch (IOException e) {
1297                 CLog.e("Failed to close the file output stream!");
1298                 CLog.e(e);
1299                 mSuccess = false;
1300             }
1301         }
1302 
isSuccess()1303         public boolean isSuccess() {
1304             return mSuccess;
1305         }
1306     }
1307 }
1308