1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.device;
17 
18 import com.android.ddmlib.AdbCommandRejectedException;
19 import com.android.ddmlib.IDevice;
20 import com.android.ddmlib.TimeoutException;
21 import com.android.helper.aoa.UsbDevice;
22 import com.android.helper.aoa.UsbException;
23 import com.android.helper.aoa.UsbHelper;
24 import com.android.tradefed.config.Option;
25 import com.android.tradefed.invoker.logger.InvocationMetricLogger;
26 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
27 import com.android.tradefed.log.LogUtil.CLog;
28 import com.android.tradefed.result.error.DeviceErrorIdentifier;
29 import com.android.tradefed.util.CommandResult;
30 import com.android.tradefed.util.CommandStatus;
31 import com.android.tradefed.util.IRunUtil;
32 import com.android.tradefed.util.RunUtil;
33 
34 import com.google.common.annotations.VisibleForTesting;
35 
36 import java.io.IOException;
37 import java.util.concurrent.ExecutionException;
38 
39 /**
40  * A simple implementation of a {@link IDeviceRecovery} that waits for device to be online and
41  * respond to simple commands.
42  */
43 public class WaitDeviceRecovery implements IDeviceRecovery {
44 
45     /** the time in ms to wait before beginning recovery attempts */
46     protected static final long INITIAL_PAUSE_TIME = 5 * 1000;
47 
48     private static final long WAIT_FOR_DEVICE_OFFLINE = 20 * 1000;
49 
50     /**
51      * The number of attempts to check if device is in bootloader.
52      * <p/>
53      * Exposed for unit testing
54      */
55     public static final int BOOTLOADER_POLL_ATTEMPTS = 3;
56 
57     // TODO: add a separate configurable timeout per operation
58     @Option(name="online-wait-time",
59             description="maximum time in ms to wait for device to come online.")
60     protected long mOnlineWaitTime = 60 * 1000;
61     @Option(name="device-wait-time",
62             description="maximum time in ms to wait for a single device recovery command.")
63     protected long mWaitTime = 4 * 60 * 1000;
64 
65     @Option(name="bootloader-wait-time",
66             description="maximum time in ms to wait for device to be in fastboot.")
67     protected long mBootloaderWaitTime = 30 * 1000;
68 
69     @Option(name="shell-wait-time",
70             description="maximum time in ms to wait for device shell to be responsive.")
71     protected long mShellWaitTime = 30 * 1000;
72 
73     @Option(name="fastboot-wait-time",
74             description="maximum time in ms to wait for a fastboot command result.")
75     protected long mFastbootWaitTime = 30 * 1000;
76 
77     @Option(name = "min-battery-after-recovery",
78             description = "require a min battery level after successful recovery, " +
79                           "default to 0 for ignoring.")
80     protected int mRequiredMinBattery = 0;
81 
82     @Option(name = "disable-unresponsive-reboot",
83             description = "If this is set, we will not attempt to reboot an unresponsive device" +
84             "that is in userspace.  Note that this will have no effect if the device is in " +
85             "fastboot or is expected to be in fastboot.")
86     protected boolean mDisableUnresponsiveReboot = false;
87 
88     @Option(
89             name = "disable-usb-reset",
90             description = "Do not attempt reset via USB in order to recover devices.")
91     protected boolean mDisableUsbReset = false;
92 
93     private String mFastbootPath = "fastboot";
94 
95     /**
96      * Get the {@link RunUtil} instance to use.
97      * <p/>
98      * Exposed for unit testing.
99      */
getRunUtil()100     protected IRunUtil getRunUtil() {
101         return RunUtil.getDefault();
102     }
103 
104     /**
105      * Sets the maximum time in ms to wait for a single device recovery command.
106      */
setWaitTime(long waitTime)107     void setWaitTime(long waitTime) {
108         mWaitTime = waitTime;
109     }
110 
111     /**
112      * {@inheritDoc}
113      */
114     @Override
setFastbootPath(String fastbootPath)115     public void setFastbootPath(String fastbootPath) {
116         mFastbootPath = fastbootPath;
117     }
118 
119     /**
120      * {@inheritDoc}
121      */
122     @Override
recoverDevice(IDeviceStateMonitor monitor, boolean recoverUntilOnline)123     public void recoverDevice(IDeviceStateMonitor monitor, boolean recoverUntilOnline)
124             throws DeviceNotAvailableException {
125         // device may have just gone offline
126         // sleep a small amount to give ddms state a chance to settle
127         // TODO - see if there is better way to handle this
128         CLog.i("Pausing for %d for %s to recover", INITIAL_PAUSE_TIME, monitor.getSerialNumber());
129         getRunUtil().sleep(INITIAL_PAUSE_TIME);
130 
131         // ensure bootloader state is updated
132         monitor.waitForDeviceBootloaderStateUpdate();
133 
134         TestDeviceState state = monitor.getDeviceState();
135         if (TestDeviceState.FASTBOOT.equals(state) || TestDeviceState.FASTBOOTD.equals(state)) {
136             CLog.i(
137                     "Found device %s in %s but expected online. Rebooting...",
138                     monitor.getSerialNumber(), state);
139             // TODO: retry if failed
140             getRunUtil()
141                     .runTimedCmd(
142                             mFastbootWaitTime,
143                             mFastbootPath,
144                             "-s",
145                             monitor.getFastbootSerialNumber(),
146                             "reboot");
147         }
148 
149         // wait for device online
150         IDevice device = monitor.waitForDeviceOnline(mOnlineWaitTime);
151         if (device == null) {
152             handleDeviceNotAvailable(monitor, recoverUntilOnline);
153             // function returning implies that recovery is successful, check battery level here
154             checkMinBatteryLevel(getDeviceAfterRecovery(monitor));
155             return;
156         }
157         // occasionally device is erroneously reported as online - double check that we can shell
158         // into device
159         if (!monitor.waitForDeviceShell(mShellWaitTime)) {
160             // treat this as a not available device
161             handleDeviceNotAvailable(monitor, recoverUntilOnline);
162             checkMinBatteryLevel(getDeviceAfterRecovery(monitor));
163             return;
164         }
165 
166         if (!recoverUntilOnline) {
167             if (monitor.waitForDeviceAvailableInRecoverPath(mWaitTime) == null) {
168                 // device is online but not responsive
169                 handleDeviceUnresponsive(device, monitor);
170             }
171         }
172         // do a final check here when all previous if blocks are skipped or the last
173         // handleDeviceUnresponsive was successful
174         checkMinBatteryLevel(getDeviceAfterRecovery(monitor));
175     }
176 
getDeviceAfterRecovery(IDeviceStateMonitor monitor)177     private IDevice getDeviceAfterRecovery(IDeviceStateMonitor monitor)
178             throws DeviceNotAvailableException {
179         IDevice device = monitor.waitForDeviceOnline(mOnlineWaitTime);
180         if (device == null) {
181             throw new DeviceNotAvailableException(
182                     "Device still not online after successful recovery",
183                     monitor.getSerialNumber(),
184                     DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
185         }
186         return device;
187     }
188 
189     /**
190      * Checks if device battery level meets min requirement
191      * @param device
192      * @throws DeviceNotAvailableException if battery level cannot be read or lower than min
193      */
checkMinBatteryLevel(IDevice device)194     protected void checkMinBatteryLevel(IDevice device) throws DeviceNotAvailableException {
195         if (mRequiredMinBattery <= 0) {
196             // don't do anything if check is not required
197             return;
198         }
199         try {
200             Integer level = device.getBattery().get();
201             if (level == null) {
202                 // can't read battery level but we are requiring a min, reject
203                 // device
204                 throw new DeviceNotAvailableException(
205                         "Cannot read battery level but a min is required",
206                         device.getSerialNumber(),
207                         DeviceErrorIdentifier.DEVICE_UNEXPECTED_RESPONSE);
208             } else if (level < mRequiredMinBattery) {
209                 throw new DeviceNotAvailableException(String.format(
210                         "After recovery, device battery level %d is lower than required minimum %d",
211                         level, mRequiredMinBattery), device.getSerialNumber());
212             }
213             return;
214         } catch (InterruptedException | ExecutionException e) {
215             throw new DeviceNotAvailableException(
216                     "exception while reading battery level",
217                     e,
218                     device.getSerialNumber(),
219                     DeviceErrorIdentifier.DEVICE_UNEXPECTED_RESPONSE);
220         }
221     }
222 
223     /**
224      * Handle situation where device is online but unresponsive.
225      * @param monitor
226      * @throws DeviceNotAvailableException
227      */
handleDeviceUnresponsive(IDevice device, IDeviceStateMonitor monitor)228     protected void handleDeviceUnresponsive(IDevice device, IDeviceStateMonitor monitor)
229             throws DeviceNotAvailableException {
230         if (!mDisableUnresponsiveReboot) {
231             CLog.i("Device %s unresponsive. Rebooting...", monitor.getSerialNumber());
232             rebootDevice(device, null);
233             IDevice newdevice = monitor.waitForDeviceOnline(mOnlineWaitTime);
234             if (newdevice == null) {
235                 handleDeviceNotAvailable(monitor, false);
236                 return;
237             }
238             if (monitor.waitForDeviceAvailable(mWaitTime) != null) {
239                 return;
240             }
241         }
242         // If no reboot was done, waitForDeviceAvailable has already been checked.
243         throw new DeviceUnresponsiveException(
244                 String.format("Device %s is online but unresponsive", monitor.getSerialNumber()),
245                 monitor.getSerialNumber(),
246                 DeviceErrorIdentifier.DEVICE_UNRESPONSIVE);
247     }
248 
249     /**
250      * Handle situation where device is not available.
251      *
252      * @param monitor the {@link IDeviceStateMonitor}
253      * @param recoverTillOnline if true this method should return if device is online, and not
254      * check for responsiveness
255      * @throws DeviceNotAvailableException
256      */
handleDeviceNotAvailable(IDeviceStateMonitor monitor, boolean recoverTillOnline)257     protected void handleDeviceNotAvailable(IDeviceStateMonitor monitor, boolean recoverTillOnline)
258             throws DeviceNotAvailableException {
259         if (attemptDeviceUnavailableRecovery(monitor, recoverTillOnline)) {
260             return;
261         }
262         String serial = monitor.getSerialNumber();
263         throw new DeviceNotAvailableException(
264                 String.format("Could not find device %s", serial),
265                 serial,
266                 DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
267     }
268 
269     /**
270      * {@inheritDoc}
271      */
272     @Override
recoverDeviceBootloader(final IDeviceStateMonitor monitor)273     public void recoverDeviceBootloader(final IDeviceStateMonitor monitor)
274             throws DeviceNotAvailableException {
275         // device may have just gone offline
276         // wait a small amount to give device state a chance to settle
277         // TODO - see if there is better way to handle this
278         CLog.i("Pausing for %d for %s to recover", INITIAL_PAUSE_TIME, monitor.getSerialNumber());
279         getRunUtil().sleep(INITIAL_PAUSE_TIME);
280 
281         // poll and wait for device to return to valid state
282         long pollTime = mBootloaderWaitTime / BOOTLOADER_POLL_ATTEMPTS;
283         for (int i=0; i < BOOTLOADER_POLL_ATTEMPTS; i++) {
284             if (monitor.waitForDeviceBootloader(pollTime)) {
285                 handleDeviceBootloaderUnresponsive(monitor);
286                 // passed above check, abort
287                 return;
288             } else if (monitor.getDeviceState() == TestDeviceState.ONLINE) {
289                 handleDeviceOnlineExpectedBootloader(monitor);
290                 return;
291             }
292         }
293         handleDeviceBootloaderOrFastbootNotAvailable(monitor, "bootloader");
294     }
295 
296     /** {@inheritDoc} */
297     @Override
recoverDeviceFastbootd(IDeviceStateMonitor monitor)298     public void recoverDeviceFastbootd(IDeviceStateMonitor monitor)
299             throws DeviceNotAvailableException {
300         // device may have just gone offline
301         // wait a small amount to give device state a chance to settle
302         // TODO - see if there is better way to handle this
303         CLog.i("Pausing for %d for %s to recover", INITIAL_PAUSE_TIME, monitor.getSerialNumber());
304         getRunUtil().sleep(INITIAL_PAUSE_TIME);
305 
306         // poll and wait for device to return to valid state
307         long pollTime = mBootloaderWaitTime / BOOTLOADER_POLL_ATTEMPTS;
308         for (int i = 0; i < BOOTLOADER_POLL_ATTEMPTS; i++) {
309             if (monitor.waitForDeviceFastbootd(mFastbootPath, pollTime)) {
310                 handleDeviceFastbootdUnresponsive(monitor);
311                 // passed above check, abort
312                 return;
313             } else if (monitor.getDeviceState() == TestDeviceState.ONLINE) {
314                 handleDeviceOnlineExpectedFasbootd(monitor);
315                 return;
316             }
317         }
318         handleDeviceBootloaderOrFastbootNotAvailable(monitor, "fastbootd");
319     }
320 
321     /**
322      * Handle condition where device is online, but should be in bootloader state.
323      *
324      * <p>If this method
325      *
326      * @param monitor
327      * @throws DeviceNotAvailableException
328      */
handleDeviceOnlineExpectedBootloader(final IDeviceStateMonitor monitor)329     private void handleDeviceOnlineExpectedBootloader(final IDeviceStateMonitor monitor)
330             throws DeviceNotAvailableException {
331         CLog.i("Found device %s online but expected bootloader.", monitor.getSerialNumber());
332         // call waitForDeviceOnline to get handle to IDevice
333         IDevice device = monitor.waitForDeviceOnline(mOnlineWaitTime);
334         if (device == null) {
335             handleDeviceBootloaderOrFastbootNotAvailable(monitor, "bootloader");
336             return;
337         }
338         rebootDevice(device, "bootloader");
339         if (!monitor.waitForDeviceBootloader(mBootloaderWaitTime)) {
340             throw new DeviceNotAvailableException(
341                     String.format(
342                             "Device %s not in bootloader after reboot", monitor.getSerialNumber()),
343                     monitor.getSerialNumber(),
344                     DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
345         }
346     }
347 
handleDeviceOnlineExpectedFasbootd(final IDeviceStateMonitor monitor)348     private void handleDeviceOnlineExpectedFasbootd(final IDeviceStateMonitor monitor)
349             throws DeviceNotAvailableException {
350         CLog.i("Found device %s online but expected fastbootd.", monitor.getSerialNumber());
351         // call waitForDeviceOnline to get handle to IDevice
352         IDevice device = monitor.waitForDeviceOnline(mOnlineWaitTime);
353         if (device == null) {
354             handleDeviceBootloaderOrFastbootNotAvailable(monitor, "fastbootd");
355             return;
356         }
357         rebootDevice(device, "fastboot");
358         if (!monitor.waitForDeviceFastbootd(mFastbootPath, mBootloaderWaitTime)) {
359             throw new DeviceNotAvailableException(
360                     String.format(
361                             "Device %s not in fastbootd after reboot", monitor.getSerialNumber()),
362                     monitor.getSerialNumber(),
363                     DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
364         }
365     }
366 
handleDeviceFastbootdUnresponsive(IDeviceStateMonitor monitor)367     private void handleDeviceFastbootdUnresponsive(IDeviceStateMonitor monitor)
368             throws DeviceNotAvailableException {
369         CLog.i(
370                 "Found device %s in fastbootd but potentially unresponsive.",
371                 monitor.getSerialNumber());
372         // TODO: retry reboot
373         getRunUtil()
374                 .runTimedCmd(
375                         mFastbootWaitTime,
376                         mFastbootPath,
377                         "-s",
378                         monitor.getSerialNumber(),
379                         "reboot",
380                         "fastboot");
381         // wait for device to reboot
382         monitor.waitForDeviceNotAvailable(WAIT_FOR_DEVICE_OFFLINE);
383         if (!monitor.waitForDeviceFastbootd(mFastbootPath, mBootloaderWaitTime)) {
384             throw new DeviceNotAvailableException(
385                     String.format(
386                             "Device %s not in fastbootd after reboot", monitor.getSerialNumber()),
387                     monitor.getSerialNumber(),
388                     DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
389         }
390         // running a meaningless command just to see whether the device is responsive.
391         CommandResult result =
392                 getRunUtil()
393                         .runTimedCmd(
394                                 mFastbootWaitTime,
395                                 mFastbootPath,
396                                 "-s",
397                                 monitor.getSerialNumber(),
398                                 "getvar",
399                                 "product");
400         if (result.getStatus().equals(CommandStatus.TIMED_OUT)) {
401             throw new DeviceNotAvailableException(
402                     String.format(
403                             "Device %s is in fastbootd but unresponsive",
404                             monitor.getSerialNumber()),
405                     monitor.getSerialNumber(),
406                     DeviceErrorIdentifier.DEVICE_UNRESPONSIVE);
407         }
408     }
409 
410     /**
411      * @param monitor
412      * @throws DeviceNotAvailableException
413      */
handleDeviceBootloaderUnresponsive(IDeviceStateMonitor monitor)414     private void handleDeviceBootloaderUnresponsive(IDeviceStateMonitor monitor)
415             throws DeviceNotAvailableException {
416         CLog.i("Found device %s in fastboot but potentially unresponsive.",
417                 monitor.getSerialNumber());
418         // TODO: retry reboot
419         getRunUtil().runTimedCmd(mFastbootWaitTime, mFastbootPath, "-s", monitor.getSerialNumber(),
420                 "reboot-bootloader");
421         // wait for device to reboot
422         monitor.waitForDeviceNotAvailable(20*1000);
423         if (!monitor.waitForDeviceBootloader(mBootloaderWaitTime)) {
424             throw new DeviceNotAvailableException(
425                     String.format(
426                             "Device %s not in bootloader after reboot", monitor.getSerialNumber()),
427                     monitor.getSerialNumber(),
428                     DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
429         }
430         // running a meaningless command just to see whether the device is responsive.
431         CommandResult result = getRunUtil().runTimedCmd(mFastbootWaitTime, mFastbootPath, "-s",
432                 monitor.getSerialNumber(), "getvar", "product");
433         if (result.getStatus().equals(CommandStatus.TIMED_OUT)) {
434             throw new DeviceNotAvailableException(
435                     String.format(
436                             "Device %s is in fastboot but unresponsive", monitor.getSerialNumber()),
437                     monitor.getSerialNumber(),
438                     DeviceErrorIdentifier.DEVICE_UNRESPONSIVE);
439         }
440     }
441 
442     /**
443      * Reboot device into given mode.
444      *
445      * @param device the {@link IDevice} to reboot.
446      * @param mode The mode into which to reboot the device. null being regular reboot.
447      */
rebootDevice(IDevice device, String mode)448     private void rebootDevice(IDevice device, String mode) throws DeviceNotAvailableException {
449         try {
450             device.reboot(mode);
451         } catch (IOException e) {
452             CLog.w(
453                     "%s: failed to reboot %s: %s",
454                     e.getClass().getSimpleName(), device.getSerialNumber(), e.getMessage());
455         } catch (TimeoutException e) {
456             CLog.w("failed to reboot %s: timeout", device.getSerialNumber());
457         } catch (AdbCommandRejectedException e) {
458             CLog.w(
459                     "%s: failed to reboot %s: %s",
460                     e.getClass().getSimpleName(), device.getSerialNumber(), e.getMessage());
461             if (e.isDeviceOffline() || e.wasErrorDuringDeviceSelection()) {
462                 // If reboot is not attempted, then fail right away
463                 throw new DeviceNotAvailableException(
464                         e.getMessage(),
465                         e,
466                         device.getSerialNumber(),
467                         DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
468             }
469         }
470     }
471 
472     /**
473      * Handle situation where device is not available when expected to be in bootloader.
474      *
475      * @param monitor the {@link IDeviceStateMonitor}
476      * @throws DeviceNotAvailableException
477      */
handleDeviceBootloaderOrFastbootNotAvailable( final IDeviceStateMonitor monitor, String mode)478     private void handleDeviceBootloaderOrFastbootNotAvailable(
479             final IDeviceStateMonitor monitor, String mode) throws DeviceNotAvailableException {
480         throw new DeviceNotAvailableException(
481                 String.format("Could not find device %s in %s", monitor.getSerialNumber(), mode),
482                 monitor.getSerialNumber(),
483                 DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
484     }
485 
486     /**
487      * {@inheritDoc}
488      */
489     @Override
recoverDeviceRecovery(IDeviceStateMonitor monitor)490     public void recoverDeviceRecovery(IDeviceStateMonitor monitor)
491             throws DeviceNotAvailableException {
492         // TODO(b/305735893): Root and capture logs
493         throw new DeviceNotAvailableException(
494                 "device unexpectedly went into recovery mode.",
495                 monitor.getSerialNumber(),
496                 DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
497     }
498 
499     /** Recovery routine for device unavailable errors. */
attemptDeviceUnavailableRecovery( IDeviceStateMonitor monitor, boolean recoverTillOnline)500     private boolean attemptDeviceUnavailableRecovery(
501             IDeviceStateMonitor monitor, boolean recoverTillOnline)
502             throws DeviceNotAvailableException {
503         TestDeviceState state = monitor.getDeviceState();
504         if (TestDeviceState.RECOVERY.equals(state)) {
505             CLog.d("Device is in '%s' state skipping USB reset attempt.", state);
506             recoverDeviceRecovery(monitor);
507             return false;
508         }
509         if (TestDeviceState.FASTBOOT.equals(state) || TestDeviceState.FASTBOOTD.equals(state)) {
510             CLog.d("Device is in '%s' state skipping USB reset attempt.", state);
511             return false;
512         }
513         if (monitor.isAdbTcp()) {
514             CLog.d("Device is connected via TCP, skipping USB reset attempt.");
515             return false;
516         }
517         boolean recoveryAttempted = false;
518         if (!mDisableUsbReset) {
519             // First try to do a USB reset to get the device back
520             try (UsbHelper usb = getUsbHelper()) {
521                 String serial = monitor.getSerialNumber();
522                 try (UsbDevice usbDevice = usb.getDevice(serial)) {
523                     if (usbDevice != null) {
524                         CLog.d("Resetting USB port for device '%s'", serial);
525                         usbDevice.reset();
526                         recoveryAttempted = true;
527                         if (waitForDevice(monitor, recoverTillOnline)) {
528                             // Success
529                             CLog.d("Device recovered from USB reset and is online.");
530                             InvocationMetricLogger.addInvocationMetrics(
531                                     InvocationMetricKey.DEVICE_RECOVERY, 1);
532                             return true;
533                         }
534                     }
535                 }
536             } catch (LinkageError e) {
537                 CLog.w("Problem initializing USB helper, skipping USB reset and disabling it.");
538                 CLog.w(e);
539                 mDisableUsbReset = true;
540             } catch (UsbException e) {
541                 CLog.w("Problem initializing USB helper, skipping USB reset.");
542                 CLog.w(e);
543             }
544         }
545         if (recoveryAttempted) {
546             // Sometimes device come back visible but in recovery
547             if (TestDeviceState.RECOVERY.equals(monitor.getDeviceState())) {
548                 IDevice device = monitor.waitForDeviceInRecovery();
549                 if (device != null) {
550                     CLog.d("Device came back in 'RECOVERY' mode when we expected 'ONLINE'");
551                     rebootDevice(
552                             device, null
553                             /** regular mode */
554                             );
555                     if (waitForDevice(monitor, recoverTillOnline)) {
556                         // Success
557                         CLog.d("Device recovered from recovery mode and is online.");
558                         InvocationMetricLogger.addInvocationMetrics(
559                                 InvocationMetricKey.DEVICE_RECOVERY, 1);
560                         // Individually track this too
561                         InvocationMetricLogger.addInvocationMetrics(
562                                 InvocationMetricKey.DEVICE_RECOVERY_FROM_RECOVERY, 1);
563                         return true;
564                     }
565                 }
566             }
567             // Track the failure
568             InvocationMetricLogger.addInvocationMetrics(
569                     InvocationMetricKey.DEVICE_RECOVERY_FAIL, 1);
570             CLog.w("USB reset recovery was unsuccessful");
571         }
572         return false;
573     }
574 
waitForDevice(IDeviceStateMonitor monitor, boolean recoverTillOnline)575     private boolean waitForDevice(IDeviceStateMonitor monitor, boolean recoverTillOnline) {
576         if (recoverTillOnline) {
577             if (monitor.waitForDeviceOnline() != null) {
578                 // Success
579                 return true;
580             }
581         } else if (monitor.waitForDeviceAvailable() != null) {
582             // Success
583             return true;
584         }
585         return false;
586     }
587 
588     @VisibleForTesting
getUsbHelper()589     UsbHelper getUsbHelper() {
590         return new UsbHelper();
591     }
592 }
593