/* * Copyright (C) 2022 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.server.telecom; import static com.android.server.telecom.LogUtils.Events.STATE_TIMEOUT; import android.provider.DeviceConfig; import android.telecom.ConnectionService; import android.telecom.DisconnectCause; import android.telecom.Log; import android.util.LocalLog; import com.android.internal.annotations.VisibleForTesting; import com.android.internal.util.IndentingPrintWriter; import com.android.server.telecom.stats.CallStateChangedAtomWriter; import java.util.Collections; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** * Watchdog class responsible for detecting potential anomalous conditions for {@link Call}s. */ public class CallAnomalyWatchdog extends CallsManagerListenerBase implements Call.Listener { private final EmergencyCallDiagnosticLogger mEmergencyCallDiagnosticLogger; /** * Class used to track the call state as it pertains to the watchdog. The watchdog cares about * both the call state and whether a {@link ConnectionService} has finished creating the * connection. */ public static class WatchdogCallState { public final int state; public final boolean isCreateConnectionComplete; public final long stateStartTimeMillis; public WatchdogCallState(int newState, boolean newIsCreateConnectionComplete, long newStateStartTimeMillis) { state = newState; isCreateConnectionComplete = newIsCreateConnectionComplete; stateStartTimeMillis = newStateStartTimeMillis; } @Override public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof WatchdogCallState)) return false; WatchdogCallState that = (WatchdogCallState) o; // don't include the state timestamp in the equality check. return state == that.state && isCreateConnectionComplete == that.isCreateConnectionComplete; } @Override public int hashCode() { return Objects.hash(state, isCreateConnectionComplete); } @Override public String toString() { return "[isCreateConnComplete=" + isCreateConnectionComplete + ", state=" + CallState.toString(state) + "]"; } /** * Determines if the current call is in a transitory state. A call is deemed to be in a * transitory state if either {@link CallState#isTransitoryState(int)} returns true, OR * if the call has been created but is not yet added to {@link CallsManager} (i.e. we are * still waiting for the {@link ConnectionService} to create the connection. * @return {@code true} if the call is in a transitory state, {@code false} otherwise. */ public boolean isInTransitoryState() { return CallState.isTransitoryState(state) // Consider it transitory if create connection hasn't completed, EXCEPT if we // are in SELECT_PHONE_ACCOUNT state since that state will depend on user input. || (!isCreateConnectionComplete && state != CallState.SELECT_PHONE_ACCOUNT); } /** * Determines if the current call is in an intermediate state. A call is deemed to be in * an intermediate state if either {@link CallState#isIntermediateState(int)} returns true, * AND the call has been created to the connection. * @return {@code true} if the call is in a intermediate state, {@code false} otherwise. */ public boolean isInIntermediateState() { return CallState.isIntermediateState(state) && isCreateConnectionComplete; } } // Handler for tracking pending timeouts. private final ScheduledExecutorService mScheduledExecutorService; private final TelecomSystem.SyncRoot mLock; private final Timeouts.Adapter mTimeoutAdapter; private final ClockProxy mClockProxy; private AnomalyReporterAdapter mAnomalyReporter = new AnomalyReporterAdapterImpl(); // Pre-allocate space for 2 calls; realistically thats all we should ever need (tm) private final Map> mScheduledFutureMap = new ConcurrentHashMap<>(2); private final Map mWatchdogCallStateMap = new ConcurrentHashMap<>(2); // Track the calls which are pending destruction. // TODO: enhance to handle the case where a call never gets destroyed. private final Set mCallsPendingDestruction = Collections.newSetFromMap( new ConcurrentHashMap<>(2)); private final LocalLog mLocalLog = new LocalLog(20); /** * Enables the action to disconnect the call when the Transitory state and Intermediate state * time expires. */ private static final String ENABLE_DISCONNECT_CALL_ON_STUCK_STATE = "enable_disconnect_call_on_stuck_state"; /** * Anomaly Report UUIDs and corresponding event descriptions specific to CallAnomalyWatchdog. */ public static final UUID WATCHDOG_DISCONNECTED_STUCK_CALL_UUID = UUID.fromString("4b093985-c78f-45e3-a9fe-5319f397b025"); public static final String WATCHDOG_DISCONNECTED_STUCK_CALL_MSG = "Telecom CallAnomalyWatchdog caught and disconnected a stuck/zombie call."; public static final UUID WATCHDOG_DISCONNECTED_STUCK_EMERGENCY_CALL_UUID = UUID.fromString("d57d8aab-d723-485e-a0dd-d1abb0f346c8"); public static final String WATCHDOG_DISCONNECTED_STUCK_EMERGENCY_CALL_MSG = "Telecom CallAnomalyWatchdog caught and disconnected a stuck/zombie emergency call."; @VisibleForTesting public void setAnomalyReporterAdapter(AnomalyReporterAdapter mAnomalyReporterAdapter){ mAnomalyReporter = mAnomalyReporterAdapter; } public CallAnomalyWatchdog(ScheduledExecutorService executorService, TelecomSystem.SyncRoot lock, Timeouts.Adapter timeoutAdapter, ClockProxy clockProxy, EmergencyCallDiagnosticLogger emergencyCallDiagnosticLogger) { mScheduledExecutorService = executorService; mLock = lock; mTimeoutAdapter = timeoutAdapter; mClockProxy = clockProxy; mEmergencyCallDiagnosticLogger = emergencyCallDiagnosticLogger; } /** * Start tracking a call that we're waiting for a ConnectionService to create. * @param call the call. */ @Override public void onStartCreateConnection(Call call) { maybeTrackCall(call); call.addListener(this); } @Override public void onCallAdded(Call call) { maybeTrackCall(call); } /** * Override of {@link CallsManagerListenerBase} to track when calls have failed to be created by * a ConnectionService. These calls should no longer be tracked by the CallAnomalyWatchdog. * @param call the call */ @Override public void onCreateConnectionFailed(Call call) { Log.i(this, "onCreateConnectionFailed: call=%s", call.toString()); stopTrackingCall(call); } /** * Override of {@link CallsManagerListenerBase} to track when calls are removed * @param call the call */ @Override public void onCallRemoved(Call call) { Log.i(this, "onCallRemoved: call=%s", call.toString()); stopTrackingCall(call); } /** * Override of {@link com.android.server.telecom.CallsManager.CallsManagerListener} to track * call state changes. * @param call the call * @param oldState its old state * @param newState the new state */ @Override public void onCallStateChanged(Call call, int oldState, int newState) { Log.i(this, "onCallStateChanged: call=%s", call.toString()); maybeTrackCall(call); } /** * Override of {@link Call.Listener} so we can capture successful creation of calls. * @param call the call * @param callState the state the call is now in */ @Override public void onSuccessfulOutgoingCall(Call call, int callState) { maybeTrackCall(call); } /** * Override of {@link Call.Listener} so we can capture failed call creation. * @param call the call * @param disconnectCause the disconnect cause */ @Override public void onFailedOutgoingCall(Call call, DisconnectCause disconnectCause) { Log.i(this, "onFailedOutgoingCall: call=%s", call.toString()); stopTrackingCall(call); } /** * Override of {@link Call.Listener} so we can capture successful creation of calls * @param call the call */ @Override public void onSuccessfulIncomingCall(Call call) { maybeTrackCall(call); } /** * Override of {@link Call.Listener} so we can capture failed call creation. * @param call the call */ @Override public void onFailedIncomingCall(Call call) { Log.i(this, "onFailedIncomingCall: call=%s", call.toString()); stopTrackingCall(call); } /** * Helper method used to stop CallAnomalyWatchdog from tracking or destroying the call. * @param call the call. */ private void stopTrackingCall(Call call) { if (mScheduledFutureMap.containsKey(call)) { ScheduledFuture existingTimeout = mScheduledFutureMap.get(call); existingTimeout.cancel(false /* cancelIfRunning */); mScheduledFutureMap.remove(call); } if (mCallsPendingDestruction.contains(call)) { mCallsPendingDestruction.remove(call); } if (mWatchdogCallStateMap.containsKey(call)) { mWatchdogCallStateMap.remove(call); } call.removeListener(this); } /** * Given a {@link Call}, potentially post a cleanup task to track when the call has been in a * transitory state too long. * @param call the call. */ private void maybeTrackCall(Call call) { final WatchdogCallState currentState = mWatchdogCallStateMap.get(call); final WatchdogCallState newState = new WatchdogCallState(call.getState(), call.isCreateConnectionComplete(), mClockProxy.elapsedRealtime()); if (Objects.equals(currentState, newState)) { // No state change; skip. return; } mWatchdogCallStateMap.put(call, newState); // The call's state has changed, so we will remove any existing state cleanup tasks. if (mScheduledFutureMap.containsKey(call)) { ScheduledFuture existingTimeout = mScheduledFutureMap.get(call); existingTimeout.cancel(false /* cancelIfRunning */); mScheduledFutureMap.remove(call); } Log.i(this, "maybePostCleanupTask; callId=%s, state=%s, createConnComplete=%b", call.getId(), CallState.toString(call.getState()), call.isCreateConnectionComplete()); long timeoutMillis = getTimeoutMillis(call, newState); boolean isEnabledDisconnect = isEnabledDisconnectForStuckCall(); // If the call is now in a transitory or intermediate state, post a new cleanup task. if (timeoutMillis > 0) { Runnable cleanupRunnable = getCleanupRunnable(call, newState, timeoutMillis, isEnabledDisconnect); // Post cleanup to the executor service and cache the future, so we can cancel it if // needed. ScheduledFuture future = mScheduledExecutorService.schedule(cleanupRunnable, timeoutMillis, TimeUnit.MILLISECONDS); mScheduledFutureMap.put(call, future); } } public long getTimeoutMillis(Call call, WatchdogCallState state) { boolean isVoip = call.getIsVoipAudioMode(); boolean isEmergency = call.isEmergencyCall(); if (state.isInTransitoryState()) { if (isVoip) { return (isEmergency) ? mTimeoutAdapter.getVoipEmergencyCallTransitoryStateTimeoutMillis() : mTimeoutAdapter.getVoipCallTransitoryStateTimeoutMillis(); } return (isEmergency) ? mTimeoutAdapter.getNonVoipEmergencyCallTransitoryStateTimeoutMillis() : mTimeoutAdapter.getNonVoipCallTransitoryStateTimeoutMillis(); } if (state.isInIntermediateState()) { if (isVoip) { return (isEmergency) ? mTimeoutAdapter.getVoipEmergencyCallIntermediateStateTimeoutMillis() : mTimeoutAdapter.getVoipCallIntermediateStateTimeoutMillis(); } return (isEmergency) ? mTimeoutAdapter.getNonVoipEmergencyCallIntermediateStateTimeoutMillis() : mTimeoutAdapter.getNonVoipCallIntermediateStateTimeoutMillis(); } return 0; } private Runnable getCleanupRunnable(Call call, WatchdogCallState newState, long timeoutMillis, boolean isEnabledDisconnect) { Runnable cleanupRunnable = new android.telecom.Logging.Runnable("CAW.mR", mLock) { @Override public void loggedRun() { // If we're already pending a cleanup due to a state violation for this call. if (mCallsPendingDestruction.contains(call)) { return; } // Ensure that at timeout we are still in the original state when we posted the // timeout. final WatchdogCallState expiredState = new WatchdogCallState(call.getState(), call.isCreateConnectionComplete(), mClockProxy.elapsedRealtime()); if (expiredState.equals(newState) && getDurationInCurrentStateMillis(newState) > timeoutMillis) { // The call has been in this transitory or intermediate state too long, // so disconnect it and destroy it. Log.addEvent(call, STATE_TIMEOUT, newState); mLocalLog.log("STATE_TIMEOUT; callId=" + call.getId() + " in state " + newState); if (call.isEmergencyCall()){ mAnomalyReporter.reportAnomaly( WATCHDOG_DISCONNECTED_STUCK_EMERGENCY_CALL_UUID, WATCHDOG_DISCONNECTED_STUCK_EMERGENCY_CALL_MSG); mEmergencyCallDiagnosticLogger.reportStuckCall(call); } else { mAnomalyReporter.reportAnomaly( WATCHDOG_DISCONNECTED_STUCK_CALL_UUID, WATCHDOG_DISCONNECTED_STUCK_CALL_MSG); } if (isEnabledDisconnect) { call.setOverrideDisconnectCauseCode( new DisconnectCause(DisconnectCause.ERROR, "state_timeout")); call.disconnect("State timeout"); } else { writeCallStateChangedAtom(call); } mCallsPendingDestruction.add(call); if (mWatchdogCallStateMap.containsKey(call)) { mWatchdogCallStateMap.remove(call); } } mScheduledFutureMap.remove(call); } }.prepare(); return cleanupRunnable; } /** * Returns whether the action to disconnect the call when the Transitory state and * Intermediate state time expires is enabled or disabled. * @return {@code true} if the action is enabled, {@code false} if the action is disabled. */ private boolean isEnabledDisconnectForStuckCall() { return DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_TELEPHONY, ENABLE_DISCONNECT_CALL_ON_STUCK_STATE, false); } /** * Determines how long a call has been in a specific state. * @param state the call state. * @return the time in the state, in millis. */ private long getDurationInCurrentStateMillis(WatchdogCallState state) { return mClockProxy.elapsedRealtime() - state.stateStartTimeMillis; } private void writeCallStateChangedAtom(Call call) { new CallStateChangedAtomWriter() .setDisconnectCause(call.getDisconnectCause()) .setSelfManaged(call.isSelfManaged()) .setExternalCall(call.isExternalCall()) .setEmergencyCall(call.isEmergencyCall()) .write(call.getState()); } /** * Dumps the state of the {@link CallAnomalyWatchdog}. * * @param pw The {@code IndentingPrintWriter} to write the state to. */ public void dump(IndentingPrintWriter pw) { pw.println("Anomaly log:"); pw.increaseIndent(); mLocalLog.dump(pw); pw.decreaseIndent(); pw.print("Pending timeouts: "); pw.println(mScheduledFutureMap.keySet().stream().map(c -> c.getId()).collect( Collectors.joining(","))); pw.print("Pending destruction: "); pw.println(mCallsPendingDestruction.stream().map(c -> c.getId()).collect( Collectors.joining(","))); } @VisibleForTesting public int getNumberOfScheduledTimeouts() { return mScheduledFutureMap.size(); } }