xref: /frameworks/base/core/java/android/accessibilityservice/InputMethod.java

/*
 * Copyright (C) 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.accessibilityservice;

import static android.os.Trace.TRACE_TAG_WINDOW_MANAGER;

import android.annotation.IntRange;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.os.RemoteException;
import android.os.Trace;
import android.util.Log;
import android.view.KeyCharacterMap;
import android.view.KeyEvent;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputMethodManager;
import android.view.inputmethod.SurroundingText;
import android.view.inputmethod.TextAttribute;

import com.android.internal.inputmethod.IAccessibilityInputMethodSessionCallback;
import com.android.internal.inputmethod.IRemoteAccessibilityInputConnection;
import com.android.internal.inputmethod.RemoteAccessibilityInputConnection;

/**
 * This class provides input method APIs. Some public methods such as
 * @link #onUpdateSelection(int, int, int, int, int, int)} do nothing by default and service
 * developers should override them as needed. Developers should also override
 * {@link AccessibilityService#onCreateInputMethod()} to return
 * their custom InputMethod implementation. Accessibility services also need to set the
 * {@link AccessibilityServiceInfo#FLAG_INPUT_METHOD_EDITOR} flag to use input method APIs.
 */
public class InputMethod {
    private static final String LOG_TAG = "A11yInputMethod";

    private final AccessibilityService mService;
    private boolean mInputStarted;
    private RemoteAccessibilityInputConnection mStartedInputConnection;
    private EditorInfo mInputEditorInfo;

    /**
     * Creates a new InputMethod instance for the given <code>service</code>, so that the
     * accessibility service can control editing.
     */
    public InputMethod(@NonNull AccessibilityService service) {
        mService = service;
    }

    /**
     * Retrieve the currently active InputConnection that is bound to
     * the input method, or null if there is none.
     */
    @Nullable
    public final AccessibilityInputConnection getCurrentInputConnection() {
        if (mStartedInputConnection != null) {
            return new AccessibilityInputConnection(mStartedInputConnection);
        }
        return null;
    }

    /**
     * Whether the input has started.
     */
    public final boolean getCurrentInputStarted() {
        return mInputStarted;
    }

    /**
     * Get the EditorInfo which describes several attributes of a text editing object
     * that an accessibility service is communicating with (typically an EditText).
     */
    @Nullable
    public final EditorInfo getCurrentInputEditorInfo() {
        return mInputEditorInfo;
    }

    /**
     * Called to inform the accessibility service that text input has started in an
     * editor.  You should use this callback to initialize the state of your
     * input to match the state of the editor given to it.
     *
     * @param attribute  The attributes of the editor that input is starting
     *                   in.
     * @param restarting Set to true if input is restarting in the same
     *                   editor such as because the application has changed the text in
     *                   the editor.  Otherwise will be false, indicating this is a new
     *                   session with the editor.
     */
    public void onStartInput(@NonNull EditorInfo attribute, boolean restarting) {
        // Intentionally empty
    }

    /**
     * Called to inform the accessibility service that text input has finished in
     * the last editor. At this point there may be a call to
     * {@link #onStartInput(EditorInfo, boolean)} to perform input in a
     * new editor, or the accessibility service may be left idle. This method is
     * <em>not</em> called when input restarts in the same editor.
     *
     * <p>The default
     * implementation uses the InputConnection to clear any active composing
     * text; you can override this (not calling the base class implementation)
     * to perform whatever behavior you would like.
     */
    public void onFinishInput() {
        // Intentionally empty
    }

    /**
     * Called when the application has reported a new selection region of
     * the text. This is called whether or not the accessibility service has requested
     * extracted text updates, although if so it will not receive this call
     * if the extracted text has changed as well.
     *
     * <p>Be careful about changing the text in reaction to this call with
     * methods such as setComposingText, commitText or
     * deleteSurroundingText. If the cursor moves as a result, this method
     * will be called again, which may result in an infinite loop.
     */
    public void onUpdateSelection(int oldSelStart, int oldSelEnd, int newSelStart,
            int newSelEnd, int candidatesStart, int candidatesEnd) {
        // Intentionally empty
    }

    final void createImeSession(IAccessibilityInputMethodSessionCallback callback) {
        final AccessibilityInputMethodSessionWrapper wrapper =
                new AccessibilityInputMethodSessionWrapper(mService.getMainLooper(),
                        new SessionImpl());
        try {
            callback.sessionCreated(wrapper, mService.getConnectionId());
        } catch (RemoteException ignored) {
        }
    }

    final void startInput(@Nullable RemoteAccessibilityInputConnection ic,
            @NonNull EditorInfo attribute) {
        Log.v(LOG_TAG, "startInput(): editor=" + attribute);
        Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "AccessibilityService.startInput");
        doStartInput(ic, attribute, false /* restarting */);
        Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER);
    }

    final void restartInput(@Nullable RemoteAccessibilityInputConnection ic,
            @NonNull EditorInfo attribute) {
        Log.v(LOG_TAG, "restartInput(): editor=" + attribute);
        Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "AccessibilityService.restartInput");
        doStartInput(ic, attribute, true /* restarting */);
        Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER);
    }


    final void doStartInput(RemoteAccessibilityInputConnection ic, EditorInfo attribute,
            boolean restarting) {
        if ((ic == null || !restarting) && mInputStarted) {
            doFinishInput();
            if (ic == null) {
                // Unlike InputMethodService, A11y IME should not observe fallback InputConnection.
                return;
            }
        }
        mInputStarted = true;
        mStartedInputConnection = ic;
        mInputEditorInfo = attribute;
        Log.v(LOG_TAG, "CALL: onStartInput");
        onStartInput(attribute, restarting);
    }

    final void doFinishInput() {
        Log.v(LOG_TAG, "CALL: doFinishInput");
        if (mInputStarted) {
            Log.v(LOG_TAG, "CALL: onFinishInput");
            onFinishInput();
        }
        mInputStarted = false;
        mStartedInputConnection = null;
        mInputEditorInfo = null;
    }

    /**
     * This class provides the allowed list of {@link InputConnection} APIs for
     * accessibility services.
     */
    public final class AccessibilityInputConnection {
        private final RemoteAccessibilityInputConnection mIc;
        AccessibilityInputConnection(RemoteAccessibilityInputConnection ic) {
            this.mIc = ic;
        }

        /**
         * Commit text to the text box and set the new cursor position. This method is
         * used to allow the IME to provide extra information while setting up text.
         *
         * <p>This method commits the contents of the currently composing text, and then
         * moves the cursor according to {@code newCursorPosition}. If there
         * is no composing text when this method is called, the new text is
         * inserted at the cursor position, removing text inside the selection
         * if any.
         *
         * <p>Calling this method will cause the editor to call
         * {@link #onUpdateSelection(int, int, int, int,
         * int, int)} on the current accessibility service after the batch input is over.
         * <strong>Editor authors</strong>, for this to happen you need to
         * make the changes known to the accessibility service by calling
         * {@link InputMethodManager#updateSelection(android.view.View, int, int, int, int)},
         * but be careful to wait until the batch edit is over if one is
         * in progress.</p>
         *
         * @param text The text to commit. This may include styles.
         * @param newCursorPosition The new cursor position around the text,
         *        in Java characters. If > 0, this is relative to the end
         *        of the text - 1; if <= 0, this is relative to the start
         *        of the text. So a value of 1 will always advance the cursor
         *        to the position after the full text being inserted. Note that
         *        this means you can't position the cursor within the text,
         *        because the editor can make modifications to the text
         *        you are providing so it is not possible to correctly specify
         *        locations there.
         * @param textAttribute The extra information about the text.
         */
        public void commitText(@NonNull CharSequence text, int newCursorPosition,
                @Nullable TextAttribute textAttribute) {
            if (mIc != null) {
                mIc.commitText(text, newCursorPosition, textAttribute);
            }
        }

        /**
         * Set the selection of the text editor. To set the cursor
         * position, start and end should have the same value.
         *
         * <p>Since this moves the cursor, calling this method will cause
         * the editor to call
         * {@link android.inputmethodservice.InputMethodService#onUpdateSelection(int, int, int,
         * int,int, int)} on the current IME after the batch input is over.
         * <strong>Editor authors</strong>, for this to happen you need to
         * make the changes known to the input method by calling
         * {@link InputMethodManager#updateSelection(android.view.View, int, int, int, int)},
         * but be careful to wait until the batch edit is over if one is
         * in progress.</p>
         *
         * <p>This has no effect on the composing region which must stay
         * unchanged. The order of start and end is not important. In
         * effect, the region from start to end and the region from end to
         * start is the same. Editor authors, be ready to accept a start
         * that is greater than end.</p>
         *
         * @param start the character index where the selection should start.
         * @param end the character index where the selection should end.
         */
        public void setSelection(int start, int end) {
            if (mIc != null) {
                mIc.setSelection(start, end);
            }
        }

        /**
         * Gets the surrounding text around the current cursor, with <var>beforeLength</var>
         * characters of text before the cursor (start of the selection), <var>afterLength</var>
         * characters of text after the cursor (end of the selection), and all of the selected
         * text. The range are for java characters, not glyphs that can be multiple characters.
         *
         * <p>This method may fail either if the input connection has become invalid (such as its
         * process crashing), or the client is taking too long to respond with the text (it is
         * given a couple seconds to return), or the protocol is not supported. In any of these
         * cases, null is returned.
         *
         * <p>This method does not affect the text in the editor in any way, nor does it affect the
         * selection or composing spans.</p>
         *
         * <p>If {@link InputConnection#GET_TEXT_WITH_STYLES} is supplied as flags, the editor
         * should return a {@link android.text.Spanned} with all the spans set on the text.</p>
         *
         * <p><strong>Accessibility service authors:</strong> please consider this will trigger an
         * IPC round-trip that will take some time. Assume this method consumes a lot of time.
         *
         * @param beforeLength The expected length of the text before the cursor.
         * @param afterLength The expected length of the text after the cursor.
         * @param flags Supplies additional options controlling how the text is returned. May be
         *              either {@code 0} or {@link InputConnection#GET_TEXT_WITH_STYLES}.
         * @return an {@link android.view.inputmethod.SurroundingText} object describing the
         * surrounding text and state of selection, or null if the input connection is no longer
         * valid, or the editor can't comply with the request for some reason, or the application
         * does not implement this method. The length of the returned text might be less than the
         * sum of <var>beforeLength</var> and <var>afterLength</var> .
         * @throws IllegalArgumentException if {@code beforeLength} or {@code afterLength} is
         * negative.
         */
        @Nullable
        public SurroundingText getSurroundingText(
                @IntRange(from = 0) int beforeLength, @IntRange(from = 0) int afterLength,
                @InputConnection.GetTextType int flags) {
            if (mIc != null) {
                return mIc.getSurroundingText(beforeLength, afterLength, flags);
            }
            return null;
        }

        /**
         * Delete <var>beforeLength</var> characters of text before the
         * current cursor position, and delete <var>afterLength</var>
         * characters of text after the current cursor position, excluding
         * the selection. Before and after refer to the order of the
         * characters in the string, not to their visual representation:
         * this means you don't have to figure out the direction of the
         * text and can just use the indices as-is.
         *
         * <p>The lengths are supplied in Java chars, not in code points
         * or in glyphs.</p>
         *
         * <p>Since this method only operates on text before and after the
         * selection, it can't affect the contents of the selection. This
         * may affect the composing span if the span includes characters
         * that are to be deleted, but otherwise will not change it. If
         * some characters in the composing span are deleted, the
         * composing span will persist but get shortened by however many
         * chars inside it have been removed.</p>
         *
         * <p><strong>Accessibility service authors:</strong> please be careful not to
         * delete only half of a surrogate pair. Also take care not to
         * delete more characters than are in the editor, as that may have
         * ill effects on the application. Calling this method will cause
         * the editor to call {@link InputMethod#onUpdateSelection(int, int, int, int, int, int)}
         * on your service after the batch input is over.</p>
         *
         * <p><strong>Editor authors:</strong> please be careful of race
         * conditions in implementing this call. An IME can make a change
         * to the text or change the selection position and use this
         * method right away; you need to make sure the effects are
         * consistent with the results of the latest edits. Also, although
         * the IME should not send lengths bigger than the contents of the
         * string, you should check the values for overflows and trim the
         * indices to the size of the contents to avoid crashes. Since
         * this changes the contents of the editor, you need to make the
         * changes known to the input method by calling
         * {@link InputMethodManager#updateSelection(android.view.View, int, int, int, int)},
         * but be careful to wait until the batch edit is over if one is
         * in progress.</p>
         *
         * @param beforeLength The number of characters before the cursor to be deleted, in code
         *        unit. If this is greater than the number of existing characters between the
         *        beginning of the text and the cursor, then this method does not fail but deletes
         *        all the characters in that range.
         * @param afterLength The number of characters after the cursor to be deleted, in code unit.
         *        If this is greater than the number of existing characters between the cursor and
         *        the end of the text, then this method does not fail but deletes all the characters
         *        in that range.
         */
        public void deleteSurroundingText(int beforeLength, int afterLength) {
            if (mIc != null) {
                mIc.deleteSurroundingText(beforeLength, afterLength);
            }
        }

        /**
         * Send a key event to the process that is currently attached
         * through this input connection. The event will be dispatched
         * like a normal key event, to the currently focused view; this
         * generally is the view that is providing this InputConnection,
         * but due to the asynchronous nature of this protocol that can
         * not be guaranteed and the focus may have changed by the time
         * the event is received.
         *
         * <p>This method can be used to send key events to the
         * application. For example, an on-screen keyboard may use this
         * method to simulate a hardware keyboard. There are three types
         * of standard keyboards, numeric (12-key), predictive (20-key)
         * and ALPHA (QWERTY). You can specify the keyboard type by
         * specify the device id of the key event.</p>
         *
         * <p>You will usually want to set the flag
         * {@link KeyEvent#FLAG_SOFT_KEYBOARD KeyEvent.FLAG_SOFT_KEYBOARD}
         * on all key event objects you give to this API; the flag will
         * not be set for you.</p>
         *
         * <p>Note that it's discouraged to send such key events in normal
         * operation; this is mainly for use with
         * {@link android.text.InputType#TYPE_NULL} type text fields. Use
         * the {@link #commitText} family of methods to send text to the
         * application instead.</p>
         *
         * @param event The key event.
         *
         * @see KeyEvent
         * @see KeyCharacterMap#NUMERIC
         * @see KeyCharacterMap#PREDICTIVE
         * @see KeyCharacterMap#ALPHA
         */
        public void sendKeyEvent(@NonNull KeyEvent event) {
            if (mIc != null) {
                mIc.sendKeyEvent(event);
            }
        }

        /**
         * Have the editor perform an action it has said it can do.
         *
         * @param editorAction This must be one of the action constants for
         * {@link EditorInfo#imeOptions EditorInfo.imeOptions}, such as
         * {@link EditorInfo#IME_ACTION_GO EditorInfo.EDITOR_ACTION_GO}, or the value of
         * {@link EditorInfo#actionId EditorInfo.actionId} if a custom action is available.
         */
        public void performEditorAction(int editorAction) {
            if (mIc != null) {
                mIc.performEditorAction(editorAction);
            }
        }

        /**
         * Perform a context menu action on the field. The given id may be one of:
         * {@link android.R.id#selectAll},
         * {@link android.R.id#startSelectingText}, {@link android.R.id#stopSelectingText},
         * {@link android.R.id#cut}, {@link android.R.id#copy},
         * {@link android.R.id#paste}, {@link android.R.id#copyUrl},
         * or {@link android.R.id#switchInputMethod}
         */
        public void performContextMenuAction(int id) {
            if (mIc != null) {
                mIc.performContextMenuAction(id);
            }
        }

        /**
         * Retrieve the current capitalization mode in effect at the
         * current cursor position in the text. See
         * {@link android.text.TextUtils#getCapsMode TextUtils.getCapsMode}
         * for more information.
         *
         * <p>This method may fail either if the input connection has
         * become invalid (such as its process crashing) or the client is
         * taking too long to respond with the text (it is given a couple
         * seconds to return). In either case, 0 is returned.</p>
         *
         * <p>This method does not affect the text in the editor in any
         * way, nor does it affect the selection or composing spans.</p>
         *
         * <p><strong>Editor authors:</strong> please be careful of race
         * conditions in implementing this call. An IME can change the
         * cursor position and use this method right away; you need to make
         * sure the returned value is consistent with the results of the
         * latest edits and changes to the cursor position.</p>
         *
         * @param reqModes The desired modes to retrieve, as defined by
         * {@link android.text.TextUtils#getCapsMode TextUtils.getCapsMode}. These
         * constants are defined so that you can simply pass the current
         * {@link EditorInfo#inputType TextBoxAttribute.contentType} value
         * directly in to here.
         * @return the caps mode flags that are in effect at the current
         * cursor position. See TYPE_TEXT_FLAG_CAPS_* in {@link android.text.InputType}.
         */
        public int getCursorCapsMode(int reqModes) {
            if (mIc != null) {
                return mIc.getCursorCapsMode(reqModes);
            }
            return 0;
        }

        /**
         * Clear the given meta key pressed states in the given input
         * connection.
         *
         * <p>This can be used by the accessibility service to clear the meta key states set
         * by a hardware keyboard with latched meta keys, if the editor
         * keeps track of these.</p>
         *
         * @param states The states to be cleared, may be one or more bits as
         * per {@link KeyEvent#getMetaState() KeyEvent.getMetaState()}.
         */
        public void clearMetaKeyStates(int states) {
            if (mIc != null) {
                mIc.clearMetaKeyStates(states);
            }
        }
    }

    /**
     * Concrete implementation of {@link AccessibilityInputMethodSession} that provides all of the
     * standard behavior for an A11y input method session.
     */
    private final class SessionImpl implements AccessibilityInputMethodSession {
        boolean mEnabled = true;

        @Override
        public void setEnabled(boolean enabled) {
            mEnabled = enabled;
        }

        @Override
        public void finishInput() {
            if (mEnabled) {
                doFinishInput();
            }
        }

        @Override
        public void updateSelection(int oldSelStart, int oldSelEnd, int newSelStart,
                int newSelEnd, int candidatesStart, int candidatesEnd) {
            if (mEnabled) {
                InputMethod.this.onUpdateSelection(oldSelEnd, oldSelEnd, newSelStart,
                        newSelEnd, candidatesStart, candidatesEnd);
            }
        }

        @Override
        public void invalidateInput(EditorInfo editorInfo,
                IRemoteAccessibilityInputConnection connection, int sessionId) {
            if (!mEnabled || mStartedInputConnection == null
                    || !mStartedInputConnection.isSameConnection(connection)) {
                // This is not an error, and can be safely ignored.
                return;
            }
            editorInfo.makeCompatible(mService.getApplicationInfo().targetSdkVersion);
            restartInput(new RemoteAccessibilityInputConnection(mStartedInputConnection, sessionId),
                    editorInfo);
        }
    }
}