android-15.0.0_r3/s

/*
 * Copyright (C) 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.service.voice;

import android.annotation.DurationMillisLong;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SdkConstant;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.app.Service;
import android.content.ContentCaptureOptions;
import android.content.Context;
import android.content.Intent;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioFormat;
import android.media.AudioSystem;
import android.os.IBinder;
import android.os.IRemoteCallback;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.speech.IRecognitionServiceManager;
import android.util.Log;
import android.view.contentcapture.ContentCaptureManager;
import android.view.contentcapture.IContentCaptureManager;

import com.android.internal.infra.AndroidFuture;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.function.IntConsumer;

/**
 * Implemented by an application that wants to offer query detection with visual signals.
 *
 * This service leverages visual signals such as camera frames to detect and stream queries from the
 * device microphone to the {@link VoiceInteractionService}, without the support of hotword. The
 * system will bind an application's {@link VoiceInteractionService} first. When
 * {@link VoiceInteractionService#createVisualQueryDetector(PersistableBundle, SharedMemory,
 * Executor, VisualQueryDetector.Callback)} is called, the system will bind the application's
 * {@link VisualQueryDetectionService}. When requested from {@link VoiceInteractionService}, the
 * system calls into the {@link VisualQueryDetectionService#onStartDetection()} to enable
 * detection. This method MUST be implemented to support visual query detection service.
 *
 * Note: Methods in this class may be called concurrently.
 *
 * @hide
 */
@SystemApi
public abstract class VisualQueryDetectionService extends Service
        implements SandboxedDetectionInitializer {

    private static final String TAG = VisualQueryDetectionService.class.getSimpleName();

    private static final long UPDATE_TIMEOUT_MILLIS = 20000;

    /**
     * The {@link Intent} that must be declared as handled by the service.
     * To be supported, the service must also require the
     * {@link android.Manifest.permission#BIND_VISUAL_QUERY_DETECTION_SERVICE} permission
     * so that other applications can not abuse it.
     */
    @SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION)
    public static final String SERVICE_INTERFACE =
            "android.service.voice.VisualQueryDetectionService";


    /** @hide */
    public static final String KEY_INITIALIZATION_STATUS = "initialization_status";

    private IDetectorSessionVisualQueryDetectionCallback mRemoteCallback = null;
    @Nullable
    private ContentCaptureManager mContentCaptureManager;
    @Nullable
    private IRecognitionServiceManager mIRecognitionServiceManager;
    @Nullable
    private IDetectorSessionStorageService mDetectorSessionStorageService;


    private final ISandboxedDetectionService mInterface = new ISandboxedDetectionService.Stub() {

        @Override
        public void detectWithVisualSignals(
                IDetectorSessionVisualQueryDetectionCallback callback) {
            Log.v(TAG, "#detectWithVisualSignals");
            mRemoteCallback = callback;
            VisualQueryDetectionService.this.onStartDetection();
        }

        @Override
        public void stopDetection() {
            Log.v(TAG, "#stopDetection");
            VisualQueryDetectionService.this.onStopDetection();
        }

        @Override
        public void updateState(PersistableBundle options, SharedMemory sharedMemory,
                IRemoteCallback callback) throws RemoteException {
            Log.v(TAG, "#updateState" + (callback != null ? " with callback" : ""));
            VisualQueryDetectionService.this.onUpdateStateInternal(
                    options,
                    sharedMemory,
                    callback);
        }

        @Override
        public void ping(IRemoteCallback callback) throws RemoteException {
            callback.sendResult(null);
        }

        @Override
        public void detectFromDspSource(
                SoundTrigger.KeyphraseRecognitionEvent event,
                AudioFormat audioFormat,
                long timeoutMillis,
                IDspHotwordDetectionCallback callback) {
            throw new UnsupportedOperationException("Not supported by VisualQueryDetectionService");
        }

        @Override
        public void detectFromMicrophoneSource(
                ParcelFileDescriptor audioStream,
                @HotwordDetectionService.AudioSource int audioSource,
                AudioFormat audioFormat,
                PersistableBundle options,
                IDspHotwordDetectionCallback callback) {
            throw new UnsupportedOperationException("Not supported by VisualQueryDetectionService");
        }

        @Override
        public void updateAudioFlinger(IBinder audioFlinger) {
            AudioSystem.setAudioFlingerBinder(audioFlinger);
        }

        @Override
        public void updateContentCaptureManager(IContentCaptureManager manager,
                ContentCaptureOptions options) {
            mContentCaptureManager = new ContentCaptureManager(
                    VisualQueryDetectionService.this, manager, options);
        }

        @Override
        public void updateRecognitionServiceManager(IRecognitionServiceManager manager) {
            mIRecognitionServiceManager = manager;
        }

        @Override
        public void registerRemoteStorageService(IDetectorSessionStorageService
                detectorSessionStorageService) {
            mDetectorSessionStorageService = detectorSessionStorageService;
        }
    };

    @Override
    @SuppressLint("OnNameExpected")
    public @Nullable Object getSystemService(@ServiceName @NonNull String name) {
        if (Context.CONTENT_CAPTURE_MANAGER_SERVICE.equals(name)) {
            return mContentCaptureManager;
        } else if (Context.SPEECH_RECOGNITION_SERVICE.equals(name)
                && mIRecognitionServiceManager != null) {
            return mIRecognitionServiceManager.asBinder();
        } else {
            return super.getSystemService(name);
        }
    }

    /**
     * {@inheritDoc}
     * @hide
     */
    @Override
    @SystemApi
    public void onUpdateState(
            @Nullable PersistableBundle options,
            @Nullable SharedMemory sharedMemory,
            @DurationMillisLong long callbackTimeoutMillis,
            @Nullable IntConsumer statusCallback) {
    }

    @Override
    @Nullable
    public IBinder onBind(@NonNull Intent intent) {
        if (SERVICE_INTERFACE.equals(intent.getAction())) {
            return mInterface.asBinder();
        }
        Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": "
                + intent);
        return null;
    }

    private void onUpdateStateInternal(@Nullable PersistableBundle options,
            @Nullable SharedMemory sharedMemory, IRemoteCallback callback) {
        IntConsumer intConsumer =
                SandboxedDetectionInitializer.createInitializationStatusConsumer(callback);
        onUpdateState(options, sharedMemory, UPDATE_TIMEOUT_MILLIS, intConsumer);
    }

    /**
     * This is called after the service is set up and the client should open the camera and the
     * microphone to start recognition. When the {@link VoiceInteractionService} requests that this
     * service {@link HotwordDetector#startRecognition()} start recognition on audio coming directly
     * from the device microphone.
     * <p>
     * Signal senders that return attention and query results are also expected to be called in this
     * method according to the detection outcomes.
     * <p>
     * On successful user attention, developers should call
     * {@link VisualQueryDetectionService#gainedAttention()} to enable the streaming of the query.
     * <p>
     * On user attention is lost, developers should call
     * {@link VisualQueryDetectionService#lostAttention()} to disable the streaming of the query.
     * <p>
     * On query is detected and ready to stream, developers should call
     * {@link VisualQueryDetectionService#streamQuery(String)} to return detected query to the
     * {@link VisualQueryDetector}.
     * <p>
     * On streamed query should be rejected, clients should call
     * {@link VisualQueryDetectionService#rejectQuery()} to abandon query streamed to the
     * {@link VisualQueryDetector}.
     * <p>
     * On streamed query is finished, clients should call
     * {@link VisualQueryDetectionService#finishQuery()} to complete query streamed to
     * {@link VisualQueryDetector}.
     * <p>
     * Before a call for {@link VisualQueryDetectionService#streamQuery(String)} is triggered,
     * {@link VisualQueryDetectionService#gainedAttention()} MUST be called to enable the streaming
     * of query. A query streaming is also expected to be finished by calling either
     * {@link VisualQueryDetectionService#finishQuery()} or
     * {@link VisualQueryDetectionService#rejectQuery()} before a new query should start streaming.
     * When the service enters the state where query streaming should be disabled,
     * {@link VisualQueryDetectionService#lostAttention()} MUST be called to block unnecessary
     * streaming.
     */
    public void onStartDetection() {
        throw new UnsupportedOperationException();
    }

    /**
     * Called when the {@link VoiceInteractionService}
     * {@link HotwordDetector#stopRecognition()} requests that recognition be stopped.
     */
    public void onStopDetection() {
    }

    /**
     * Informs the system that the attention is gained for the interaction intention
     * {@link VisualQueryAttentionResult#INTERACTION_INTENTION_AUDIO_VISUAL} with
     * engagement level equals to the maximum value possible so queries can be streamed.
     *
     * Usage of this method is not recommended, please use
     * {@link VisualQueryDetectionService#gainedAttention(VisualQueryAttentionResult)} instead.
     *
     */
    public final void gainedAttention() {
        try {
            mRemoteCallback.onAttentionGained(null);
        } catch (RemoteException e) {
            throw e.rethrowFromSystemServer();
        }
    }

    /**
     * Puts the device into an attention state that will listen to certain interaction intention
     * based on the {@link VisualQueryAttentionResult} provided.
     *
     * Different type and levels of engagement will lead to corresponding UI icons showing. See
     * {@link VisualQueryAttentionResult#setInteractionIntention(int)} for details.
     *
     * Exactly one {@link VisualQueryAttentionResult} can be set at a time with this method at
     * the moment. Multiple attention results will be supported to set the device into with this
     * API before {@link android.os.Build.VERSION_CODES#VANILLA_ICE_CREAM} is finalized.
     *
     * Latest call will override the {@link VisualQueryAttentionResult} of previous calls. Queries
     * streamed are independent of the attention interactionIntention.
     *
     * @param attentionResult Attention result of type {@link VisualQueryAttentionResult}.
     */
    @SuppressLint("UnflaggedApi") // b/325678077 flags not supported in isolated process
    public final void gainedAttention(@NonNull VisualQueryAttentionResult attentionResult) {
        try {
            mRemoteCallback.onAttentionGained(attentionResult);
        } catch (RemoteException e) {
            throw e.rethrowFromSystemServer();
        }
    }

    /**
     * Informs the system that all attention has lost to stop streaming.
     */
    public final void lostAttention() {
        try {
            mRemoteCallback.onAttentionLost(0); // placeholder
        } catch (RemoteException e) {
            throw e.rethrowFromSystemServer();
        }
    }

    /**
     * This will cancel the corresponding attention if the provided interaction intention is the
     * same as which of the object called with
     * {@link VisualQueryDetectionService#gainedAttention(VisualQueryAttentionResult)}.
     *
     * @param interactionIntention Interaction intention, one of
     *        {@link VisualQueryAttentionResult#InteractionIntention}.
     */
    @SuppressLint("UnflaggedApi") // b/325678077 flags not supported in isolated process
    public final void lostAttention(
            @VisualQueryAttentionResult.InteractionIntention int interactionIntention) {
        try {
            mRemoteCallback.onAttentionLost(interactionIntention);
        } catch (RemoteException e) {
            throw e.rethrowFromSystemServer();
        }
    }

    /**
     * Informs the {@link VisualQueryDetector} with the text content being captured about the
     * query from the audio source. {@code partialQuery} is provided to the
     * {@link VisualQueryDetector}. This method is expected to be only triggered if
     * {@link VisualQueryDetectionService#gainedAttention()} is called to put the service into the
     * attention gained state.
     *
     * Usage of this method is not recommended, please use
     * {@link VisualQueryDetectionService#streamQuery(VisualQueryDetectedResult)} instead.
     *
     * @param partialQuery Partially detected query in string.
     * @throws IllegalStateException if method called without attention gained.
     */
    public final void streamQuery(@NonNull String partialQuery) throws IllegalStateException {
        Objects.requireNonNull(partialQuery);
        try {
            mRemoteCallback.onQueryDetected(partialQuery);
        } catch (RemoteException e) {
            throw new IllegalStateException("#streamQuery must be only be triggered after "
                    + "calling #gainedAttention to be in the attention gained state.");
        }
    }

    /**
     * Informs the {@link VisualQueryDetector} with the text content being captured about the
     * query from the audio source. {@code partialResult} is provided to the
     * {@link VisualQueryDetector}. This method is expected to be only triggered if
     * {@link VisualQueryDetectionService#gainedAttention()} is called to put the service into
     * the attention gained state.
     *
     * @param partialResult Partially detected result in the format of
     * {@link VisualQueryDetectedResult}.
     */
    @SuppressLint("UnflaggedApi") // b/325678077 flags not supported in isolated process
    public final void streamQuery(@NonNull VisualQueryDetectedResult partialResult) {
        Objects.requireNonNull(partialResult);
        try {
            mRemoteCallback.onResultDetected(partialResult);
        } catch (RemoteException e) {
            throw new IllegalStateException("#streamQuery must be only be triggered after "
                    + "calling #gainedAttention to be in the attention gained state.");
        }
    }

    /**
     * Informs the {@link VisualQueryDetector} to abandon the streamed partial query that has
     * been sent to {@link VisualQueryDetector}.This method is expected to be only triggered if
     * {@link VisualQueryDetectionService#streamQuery(String)} is called to put the service into
     * the query streaming state.
     *
     * @throws IllegalStateException if method called without query streamed.
     */
    public final void rejectQuery() throws IllegalStateException {
        try {
            mRemoteCallback.onQueryRejected();
        } catch (RemoteException e) {
            throw new IllegalStateException("#rejectQuery must be only be triggered after "
                    + "calling #streamQuery to be in the query streaming state.");
        }
    }

    /**
     * Informs {@link VisualQueryDetector} with the metadata to complete the streamed partial
     * query that has been sent to {@link VisualQueryDetector}. This method is expected to be
     * only triggered if {@link VisualQueryDetectionService#streamQuery(String)} is called to put
     * the service into the query streaming state.
     *
     * @throws IllegalStateException if method called without query streamed.
     */
    public final void finishQuery() throws IllegalStateException {
        try {
            mRemoteCallback.onQueryFinished();
        } catch (RemoteException e) {
            throw new IllegalStateException("#finishQuery must be only be triggered after "
                    + "calling #streamQuery to be in the query streaming state.");
        }
    }

    /**
     * Overrides {@link Context#openFileInput} to read files with the given file names under the
     * internal app storage of the {@link VoiceInteractionService}, i.e., the input file path would
     * be added with {@link Context#getFilesDir()} as prefix.
     *
     * @param filename Relative path of a file under {@link Context#getFilesDir()}.
     * @throws FileNotFoundException if the file does not exist or cannot be open.
     */
    @Override
    public @NonNull FileInputStream openFileInput(@NonNull String filename) throws
            FileNotFoundException {
        try {
            AndroidFuture<ParcelFileDescriptor> future = new AndroidFuture<>();
            assert mDetectorSessionStorageService != null;
            mDetectorSessionStorageService.openFile(filename, future);
            ParcelFileDescriptor pfd = future.get();
            if (pfd == null) {
                throw new FileNotFoundException(
                        "File does not exist. Unable to open " + filename + ".");
            }
            return new FileInputStream(pfd.getFileDescriptor());
        } catch (RemoteException | ExecutionException | InterruptedException e) {
            Log.w(TAG, "Cannot open file due to remote service failure");
            throw new FileNotFoundException(e.getMessage());
        }
    }

}