1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.test.voiceinteraction;
18 
19 import android.app.ActivityManager;
20 import android.app.VoiceInteractor;
21 import android.app.assist.AssistContent;
22 import android.app.assist.AssistStructure;
23 import android.content.Context;
24 import android.content.Intent;
25 import android.graphics.Bitmap;
26 import android.net.Uri;
27 import android.os.Bundle;
28 import android.service.voice.VoiceInteractionSession;
29 import android.util.Log;
30 import android.view.View;
31 import android.widget.Button;
32 import android.widget.CheckBox;
33 import android.widget.ImageView;
34 import android.widget.TextView;
35 
36 import java.util.Arrays;
37 
38 public class MainInteractionSession extends VoiceInteractionSession
39         implements View.OnClickListener {
40     static final String TAG = "MainInteractionSession";
41 
42     Intent mStartIntent;
43     View mContentView;
44     AssistVisualizer mAssistVisualizer;
45     View mTopContent;
46     View mBottomContent;
47     TextView mText;
48     Button mTreeButton;
49     Button mTextButton;
50     Button mStartButton;
51     CheckBox mOptionsCheck;
52     View mOptionsContainer;
53     CheckBox mDisallowAssist;
54     CheckBox mDisallowScreenshot;
55     TextView mOptionsText;
56     ImageView mScreenshot;
57     ImageView mFullScreenshot;
58     Button mConfirmButton;
59     Button mCompleteButton;
60     Button mAbortButton;
61 
62     Bundle mAssistData;
63     AssistStructure mAssistStructure;
64     AssistContent mAssistContent;
65 
66     static final int STATE_IDLE = 0;
67     static final int STATE_LAUNCHING = 1;
68     static final int STATE_CONFIRM = 2;
69     static final int STATE_PICK_OPTION = 3;
70     static final int STATE_COMMAND = 4;
71     static final int STATE_ABORT_VOICE = 5;
72     static final int STATE_COMPLETE_VOICE = 6;
73     static final int STATE_DONE = 7;
74 
75     int mState = STATE_IDLE;
76     VoiceInteractor.PickOptionRequest.Option[] mPendingOptions;
77     CharSequence mPendingPrompt;
78     Request mPendingRequest;
79     int mCurrentTask = -1;
80     int mShowFlags;
81 
MainInteractionSession(Context context)82     MainInteractionSession(Context context) {
83         super(context);
84     }
85 
86     @Override
onCreate()87     public void onCreate() {
88         super.onCreate();
89         ActivityManager am = getContext().getSystemService(ActivityManager.class);
90         am.setWatchHeapLimit(40 * 1024 * 1024);
91     }
92 
93     @Override
onShow(Bundle args, int showFlags)94     public void onShow(Bundle args, int showFlags) {
95         super.onShow(args, showFlags);
96         mShowFlags = showFlags;
97         Log.i(TAG, "onShow: flags=0x" + Integer.toHexString(showFlags) + " args=" + args);
98         mState = STATE_IDLE;
99         mStartIntent = args != null ? (Intent)args.getParcelable("intent") : null;
100         if (mStartIntent == null) {
101             mStartIntent = new Intent(getContext(), TestInteractionActivity.class);
102         }
103         if (mAssistVisualizer != null) {
104             mAssistVisualizer.clearAssistData();
105         }
106         onHandleScreenshot(null);
107         updateState();
108         refreshOptions();
109     }
110 
111     @Override
onHide()112     public void onHide() {
113         super.onHide();
114         if (mAssistVisualizer != null) {
115             mAssistVisualizer.clearAssistData();
116         }
117         mState = STATE_DONE;
118         updateState();
119     }
120 
121     @Override
onCreateContentView()122     public View onCreateContentView() {
123         mContentView = getLayoutInflater().inflate(R.layout.voice_interaction_session, null);
124         mAssistVisualizer = (AssistVisualizer)mContentView.findViewById(R.id.assist_visualizer);
125         if (mAssistStructure != null) {
126             mAssistVisualizer.setAssistStructure(mAssistStructure);
127         }
128         mTopContent = mContentView.findViewById(R.id.top_content);
129         mBottomContent = mContentView.findViewById(R.id.bottom_content);
130         mText = (TextView)mContentView.findViewById(R.id.text);
131         mTreeButton = (Button)mContentView.findViewById(R.id.do_tree);
132         mTreeButton.setOnClickListener(this);
133         mTextButton = (Button)mContentView.findViewById(R.id.do_text);
134         mTextButton.setOnClickListener(this);
135         mStartButton = (Button)mContentView.findViewById(R.id.start);
136         mStartButton.setOnClickListener(this);
137         mScreenshot = (ImageView)mContentView.findViewById(R.id.screenshot);
138         mScreenshot.setOnClickListener(this);
139         mFullScreenshot = (ImageView)mContentView.findViewById(R.id.full_screenshot);
140         mOptionsCheck = (CheckBox)mContentView.findViewById(R.id.show_options);
141         mOptionsCheck.setOnClickListener(this);
142         mOptionsContainer = mContentView.findViewById(R.id.options);
143         mDisallowAssist = (CheckBox)mContentView.findViewById(R.id.disallow_structure);
144         mDisallowAssist.setOnClickListener(this);
145         mDisallowScreenshot = (CheckBox)mContentView.findViewById(R.id.disallow_screenshot);
146         mDisallowScreenshot.setOnClickListener(this);
147         mOptionsText = (TextView)mContentView.findViewById(R.id.options_text);
148         mConfirmButton = (Button)mContentView.findViewById(R.id.confirm);
149         mConfirmButton.setOnClickListener(this);
150         mCompleteButton = (Button)mContentView.findViewById(R.id.complete);
151         mCompleteButton.setOnClickListener(this);
152         mAbortButton = (Button)mContentView.findViewById(R.id.abort);
153         mAbortButton.setOnClickListener(this);
154         refreshOptions();
155         return mContentView;
156     }
157 
refreshOptions()158     void refreshOptions() {
159         if (mOptionsContainer != null) {
160             if (mOptionsCheck.isChecked()) {
161                 mOptionsContainer.setVisibility(View.VISIBLE);
162                 int flags = getDisabledShowContext();
163                 mDisallowAssist.setChecked((flags & SHOW_WITH_ASSIST) != 0);
164                 mDisallowScreenshot.setChecked((flags & SHOW_WITH_SCREENSHOT) != 0);
165                 int disabled = getUserDisabledShowContext();
166                 mOptionsText.setText("Disabled: 0x" + Integer.toHexString(disabled));
167             } else {
168                 mOptionsContainer.setVisibility(View.GONE);
169             }
170         }
171     }
172 
onHandleAssist(Bundle assistBundle)173     public void onHandleAssist(Bundle assistBundle) {
174     }
175 
logAssistContentAndData(AssistContent content, Bundle data)176     private void logAssistContentAndData(AssistContent content, Bundle data) {
177         if (content != null) {
178             Log.i(TAG, "Assist intent: " + content.getIntent());
179             Log.i(TAG, "Assist intent from app: " + content.isAppProvidedIntent());
180             Log.i(TAG, "Assist clipdata: " + content.getClipData());
181             Log.i(TAG, "Assist structured data: " + content.getStructuredData());
182             Log.i(TAG, "Assist web uri: " + content.getWebUri());
183             Log.i(TAG, "Assist web uri from app: " + content.isAppProvidedWebUri());
184             Log.i(TAG, "Assist extras: " + content.getExtras());
185         }
186         if (data != null) {
187             Uri referrer = data.getParcelable(Intent.EXTRA_REFERRER);
188             if (referrer != null) {
189                 Log.i(TAG, "Referrer: " + referrer);
190             }
191         }
192     }
193 
194     @Override
onHandleAssist(Bundle data, AssistStructure structure, AssistContent content)195     public void onHandleAssist(Bundle data, AssistStructure structure, AssistContent content) {
196         mAssistData = data;
197         mAssistStructure = structure;
198         mAssistContent = content;
199         if (mAssistVisualizer != null) {
200             if (mAssistStructure != null) {
201                 mAssistVisualizer.setAssistStructure(mAssistStructure);
202             } else {
203                 mAssistVisualizer.clearAssistData();
204             }
205         }
206         logAssistContentAndData(content, data);
207     }
208 
209     @Override
onHandleAssistSecondary(final Bundle data, final AssistStructure structure, final AssistContent content, int index, int count)210     public void onHandleAssistSecondary(final Bundle data, final AssistStructure structure,
211             final AssistContent content, int index, int count) {
212         Log.i(TAG, "Got secondary activity assist data " + index + " of " + count);
213         Log.i(TAG, "Showing assist structure after a few seconds...");
214         mContentView.postDelayed(new Runnable() {
215             public void run() {
216                 onHandleAssist(data, structure, content);
217             }
218         }, 2000 * index);
219     }
220 
221     @Override
onHandleScreenshot(Bitmap screenshot)222     public void onHandleScreenshot(Bitmap screenshot) {
223         if (mScreenshot != null) {
224             if (screenshot != null) {
225                 mScreenshot.setImageBitmap(screenshot);
226                 mScreenshot.setAdjustViewBounds(true);
227                 mScreenshot.setMaxWidth(screenshot.getWidth() / 3);
228                 mScreenshot.setMaxHeight(screenshot.getHeight() / 3);
229                 mFullScreenshot.setImageBitmap(screenshot);
230             } else {
231                 mScreenshot.setImageDrawable(null);
232                 mFullScreenshot.setImageDrawable(null);
233             }
234         }
235     }
236 
updateState()237     void updateState() {
238         if (mTopContent == null) {
239             return;
240         }
241         if (mState == STATE_IDLE) {
242             mTopContent.setVisibility(View.VISIBLE);
243             mBottomContent.setVisibility(View.GONE);
244             mAssistVisualizer.setVisibility(View.VISIBLE);
245         } else if (mState == STATE_DONE) {
246             mTopContent.setVisibility(View.GONE);
247             mBottomContent.setVisibility(View.GONE);
248             mAssistVisualizer.setVisibility(View.GONE);
249         } else {
250             mTopContent.setVisibility(View.GONE);
251             mBottomContent.setVisibility(View.VISIBLE);
252             mAssistVisualizer.setVisibility(View.GONE);
253         }
254         mStartButton.setEnabled(mState == STATE_IDLE);
255         mConfirmButton.setEnabled(mState == STATE_CONFIRM || mState == STATE_PICK_OPTION
256                 || mState == STATE_COMMAND);
257         mAbortButton.setEnabled(mState == STATE_ABORT_VOICE);
258         mCompleteButton.setEnabled(mState == STATE_COMPLETE_VOICE);
259     }
260 
onClick(View v)261     public void onClick(View v) {
262         if (v == mTreeButton) {
263             if (mAssistVisualizer != null) {
264                 logAssistContentAndData(mAssistContent, mAssistData);
265                 mAssistVisualizer.logTree();
266             }
267         } else if (v == mTextButton) {
268             if (mAssistVisualizer != null) {
269                 mAssistVisualizer.logText();
270             }
271         } else if (v == mOptionsCheck) {
272             refreshOptions();
273         } else if (v == mDisallowAssist) {
274             int flags = getDisabledShowContext();
275             if (mDisallowAssist.isChecked()) {
276                 flags |= SHOW_WITH_ASSIST;
277             } else {
278                 flags &= ~SHOW_WITH_ASSIST;
279             }
280             setDisabledShowContext(flags);
281         } else if (v == mDisallowScreenshot) {
282             int flags = getDisabledShowContext();
283             if (mDisallowScreenshot.isChecked()) {
284                 flags |= SHOW_WITH_SCREENSHOT;
285             } else {
286                 flags &= ~SHOW_WITH_SCREENSHOT;
287             }
288             setDisabledShowContext(flags);
289         } else if (v == mStartButton) {
290             mState = STATE_LAUNCHING;
291             updateState();
292             startVoiceActivity(mStartIntent);
293         } else if (v == mConfirmButton) {
294             if (mPendingRequest instanceof ConfirmationRequest) {
295                 ((ConfirmationRequest)mPendingRequest).sendConfirmationResult(true, null);
296                 mPendingRequest = null;
297                 mState = STATE_LAUNCHING;
298             } else if (mPendingRequest instanceof PickOptionRequest) {
299                 PickOptionRequest pick = (PickOptionRequest)mPendingRequest;
300                 int numReturn = mPendingOptions.length/2;
301                 if (numReturn <= 0) {
302                     numReturn = 1;
303                 }
304                 VoiceInteractor.PickOptionRequest.Option[] picked
305                         = new VoiceInteractor.PickOptionRequest.Option[numReturn];
306                 for (int i=0; i<picked.length; i++) {
307                     picked[i] = mPendingOptions[i*2];
308                 }
309                 mPendingOptions = picked;
310                 if (picked.length <= 1) {
311                     pick.sendPickOptionResult(picked, null);
312                     mPendingRequest = null;
313                     mState = STATE_LAUNCHING;
314                 } else {
315                     pick.sendIntermediatePickOptionResult(picked, null);
316                     updatePickText();
317                 }
318             } else if (mPendingRequest instanceof CommandRequest) {
319                 Bundle result = new Bundle();
320                 result.putString("key", "a result!");
321                 ((CommandRequest)mPendingRequest).sendResult(result);
322                 mPendingRequest = null;
323                 mState = STATE_LAUNCHING;
324             }
325         } else if (v == mAbortButton && mPendingRequest instanceof AbortVoiceRequest) {
326             ((AbortVoiceRequest)mPendingRequest).sendAbortResult(null);
327             mPendingRequest = null;
328         } else if (v == mCompleteButton && mPendingRequest instanceof CompleteVoiceRequest) {
329             ((CompleteVoiceRequest)mPendingRequest).sendCompleteResult(null);
330             mPendingRequest = null;
331         } else if (v == mScreenshot) {
332             if (mFullScreenshot.getVisibility() != View.VISIBLE) {
333                 mFullScreenshot.setVisibility(View.VISIBLE);
334             } else {
335                 mFullScreenshot.setVisibility(View.INVISIBLE);
336             }
337         }
338         updateState();
339     }
340 
341     @Override
onComputeInsets(Insets outInsets)342     public void onComputeInsets(Insets outInsets) {
343         super.onComputeInsets(outInsets);
344         if (mState != STATE_IDLE) {
345             outInsets.contentInsets.top = mBottomContent.getTop();
346             outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT;
347         } else if ((mShowFlags & SHOW_SOURCE_ACTIVITY) != 0) {
348             outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT;
349         }
350     }
351 
352     @Override
onTaskStarted(Intent intent, int taskId)353     public void onTaskStarted(Intent intent, int taskId) {
354         super.onTaskStarted(intent, taskId);
355         mCurrentTask = taskId;
356     }
357 
358     @Override
onTaskFinished(Intent intent, int taskId)359     public void onTaskFinished(Intent intent, int taskId) {
360         super.onTaskFinished(intent, taskId);
361         if (mCurrentTask == taskId) {
362             mCurrentTask = -1;
363         }
364     }
365 
366     @Override
onLockscreenShown()367     public void onLockscreenShown() {
368         if (mCurrentTask < 0) {
369             hide();
370         }
371     }
372 
373     @Override
onGetSupportedCommands(String[] commands)374     public boolean[] onGetSupportedCommands(String[] commands) {
375         boolean[] res = new boolean[commands.length];
376         for (int i=0; i<commands.length; i++) {
377             if ("com.android.test.voiceinteraction.COMMAND".equals(commands[i])) {
378                 res[i] = true;
379             }
380         }
381         return res;
382     }
383 
setPrompt(VoiceInteractor.Prompt prompt)384     void setPrompt(VoiceInteractor.Prompt prompt) {
385         if (prompt == null) {
386             mText.setText("(null)");
387             mPendingPrompt = "";
388         } else {
389             mText.setText(prompt.getVisualPrompt());
390             mPendingPrompt = prompt.getVisualPrompt();
391         }
392     }
393 
394     @Override
onRequestConfirmation(ConfirmationRequest request)395     public void onRequestConfirmation(ConfirmationRequest request) {
396         Log.i(TAG, "onConfirm: prompt=" + request.getVoicePrompt() + " extras="
397                 + request.getExtras());
398         setPrompt(request.getVoicePrompt());
399         mConfirmButton.setText("Confirm");
400         mPendingRequest = request;
401         mState = STATE_CONFIRM;
402         updateState();
403     }
404 
405     @Override
onRequestPickOption(PickOptionRequest request)406     public void onRequestPickOption(PickOptionRequest request) {
407         Log.i(TAG, "onPickOption: prompt=" + request.getVoicePrompt() + " options="
408                 + Arrays.toString(request.getOptions()) + " extras=" + request.getExtras());
409         mConfirmButton.setText("Pick Option");
410         mPendingRequest = request;
411         setPrompt(request.getVoicePrompt());
412         mPendingOptions = request.getOptions();
413         mState = STATE_PICK_OPTION;
414         updatePickText();
415         updateState();
416     }
417 
updatePickText()418     void updatePickText() {
419         StringBuilder sb = new StringBuilder();
420         sb.append(mPendingPrompt);
421         sb.append(": ");
422         for (int i=0; i<mPendingOptions.length; i++) {
423             if (i > 0) {
424                 sb.append(", ");
425             }
426             sb.append(mPendingOptions[i].getLabel());
427         }
428         mText.setText(sb.toString());
429     }
430 
431     @Override
onRequestCompleteVoice(CompleteVoiceRequest request)432     public void onRequestCompleteVoice(CompleteVoiceRequest request) {
433         Log.i(TAG, "onCompleteVoice: message=" + request.getVoicePrompt() + " extras="
434                 + request.getExtras());
435         setPrompt(request.getVoicePrompt());
436         mPendingRequest = request;
437         mState = STATE_COMPLETE_VOICE;
438         updateState();
439     }
440 
441     @Override
onRequestAbortVoice(AbortVoiceRequest request)442     public void onRequestAbortVoice(AbortVoiceRequest request) {
443         Log.i(TAG, "onAbortVoice: message=" + request.getVoicePrompt() + " extras="
444                 + request.getExtras());
445         setPrompt(request.getVoicePrompt());
446         mPendingRequest = request;
447         mState = STATE_ABORT_VOICE;
448         updateState();
449     }
450 
451     @Override
onRequestCommand(CommandRequest request)452     public void onRequestCommand(CommandRequest request) {
453         Bundle extras = request.getExtras();
454         if (extras != null) {
455             extras.getString("arg");
456         }
457         Log.i(TAG, "onCommand: command=" + request.getCommand() + " extras=" + extras);
458         mText.setText("Command: " + request.getCommand() + ", " + extras);
459         mConfirmButton.setText("Finish Command");
460         mPendingRequest = request;
461         mState = STATE_COMMAND;
462         updateState();
463     }
464 
465     @Override
onCancelRequest(Request request)466     public void onCancelRequest(Request request) {
467         Log.i(TAG, "onCancel");
468         if (mPendingRequest == request) {
469             mPendingRequest = null;
470             mState = STATE_LAUNCHING;
471             updateState();
472         }
473         request.cancel();
474     }
475 }
476