1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.assist.service; 18 19 import static android.view.WindowInsets.Type.displayCutout; 20 import static android.view.WindowInsets.Type.statusBars; 21 22 import android.app.assist.AssistContent; 23 import android.app.assist.AssistStructure; 24 import android.assist.common.Utils; 25 import android.content.BroadcastReceiver; 26 import android.content.ComponentName; 27 import android.content.Context; 28 import android.content.Intent; 29 import android.content.IntentFilter; 30 import android.graphics.Bitmap; 31 import android.graphics.Color; 32 import android.graphics.Point; 33 import android.graphics.Rect; 34 import android.os.Bundle; 35 import android.os.RemoteCallback; 36 import android.service.voice.VoiceInteractionSession; 37 import android.util.Log; 38 import android.view.Display; 39 import android.view.DisplayCutout; 40 import android.view.LayoutInflater; 41 import android.view.View; 42 import android.view.ViewTreeObserver; 43 import android.view.WindowInsets; 44 import android.view.WindowManager; 45 import android.view.WindowMetrics; 46 47 public class MainInteractionSession extends VoiceInteractionSession { 48 static final String TAG = "MainInteractionSession"; 49 50 Context mContext; 51 Bundle mAssistData = new Bundle(); 52 53 private boolean hasReceivedAssistData = false; 54 private boolean hasReceivedScreenshot = false; 55 private boolean mScreenshotNeeded = true; 56 private int mCurColor; 57 private int mDisplayHeight; 58 private int mDisplayWidth; 59 private Rect mDisplayAreaBounds; 60 private BroadcastReceiver mReceiver; 61 private String mTestName; 62 private View mContentView; 63 private RemoteCallback mRemoteCallback; 64 private Bundle mOnShowArgs; 65 MainInteractionSession(Context context)66 MainInteractionSession(Context context) { 67 super(context); 68 mContext = context; 69 } 70 71 @Override onCreate()72 public void onCreate() { 73 super.onCreate(); 74 mReceiver = new BroadcastReceiver() { 75 @Override 76 public void onReceive(Context context, Intent intent) { 77 String action = intent.getAction(); 78 if (action.equals(Utils.HIDE_SESSION)) { 79 hide(); 80 } 81 82 Bundle bundle = new Bundle(); 83 bundle.putString(Utils.EXTRA_REMOTE_CALLBACK_ACTION, Utils.HIDE_SESSION_COMPLETE); 84 mRemoteCallback.sendResult(bundle); 85 } 86 }; 87 IntentFilter filter = new IntentFilter(); 88 filter.addAction(Utils.HIDE_SESSION); 89 mContext.registerReceiver(mReceiver, filter, 90 Context.RECEIVER_VISIBLE_TO_INSTANT_APPS | Context.RECEIVER_EXPORTED); 91 } 92 93 @Override onDestroy()94 public void onDestroy() { 95 Log.i(TAG, "onDestroy()"); 96 super.onDestroy(); 97 if (mReceiver != null) { 98 try { 99 mContext.unregisterReceiver(mReceiver); 100 } catch (IllegalArgumentException e) { 101 // Ignore this exception when unregisterReceiver fails. Due to there will be timing 102 // case to destroy VoiceInteractionSessionService before VoiceInteractionSession. 103 Log.e(TAG, "Failed to unregister receiver in onDestroy.", e); 104 } 105 } 106 } 107 108 @Override onPrepareShow(Bundle args, int showFlags)109 public void onPrepareShow(Bundle args, int showFlags) { 110 if (Utils.LIFECYCLE_NOUI.equals(args.getString(Utils.TESTCASE_TYPE, ""))) { 111 setUiEnabled(false); 112 } else { 113 setUiEnabled(true); 114 } 115 } 116 117 @Override onShow(Bundle args, int showFlags)118 public void onShow(Bundle args, int showFlags) { 119 if (args == null) { 120 Log.e(TAG, "onshow() received null args"); 121 return; 122 } 123 mOnShowArgs = args; 124 mScreenshotNeeded = (showFlags & SHOW_WITH_SCREENSHOT) != 0; 125 mTestName = args.getString(Utils.TESTCASE_TYPE, ""); 126 mCurColor = args.getInt(Utils.SCREENSHOT_COLOR_KEY); 127 mDisplayHeight = args.getInt(Utils.DISPLAY_HEIGHT_KEY); 128 mDisplayWidth = args.getInt(Utils.DISPLAY_WIDTH_KEY); 129 mDisplayAreaBounds = args.getParcelable(Utils.DISPLAY_AREA_BOUNDS_KEY); 130 mRemoteCallback = args.getParcelable(Utils.EXTRA_REMOTE_CALLBACK); 131 super.onShow(args, showFlags); 132 if (mContentView == null) return; // Happens when ui is not enabled. 133 mContentView.getViewTreeObserver().addOnPreDrawListener( 134 new ViewTreeObserver.OnPreDrawListener() { 135 @Override 136 public boolean onPreDraw() { 137 mContentView.getViewTreeObserver().removeOnPreDrawListener(this); 138 Display d = mContentView.getDisplay(); 139 Point displayPoint = new Point(); 140 // The voice interaction window layer is higher than keyguard, status bar, 141 // nav bar now. So we should take both status bar, nav bar into consideration. 142 // The voice interaction hide the nav bar, so the height only need to consider 143 // status bar. The status bar may contain display cutout but the display cutout 144 // is device specific, we need to check it. 145 WindowManager wm = mContext.getSystemService(WindowManager.class); 146 WindowMetrics windowMetrics = wm.getCurrentWindowMetrics(); 147 Rect bound = windowMetrics.getBounds(); 148 WindowInsets windowInsets = windowMetrics.getWindowInsets(); 149 android.graphics.Insets statusBarInsets = 150 windowInsets.getInsets(statusBars()); 151 android.graphics.Insets displayCutoutInsets = 152 windowInsets.getInsets(displayCutout()); 153 android.graphics.Insets min = 154 android.graphics.Insets.min(statusBarInsets, displayCutoutInsets); 155 boolean statusBarContainsCutout = !android.graphics.Insets.NONE.equals(min); 156 Log.d(TAG, "statusBarContainsCutout=" + statusBarContainsCutout); 157 displayPoint.y = statusBarContainsCutout 158 ? bound.height() - min.top - min.bottom : 159 bound.height() - displayCutoutInsets.top - displayCutoutInsets.bottom; 160 displayPoint.x = statusBarContainsCutout ? 161 bound.width() - min.left - min.right : 162 bound.width() - displayCutoutInsets.left - displayCutoutInsets.right; 163 DisplayCutout dc = d.getCutout(); 164 if (dc != null) { 165 // Means the device has a cutout area 166 android.graphics.Insets wi = d.getCutout().getWaterfallInsets(); 167 168 if (wi != android.graphics.Insets.NONE) { 169 // Waterfall cutout. Considers only the display 170 // useful area discarding the cutout. 171 displayPoint.x -= (wi.left + wi.right); 172 } 173 } 174 Bundle bundle = new Bundle(); 175 bundle.putString(Utils.EXTRA_REMOTE_CALLBACK_ACTION, 176 Utils.BROADCAST_CONTENT_VIEW_HEIGHT); 177 bundle.putInt(Utils.EXTRA_CONTENT_VIEW_HEIGHT, mContentView.getHeight()); 178 bundle.putInt(Utils.EXTRA_CONTENT_VIEW_WIDTH, mContentView.getWidth()); 179 bundle.putParcelable(Utils.EXTRA_DISPLAY_POINT, displayPoint); 180 mRemoteCallback.sendResult(bundle); 181 return true; 182 } 183 }); 184 } 185 186 @Override onHandleAssist(AssistState state)187 public void onHandleAssist(AssistState state) { 188 super.onHandleAssist(state); 189 Bundle data = state.getAssistData(); 190 AssistStructure structure = state.getAssistStructure(); 191 AssistContent content = state.getAssistContent(); 192 ComponentName activity = structure == null ? null : structure.getActivityComponent(); 193 Log.i(TAG, "onHandleAssist()"); 194 Log.i(TAG, String.format("Bundle: %s, Activity: %s, Structure: %s, Content: %s", 195 data, activity, structure, content)); 196 197 // The structure becomes null under following conditions 198 // May be null if assist data has been disabled by the user or device policy; 199 // Will be an empty stub if the application has disabled assist by marking its window as secure. 200 // The CTS testcase will fail under the condition(automotive usecases) where 201 // there are multiple displays and some of the displays are marked with FLAG_SECURE 202 203 if ((Utils.isAutomotive(mContext)) && (structure == null)) { 204 Log.i(TAG, "Ignoring... Structure is null"); 205 return; 206 } 207 208 if (activity != null && Utils.isAutomotive(mContext) 209 && !activity.getPackageName().startsWith("android.assist")) { 210 // TODO: automotive has multiple activities / displays, so the test might fail if it 211 // receives one of them (like the cluster activity) instead of what's expecting. This is 212 // a quick fix for the issue; a better solution would be refactoring the infra to 213 // either send all events, or let the test specifify which activity it's waiting for 214 Log.i(TAG, "Ignoring " + activity.flattenToShortString() + " on automotive"); 215 return; 216 } 217 218 if (structure != null && structure.isHomeActivity() && !state.isFocused()) { 219 // If the system has multiple display areas, the launcher may be visible and resumed 220 // when the tests are in progress, so the tests might fail if they receives unexpected 221 // state from the launcher. Ignore the states from unfocused launcher to avoid this 222 // failure. 223 Log.i(TAG, "Ignoring the state from unfocused launcher"); 224 return; 225 } 226 227 // send to test to verify that this is accurate. 228 mAssistData.putBoolean(Utils.ASSIST_IS_ACTIVITY_ID_NULL, state.getActivityId() == null); 229 mAssistData.putParcelable(Utils.ASSIST_STRUCTURE_KEY, structure); 230 mAssistData.putParcelable(Utils.ASSIST_CONTENT_KEY, content); 231 mAssistData.putBundle(Utils.ASSIST_BUNDLE_KEY, data); 232 hasReceivedAssistData = true; 233 maybeBroadcastResults(); 234 } 235 236 @Override onAssistStructureFailure(Throwable failure)237 public void onAssistStructureFailure(Throwable failure) { 238 Log.e(TAG, "onAssistStructureFailure(): D'OH!!!", failure); 239 } 240 241 @Override onHandleScreenshot( Bitmap screenshot)242 public void onHandleScreenshot(/*@Nullable*/ Bitmap screenshot) { 243 Log.i(TAG, String.format("onHandleScreenshot - Screenshot: %s", screenshot)); 244 super.onHandleScreenshot(screenshot); 245 246 if (screenshot != null) { 247 mAssistData.putBoolean(Utils.ASSIST_SCREENSHOT_KEY, true); 248 249 if (mTestName.equals(Utils.SCREENSHOT)) { 250 boolean screenshotMatches = compareScreenshot(screenshot, mCurColor); 251 Log.i(TAG, "this is a screenshot test. Matches? " + screenshotMatches); 252 mAssistData.putBoolean( 253 Utils.COMPARE_SCREENSHOT_KEY, screenshotMatches); 254 } 255 } else { 256 mAssistData.putBoolean(Utils.ASSIST_SCREENSHOT_KEY, false); 257 } 258 hasReceivedScreenshot = true; 259 maybeBroadcastResults(); 260 } 261 compareScreenshot(Bitmap screenshot, int color)262 private boolean compareScreenshot(Bitmap screenshot, int color) { 263 // TODO(b/215668037): Uncomment when we find a reliable approach across different form 264 // factors. 265 // The current approach does not handle overridden screen sizes, and there's no clear way 266 // to handle that and multiple display areas at the same time. 267 // Point size = new Point(mDisplayWidth, mDisplayHeight); 268 269 // if (screenshot.getWidth() != size.x || screenshot.getHeight() != size.y) { 270 // Log.i(TAG, "width or height didn't match: " + size + " vs " + screenshot.getWidth() 271 // + "," + screenshot.getHeight()); 272 // return false; 273 // } 274 Point size = new Point(screenshot.getWidth(), screenshot.getHeight()); 275 int[] pixels = new int[size.x * size.y]; 276 screenshot.getPixels(pixels, 0, size.x, 0, 0, size.x, size.y); 277 278 // screenshot bitmap contains the screenshot for the entire physical display. A single 279 // physical display could have multiple display area with different applications. 280 // Let's grab the region of the display area from the original screenshot. 281 Bitmap displayAreaScreenshot = Bitmap.createBitmap(screenshot, mDisplayAreaBounds.left, 282 mDisplayAreaBounds.top, mDisplayAreaBounds.width(), mDisplayAreaBounds.height()); 283 int expectedColor = 0; 284 for (int pixel : pixels) { 285 // Check for roughly the same because there are rounding errors converting from the 286 // screenshot's color space to SRGB, which is what getPixels does. 287 if ((Color.red(pixel) - Color.red(color) < 5) 288 && (Color.green(pixel) - Color.green(color) < 5) 289 && (Color.blue(pixel) - Color.blue(color) < 5)) { 290 expectedColor += 1; 291 } 292 } 293 294 int pixelCount = displayAreaScreenshot.getWidth() * displayAreaScreenshot.getHeight(); 295 double colorRatio = (double) expectedColor / pixelCount; 296 Log.i(TAG, "the ratio is " + colorRatio); 297 return colorRatio >= 0.6; 298 } 299 maybeBroadcastResults()300 private void maybeBroadcastResults() { 301 if (!hasReceivedAssistData) { 302 Log.i(TAG, "waiting for assist data before broadcasting results"); 303 } else if (mScreenshotNeeded && !hasReceivedScreenshot) { 304 Log.i(TAG, "waiting for screenshot before broadcasting results"); 305 } else { 306 Bundle bundle = new Bundle(); 307 bundle.putString(Utils.EXTRA_REMOTE_CALLBACK_ACTION, Utils.BROADCAST_ASSIST_DATA_INTENT); 308 bundle.putBundle(Utils.ON_SHOW_ARGS_KEY, mOnShowArgs); 309 bundle.putAll(mAssistData); 310 mRemoteCallback.sendResult(bundle); 311 312 hasReceivedAssistData = false; 313 hasReceivedScreenshot = false; 314 } 315 } 316 317 @Override onCreateContentView()318 public View onCreateContentView() { 319 LayoutInflater f = getLayoutInflater(); 320 if (f == null) { 321 Log.wtf(TAG, "layout inflater was null"); 322 } 323 mContentView = f.inflate(R.layout.assist_layer,null); 324 Log.i(TAG, "onCreateContentView"); 325 return mContentView; 326 } 327 } 328