1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.accessibilityservice.cts;
18 
19 import static android.accessibilityservice.AccessibilityServiceInfo.CAPABILITY_CAN_RETRIEVE_WINDOW_CONTENT;
20 import static android.accessibilityservice.AccessibilityServiceInfo.CAPABILITY_CAN_TAKE_SCREENSHOT;
21 import static android.accessibilityservice.cts.utils.AccessibilityEventFilterUtils.filterWindowsChangedWithChangeTypes;
22 import static android.accessibilityservice.cts.utils.ActivityLaunchUtils.launchActivityOnSpecifiedDisplayAndWaitForItToBeOnscreen;
23 import static android.accessibilityservice.cts.utils.ActivityLaunchUtils.supportsMultiDisplay;
24 import static android.accessibilityservice.cts.utils.AsyncUtils.DEFAULT_TIMEOUT_MS;
25 import static android.accessibilityservice.cts.utils.DisplayUtils.VirtualDisplaySession;
26 import static android.view.WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES;
27 import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ADDED;
28 
29 import static com.google.common.truth.Truth.assertThat;
30 
31 import static org.junit.Assume.assumeTrue;
32 import static org.mockito.Mockito.timeout;
33 import static org.mockito.Mockito.verify;
34 
35 import android.accessibility.cts.common.AccessibilityDumpOnFailureRule;
36 import android.accessibility.cts.common.InstrumentedAccessibilityService;
37 import android.accessibility.cts.common.InstrumentedAccessibilityServiceTestRule;
38 import android.accessibilityservice.AccessibilityService;
39 import android.accessibilityservice.AccessibilityService.ScreenshotResult;
40 import android.accessibilityservice.AccessibilityService.TakeScreenshotCallback;
41 import android.accessibilityservice.cts.activities.AccessibilityWindowQueryActivity;
42 import android.accessibilityservice.cts.utils.ActivityLaunchUtils;
43 import android.app.Activity;
44 import android.app.Instrumentation;
45 import android.app.UiAutomation;
46 import android.content.Context;
47 import android.graphics.Bitmap;
48 import android.graphics.Color;
49 import android.graphics.ColorSpace;
50 import android.graphics.Point;
51 import android.graphics.drawable.ColorDrawable;
52 import android.hardware.HardwareBuffer;
53 import android.os.SystemClock;
54 import android.platform.test.annotations.Presubmit;
55 import android.view.Display;
56 import android.view.View;
57 import android.view.WindowManager;
58 import android.view.accessibility.AccessibilityWindowInfo;
59 import android.widget.ImageView;
60 
61 import androidx.test.InstrumentationRegistry;
62 import androidx.test.filters.FlakyTest;
63 import androidx.test.runner.AndroidJUnit4;
64 
65 import com.android.compatibility.common.util.ApiTest;
66 import com.android.compatibility.common.util.CddTest;
67 
68 import org.junit.AfterClass;
69 import org.junit.Before;
70 import org.junit.BeforeClass;
71 import org.junit.Rule;
72 import org.junit.Test;
73 import org.junit.rules.RuleChain;
74 import org.junit.runner.RunWith;
75 import org.mockito.ArgumentCaptor;
76 import org.mockito.Captor;
77 import org.mockito.Mock;
78 import org.mockito.Mockito;
79 import org.mockito.MockitoAnnotations;
80 
81 import java.util.ArrayList;
82 import java.util.List;
83 import java.util.stream.Collectors;
84 
85 /**
86  * Test cases for accessibility service takeScreenshot API.
87  */
88 @RunWith(AndroidJUnit4.class)
89 @CddTest(requirements = {"3.10/C-1-1,C-1-2"})
90 @Presubmit
91 public class AccessibilityTakeScreenshotTest {
92     /**
93      * The timeout for waiting screenshot had been taken done.
94      */
95     private static final long TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS = 1000;
96     public static final int SECURE_WINDOW_CONTENT_COLOR = Color.BLUE;
97 
98     private static Instrumentation sInstrumentation;
99     private static UiAutomation sUiAutomation;
100 
101     private InstrumentedAccessibilityServiceTestRule<StubTakeScreenshotService> mServiceRule =
102             new InstrumentedAccessibilityServiceTestRule<>(StubTakeScreenshotService.class);
103 
104     private AccessibilityDumpOnFailureRule mDumpOnFailureRule =
105             new AccessibilityDumpOnFailureRule();
106 
107     @Rule
108     public final RuleChain mRuleChain = RuleChain
109             .outerRule(mServiceRule)
110             .around(mDumpOnFailureRule);
111 
112     private StubTakeScreenshotService mService;
113     private Context mContext;
114     private Point mDisplaySize;
115     private long mStartTestingTime;
116     @Mock
117     private TakeScreenshotCallback mCallback;
118     @Captor
119     private ArgumentCaptor<ScreenshotResult> mSuccessResultArgumentCaptor;
120 
121     @BeforeClass
oneTimeSetup()122     public static void oneTimeSetup() {
123         sInstrumentation = InstrumentationRegistry.getInstrumentation();
124         sUiAutomation = sInstrumentation.getUiAutomation();
125     }
126 
127     @AfterClass
finalTearDown()128     public static void finalTearDown() {
129         sUiAutomation.destroy();
130     }
131 
132     @Before
setUp()133     public void setUp() throws Exception {
134         MockitoAnnotations.initMocks(this);
135         mService = mServiceRule.getService();
136         mContext = mService.getApplicationContext();
137 
138         WindowManager windowManager =
139                 (WindowManager) mContext.getSystemService(Context.WINDOW_SERVICE);
140         final Display display = windowManager.getDefaultDisplay();
141 
142         mDisplaySize = new Point();
143         display.getRealSize(mDisplaySize);
144     }
145 
146     @Test
testTakeScreenshot_GetScreenshotResult()147     public void testTakeScreenshot_GetScreenshotResult() {
148         takeScreenshot(Display.DEFAULT_DISPLAY);
149         verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onSuccess(
150                 mSuccessResultArgumentCaptor.capture());
151 
152         verifyScreenshotResult(mSuccessResultArgumentCaptor.getValue());
153     }
154 
155     @Test
testTakeScreenshot_RequestIntervalTime()156     public void testTakeScreenshot_RequestIntervalTime() throws Exception {
157         takeScreenshot(Display.DEFAULT_DISPLAY);
158         verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onSuccess(
159                 mSuccessResultArgumentCaptor.capture());
160 
161         Thread.sleep(
162                 AccessibilityService.ACCESSIBILITY_TAKE_SCREENSHOT_REQUEST_INTERVAL_TIMES_MS / 2);
163         // Requests the API again during interval time from calling the first time.
164         takeScreenshot(Display.DEFAULT_DISPLAY);
165         verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onFailure(
166                 AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT);
167 
168         Thread.sleep(
169                 AccessibilityService.ACCESSIBILITY_TAKE_SCREENSHOT_REQUEST_INTERVAL_TIMES_MS / 2 +
170                         1);
171         // Requests the API again after interval time from calling the first time.
172         takeScreenshot(Display.DEFAULT_DISPLAY);
173         verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onSuccess(
174                 mSuccessResultArgumentCaptor.capture());
175     }
176 
177     @Test
testTakeScreenshotOnVirtualDisplay_GetScreenshotResult()178     public void testTakeScreenshotOnVirtualDisplay_GetScreenshotResult() throws Exception {
179         assumeTrue(supportsMultiDisplay(sInstrumentation.getContext()));
180         try (VirtualDisplaySession displaySession = new VirtualDisplaySession()) {
181             final int virtualDisplayId =
182                     displaySession.createDisplayWithDefaultDisplayMetricsAndWait(mContext,
183                             false).getDisplayId();
184             // Launches an activity on virtual display.
185             final Activity activity = launchActivityOnSpecifiedDisplayAndWaitForItToBeOnscreen(
186                     sInstrumentation, sUiAutomation, AccessibilityWindowQueryActivity.class,
187                     virtualDisplayId);
188             try {
189                 takeScreenshot(virtualDisplayId);
190                 verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onSuccess(
191                         mSuccessResultArgumentCaptor.capture());
192 
193                 verifyScreenshotResult(mSuccessResultArgumentCaptor.getValue());
194             } finally {
195                 sInstrumentation.runOnMainSync(() -> {
196                     activity.finish();
197                 });
198             }
199         }
200     }
201 
202     @Test
testTakeScreenshotOnPrivateDisplay_GetErrorCode()203     public void testTakeScreenshotOnPrivateDisplay_GetErrorCode() {
204         try (VirtualDisplaySession displaySession = new VirtualDisplaySession()) {
205             final int virtualDisplayId =
206                     displaySession.createDisplayWithDefaultDisplayMetricsAndWait(mContext,
207                             true).getDisplayId();
208             takeScreenshot(virtualDisplayId);
209             verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onFailure(
210                     AccessibilityService.ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY);
211         }
212     }
213 
214     @Test
testTakeScreenshotWithSecureWindow_GetScreenshotAndVerifyBitmap()215     public void testTakeScreenshotWithSecureWindow_GetScreenshotAndVerifyBitmap() throws Throwable {
216         final Activity activity = launchActivityOnSpecifiedDisplayAndWaitForItToBeOnscreen(
217                 sInstrumentation, sUiAutomation, AccessibilityWindowQueryActivity.class,
218                 Display.DEFAULT_DISPLAY);
219 
220         final ImageView image = new ImageView(activity);
221         image.setSystemUiVisibility(View.SYSTEM_UI_FLAG_LAYOUT_STABLE
222                 | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION
223                 | View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN
224                 | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION
225                 | View.SYSTEM_UI_FLAG_FULLSCREEN);
226         image.setImageDrawable(new ColorDrawable(SECURE_WINDOW_CONTENT_COLOR));
227 
228         final WindowManager.LayoutParams params = new WindowManager.LayoutParams();
229         params.width = WindowManager.LayoutParams.MATCH_PARENT;
230         params.height = WindowManager.LayoutParams.MATCH_PARENT;
231         params.layoutInDisplayCutoutMode = LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES;
232         params.flags = WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN
233                 | WindowManager.LayoutParams.FLAG_SECURE;
234 
235         sUiAutomation.executeAndWaitForEvent(() -> sInstrumentation.runOnMainSync(
236                 () -> {
237                     activity.getWindowManager().addView(image, params);
238                 }),
239                 filterWindowsChangedWithChangeTypes(WINDOWS_CHANGE_ADDED),
240                 DEFAULT_TIMEOUT_MS);
241         takeScreenshot(Display.DEFAULT_DISPLAY);
242 
243         verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onSuccess(
244                 mSuccessResultArgumentCaptor.capture());
245 
246         assertThat(doesScreenshotContainColor(mSuccessResultArgumentCaptor.getValue(),
247                 SECURE_WINDOW_CONTENT_COLOR)).isFalse();
248     }
249 
250     @Test
251     @ApiTest(apis = {"android.accessibilityservice.AccessibilityService#takeScreenshotOfWindow"})
testTakeScreenshotOfWindow_GetScreenshotResult()252     public void testTakeScreenshotOfWindow_GetScreenshotResult() throws Throwable {
253         final Activity activity = launchActivityOnSpecifiedDisplayAndWaitForItToBeOnscreen(
254                 sInstrumentation, sUiAutomation, AccessibilityWindowQueryActivity.class,
255                 Display.DEFAULT_DISPLAY);
256         try {
257             final AccessibilityWindowInfo activityWindowInfo =
258                     ActivityLaunchUtils.findWindowByTitle(sUiAutomation, activity.getTitle());
259             assertThat(activityWindowInfo).isNotNull();
260 
261             final long timestampBeforeTakeScreenshot = SystemClock.uptimeMillis();
262             mService.takeScreenshotOfWindow(activityWindowInfo.getId(),
263                     mContext.getMainExecutor(), mCallback);
264             verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onSuccess(
265                     mSuccessResultArgumentCaptor.capture());
266 
267             final View activityRootView = activity.getWindow().getDecorView();
268             verifyScreenshotResult(mSuccessResultArgumentCaptor.getValue(),
269                     activityRootView.getWidth(), activityRootView.getHeight(),
270                     timestampBeforeTakeScreenshot);
271         } finally {
272             if (activity != null) {
273                 activity.finish();
274             }
275         }
276     }
277 
278     @Test
279     @FlakyTest
280     @ApiTest(apis = {"android.accessibilityservice.AccessibilityService#takeScreenshotOfWindow"})
testTakeScreenshotOfWindow_ErrorForSecureWindow()281     public void testTakeScreenshotOfWindow_ErrorForSecureWindow() throws Throwable {
282         final Activity activity = launchActivityOnSpecifiedDisplayAndWaitForItToBeOnscreen(
283                 sInstrumentation, sUiAutomation, AccessibilityWindowQueryActivity.class,
284                 Display.DEFAULT_DISPLAY);
285         try {
286             final ImageView image = new ImageView(activity);
287             image.setImageDrawable(new ColorDrawable(SECURE_WINDOW_CONTENT_COLOR));
288             final String secureWindowTitle = "Secure Window";
289             final WindowManager.LayoutParams params = new WindowManager.LayoutParams();
290             params.width = WindowManager.LayoutParams.MATCH_PARENT;
291             params.height = WindowManager.LayoutParams.MATCH_PARENT;
292             params.flags = WindowManager.LayoutParams.FLAG_SECURE;
293             params.accessibilityTitle = secureWindowTitle;
294             sUiAutomation.executeAndWaitForEvent(
295                     () -> sInstrumentation.runOnMainSync(
296                             () -> activity.getWindowManager().addView(image, params)),
297                     filterWindowsChangedWithChangeTypes(WINDOWS_CHANGE_ADDED),
298                     DEFAULT_TIMEOUT_MS);
299 
300             final AccessibilityWindowInfo secureWindowInfo =
301                     ActivityLaunchUtils.findWindowByTitle(sUiAutomation, secureWindowTitle);
302             assertThat(secureWindowInfo).isNotNull();
303 
304             mService.takeScreenshotOfWindow(secureWindowInfo.getId(),
305                     mContext.getMainExecutor(), mCallback);
306             verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onFailure(
307                     AccessibilityService.ERROR_TAKE_SCREENSHOT_SECURE_WINDOW);
308         } finally {
309             if (activity != null) {
310                 activity.finish();
311             }
312         }
313     }
314 
315     @Test
316     @ApiTest(apis = {"android.accessibilityservice.AccessibilityService#takeScreenshotOfWindow"})
testTakeScreenshotOfWindow_MultipleWindowsIntervalTime()317     public void testTakeScreenshotOfWindow_MultipleWindowsIntervalTime() throws Throwable {
318         final long halfInterval =
319                 AccessibilityService.ACCESSIBILITY_TAKE_SCREENSHOT_REQUEST_INTERVAL_TIMES_MS / 2;
320         final List<Integer> accessibilityWindowIds = sUiAutomation.getWindows()
321                 .stream().map(AccessibilityWindowInfo::getId).collect(Collectors.toList());
322 
323         // The initial batch of window screenshots should succeed.
324         for (TakeScreenshotCallback callback : takeScreenshotsOfWindows(accessibilityWindowIds)) {
325             verify(callback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onSuccess(
326                     Mockito.any());
327         }
328 
329         // The next batch of window screenshots, taken during the interval time, should fail.
330         Thread.sleep(halfInterval);
331         for (TakeScreenshotCallback callback : takeScreenshotsOfWindows(accessibilityWindowIds)) {
332             verify(callback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onFailure(
333                     AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT);
334         }
335 
336         // The next batch of window screenshots, taken after the interval time, should succeed.
337         Thread.sleep(halfInterval + 1);
338         for (TakeScreenshotCallback callback : takeScreenshotsOfWindows(accessibilityWindowIds)) {
339             verify(callback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onSuccess(
340                     Mockito.any());
341         }
342     }
343 
344     @Test
345     @ApiTest(apis = {"android.accessibilityservice.AccessibilityService#takeScreenshotOfWindow"})
testTakeScreenshotOfWindow_InvalidWindow()346     public void testTakeScreenshotOfWindow_InvalidWindow() {
347         mService.takeScreenshotOfWindow(-1,
348                 mContext.getMainExecutor(), mCallback);
349         verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onFailure(
350                 AccessibilityService.ERROR_TAKE_SCREENSHOT_INVALID_WINDOW);
351     }
352 
353     @Test
354     @ApiTest(apis = {"android.accessibilityservice.AccessibilityService#takeScreenshotOfWindow"})
testTakeScreenshotOfWindow_ErrorForServiceWithoutScreenshotCapability()355     public void testTakeScreenshotOfWindow_ErrorForServiceWithoutScreenshotCapability() {
356         final InstrumentedAccessibilityService serviceWithoutScreenshotCapability =
357                 InstrumentedAccessibilityService.enableService(
358                         TouchExplorationStubAccessibilityService.class);
359         try {
360             // Make sure this service can retrieve windows but not take screenshots before we
361             // go forward with the test.
362             final int capabilities =
363                     serviceWithoutScreenshotCapability.getServiceInfo().getCapabilities();
364             assertThat(capabilities & CAPABILITY_CAN_RETRIEVE_WINDOW_CONTENT).isNotEqualTo(0);
365             assertThat(capabilities & CAPABILITY_CAN_TAKE_SCREENSHOT).isEqualTo(0);
366 
367             final int accessibilityWindowId = serviceWithoutScreenshotCapability
368                     .getWindows().get(0).getId();
369             serviceWithoutScreenshotCapability.takeScreenshotOfWindow(accessibilityWindowId,
370                     mContext.getMainExecutor(), mCallback);
371             verify(mCallback, timeout(TIMEOUT_TAKE_SCREENSHOT_DONE_MILLIS)).onFailure(
372                     AccessibilityService.ERROR_TAKE_SCREENSHOT_NO_ACCESSIBILITY_ACCESS);
373         } finally {
374             serviceWithoutScreenshotCapability.disableSelfAndRemove();
375         }
376     }
377 
takeScreenshotsOfWindows( List<Integer> accessibilityWindowIds)378     private List<TakeScreenshotCallback> takeScreenshotsOfWindows(
379             List<Integer> accessibilityWindowIds) {
380         final List<TakeScreenshotCallback> result = new ArrayList<>();
381         for (Integer id : accessibilityWindowIds) {
382             TakeScreenshotCallback callback = Mockito.mock(TakeScreenshotCallback.class);
383             mService.takeScreenshotOfWindow(id, mContext.getMainExecutor(), callback);
384             result.add(callback);
385         }
386         return result;
387     }
388 
doesScreenshotContainColor(ScreenshotResult screenshot, int color)389     private boolean doesScreenshotContainColor(ScreenshotResult screenshot, int color) {
390         final Bitmap bitmap = Bitmap.wrapHardwareBuffer(screenshot.getHardwareBuffer(),
391                 screenshot.getColorSpace()).copy(Bitmap.Config.ARGB_8888, false);
392         final int[] pixels = new int[bitmap.getWidth() * bitmap.getHeight()];
393         bitmap.getPixels(pixels, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(),
394                 bitmap.getHeight());
395         final int r = Color.red(color);
396         final int g = Color.green(color);
397         final int b = Color.blue(color);
398         for (int pixel : pixels) {
399             if (Color.red(pixel) == r && Color.green(pixel) == g && Color.blue(pixel) == b) {
400                 return true;
401             }
402         }
403         return false;
404     }
405 
takeScreenshot(int displayId)406     private void takeScreenshot(int displayId) {
407         mStartTestingTime = SystemClock.uptimeMillis();
408         mService.takeScreenshot(displayId, mContext.getMainExecutor(),
409                 mCallback);
410     }
411 
verifyScreenshotResult(AccessibilityService.ScreenshotResult screenshot)412     private void verifyScreenshotResult(AccessibilityService.ScreenshotResult screenshot) {
413         verifyScreenshotResult(screenshot, mDisplaySize.x, mDisplaySize.y, mStartTestingTime);
414     }
415 
verifyScreenshotResult(AccessibilityService.ScreenshotResult screenshot, int expectedWidth, int expectedHeight, long timestampBeforeTakeScreenshot)416     private void verifyScreenshotResult(AccessibilityService.ScreenshotResult screenshot,
417             int expectedWidth, int expectedHeight, long timestampBeforeTakeScreenshot) {
418         assertThat(screenshot).isNotNull();
419         assertThat(screenshot.getTimestamp()).isGreaterThan(timestampBeforeTakeScreenshot);
420 
421         final HardwareBuffer hardwareBuffer = screenshot.getHardwareBuffer();
422         assertThat(hardwareBuffer).isNotNull();
423         assertThat(hardwareBuffer.getWidth()).isEqualTo(expectedWidth);
424         assertThat(hardwareBuffer.getHeight()).isEqualTo(expectedHeight);
425 
426         final ColorSpace colorSpace = screenshot.getColorSpace();
427         assertThat(colorSpace).isNotNull();
428         assertThat(Bitmap.wrapHardwareBuffer(hardwareBuffer, colorSpace)).isNotNull();
429     }
430 }
431