1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.csuite.core;
18 
19 import com.android.csuite.core.TestUtils.TestArtifactReceiver;
20 import com.android.tradefed.log.LogUtil.CLog;
21 import com.android.tradefed.result.LogDataType;
22 
23 import com.google.common.annotations.VisibleForTesting;
24 
25 import java.awt.Color;
26 import java.awt.Graphics;
27 import java.awt.Rectangle;
28 import java.awt.image.BufferedImage;
29 import java.io.ByteArrayOutputStream;
30 import java.io.IOException;
31 import java.util.ArrayDeque;
32 
33 import javax.imageio.ImageIO;
34 
35 /**
36  * A class that helps detect the presence of a blank screen in an app using the approach of first
37  * finding the largest same-color rectangle area, and then comparing it to the total area of the
38  * original image.
39  */
40 public class BlankScreenDetectorWithSameColorRectangle {
41     /**
42      * @param image against which to calculate the blank screen area.
43      * @return a BlankScreen object which represents the largest same-color rectangle within the
44      *     given image.
45      */
getBlankScreen(BufferedImage image)46     public static BlankScreen getBlankScreen(BufferedImage image) {
47         return new BlankScreen(maxSameColorRectangle(image), image);
48     }
49 
50     /**
51      * Given an RGB image, finds the biggest same-color rectangle.
52      *
53      * @param image within which to look for the largest same-color rectangle.
54      * @return the Rectangle object representing the largest same-color rectangle.
55      */
56     @VisibleForTesting
maxSameColorRectangle(BufferedImage image)57     static Rectangle maxSameColorRectangle(BufferedImage image) {
58         int[][] imageMatrix = getPixels(image);
59         int[][] similarityMatrix = new int[imageMatrix.length][imageMatrix[0].length];
60         for (int i = 0; i < similarityMatrix.length; i++) {
61             for (int j = 0; j < similarityMatrix[0].length; j++) {
62                 similarityMatrix[i][j] = 0;
63             }
64         }
65         Rectangle maxRectangle = new Rectangle();
66 
67         for (int i = 0; i < similarityMatrix.length; i++) {
68             for (int j = 0; j < similarityMatrix[0].length; j++) {
69                 if (i == 0) {
70                     similarityMatrix[i][j] = 1;
71                 } else if (imageMatrix[i][j] == imageMatrix[i - 1][j]) {
72                     similarityMatrix[i][j] = similarityMatrix[i - 1][j] + 1;
73                 } else {
74                     similarityMatrix[i][j] = 1;
75                 }
76             }
77             Rectangle currentBiggestRectangle = maxSubRectangle(similarityMatrix[i], i);
78             if (getRectangleArea(currentBiggestRectangle) > getRectangleArea(maxRectangle)) {
79                 maxRectangle = currentBiggestRectangle;
80             }
81         }
82         return maxRectangle;
83     }
84 
85     /**
86      * Finds the SubRectangle with the largest possible area given a row of column heights and its
87      * index in a larger matrix.
88      *
89      * @param heightsRow an array representing the height of each column of same-colorex pixels.
90      * @param index the index of the given array in the larger two-dimensional matrix.
91      * @return the Rectangle object representing the largest same-color rectangle.
92      */
93     @VisibleForTesting
maxSubRectangle(int[] heightsRow, int index)94     static Rectangle maxSubRectangle(int[] heightsRow, int index) {
95         ArrayDeque<Integer> stack = new ArrayDeque<>();
96         Rectangle maxRectangle = new Rectangle();
97 
98         for (int i = 0; i < heightsRow.length; i++) {
99             while (!stack.isEmpty() && heightsRow[stack.peek()] > heightsRow[i]) {
100                 int height = heightsRow[stack.pop()];
101                 int width = stack.isEmpty() ? i : i - stack.peek() - 1;
102                 int area = height * width;
103                 if (area > getRectangleArea(maxRectangle)) {
104                     int leftCornerXCoord = stack.isEmpty() ? 0 : stack.peek() + 1;
105                     int leftCornerYCoord = index - height + 1;
106                     maxRectangle.setRect(leftCornerXCoord, leftCornerYCoord, width, height);
107                 }
108             }
109             stack.push(i);
110         }
111 
112         while (!stack.isEmpty()) {
113             int height = heightsRow[stack.pop()];
114             int width =
115                     stack.isEmpty() ? heightsRow.length : (heightsRow.length - stack.peek() - 1);
116             int area = height * width;
117             if (area > getRectangleArea(maxRectangle)) {
118                 int leftCornerXCoord = stack.isEmpty() ? 0 : stack.peek() + 1;
119                 int leftCornerYCoord = index - height + 1;
120                 maxRectangle.setRect(leftCornerXCoord, leftCornerYCoord, width, height);
121             }
122         }
123         return maxRectangle;
124     }
125 
126     /**
127      * Converts a BufferedImage to a two-dimensional array containing the int representation of its
128      * pixels.
129      *
130      * @param image which to convert.
131      * @return a two-dimensional array containing the int representation of the image's pixels.
132      */
getPixels(BufferedImage image)133     private static int[][] getPixels(BufferedImage image) {
134         int[][] pixels = new int[image.getHeight()][image.getWidth()];
135         for (int i = 0; i < pixels.length; i++) {
136             for (int j = 0; j < pixels[0].length; j++) {
137                 pixels[i][j] = image.getRGB(j, i);
138             }
139         }
140         return pixels;
141     }
142 
getRectangleArea(Rectangle rectangle)143     private static long getRectangleArea(Rectangle rectangle) {
144         return (long) rectangle.width * (long) rectangle.height;
145     }
146 
147     /**
148      * Saves the image containing the screenshot and drawing of the blank screen rectangle to the
149      * test artifacts.
150      *
151      * @param prefix the file name prefix.
152      * @param blankScreen the representation of the blank screen to write to artifacts.
153      * @param testArtifactReceiver the interface which allows to save test artifacts.
154      * @param deviceSerial the serial number of the device which will be used to name the artifacts.
155      */
saveBlankScreenArtifact( String prefix, BlankScreen blankScreen, TestArtifactReceiver testArtifactReceiver, String deviceSerial)156     public static void saveBlankScreenArtifact(
157             String prefix,
158             BlankScreen blankScreen,
159             TestArtifactReceiver testArtifactReceiver,
160             String deviceSerial) {
161         ByteArrayOutputStream baos = new ByteArrayOutputStream();
162         try {
163             ImageIO.write(blankScreen.getScreenImage(), "png", baos);
164         } catch (IOException e) {
165             CLog.e(
166                     "Failed to write the screenshot to byte array when saving blank screen"
167                             + " artifact.");
168         }
169         testArtifactReceiver.addTestArtifact(
170                 prefix + "_screenshot_blankscreen_" + deviceSerial,
171                 LogDataType.PNG,
172                 baos.toByteArray());
173     }
174 
175     /**
176      * A representation of a same-color rectangle within a larger image which may represent a blank
177      * screen in a mobile app.
178      */
179     public static final class BlankScreen {
180         // The screen capture within which the blank screen rectangle is computed.
181         private BufferedImage mScreenImage;
182         // The same-color rectangle arearepresenting the blank screen.
183         private Rectangle mBlankScreenRectangle;
184 
getScreenImage()185         public BufferedImage getScreenImage() {
186             return mScreenImage;
187         }
188 
getBlankScreenRectangle()189         public Rectangle getBlankScreenRectangle() {
190             return mBlankScreenRectangle;
191         }
192 
BlankScreen(Rectangle r, BufferedImage image)193         public BlankScreen(Rectangle r, BufferedImage image) {
194             this.mScreenImage = image;
195             this.mBlankScreenRectangle = r;
196             drawBlankScreenRectangle();
197         }
198 
199         // Draws the outline of the rectangle which is interpreted as a blank screen within the
200         // context of the larger image.
drawBlankScreenRectangle()201         private void drawBlankScreenRectangle() {
202             Graphics g = mScreenImage.getGraphics();
203             g.setColor(Color.MAGENTA);
204             g.drawRect(
205                     mBlankScreenRectangle.x,
206                     mBlankScreenRectangle.y,
207                     mBlankScreenRectangle.width,
208                     mBlankScreenRectangle.height);
209             g.dispose();
210         }
211 
212         // Returns the percentage of the larger image occupied by the blank screen rectangle.
getBlankScreenPercent()213         public double getBlankScreenPercent() {
214             return ((double) mBlankScreenRectangle.width * mBlankScreenRectangle.height)
215                     / ((double) mScreenImage.getHeight() * mScreenImage.getWidth());
216         }
217     }
218 }
219