1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.compatibility.common.tradefed.util;
18 
19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
20 import com.android.compatibility.common.tradefed.result.SubPlanHelper;
21 import com.android.compatibility.common.tradefed.testtype.ISubPlan;
22 import com.android.compatibility.common.util.IInvocationResult;
23 import com.android.compatibility.common.util.LightInvocationResult;
24 import com.android.compatibility.common.util.ResultHandler;
25 import com.android.compatibility.common.util.TestFilter;
26 import com.android.tradefed.build.IBuildInfo;
27 import com.android.tradefed.config.ArgsOptionParser;
28 import com.android.tradefed.config.ConfigurationException;
29 import com.android.tradefed.device.DeviceNotAvailableException;
30 import com.android.tradefed.device.ITestDevice;
31 import com.android.tradefed.util.ArrayUtil;
32 
33 import com.google.common.annotations.VisibleForTesting;
34 
35 import java.io.File;
36 import java.io.FileNotFoundException;
37 import java.io.FilenameFilter;
38 import java.util.ArrayList;
39 import java.util.HashSet;
40 import java.util.List;
41 import java.util.Set;
42 
43 /**
44  * Helper for generating --include-filter and --exclude-filter values on compatibility retry.
45  */
46 public class RetryFilterHelper {
47 
48     protected String mSubPlan;
49     protected Set<String> mIncludeFilters = new HashSet<>();
50     protected Set<String> mExcludeFilters = new HashSet<>();
51     protected String mAbiName = null;
52     protected String mModuleName = null;
53     protected String mTestName = null;
54     protected RetryType mRetryType = null;
55 
56     /* Instance variables handy for retreiving the result to be retried */
57     private CompatibilityBuildHelper mBuild = null;
58     private int mSessionId;
59 
60     /* Sets to be populated by retry logic and returned by getter methods */
61     private Set<String> mRetryIncludes;
62     private Set<String> mRetryExcludes;
63 
RetryFilterHelper()64     public RetryFilterHelper() {}
65 
66     /**
67      * Constructor for a {@link RetryFilterHelper}. Requires a CompatibilityBuildHelper for
68      * retrieving previous sessions and the ID of the session to retry.
69      */
RetryFilterHelper(CompatibilityBuildHelper build, int sessionId)70     public RetryFilterHelper(CompatibilityBuildHelper build, int sessionId) {
71         mBuild = build;
72         mSessionId = sessionId;
73     }
74 
75     /**
76      * Constructor for a {@link RetryFilterHelper}.
77      *
78      * @param build a {@link CompatibilityBuildHelper} describing the build.
79      * @param sessionId The ID of the session to retry.
80      * @param subPlan The name of a subPlan to be used. Can be null.
81      * @param includeFilters The include module filters to apply
82      * @param excludeFilters The exclude module filters to apply
83      * @param abiName The name of abi to use. Can be null.
84      * @param moduleName The name of the module to run. Can be null.
85      * @param testName The name of the test to run. Can be null.
86      * @param retryType The type of results to retry. Can be null.
87      */
RetryFilterHelper(CompatibilityBuildHelper build, int sessionId, String subPlan, Set<String> includeFilters, Set<String> excludeFilters, String abiName, String moduleName, String testName, RetryType retryType)88     public RetryFilterHelper(CompatibilityBuildHelper build, int sessionId, String subPlan,
89             Set<String> includeFilters, Set<String> excludeFilters, String abiName,
90             String moduleName, String testName, RetryType retryType) {
91         this(build, sessionId);
92         mSubPlan = subPlan;
93         mIncludeFilters.addAll(includeFilters);
94         mExcludeFilters.addAll(excludeFilters);
95         mAbiName = abiName;
96         mModuleName = moduleName;
97         mTestName = testName;
98         mRetryType = retryType;
99     }
100 
101     /**
102      * Throws an {@link IllegalArgumentException} if the device build fingerprint doesn't match
103      * the fingerprint recorded in the previous session's result.
104      */
validateBuildFingerprint(ITestDevice device)105     public void validateBuildFingerprint(ITestDevice device) throws DeviceNotAvailableException {
106         String oldBuildFingerprint = new LightInvocationResult(getResult()).getBuildFingerprint();
107         if (oldBuildFingerprint == null) {
108             throw new FingerprintComparisonException(
109                     "Could not find the build_fingerprint field in the result xml.");
110         }
111         String currentBuildFingerprint = device.getProperty("ro.build.fingerprint");
112         if (!oldBuildFingerprint.equals(currentBuildFingerprint)) {
113             throw new FingerprintComparisonException(String.format(
114                     "Device build fingerprint must match %s to retry session %d",
115                     oldBuildFingerprint, mSessionId));
116         }
117     }
118 
119     /**
120      * Copy all applicable options from an input object to this instance of RetryFilterHelper.
121      */
122     @VisibleForTesting
setAllOptionsFrom(RetryFilterHelper obj)123     void setAllOptionsFrom(RetryFilterHelper obj) {
124         clearOptions(); // Remove existing options first
125         mSubPlan = obj.mSubPlan;
126         mIncludeFilters.addAll(obj.mIncludeFilters);
127         mExcludeFilters.addAll(obj.mExcludeFilters);
128         mAbiName = obj.mAbiName;
129         mModuleName = obj.mModuleName;
130         mTestName = obj.mTestName;
131         mRetryType = obj.mRetryType;
132     }
133 
134     /**
135      * Clear all option values of this RetryFilterHelper.
136      */
clearOptions()137     public void clearOptions() {
138         mSubPlan = null;
139         mIncludeFilters.clear();
140         mExcludeFilters.clear();
141         mModuleName = null;
142         mTestName = null;
143         mRetryType = null;
144         mAbiName = null;
145     }
146 
147     /**
148      * Using command-line arguments from the previous session's result, set the input object's
149      * option values to the values applied in the previous session.
150      */
setCommandLineOptionsFor(Object obj)151     public void setCommandLineOptionsFor(Object obj) {
152         // only need light version to retrieve command-line args
153         IInvocationResult result = new LightInvocationResult(getResult());
154         String retryCommandLineArgs = result.getCommandLineArgs();
155         if (retryCommandLineArgs != null) {
156             try {
157                 // parse the command-line string from the result file and set options
158                 ArgsOptionParser parser = new ArgsOptionParser(obj);
159                 parser.parse(OptionHelper.getValidCliArgs(retryCommandLineArgs, obj));
160             } catch (ConfigurationException e) {
161                 throw new RuntimeException(e);
162             }
163         }
164     }
165 
166     /**
167      * Set the retry command line args on the {@link IBuildInfo} to carry the original command
168      * across retries.
169      */
setBuildInfoRetryCommand(IBuildInfo info)170     public void setBuildInfoRetryCommand(IBuildInfo info) {
171         IInvocationResult result = new LightInvocationResult(getResult());
172         String retryCommandLineArgs = result.getCommandLineArgs();
173         new CompatibilityBuildHelper(info).setRetryCommandLineArgs(retryCommandLineArgs);
174     }
175 
176     /**
177      * Retrieve an instance of the result to retry using the instance variables referencing
178      * the build and the desired session ID. While it is faster to load this result once and
179      * store it as an instance variable, {@link IInvocationResult} objects are large, and
180      * memory is of greater concern.
181      */
getResult()182     public IInvocationResult getResult() {
183         IInvocationResult result = null;
184         try {
185             result = ResultHandler.findResult(mBuild.getResultsDir(), mSessionId);
186         } catch (FileNotFoundException e) {
187             throw new RuntimeException(e);
188         }
189         if (result == null) {
190             throw new IllegalArgumentException(String.format(
191                     "Could not find session with id %d", mSessionId));
192         }
193         return result;
194     }
195 
196     /**
197      * Populate mRetryIncludes and mRetryExcludes based on the options and the result set for
198      * this instance of RetryFilterHelper.
199      */
populateRetryFilters()200     public void populateRetryFilters() {
201         mRetryIncludes = new HashSet<>(mIncludeFilters); // reset for each population
202         mRetryExcludes = new HashSet<>(mExcludeFilters); // reset for each population
203         if (RetryType.CUSTOM.equals(mRetryType)) {
204             Set<String> customIncludes = new HashSet<>(mIncludeFilters);
205             Set<String> customExcludes = new HashSet<>(mExcludeFilters);
206             if (mSubPlan != null) {
207                 ISubPlan retrySubPlan = SubPlanHelper.getSubPlanByName(mBuild, mSubPlan);
208                 customIncludes.addAll(retrySubPlan.getIncludeFilters());
209                 customExcludes.addAll(retrySubPlan.getExcludeFilters());
210             }
211             // If includes were added, only use those includes. Also use excludes added directly
212             // or by subplan. Otherwise, default to normal retry.
213             if (!customIncludes.isEmpty()) {
214                 mRetryIncludes.clear();
215                 mRetryIncludes.addAll(customIncludes);
216                 mRetryExcludes.addAll(customExcludes);
217                 return;
218             }
219         }
220         // remove any extra filtering options
221         // TODO(aaronholden) remove non-plan includes (e.g. those in cts-vendor-interface)
222         // TODO(aaronholden) remove non-known-failure excludes
223         mModuleName = null;
224         mTestName = null;
225         mSubPlan = null;
226         populateFiltersBySubPlan();
227         populatePreviousSessionFilters();
228     }
229 
230     /* Generation of filters based on previous sessions is implemented thoroughly in SubPlanHelper,
231      * and retry filter generation is just a subset of the use cases for the subplan retry logic.
232      * Use retry type to determine which result types SubPlanHelper targets. */
populateFiltersBySubPlan()233     public void populateFiltersBySubPlan() {
234         SubPlanHelper retryPlanCreator = new SubPlanHelper();
235         retryPlanCreator.setResult(getResult());
236         if (RetryType.FAILED.equals(mRetryType)) {
237             // retry only failed tests
238             retryPlanCreator.addResultType(SubPlanHelper.FAILED);
239         } else if (RetryType.NOT_EXECUTED.equals(mRetryType)){
240             // retry only not executed tests
241             retryPlanCreator.addResultType(SubPlanHelper.NOT_EXECUTED);
242         } else {
243             // retry both failed and not executed tests
244             retryPlanCreator.addResultType(SubPlanHelper.FAILED);
245             retryPlanCreator.addResultType(SubPlanHelper.NOT_EXECUTED);
246         }
247         try {
248             ISubPlan retryPlan = retryPlanCreator.createSubPlan(mBuild);
249             mRetryIncludes.addAll(retryPlan.getIncludeFilters());
250             mRetryExcludes.addAll(retryPlan.getExcludeFilters());
251         } catch (ConfigurationException e) {
252             throw new RuntimeException ("Failed to create subplan for retry", e);
253         }
254     }
255 
256     /* Retrieves the options set via command-line on the previous session, and generates/adds
257      * filters accordingly */
populatePreviousSessionFilters()258     private void populatePreviousSessionFilters() {
259         // Temporarily store options from this instance in another instance
260         RetryFilterHelper tmpHelper = new RetryFilterHelper(mBuild, mSessionId);
261         tmpHelper.setAllOptionsFrom(this);
262         // Copy command-line args from previous session to this RetryFilterHelper's options
263         setCommandLineOptionsFor(this);
264 
265         mRetryIncludes.addAll(mIncludeFilters);
266         mRetryExcludes.addAll(mExcludeFilters);
267         if (mSubPlan != null) {
268             ISubPlan retrySubPlan = SubPlanHelper.getSubPlanByName(mBuild, mSubPlan);
269             mRetryIncludes.addAll(retrySubPlan.getIncludeFilters());
270             mRetryExcludes.addAll(retrySubPlan.getExcludeFilters());
271         }
272         if (mModuleName != null) {
273             try {
274                 List<String> modules = getModuleNamesMatching(mBuild.getTestsDir(), mModuleName);
275                 if (modules.size() == 0) {
276                     throw new IllegalArgumentException(
277                             String.format("No modules found matching %s", mModuleName));
278                 } else if (modules.size() > 1) {
279                     throw new IllegalArgumentException(String.format(
280                             "Multiple modules found matching %s:\n%s\nWhich one did you mean?\n",
281                             mModuleName, ArrayUtil.join("\n", modules)));
282                 } else {
283                     String module = modules.get(0);
284                     cleanFilters(mRetryIncludes, module);
285                     cleanFilters(mRetryExcludes, module);
286                     mRetryIncludes.add(new TestFilter(mAbiName, module, mTestName).toString());
287                 }
288             } catch (FileNotFoundException e) {
289                 throw new RuntimeException(e);
290             }
291         } else if (mTestName != null) {
292             throw new IllegalArgumentException(
293                 "Test name given without module name. Add --module <module-name>");
294         }
295 
296         // Copy options for current session back to this instance
297         setAllOptionsFrom(tmpHelper);
298     }
299 
300     /* Helper method designed to remove filters in a list not applicable to the given module */
cleanFilters(Set<String> filters, String module)301     private static void cleanFilters(Set<String> filters, String module) {
302         Set<String> cleanedFilters = new HashSet<String>();
303         for (String filter : filters) {
304             if (module.equals(TestFilter.createFrom(filter).getName())) {
305                 cleanedFilters.add(filter); // Module name matches, filter passes
306             }
307         }
308         filters.clear();
309         filters.addAll(cleanedFilters);
310     }
311 
312     /** Retrieve include filters to be applied on retry */
getIncludeFilters()313     public Set<String> getIncludeFilters() {
314         return new HashSet<>(mRetryIncludes);
315     }
316 
317     /** Retrieve exclude filters to be applied on retry */
getExcludeFilters()318     public Set<String> getExcludeFilters() {
319         return new HashSet<>(mRetryExcludes);
320     }
321 
322     /** Clears retry filters and internal storage of options, except buildInfo and session ID */
tearDown()323     public void tearDown() {
324         clearOptions();
325         mRetryIncludes = null;
326         mRetryExcludes = null;
327         // keep references to buildInfo and session ID
328     }
329 
330     /** @return the {@link List} of modules whose name contains the given pattern. */
getModuleNamesMatching(File directory, String pattern)331     public static List<String> getModuleNamesMatching(File directory, String pattern) {
332         String[] names = directory.list(new NameFilter(pattern));
333         List<String> modules = new ArrayList<String>(names.length);
334         for (String name : names) {
335             int index = name.indexOf(".config");
336             if (index > 0) {
337                 String module = name.substring(0, index);
338                 if (module.equals(pattern)) {
339                     // Pattern represents a single module, just return a single-item list
340                     modules = new ArrayList<>(1);
341                     modules.add(module);
342                     return modules;
343                 }
344                 modules.add(module);
345             }
346         }
347         return modules;
348     }
349 
350     /** A {@link FilenameFilter} to find all modules in a directory who match the given pattern. */
351     public static class NameFilter implements FilenameFilter {
352 
353         private String mPattern;
354 
NameFilter(String pattern)355         public NameFilter(String pattern) {
356             mPattern = pattern;
357         }
358 
359         /** {@inheritDoc} */
360         @Override
accept(File dir, String name)361         public boolean accept(File dir, String name) {
362             return name.contains(mPattern) && name.endsWith(".config");
363         }
364     }
365 }
366