1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.result.skipped;
17 
18 import com.android.tradefed.build.content.ContentAnalysisContext;
19 import com.android.tradefed.config.IConfiguration;
20 import com.android.tradefed.config.Option;
21 import com.android.tradefed.config.OptionClass;
22 import com.android.tradefed.device.ITestDevice;
23 import com.android.tradefed.invoker.IInvocationContext;
24 import com.android.tradefed.invoker.TestInformation;
25 import com.android.tradefed.invoker.logger.InvocationMetricLogger;
26 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
27 import com.android.tradefed.log.LogUtil.CLog;
28 import com.android.tradefed.result.skipped.SkipReason.DemotionTrigger;
29 import com.android.tradefed.service.TradefedFeatureClient;
30 import com.android.tradefed.util.IDisableable;
31 import com.android.tradefed.util.MultiMap;
32 
33 import com.proto.tradefed.feature.FeatureResponse;
34 import com.proto.tradefed.feature.PartResponse;
35 
36 import java.util.ArrayList;
37 import java.util.HashMap;
38 import java.util.LinkedHashMap;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.Map.Entry;
42 
43 /**
44  * Based on a variety of criteria the skip manager helps to decide what should be skipped at
45  * different levels: invocation, modules and tests.
46  */
47 @OptionClass(alias = "skip-manager")
48 public class SkipManager implements IDisableable {
49 
50     @Option(name = "disable-skip-manager", description = "Disable the skip manager feature.")
51     private boolean mIsDisabled = false;
52 
53     @Option(
54             name = "demotion-filters",
55             description =
56                     "An option to manually inject demotion filters. Intended for testing and"
57                             + " validation, not for production demotion.")
58     private Map<String, String> mDemotionFilterOption = new LinkedHashMap<>();
59 
60     @Option(
61             name = "skip-on-no-change",
62             description = "Enable the layer of skipping when there is no changes to artifacts.")
63     private boolean mSkipOnNoChange = false;
64 
65     @Option(
66             name = "skip-on-no-tests-discovered",
67             description = "Enable the layer of skipping when there is no discovered tests to run.")
68     private boolean mSkipOnNoTestsDiscovered = true;
69 
70     @Option(
71             name = "skip-on-no-change-presubmit-only",
72             description = "Allow enabling the skip logic only in presubmit.")
73     private boolean mSkipOnNoChangePresubmitOnly = true;
74 
75     @Option(
76             name = "considered-for-content-analysis",
77             description = "Some tests do not directly rely on content for being relevant.")
78     private boolean mConsideredForContent = true;
79 
80     @Option(name = "analysis-level", description = "Alter assumptions level of the analysis.")
81     private AnalysisHeuristic mAnalysisLevel = AnalysisHeuristic.REMOVE_EXEMPTION;
82 
83     // Contains the filter and reason for demotion
84     private final Map<String, SkipReason> mDemotionFilters = new LinkedHashMap<>();
85 
86     private boolean mNoTestsDiscovered = false;
87     private MultiMap<ITestDevice, ContentAnalysisContext> mImageAnalysis = new MultiMap<>();
88     private List<ContentAnalysisContext> mTestArtifactsAnalysisContent = new ArrayList<>();
89     private List<String> mModulesDiscovered = new ArrayList<String>();
90     private List<String> mDependencyFiles = new ArrayList<String>();
91 
92     private String mReasonForSkippingInvocation = "SkipManager decided to skip.";
93 
94     /** Setup and initialize the skip manager. */
setup(IConfiguration config, IInvocationContext context)95     public void setup(IConfiguration config, IInvocationContext context) {
96         if (config.getCommandOptions().getInvocationData().containsKey("subprocess")) {
97             // Information is going to flow through GlobalFilters mechanism
98             return;
99         }
100         for (Entry<String, String> filterReason : mDemotionFilterOption.entrySet()) {
101             mDemotionFilters.put(
102                     filterReason.getKey(),
103                     new SkipReason(filterReason.getValue(), DemotionTrigger.UNKNOWN_TRIGGER));
104         }
105         fetchDemotionInformation(context);
106     }
107 
108     /** Returns the demoted tests and the reason for demotion */
getDemotedTests()109     public Map<String, SkipReason> getDemotedTests() {
110         return mDemotionFilters;
111     }
112 
setImageAnalysis(ITestDevice device, ContentAnalysisContext analysisContext)113     public void setImageAnalysis(ITestDevice device, ContentAnalysisContext analysisContext) {
114         CLog.d(
115                 "Received image artifact analysis '%s' for %s",
116                 analysisContext.contentEntry(), device.getSerialNumber());
117         mImageAnalysis.put(device, analysisContext);
118     }
119 
setTestArtifactsAnalysis(ContentAnalysisContext analysisContext)120     public void setTestArtifactsAnalysis(ContentAnalysisContext analysisContext) {
121         CLog.d("Received test artifact analysis '%s'", analysisContext.contentEntry());
122         mTestArtifactsAnalysisContent.add(analysisContext);
123     }
124 
125     /**
126      * In the early download and discovery process, report to the skip manager that no tests are
127      * expected to be run. This should lead to skipping the invocation.
128      */
reportDiscoveryWithNoTests()129     public void reportDiscoveryWithNoTests() {
130         CLog.d("Test discovery reported that no tests were found.");
131         mNoTestsDiscovered = true;
132     }
133 
reportDiscoveryDependencies(List<String> modules, List<String> depFiles)134     public void reportDiscoveryDependencies(List<String> modules, List<String> depFiles) {
135         mModulesDiscovered.addAll(modules);
136         mDependencyFiles.addAll(depFiles);
137     }
138 
139     /** Reports whether we should skip the current invocation. */
shouldSkipInvocation(TestInformation information)140     public boolean shouldSkipInvocation(TestInformation information) {
141         // Build heuristic for skipping invocation
142         if (mNoTestsDiscovered) {
143             InvocationMetricLogger.addInvocationMetrics(
144                     InvocationMetricKey.SKIP_NO_TESTS_DISCOVERED, 1);
145             if (mSkipOnNoTestsDiscovered) {
146                 mReasonForSkippingInvocation =
147                         "No tests to be executed where found in the configuration.";
148                 return true;
149             } else {
150                 InvocationMetricLogger.addInvocationMetrics(
151                         InvocationMetricKey.SILENT_INVOCATION_SKIP_COUNT, 1);
152                 return false;
153             }
154         }
155         ArtifactsAnalyzer analyzer =
156                 new ArtifactsAnalyzer(
157                         information,
158                         mImageAnalysis,
159                         mTestArtifactsAnalysisContent,
160                         mModulesDiscovered,
161                         mDependencyFiles,
162                         mAnalysisLevel);
163         return buildAnalysisDecision(information, analyzer.analyzeArtifacts());
164     }
165 
166     /**
167      * Request to fetch the demotion information for the invocation. This should only be done once
168      * in the parent process.
169      */
fetchDemotionInformation(IInvocationContext context)170     private void fetchDemotionInformation(IInvocationContext context) {
171         if (isDisabled()) {
172             return;
173         }
174         if ("WORK_NODE".equals(context.getAttribute("trigger"))) {
175             try (TradefedFeatureClient client = new TradefedFeatureClient()) {
176                 Map<String, String> args = new HashMap<>();
177                 FeatureResponse response = client.triggerFeature("FetchDemotionInformation", args);
178                 if (response.hasErrorInfo()) {
179                     InvocationMetricLogger.addInvocationMetrics(
180                             InvocationMetricKey.DEMOTION_ERROR_RESPONSE, 1);
181                 } else {
182                     for (PartResponse part :
183                             response.getMultiPartResponse().getResponsePartList()) {
184                         String filter = part.getKey();
185                         mDemotionFilters.put(filter, SkipReason.fromString(part.getValue()));
186                     }
187                 }
188             }
189         }
190         if (!mDemotionFilters.isEmpty()) {
191             CLog.d("Demotion filters size '%s': %s", mDemotionFilters.size(), mDemotionFilters);
192             InvocationMetricLogger.addInvocationMetrics(
193                     InvocationMetricKey.DEMOTION_FILTERS_RECEIVED_COUNT, mDemotionFilters.size());
194         }
195     }
196 
197     /** Based on environment of the run and the build analysis, decide to skip or not. */
buildAnalysisDecision(TestInformation information, BuildAnalysis results)198     private boolean buildAnalysisDecision(TestInformation information, BuildAnalysis results) {
199         if (results == null) {
200             return false;
201         }
202         boolean presubmit = "WORK_NODE".equals(information.getContext().getAttribute("trigger"));
203         if (results.deviceImageChanged()) {
204             return false;
205         }
206         InvocationMetricLogger.addInvocationMetrics(
207                 InvocationMetricKey.DEVICE_IMAGE_NOT_CHANGED, 1);
208         if (results.hasTestsArtifacts()) {
209             if (results.hasChangesInTestsArtifacts()) {
210                 InvocationMetricLogger.addInvocationMetrics(
211                         InvocationMetricKey.TEST_ARTIFACT_CHANGE_ONLY, 1);
212                 return false;
213             } else {
214                 InvocationMetricLogger.addInvocationMetrics(
215                         InvocationMetricKey.TEST_ARTIFACT_NOT_CHANGED, 1);
216             }
217         } else {
218             InvocationMetricLogger.addInvocationMetrics(
219                     InvocationMetricKey.PURE_DEVICE_IMAGE_UNCHANGED, 1);
220         }
221         // If we get here, it means both device image and test artifacts are unaffected.
222         if (!mConsideredForContent) {
223             return false;
224         }
225         if (!presubmit) {
226             // Eventually support postsubmit analysis.
227             InvocationMetricLogger.addInvocationMetrics(
228                     InvocationMetricKey.NO_CHANGES_POSTSUBMIT, 1);
229             return false;
230         }
231         // Currently only consider skipping in presubmit
232         InvocationMetricLogger.addInvocationMetrics(InvocationMetricKey.SKIP_NO_CHANGES, 1);
233         if (mSkipOnNoChange) {
234             mReasonForSkippingInvocation =
235                     "No relevant changes to device image or test artifacts detected.";
236             return true;
237         }
238         if (presubmit && mSkipOnNoChangePresubmitOnly) {
239             mReasonForSkippingInvocation =
240                     "No relevant changes to device image or test artifacts detected.";
241             return true;
242         }
243         InvocationMetricLogger.addInvocationMetrics(
244                 InvocationMetricKey.SILENT_INVOCATION_SKIP_COUNT, 1);
245         return false;
246     }
247 
clearManager()248     public void clearManager() {
249         mDemotionFilters.clear();
250         mDemotionFilterOption.clear();
251         mModulesDiscovered.clear();
252         mDependencyFiles.clear();
253         for (ContentAnalysisContext request : mTestArtifactsAnalysisContent) {
254             if (request.contentInformation() != null) {
255                 request.contentInformation().clean();
256             }
257         }
258         for (ContentAnalysisContext request : mImageAnalysis.values()) {
259             if (request.contentInformation() != null) {
260                 request.contentInformation().clean();
261             }
262         }
263         mTestArtifactsAnalysisContent.clear();
264         mImageAnalysis.clear();
265     }
266 
267     @Override
isDisabled()268     public boolean isDisabled() {
269         return mIsDisabled;
270     }
271 
272     @Override
setDisable(boolean isDisabled)273     public void setDisable(boolean isDisabled) {
274         mIsDisabled = isDisabled;
275     }
276 
setSkipDecision(boolean shouldSkip)277     public void setSkipDecision(boolean shouldSkip) {
278         mSkipOnNoChange = shouldSkip;
279         mSkipOnNoTestsDiscovered = shouldSkip;
280     }
281 
getInvocationSkipReason()282     public String getInvocationSkipReason() {
283         return mReasonForSkippingInvocation;
284     }
285 }
286