1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.observatory;
18 
19 import com.android.annotations.VisibleForTesting;
20 import com.android.ddmlib.DdmPreferences;
21 import com.android.ddmlib.Log;
22 import com.android.ddmlib.Log.LogLevel;
23 import com.android.tradefed.config.Configuration;
24 import com.android.tradefed.config.ConfigurationException;
25 import com.android.tradefed.config.ConfigurationFactory;
26 import com.android.tradefed.config.IConfiguration;
27 import com.android.tradefed.config.IConfigurationFactory;
28 import com.android.tradefed.invoker.tracing.ActiveTrace;
29 import com.android.tradefed.invoker.tracing.CloseableTraceScope;
30 import com.android.tradefed.invoker.tracing.TracingLogger;
31 import com.android.tradefed.log.LogRegistry;
32 import com.android.tradefed.log.LogUtil.CLog;
33 import com.android.tradefed.log.StdoutLogger;
34 import com.android.tradefed.testtype.IRemoteTest;
35 import com.android.tradefed.testtype.suite.BaseTestSuite;
36 import com.android.tradefed.testtype.suite.ITestSuite.MultiDeviceModuleStrategy;
37 import com.android.tradefed.testtype.suite.SuiteTestFilter;
38 import com.android.tradefed.testtype.suite.TestMappingSuiteRunner;
39 import com.android.tradefed.util.FileUtil;
40 import com.android.tradefed.util.IDisableable;
41 import com.android.tradefed.util.MultiMap;
42 import com.android.tradefed.util.keystore.DryRunKeyStore;
43 
44 import com.google.common.base.Strings;
45 import com.google.common.collect.ImmutableSet;
46 
47 import org.json.JSONArray;
48 import org.json.JSONException;
49 import org.json.JSONObject;
50 
51 import java.io.File;
52 import java.io.IOException;
53 import java.util.ArrayList;
54 import java.util.Arrays;
55 import java.util.Collections;
56 import java.util.HashSet;
57 import java.util.LinkedHashSet;
58 import java.util.List;
59 import java.util.Set;
60 import java.util.stream.Collectors;
61 
62 /**
63  * A class for getting test modules and target preparers for a given command line args.
64  *
65  * <p>TestDiscoveryExecutor will consume the command line args and print test module names and
66  * target preparer apks on stdout for the parent TradeFed process to receive and parse it.
67  *
68  * <p>
69  */
70 public class TestDiscoveryExecutor {
71 
getConfigurationFactory()72     IConfigurationFactory getConfigurationFactory() {
73         return ConfigurationFactory.getInstance();
74     }
75 
76     private boolean mReportPartialFallback = false;
77     private boolean mReportNoPossibleDiscovery = false;
78 
hasOutputResultFile()79     private static boolean hasOutputResultFile() {
80         return System.getenv(TestDiscoveryInvoker.OUTPUT_FILE) != null;
81     }
82 
83     /**
84      * An TradeFederation entry point that will use command args to discover test artifact
85      * information.
86      *
87      * <p>Intended for use with cts partial download to only download necessary files for the test
88      * run.
89      *
90      * <p>Will only exit with 0 when successfully discovered test modules.
91      *
92      * <p>Expected arguments: [commands options] (config to run)
93      */
main(String[] args)94     public static void main(String[] args) {
95         long pid = ProcessHandle.current().pid();
96         long tid = Thread.currentThread().getId();
97         ActiveTrace trace = TracingLogger.createActiveTrace(pid, tid);
98         trace.startTracing(false);
99         DiscoveryExitCode exitCode = DiscoveryExitCode.SUCCESS;
100         TestDiscoveryExecutor testDiscoveryExecutor = new TestDiscoveryExecutor();
101         try (CloseableTraceScope ignored = new CloseableTraceScope("main_discovery")) {
102             String testModules = testDiscoveryExecutor.discoverDependencies(args);
103             if (hasOutputResultFile()) {
104                 FileUtil.writeToFile(
105                         testModules, new File(System.getenv(TestDiscoveryInvoker.OUTPUT_FILE)));
106             }
107             System.out.print(testModules);
108         } catch (TestDiscoveryException e) {
109             System.err.print(e.getMessage());
110             if (e.exitCode() != null) {
111                 exitCode = e.exitCode();
112             } else {
113                 exitCode = DiscoveryExitCode.ERROR;
114             }
115         } catch (ConfigurationException e) {
116             System.err.print(e.getMessage());
117             exitCode = DiscoveryExitCode.CONFIGURATION_EXCEPTION;
118         } catch (Exception e) {
119             System.err.print(e.getMessage());
120             exitCode = DiscoveryExitCode.ERROR;
121         }
122         File traceFile = trace.finalizeTracing();
123         if (traceFile != null) {
124             if (System.getenv(TestDiscoveryInvoker.DISCOVERY_TRACE_FILE) != null) {
125                 try {
126                     FileUtil.copyFile(
127                             traceFile,
128                             new File(System.getenv(TestDiscoveryInvoker.DISCOVERY_TRACE_FILE)));
129                 } catch (IOException | RuntimeException e) {
130                     System.err.print(e.getMessage());
131                 }
132             }
133             FileUtil.deleteFile(traceFile);
134         }
135         System.exit(exitCode.exitCode());
136     }
137 
138     /**
139      * Discover test dependencies base on command line args.
140      *
141      * @param args the command line args of the test.
142      * @return A JSON string with one test module names array and one other test dependency array.
143      */
discoverDependencies(String[] args)144     public String discoverDependencies(String[] args)
145             throws TestDiscoveryException, ConfigurationException, JSONException {
146         // Create IConfiguration base on command line args.
147         IConfiguration config = getConfiguration(args);
148 
149         if (hasOutputResultFile()) {
150             DdmPreferences.setLogLevel(LogLevel.VERBOSE.getStringValue());
151             Log.setLogOutput(LogRegistry.getLogRegistry());
152             StdoutLogger logger = new StdoutLogger();
153             logger.setLogLevel(LogLevel.VERBOSE);
154             LogRegistry.getLogRegistry().registerLogger(logger);
155         }
156         try {
157             // Get tests from the configuration.
158             List<IRemoteTest> tests = config.getTests();
159 
160             // Tests could be empty if input args are corrupted.
161             if (tests == null || tests.isEmpty()) {
162                 throw new TestDiscoveryException(
163                         "Tradefed Observatory discovered no tests from the IConfiguration created"
164                                 + " from command line args.",
165                         null,
166                         DiscoveryExitCode.ERROR);
167             }
168 
169             List<String> testModules = new ArrayList<>(discoverTestModulesFromTests(tests));
170             List<String> testDependencies = new ArrayList<>(discoverDependencies(config));
171             Collections.sort(testModules);
172             Collections.sort(testDependencies);
173 
174             try (CloseableTraceScope ignored = new CloseableTraceScope("format_results")) {
175                 JSONObject j = new JSONObject();
176                 j.put(TestDiscoveryInvoker.TEST_MODULES_LIST_KEY, new JSONArray(testModules));
177                 j.put(
178                         TestDiscoveryInvoker.TEST_DEPENDENCIES_LIST_KEY,
179                         new JSONArray(testDependencies));
180                 if (mReportPartialFallback) {
181                     j.put(TestDiscoveryInvoker.PARTIAL_FALLBACK_KEY, "true");
182                 }
183                 if (mReportNoPossibleDiscovery) {
184                     j.put(TestDiscoveryInvoker.NO_POSSIBLE_TEST_DISCOVERY_KEY, "true");
185                 }
186                 return j.toString();
187             }
188         } finally {
189             if (hasOutputResultFile()) {
190                 LogRegistry.getLogRegistry().unregisterLogger();
191             }
192         }
193     }
194 
195     /**
196      * Retrieve configuration base on command line args.
197      *
198      * @param args the command line args of the test.
199      * @return A {@link IConfiguration} which constructed based on command line args.
200      */
getConfiguration(String[] args)201     private IConfiguration getConfiguration(String[] args) throws ConfigurationException {
202         try (CloseableTraceScope ignored = new CloseableTraceScope("create_configuration")) {
203             IConfigurationFactory configurationFactory = getConfigurationFactory();
204             return configurationFactory.createPartialConfigurationFromArgs(
205                     args,
206                     new DryRunKeyStore(),
207                     Set.of(Configuration.TEST_TYPE_NAME, Configuration.TARGET_PREPARER_TYPE_NAME),
208                     null);
209         }
210     }
211 
212     /**
213      * Discover configuration by a list of {@link IRemoteTest}.
214      *
215      * @param testList a list of {@link IRemoteTest}.
216      * @return A set of test module names.
217      */
discoverTestModulesFromTests(List<IRemoteTest> testList)218     private Set<String> discoverTestModulesFromTests(List<IRemoteTest> testList)
219             throws IllegalStateException, TestDiscoveryException {
220         try (CloseableTraceScope ignored =
221                 new CloseableTraceScope("discoverTestModulesFromTests")) {
222             Set<String> testModules = new LinkedHashSet<String>();
223             Set<String> includeFilters = new LinkedHashSet<String>();
224             Set<String> excludeFilters = new LinkedHashSet<String>();
225             // Collect include filters from every test.
226             boolean discoveredLogic = true;
227             for (IRemoteTest test : testList) {
228                 if (!(test instanceof BaseTestSuite)) {
229                     throw new TestDiscoveryException(
230                             "Tradefed Observatory can't do test discovery on non suite-based test"
231                                     + " runner.",
232                             null,
233                             DiscoveryExitCode.ERROR);
234                 }
235                 if (test instanceof TestMappingSuiteRunner) {
236                     ((TestMappingSuiteRunner) test).loadTestInfos();
237                 }
238                 Set<String> suiteIncludeFilters = ((BaseTestSuite) test).getIncludeFilter();
239                 excludeFilters.addAll(((BaseTestSuite) test).getExcludeFilter());
240                 MultiMap<String, String> moduleMetadataIncludeFilters =
241                         ((BaseTestSuite) test).getModuleMetadataIncludeFilters();
242                 // Include/Exclude filters in suites are evaluated first,
243                 // then metadata are applied on top, so having metadata filters
244                 // and include-filters can actually be resolved to a super-set
245                 // which is better than falling back.
246                 if (!excludeFilters.isEmpty() && ((BaseTestSuite) test).reverseExcludeFilters()) {
247                     includeFilters.addAll(excludeFilters);
248                     excludeFilters.clear();
249                 } else if (!suiteIncludeFilters.isEmpty()) {
250                     includeFilters.addAll(suiteIncludeFilters);
251                 } else if (!moduleMetadataIncludeFilters.isEmpty()) {
252                     String rootDirPath =
253                             getEnvironment(TestDiscoveryInvoker.ROOT_DIRECTORY_ENV_VARIABLE_KEY);
254                     boolean throwException = true;
255                     if (rootDirPath != null) {
256                         File rootDir = new File(rootDirPath);
257                         if (rootDir.exists() && rootDir.isDirectory()) {
258                             Set<String> configs =
259                                     searchConfigsForMetadata(rootDir, moduleMetadataIncludeFilters);
260                             if (configs != null) {
261                                 testModules.addAll(configs);
262                                 throwException = false;
263                                 mReportPartialFallback = true;
264                             }
265                         }
266                     }
267                     if (throwException) {
268                         throw new TestDiscoveryException(
269                                 "Tradefed Observatory can't do test discovery because the existence"
270                                         + " of metadata include filter option.",
271                                 null,
272                                 DiscoveryExitCode.COMPONENT_METADATA);
273                     }
274                 } else if (MultiDeviceModuleStrategy.ONLY_MULTI_DEVICES.equals(
275                         ((BaseTestSuite) test).getMultiDeviceStrategy())) {
276                     String rootDirPath =
277                             getEnvironment(TestDiscoveryInvoker.ROOT_DIRECTORY_ENV_VARIABLE_KEY);
278                     boolean throwException = true;
279                     if (rootDirPath != null) {
280                         File rootDir = new File(rootDirPath);
281                         if (rootDir.exists() && rootDir.isDirectory()) {
282                             Set<String> configs = searchForMultiDevicesConfig(rootDir);
283                             if (configs != null) {
284                                 testModules.addAll(configs);
285                                 throwException = false;
286                                 mReportPartialFallback = true;
287                             }
288                         }
289                     }
290                     if (throwException) {
291                         throw new TestDiscoveryException(
292                                 "Tradefed Observatory can't do test discovery because the existence"
293                                         + " of multi-devices option.",
294                                 null,
295                                 DiscoveryExitCode.COMPONENT_METADATA);
296                     }
297                 } else if (!Strings.isNullOrEmpty(((BaseTestSuite) test).getRunSuiteTag())) {
298                     String rootDirPath =
299                             getEnvironment(TestDiscoveryInvoker.ROOT_DIRECTORY_ENV_VARIABLE_KEY);
300                     boolean throwException = true;
301                     if (rootDirPath != null) {
302                         File rootDir = new File(rootDirPath);
303                         if (rootDir.exists() && rootDir.isDirectory()) {
304                             Set<String> configs =
305                                     searchConfigsForSuiteTag(
306                                             rootDir, ((BaseTestSuite) test).getRunSuiteTag());
307                             if (configs != null) {
308                                 testModules.addAll(configs);
309                                 throwException = false;
310                                 mReportPartialFallback = true;
311                             }
312                         }
313                     }
314                     if (throwException) {
315                         throw new TestDiscoveryException(
316                                 "Tradefed Observatory can't do test discovery because the existence"
317                                         + " of run-suite-tag option.",
318                                 null,
319                                 DiscoveryExitCode.COMPONENT_METADATA);
320                     }
321                 } else {
322                     discoveredLogic = false;
323                 }
324             }
325             if (!discoveredLogic) {
326                 mReportNoPossibleDiscovery = true;
327             }
328             // Extract test module names from included filters.
329             if (hasOutputResultFile()) {
330                 System.out.println(String.format("include filters: %s", includeFilters));
331             }
332             Set<String> moduleOnlyIncludeFilters =
333                     extractTestModulesFromIncludeFilters(includeFilters);
334             testModules.addAll(moduleOnlyIncludeFilters);
335             testModules.addAll(findExtraConfigsParents(moduleOnlyIncludeFilters));
336             // Any directly excluded won't be discovered since it shouldn't run
337             testModules.removeAll(excludeFilters);
338             return testModules;
339         }
340     }
341 
342     /**
343      * Extract test module names from include filters.
344      *
345      * @param includeFilters a set of include filters.
346      * @return A set of test module names.
347      */
extractTestModulesFromIncludeFilters(Set<String> includeFilters)348     private Set<String> extractTestModulesFromIncludeFilters(Set<String> includeFilters)
349             throws IllegalStateException {
350         Set<String> testModuleNames = new LinkedHashSet<>();
351         // Extract module name from each include filter.
352         // TODO: Ensure if a module is fully excluded then it's excluded.
353         for (String includeFilter : includeFilters) {
354             String testModuleName = SuiteTestFilter.createFrom(includeFilter).getBaseName();
355             if (testModuleName == null) {
356                 // If unable to parse an include filter, throw exception to exit.
357                 throw new IllegalStateException(
358                         String.format(
359                                 "Unable to parse test module name from include filter %s",
360                                 includeFilter));
361             } else {
362                 testModuleNames.add(testModuleName);
363             }
364         }
365         return testModuleNames;
366     }
367 
368     /**
369      * When using extra_test_configs in Soong, they end up in the original module named folder, so
370      * search for it to backfill the download
371      */
findExtraConfigsParents(Set<String> moduleNames)372     private Set<String> findExtraConfigsParents(Set<String> moduleNames) {
373         Set<String> parentModules = Collections.synchronizedSet(new HashSet<>());
374         String rootDirPath = getEnvironment(TestDiscoveryInvoker.ROOT_DIRECTORY_ENV_VARIABLE_KEY);
375         if (rootDirPath == null) {
376             CLog.w("root dir env not set.");
377             return parentModules;
378         }
379         CLog.d("Seaching parent configs.");
380         try (CloseableTraceScope ignored = new CloseableTraceScope("find parent configs")) {
381             Set<File> testCasesDirs = FileUtil.findFilesObject(new File(rootDirPath), "testcases");
382             Set<String> moduleDirs = Collections.synchronizedSet(new HashSet<>());
383             testCasesDirs.parallelStream()
384                     .forEach(
385                             f -> {
386                                 String[] modules = f.list();
387                                 if (modules != null) {
388                                     moduleDirs.addAll(Arrays.asList(modules));
389                                 }
390                             });
391             Set<String> moduleNameMismatch =
392                     moduleNames.parallelStream()
393                             .filter(m -> !moduleDirs.contains(m))
394                             .collect(Collectors.toSet());
395             // Only search the mismatch
396             moduleNameMismatch.parallelStream()
397                     .forEach(
398                             name -> {
399                                 File config =
400                                         FileUtil.findFile(new File(rootDirPath), name + ".config");
401                                 if (config != null) {
402                                     if (!config.getParentFile().getName().equals(name)) {
403                                         CLog.d(
404                                                 "Parent: %s being added for the extra configs",
405                                                 config.getParentFile().getName());
406                                         parentModules.add(config.getParentFile().getName());
407                                     }
408                                 }
409                             });
410         } catch (IOException e) {
411             CLog.e(e);
412         }
413         CLog.d("Done searching parent configs.");
414         return parentModules;
415     }
416 
discoverDependencies(IConfiguration config)417     private Set<String> discoverDependencies(IConfiguration config) {
418         Set<String> dependencies = new HashSet<>();
419         try (CloseableTraceScope ignored = new CloseableTraceScope("discoverDependencies")) {
420             for (Object o :
421                     config.getAllConfigurationObjectsOfType(
422                             Configuration.TARGET_PREPARER_TYPE_NAME)) {
423                 if (o instanceof IDisableable) {
424                     if (((IDisableable) o).isDisabled()) {
425                         continue;
426                     }
427                 }
428                 if (o instanceof IDiscoverDependencies) {
429                     dependencies.addAll(((IDiscoverDependencies) o).reportDependencies());
430                 }
431             }
432             return dependencies;
433         }
434     }
435 
searchForMultiDevicesConfig(File rootDir)436     private Set<String> searchForMultiDevicesConfig(File rootDir) {
437         try {
438             Set<File> configFiles = FileUtil.findFilesObject(rootDir, ".*\\.config$");
439             if (configFiles.isEmpty()) {
440                 return null;
441             }
442             Set<File> shouldRunFiles =
443                     configFiles.stream()
444                             .filter(
445                                     f -> {
446                                         try {
447                                             IConfiguration c =
448                                                     getConfigurationFactory()
449                                                             .createPartialConfigurationFromArgs(
450                                                                     new String[] {
451                                                                         f.getAbsolutePath()
452                                                                     },
453                                                                     new DryRunKeyStore(),
454                                                                     ImmutableSet.of(
455                                                                             Configuration
456                                                                                     .CONFIGURATION_DESCRIPTION_TYPE_NAME),
457                                                                     null);
458                                             return c.getDeviceConfig().size() > 1;
459                                         } catch (ConfigurationException e) {
460                                             return false;
461                                         }
462                                     })
463                             .collect(Collectors.toSet());
464             return shouldRunFiles.stream()
465                     .map(c -> FileUtil.getBaseName(c.getName()))
466                     .collect(Collectors.toSet());
467         } catch (IOException e) {
468             System.err.println(e);
469         }
470         return null;
471     }
472 
searchConfigsForMetadata( File rootDir, MultiMap<String, String> moduleMetadataIncludeFilters)473     private Set<String> searchConfigsForMetadata(
474             File rootDir, MultiMap<String, String> moduleMetadataIncludeFilters) {
475         try {
476             Set<File> configFiles = FileUtil.findFilesObject(rootDir, ".*\\.config$");
477             if (configFiles.isEmpty()) {
478                 return null;
479             }
480             Set<File> shouldRunFiles =
481                     configFiles.stream()
482                             .filter(
483                                     f -> {
484                                         try {
485                                             IConfiguration c =
486                                                     getConfigurationFactory()
487                                                             .createPartialConfigurationFromArgs(
488                                                                     new String[] {
489                                                                         f.getAbsolutePath()
490                                                                     },
491                                                                     new DryRunKeyStore(),
492                                                                     ImmutableSet.of(
493                                                                             Configuration
494                                                                                     .CONFIGURATION_DESCRIPTION_TYPE_NAME),
495                                                                     null);
496                                             return new BaseTestSuite()
497                                                     .filterByConfigMetadata(
498                                                             c,
499                                                             moduleMetadataIncludeFilters,
500                                                             new MultiMap<String, String>());
501                                         } catch (ConfigurationException e) {
502                                             return false;
503                                         }
504                                     })
505                             .collect(Collectors.toSet());
506             return shouldRunFiles.stream()
507                     .map(c -> FileUtil.getBaseName(c.getName()))
508                     .collect(Collectors.toSet());
509         } catch (IOException e) {
510             System.err.println(e);
511         }
512         return null;
513     }
514 
searchConfigsForSuiteTag(File rootDir, String suiteTag)515     private Set<String> searchConfigsForSuiteTag(File rootDir, String suiteTag) {
516         try {
517             Set<File> configFiles = FileUtil.findFilesObject(rootDir, ".*\\.config$");
518             if (configFiles.isEmpty()) {
519                 return null;
520             }
521             Set<File> shouldRunFiles =
522                     configFiles.stream()
523                             .filter(
524                                     f -> {
525                                         try {
526                                             // TODO: make it more robust to detect
527                                             String content = FileUtil.readStringFromFile(f);
528                                             return content.contains("test-suite-tag")
529                                                     && content.contains(suiteTag);
530                                         } catch (IOException e) {
531                                             return false;
532                                         }
533                                     })
534                             .collect(Collectors.toSet());
535             return shouldRunFiles.stream()
536                     .map(c -> FileUtil.getBaseName(c.getName()))
537                     .collect(Collectors.toSet());
538         } catch (IOException e) {
539             System.err.println(e);
540         }
541         return null;
542     }
543 
544     @VisibleForTesting
getEnvironment(String var)545     protected String getEnvironment(String var) {
546         return System.getenv(var);
547     }
548 }
549