1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.binary;
17 
18 import com.google.common.annotations.VisibleForTesting;
19 import com.android.tradefed.config.Option;
20 import com.android.tradefed.config.OptionCopier;
21 import com.android.tradefed.device.DeviceNotAvailableException;
22 import com.android.tradefed.invoker.IInvocationContext;
23 import com.android.tradefed.invoker.TestInformation;
24 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
25 import com.android.tradefed.observatory.IDiscoverDependencies;
26 import com.android.tradefed.result.FailureDescription;
27 import com.android.tradefed.result.ITestInvocationListener;
28 import com.android.tradefed.result.TestDescription;
29 import com.android.tradefed.testtype.IAbi;
30 import com.android.tradefed.testtype.IAbiReceiver;
31 import com.android.tradefed.testtype.IRemoteTest;
32 import com.android.tradefed.testtype.IRuntimeHintProvider;
33 import com.android.tradefed.testtype.IShardableTest;
34 import com.android.tradefed.testtype.ITestCollector;
35 import com.android.tradefed.testtype.ITestFilterReceiver;
36 import com.android.tradefed.testtype.suite.ModuleDefinition;
37 import com.android.tradefed.result.error.InfraErrorIdentifier;
38 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
39 import com.android.tradefed.util.StreamUtil;
40 
41 import com.google.common.collect.ImmutableList;
42 import com.google.common.collect.ImmutableMap;
43 
44 import java.io.File;
45 import java.io.IOException;
46 import java.lang.reflect.InvocationTargetException;
47 import java.util.ArrayList;
48 import java.util.Collection;
49 import java.util.HashMap;
50 import java.util.HashSet;
51 import java.util.LinkedHashMap;
52 import java.util.LinkedHashSet;
53 import java.util.List;
54 import java.util.Map;
55 import java.util.Set;
56 
57 /** Base class for executable style of tests. For example: binaries, shell scripts. */
58 public abstract class ExecutableBaseTest
59         implements IRemoteTest,
60                 IRuntimeHintProvider,
61                 ITestCollector,
62                 IShardableTest,
63                 IAbiReceiver,
64                 ITestFilterReceiver,
65                 IDiscoverDependencies {
66 
67     public static final String NO_BINARY_ERROR = "Binary %s does not exist.";
68 
69     @Option(
70             name = "per-binary-timeout",
71             isTimeVal = true,
72             description = "Timeout applied to each binary for their execution.")
73     private long mTimeoutPerBinaryMs = 5 * 60 * 1000L;
74 
75     @Option(name = "binary", description = "Path to the binary to be run. Can be repeated.")
76     private List<String> mBinaryPaths = new ArrayList<>();
77 
78     @Option(
79             name = "test-command-line",
80             description = "The test commands of each test names.",
81             requiredForRerun = true)
82     private Map<String, String> mTestCommands = new LinkedHashMap<>();
83 
84     @Option(
85             name = "collect-tests-only",
86             description = "Only dry-run through the tests, do not actually run them.")
87     private boolean mCollectTestsOnly = false;
88 
89     @Option(
90         name = "runtime-hint",
91         description = "The hint about the test's runtime.",
92         isTimeVal = true
93     )
94     private long mRuntimeHintMs = 60000L; // 1 minute
95 
96     enum ShardSplit {
97         PER_TEST_CMD,
98         PER_SHARD;
99     }
100 
101     @Option(name = "shard-split", description = "Shard by test command or shard count")
102     private ShardSplit mShardSplit = ShardSplit.PER_TEST_CMD;
103 
104     private IAbi mAbi;
105     private TestInformation mTestInfo;
106     private Set<String> mIncludeFilters = new LinkedHashSet<>();
107     private Set<String> mExcludeFilters = new LinkedHashSet<>();
108 
109     /**
110      * Get test commands.
111      *
112      * @return the test commands.
113      */
114     @VisibleForTesting
getTestCommands()115     Map<String, String> getTestCommands() {
116         return mTestCommands;
117     }
118 
119     /** @return the timeout applied to each binary for their execution. */
getTimeoutPerBinaryMs()120     protected long getTimeoutPerBinaryMs() {
121         return mTimeoutPerBinaryMs;
122     }
123 
getModuleId(IInvocationContext context)124     protected String getModuleId(IInvocationContext context) {
125         return context != null
126                 ? context.getAttributes().getUniqueMap().get(ModuleDefinition.MODULE_ID)
127                 : getClass().getName();
128     }
129 
getFilterDescriptions(Map<String, String> testCommands)130     protected TestDescription[] getFilterDescriptions(Map<String, String> testCommands) {
131         return testCommands.keySet().stream()
132                 .map(testName -> new TestDescription(testName, testName))
133                 .filter(description -> !shouldSkipCurrentTest(description))
134                 .toArray(TestDescription[]::new);
135     }
136 
doesRunBinaryGenerateTestResults()137     protected boolean doesRunBinaryGenerateTestResults() {
138         return false;
139     }
140 
141     /** {@inheritDoc} */
142     @Override
addIncludeFilter(String filter)143     public void addIncludeFilter(String filter) {
144         mIncludeFilters.add(filter);
145     }
146 
147     /** {@inheritDoc} */
148     @Override
addExcludeFilter(String filter)149     public void addExcludeFilter(String filter) {
150         mExcludeFilters.add(filter);
151     }
152 
153     /** {@inheritDoc} */
154     @Override
addAllIncludeFilters(Set<String> filters)155     public void addAllIncludeFilters(Set<String> filters) {
156         mIncludeFilters.addAll(filters);
157     }
158 
159     /** {@inheritDoc} */
160     @Override
addAllExcludeFilters(Set<String> filters)161     public void addAllExcludeFilters(Set<String> filters) {
162         mExcludeFilters.addAll(filters);
163     }
164 
165     /** {@inheritDoc} */
166     @Override
clearIncludeFilters()167     public void clearIncludeFilters() {
168         mIncludeFilters.clear();
169     }
170 
171     /** {@inheritDoc} */
172     @Override
clearExcludeFilters()173     public void clearExcludeFilters() {
174         mExcludeFilters.clear();
175     }
176 
177     /** {@inheritDoc} */
178     @Override
getIncludeFilters()179     public Set<String> getIncludeFilters() {
180         return mIncludeFilters;
181     }
182 
183     /** {@inheritDoc} */
184     @Override
getExcludeFilters()185     public Set<String> getExcludeFilters() {
186         return mExcludeFilters;
187     }
188 
189     @Override
run(TestInformation testInfo, ITestInvocationListener listener)190     public void run(TestInformation testInfo, ITestInvocationListener listener)
191             throws DeviceNotAvailableException {
192         setTestInfo(testInfo);
193         String moduleId = getModuleId(testInfo.getContext());
194         Map<String, String> testCommands = getAllTestCommands();
195         TestDescription[] testDescriptions = getFilterDescriptions(testCommands);
196 
197         if (testDescriptions.length == 0) {
198             return;
199         }
200 
201         String testRunName =
202                 testDescriptions.length == 1 ? testDescriptions[0].getTestName() : moduleId;
203         long startTimeMs = System.currentTimeMillis();
204 
205         try {
206             listener.testRunStarted(testRunName, testDescriptions.length);
207 
208             for (TestDescription description : testDescriptions) {
209                 String testName = description.getTestName();
210                 String cmd = testCommands.get(testName);
211                 String path = findBinary(cmd);
212                 try {
213                     if (path == null) {
214                         listener.testFailed(
215                                 description,
216                                 FailureDescription.create(
217                                                 String.format(NO_BINARY_ERROR, cmd),
218                                                 FailureStatus.TEST_FAILURE)
219                                         .setErrorIdentifier(
220                                                 InfraErrorIdentifier
221                                                         .CONFIGURED_ARTIFACT_NOT_FOUND));
222                     } else {
223                         try {
224                             if (!doesRunBinaryGenerateTestResults()) {
225                                 listener.testStarted(description);
226                             }
227 
228                             if (!getCollectTestsOnly()) {
229                                 // Do not actually run the test if we are dry running it.
230                                 runBinary(path, listener, description);
231                             }
232                         } catch (IOException e) {
233                             listener.testFailed(
234                                     description,
235                                     FailureDescription.create(StreamUtil.getStackTrace(e)));
236                             if (doesRunBinaryGenerateTestResults()) {
237                                 // We can't rely on the `testEnded()` call in the finally
238                                 // clause if `runBinary()` is responsible for generating test
239                                 // results, therefore we call it here.
240                                 listener.testEnded(description, new HashMap<String, Metric>());
241                             }
242                         }
243                     }
244                 } finally {
245                     if (!doesRunBinaryGenerateTestResults()) {
246                         listener.testEnded(description, new HashMap<String, Metric>());
247                     }
248                 }
249             }
250         } finally {
251             listener.testRunEnded(
252                     System.currentTimeMillis() - startTimeMs, new HashMap<String, Metric>());
253         }
254     }
255 
256     /**
257      * Check if current test should be skipped.
258      *
259      * @param description The test in progress.
260      * @return true if the test should be skipped.
261      */
shouldSkipCurrentTest(TestDescription description)262     private boolean shouldSkipCurrentTest(TestDescription description) {
263         // Force to skip any test not listed in include filters, or listed in exclude filters.
264         // exclude filters have highest priority.
265         String testName = description.getTestName();
266         if (mExcludeFilters.contains(testName)
267                 || mExcludeFilters.contains(description.toString())) {
268             return true;
269         }
270         if (!mIncludeFilters.isEmpty()) {
271             return !mIncludeFilters.contains(testName)
272                     && !mIncludeFilters.contains(description.toString());
273         }
274         return false;
275     }
276 
277     /**
278      * Search for the binary to be able to run it.
279      *
280      * @param binary the path of the binary or simply the binary name.
281      * @return The path to the binary, or null if not found.
282      */
findBinary(String binary)283     public abstract String findBinary(String binary) throws DeviceNotAvailableException;
284 
285     /**
286      * Actually run the binary at the given path.
287      *
288      * @param binaryPath The path of the binary.
289      * @param listener The listener where to report the results.
290      * @param description The test in progress.
291      */
runBinary( String binaryPath, ITestInvocationListener listener, TestDescription description)292     public abstract void runBinary(
293             String binaryPath, ITestInvocationListener listener, TestDescription description)
294             throws DeviceNotAvailableException, IOException;
295 
296     /** {@inheritDoc} */
297     @Override
setCollectTestsOnly(boolean shouldCollectTest)298     public final void setCollectTestsOnly(boolean shouldCollectTest) {
299         mCollectTestsOnly = shouldCollectTest;
300     }
301 
getCollectTestsOnly()302     public boolean getCollectTestsOnly() {
303         return mCollectTestsOnly;
304     }
305 
306     /** {@inheritDoc} */
307     @Override
getRuntimeHint()308     public final long getRuntimeHint() {
309         return mRuntimeHintMs;
310     }
311 
312     /** {@inheritDoc} */
313     @Override
setAbi(IAbi abi)314     public final void setAbi(IAbi abi) {
315         mAbi = abi;
316     }
317 
318     /** {@inheritDoc} */
319     @Override
getAbi()320     public IAbi getAbi() {
321         return mAbi;
322     }
323 
getTestInfo()324     TestInformation getTestInfo() {
325         return mTestInfo;
326     }
327 
setTestInfo(TestInformation testInfo)328     void setTestInfo(TestInformation testInfo) {
329         mTestInfo = testInfo;
330     }
331 
332     /** {@inheritDoc} */
333     @Override
split(int shardHint)334     public final Collection<IRemoteTest> split(int shardHint) {
335         if (shardHint <= 1) {
336             return null;
337         }
338         int testCount = mBinaryPaths.size() + mTestCommands.size();
339         if (testCount <= 2) {
340             return null;
341         }
342 
343         if (mShardSplit == ShardSplit.PER_TEST_CMD) {
344             return splitByTestCommand();
345         } else if (mShardSplit == ShardSplit.PER_SHARD) {
346             return splitByShardCount(testCount, shardHint);
347         }
348 
349         return null;
350     }
351 
splitByTestCommand()352     private Collection<IRemoteTest> splitByTestCommand() {
353         Collection<IRemoteTest> tests = new ArrayList<>();
354         for (String path : mBinaryPaths) {
355             tests.add(getTestShard(ImmutableList.of(path), null));
356         }
357         Map<String, String> testCommands = new LinkedHashMap<>(mTestCommands);
358         for (String testName : testCommands.keySet()) {
359             String cmd = testCommands.get(testName);
360             tests.add(getTestShard(null, ImmutableMap.of(testName, cmd)));
361         }
362         return tests;
363     }
364 
splitByShardCount(int testCount, int shardCount)365     private Collection<IRemoteTest> splitByShardCount(int testCount, int shardCount) {
366         int maxTestCntPerShard = (int) Math.ceil((double) testCount / shardCount);
367         int numFullSizeShards = testCount % maxTestCntPerShard;
368         List<Map.Entry<String, String>> testCommands = new ArrayList<>(mTestCommands.entrySet());
369 
370         int runningTestCount = 0;
371         int runningTestCountInShard = 0;
372 
373         Collection<IRemoteTest> tests = new ArrayList<>();
374         List<String> binaryPathsInShard = new ArrayList<String>();
375         HashMap<String, String> testCommandsInShard = new HashMap<String, String>();
376         while (runningTestCount < testCount) {
377             if (runningTestCount < mBinaryPaths.size()) {
378                 binaryPathsInShard.add(mBinaryPaths.get(runningTestCount));
379             } else {
380                 Map.Entry<String, String> entry =
381                         testCommands.get(runningTestCount - mBinaryPaths.size());
382                 testCommandsInShard.put(entry.getKey(), entry.getValue());
383             }
384             ++runningTestCountInShard;
385 
386             if ((tests.size() < numFullSizeShards && runningTestCountInShard == maxTestCntPerShard)
387                     || (tests.size() >= numFullSizeShards
388                             && (runningTestCountInShard >= (maxTestCntPerShard - 1)))) {
389                 tests.add(getTestShard(binaryPathsInShard, testCommandsInShard));
390                 binaryPathsInShard.clear();
391                 testCommandsInShard.clear();
392                 runningTestCountInShard = 0;
393             }
394             ++runningTestCount;
395         }
396         return tests;
397     }
398 
399     /**
400      * Get a testShard of ExecutableBaseTest.
401      *
402      * @param binaryPath the binary path for ExecutableHostTest.
403      * @param testName the test name for ExecutableTargetTest.
404      * @param cmd the test command for ExecutableTargetTest.
405      * @return a shard{@link IRemoteTest} of ExecutableBaseTest{@link ExecutableBaseTest}
406      */
getTestShard(List<String> binaryPaths, Map<String, String> testCmds)407     private IRemoteTest getTestShard(List<String> binaryPaths, Map<String, String> testCmds) {
408         ExecutableBaseTest shard = null;
409         try {
410             shard = this.getClass().getDeclaredConstructor().newInstance();
411             OptionCopier.copyOptionsNoThrow(this, shard);
412             shard.mBinaryPaths.clear();
413             shard.mTestCommands.clear();
414             if (binaryPaths != null) {
415                 for (String binaryPath : binaryPaths) {
416                     shard.mBinaryPaths.add(binaryPath);
417                 }
418             }
419             if (testCmds != null) {
420                 for (Map.Entry<String, String> entry : testCmds.entrySet()) {
421                     shard.mTestCommands.put(entry.getKey(), entry.getValue());
422                 }
423             }
424             // Copy the filters to each shard
425             shard.mExcludeFilters.addAll(mExcludeFilters);
426             shard.mIncludeFilters.addAll(mIncludeFilters);
427         } catch (InstantiationException
428                 | IllegalAccessException
429                 | InvocationTargetException
430                 | NoSuchMethodException e) {
431             // This cannot happen because the class was already created once at that point.
432             throw new RuntimeException(
433                     String.format(
434                             "%s (%s) when attempting to create shard object",
435                             e.getClass().getSimpleName(), e.getMessage()));
436         }
437         return shard;
438     }
439 
440     /**
441      * Convert mBinaryPaths to mTestCommands for consistency.
442      *
443      * @return a Map{@link LinkedHashMap}<String, String> of testCommands.
444      */
445     @VisibleForTesting
getAllTestCommands()446     Map<String, String> getAllTestCommands() {
447         Map<String, String> testCommands = new LinkedHashMap<>(mTestCommands);
448         for (String binary : mBinaryPaths) {
449             testCommands.put(new File(binary).getName(), binary);
450         }
451         return testCommands;
452     }
453 
454     @Override
reportDependencies()455     public Set<String> reportDependencies() {
456         Set<String> deps = new HashSet<String>();
457         deps.addAll(mBinaryPaths);
458         return deps;
459     }
460 }
461