1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.testtype;
18 
19 import com.android.tradefed.config.Option;
20 import com.android.tradefed.config.OptionClass;
21 import com.android.tradefed.device.DeviceNotAvailableException;
22 import com.android.tradefed.device.IFileEntry;
23 import com.android.tradefed.device.ITestDevice;
24 import com.android.tradefed.invoker.TestInformation;
25 import com.android.tradefed.log.LogUtil.CLog;
26 import com.android.tradefed.result.ITestInvocationListener;
27 import com.android.tradefed.util.proto.TfMetricProtoUtil;
28 
29 import com.google.common.annotations.VisibleForTesting;
30 
31 import java.util.ArrayList;
32 import java.util.Collection;
33 import java.util.HashMap;
34 import java.util.Map;
35 import java.util.concurrent.TimeUnit;
36 
37 /**
38  * A Test that runs a native benchmark test executable on given device.
39  * <p/>
40  * It uses {@link NativeBenchmarkTestParser} to parse out the average operation time vs delay
41  * between operations those results to the {@link ITestInvocationListener}s.
42  */
43 @OptionClass(alias = "native-benchmark")
44 public class NativeBenchmarkTest implements IDeviceTest, IRemoteTest {
45 
46     static final String DEFAULT_TEST_PATH = "data/nativebenchmark";
47 
48     // The metrics key names to report to listeners
49     static final String AVG_OP_TIME_KEY_PREFIX = "avg-operation-time";
50     static final String ITERATION_KEY = "iterations";
51 
52     private ITestDevice mDevice = null;
53 
54     @Option(name = "native-benchmark-device-path",
55       description="The path on the device where native stress tests are located.")
56     private String mDeviceTestPath = DEFAULT_TEST_PATH;
57 
58     @Option(name = "benchmark-module-name",
59             description="The name of the native benchmark test module to run. " +
60             "If not specified all tests in --native-benchmark-device-path will be run.")
61     private String mTestModule = null;
62 
63     @Option(name = "benchmark-run-name",
64             description="Optional name to pass to test reporters. If unspecified, will use" +
65             "--benchmark-module-name.")
66     private String mReportRunName = null;
67 
68     @Option(name = "iterations",
69             description="The number of benchmark test iterations per run.")
70     private int mNumIterations = 1000;
71 
72     @Option(
73             name = "delay-per-run",
74             description =
75                     "The delay between each benchmark iteration, in micro seconds.Multiple values"
76                             + " may be given to specify multiple runs with different delay values.")
77     // TODO: change units to seconds for consistency with native benchmark module input
78     private Collection<Integer> mDelays = new ArrayList<Integer>();
79 
80     @Option(name = "max-run-time", description =
81          "The maximum time to allow for one benchmark run in ms.")
82     private int mMaxRunTime = 5 * 60 * 1000;
83 
84     @Option(name = "server-cpu",
85             description="Optionally specify a server cpu.")
86     private int mServerCpu = 1;
87 
88     @Option(name = "client-cpu",
89             description="Optionally specify a client cpu.")
90     private int mClientCpu = 1;
91 
92     @Option(name = "max-cpu-freq",
93             description="Flag to force device cpu to run at maximum frequency.")
94     private boolean mMaxCpuFreq = false;
95 
96 
97     // TODO: consider sharing code with {@link GTest} and {@link NativeStressTest}
98 
99     /**
100      * {@inheritDoc}
101      */
102     @Override
setDevice(ITestDevice device)103     public void setDevice(ITestDevice device) {
104         mDevice = device;
105     }
106 
107     /**
108      * {@inheritDoc}
109      */
110     @Override
getDevice()111     public ITestDevice getDevice() {
112         return mDevice;
113     }
114 
115     /**
116      * Set the Android native benchmark test module to run.
117      *
118      * @param moduleName The name of the native test module to run
119      */
setModuleName(String moduleName)120     public void setModuleName(String moduleName) {
121         mTestModule = moduleName;
122     }
123 
124     /**
125      * Get the Android native benchmark test module to run.
126      *
127      * @return the name of the native test module to run, or null if not set
128      */
getModuleName()129     public String getModuleName() {
130         return mTestModule;
131     }
132 
133     /**
134      * Set the number of iterations to execute per run
135      */
setNumIterations(int iterations)136     void setNumIterations(int iterations) {
137         mNumIterations = iterations;
138     }
139 
140     /**
141      * Set the delay values per run
142      */
addDelaysPerRun(Collection<Integer> delays)143     void addDelaysPerRun(Collection<Integer> delays) {
144         mDelays.addAll(delays);
145     }
146 
147     /**
148      * Gets the path where native benchmark tests live on the device.
149      *
150      * @return The path on the device where the native tests live.
151      */
152     @VisibleForTesting
getTestPath()153     String getTestPath() {
154         StringBuilder testPath = new StringBuilder(mDeviceTestPath);
155         if (mTestModule != null) {
156             testPath.append("/");
157             testPath.append(mTestModule);
158         }
159         return testPath.toString();
160     }
161 
162     /**
163      * Executes all native benchmark tests in a folder as well as in all subfolders recursively.
164      *
165      * @param rootEntry The root folder to begin searching for native tests
166      * @param testDevice The device to run tests on
167      * @param listener the run listener
168      * @throws DeviceNotAvailableException
169      */
170     @VisibleForTesting
doRunAllTestsInSubdirectory( IFileEntry rootEntry, ITestDevice testDevice, ITestInvocationListener listener)171     void doRunAllTestsInSubdirectory(
172             IFileEntry rootEntry, ITestDevice testDevice, ITestInvocationListener listener)
173             throws DeviceNotAvailableException {
174 
175         if (rootEntry.isDirectory()) {
176             // recursively run tests in all subdirectories
177             for (IFileEntry childEntry : rootEntry.getChildren(true)) {
178                 doRunAllTestsInSubdirectory(childEntry, testDevice, listener);
179             }
180         } else {
181             // assume every file is a valid benchmark test binary.
182             // use name of file as run name
183             String runName = (mReportRunName == null ? rootEntry.getName() : mReportRunName);
184             String fullPath = rootEntry.getFullEscapedPath();
185             if (mDelays.size() == 0) {
186                 // default to one run with no delay
187                 mDelays.add(0);
188             }
189 
190             // force file to be executable
191             testDevice.executeShellCommand(String.format("chmod 755 %s", fullPath));
192             long startTime = System.currentTimeMillis();
193 
194             listener.testRunStarted(runName, 0);
195             Map<String, String> metricMap = new HashMap<String, String>();
196             metricMap.put(ITERATION_KEY, Integer.toString(mNumIterations));
197             try {
198                 for (Integer delay : mDelays) {
199                     NativeBenchmarkTestParser resultParser = createResultParser(runName);
200                     // convert delay to seconds
201                     double delayFloat = ((double)delay)/1000000;
202                     CLog.i(
203                             "Running %s for %d iterations with delay %f",
204                             rootEntry.getName(), mNumIterations, delayFloat);
205                     String cmd = String.format("%s -n %d -d %f -c %d -s %d", fullPath,
206                             mNumIterations, delayFloat, mClientCpu, mServerCpu);
207                     CLog.i(
208                             "Running native benchmark test on %s: %s",
209                             mDevice.getSerialNumber(), cmd);
210                     testDevice.executeShellCommand(cmd, resultParser,
211                             mMaxRunTime, TimeUnit.MILLISECONDS, 0);
212                     addMetric(metricMap, resultParser, delay);
213                 }
214                 // TODO: is catching exceptions, and reporting testRunFailed necessary?
215             } finally {
216                 final long elapsedTime = System.currentTimeMillis() - startTime;
217                 listener.testRunEnded(elapsedTime, TfMetricProtoUtil.upgradeConvert(metricMap));
218             }
219         }
220     }
221 
222     /**
223      * Adds the operation time metric for a run with given delay
224      *
225      * @param metricMap
226      * @param resultParser
227      * @param delay
228      */
addMetric(Map<String, String> metricMap, NativeBenchmarkTestParser resultParser, Integer delay)229     private void addMetric(Map<String, String> metricMap, NativeBenchmarkTestParser resultParser,
230             Integer delay) {
231         String metricKey = String.format("%s-delay%d", AVG_OP_TIME_KEY_PREFIX, delay);
232         // temporarily convert seconds to microseconds, as some reporters cannot handle small values
233         metricMap.put(metricKey, Double.toString(resultParser.getAvgOperationTime()*1000000));
234     }
235 
236     /**
237      * Factory method for creating a {@link NativeBenchmarkTestParser} that parses test output
238      * <p/>
239      * Exposed so unit tests can mock.
240      *
241      * @param runName
242      * @return a {@link NativeBenchmarkTestParser}
243      */
createResultParser(String runName)244     NativeBenchmarkTestParser createResultParser(String runName) {
245         return new NativeBenchmarkTestParser(runName);
246     }
247 
248     /** {@inheritDoc} */
249     @Override
run(TestInformation testInfo, ITestInvocationListener listener)250     public void run(TestInformation testInfo, ITestInvocationListener listener)
251             throws DeviceNotAvailableException {
252         if (mDevice == null) {
253             throw new IllegalArgumentException("Device has not been set");
254         }
255 
256         String testPath = getTestPath();
257         IFileEntry nativeTestDirectory = mDevice.getFileEntry(testPath);
258         if (nativeTestDirectory == null) {
259             CLog.w(
260                     "Could not find native benchmark test directory %s in %s!",
261                     testPath, mDevice.getSerialNumber());
262             return;
263         }
264         if (mMaxCpuFreq) {
265             mDevice.executeShellCommand(
266                     "cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq > " +
267                     "/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq");
268         }
269         doRunAllTestsInSubdirectory(nativeTestDirectory, mDevice, listener);
270         if (mMaxCpuFreq) {
271             // revert to normal
272             mDevice.executeShellCommand(
273                     "cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_min_freq > " +
274                     "/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq");
275         }
276 
277     }
278 }
279