1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.performance;
17 
18 import com.android.tradefed.config.Option;
19 import com.android.tradefed.config.Option.Importance;
20 import com.android.tradefed.device.DeviceNotAvailableException;
21 import com.android.tradefed.device.ITestDevice;
22 import com.android.tradefed.invoker.TestInformation;
23 import com.android.tradefed.log.LogUtil.CLog;
24 import com.android.tradefed.result.ITestInvocationListener;
25 import com.android.tradefed.testtype.IDeviceTest;
26 import com.android.tradefed.testtype.IRemoteTest;
27 import com.android.tradefed.util.SimpleStats;
28 import java.util.ArrayList;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 
33 /** inodeop benchmark runner. */
34 public class InodeopBenchmarkTest implements IRemoteTest, IDeviceTest {
35     @Option(name = "inodeop-bin", description = "Path to inodeop binary.", mandatory = true)
36     private String inodeopBinary = null;
37 
38     @Option(
39             name = "name",
40             description = "ASCII name of the benchmark.",
41             mandatory = true,
42             importance = Importance.ALWAYS)
43     private String benchmarkName = "inodeop_benchmark";
44 
45     @Option(
46             name = "iter",
47             description = "Number of times the inodeop benchmark is executed.",
48             mandatory = true,
49             importance = Importance.ALWAYS)
50     private int iterations = 1;
51 
52     @Option(name = "workload", description = "Type of workload to execute.", mandatory = true)
53     private String workload = null;
54 
55     @Option(name = "dir", description = "Working directory.")
56     private String directory = null;
57 
58     @Option(name = "from-dir", description = "Source directory.")
59     private String fromDirectory = null;
60 
61     @Option(name = "to-dir", description = "Destination directory.")
62     private String toDirectory = null;
63 
64     @Option(
65             name = "n-files",
66             description = "Number of files to create/delete etc.",
67             mandatory = true)
68     private int nFiles = 0;
69 
70     @Option(
71             name = "maintain-state",
72             description = "Do not drop state (caches) before running the workload.")
73     private boolean maintainState = false;
74 
75     @Option(name = "reboot-between-runs", description = "Reboot the device before each run.")
76     private boolean rebootBetweenRuns = false;
77 
78     private Map<String, String> metrics = new HashMap<>();
79     private SimpleStats executionTimeStats = new SimpleStats();
80     private boolean hasCollectedMetrics = false;
81     private ITestDevice mDevice;
82 
83     @Override
run(TestInformation testInfo, ITestInvocationListener listener)84     public void run(TestInformation testInfo, ITestInvocationListener listener)
85             throws DeviceNotAvailableException {
86         List<String> results = runInodeopBenchmark();
87         for (String result : results)
88             hasCollectedMetrics = parseInodeopOutput(result) || hasCollectedMetrics;
89         reportInodeopMetrics(listener);
90     }
91 
runInodeopBenchmark()92     private List<String> runInodeopBenchmark() throws DeviceNotAvailableException {
93         List<String> results = new ArrayList<String>();
94         String inodeopCommand =
95                 buildInodeopCommand(
96                         inodeopBinary,
97                         directory,
98                         fromDirectory,
99                         toDirectory,
100                         nFiles,
101                         maintainState,
102                         workload);
103 
104         for (int i = 0; i < iterations; i++) {
105             dropState();
106             results.add(getDevice().executeShellCommand(inodeopCommand));
107         }
108 
109         return results;
110     }
111 
getInodeopVersion()112     private String getInodeopVersion() throws DeviceNotAvailableException {
113         String inodeopVersionCommand = String.format("%s -v", inodeopBinary);
114         return getDevice().executeShellCommand(inodeopVersionCommand).trim();
115     }
116 
reportInodeopMetrics(ITestInvocationListener listener)117     private void reportInodeopMetrics(ITestInvocationListener listener)
118             throws DeviceNotAvailableException {
119         listener.testRunStarted(benchmarkName, 0);
120         if (!hasCollectedMetrics) {
121             String errorMessage = "Failed to collect inodeop benchmark metrics";
122             CLog.i(errorMessage);
123             listener.testRunFailed(errorMessage);
124             return;
125         }
126         String inodeopVersion = getInodeopVersion();
127         String meanMetricName = String.format("%s-%s", inodeopVersion, "exec_time_avg_ms");
128         String stdevMetricName = String.format("%s-%s", inodeopVersion, "exec_time_stdev_ms");
129         metrics.put(meanMetricName, String.format("%.4f", executionTimeStats.mean()));
130         metrics.put(stdevMetricName, String.format("%.4f", executionTimeStats.stdev()));
131         listener.testRunEnded(0, metrics);
132     }
133 
dropState()134     private void dropState() throws DeviceNotAvailableException {
135         if (rebootBetweenRuns) getDevice().reboot();
136         else getDevice().executeShellCommand("sync; echo 3 > /proc/sys/vm/drop_caches");
137     }
138 
139     /** Parse inodeop output assuming the `0` version output format. */
parseInodeopOutput(String output)140     private boolean parseInodeopOutput(String output) {
141         /* In case of failure the output is empty and the error message is written to stderr.
142          * Instead, in case of success the output format is the following:
143          * 0;WORKLOAD_NAME;EXEC_TIME;ms
144          */
145         if (output == null || output.equals("")) return false;
146         String[] fields = output.split(";");
147         if (fields.length != InodeopOutputV0.values().length) return false;
148         int outputVersion =
149                 Integer.parseInt(getInodeopOutputField(fields, InodeopOutputV0.VERSION));
150         if (outputVersion != 0) return false;
151 
152         String timeUnit = getInodeopOutputField(fields, InodeopOutputV0.TIME_UNIT);
153         if (!timeUnit.equals("ms")) return false;
154         double executionTime =
155                 Double.parseDouble(getInodeopOutputField(fields, InodeopOutputV0.EXEC_TIME));
156         executionTimeStats.add(executionTime);
157         return true;
158     }
159 
160     public enum InodeopOutputV0 {
161         VERSION,
162         WORKLOAD,
163         EXEC_TIME,
164         TIME_UNIT;
165     }
166 
getInodeopOutputField(String[] inodeOutputFields, InodeopOutputV0 field)167     public static String getInodeopOutputField(String[] inodeOutputFields, InodeopOutputV0 field) {
168         return inodeOutputFields[field.ordinal()].trim();
169     }
170 
buildInodeopCommand( String inodeopBinary, String directory, String fromDirectory, String toDirectory, int nFiles, boolean maintainState, String workload)171     public static String buildInodeopCommand(
172             String inodeopBinary,
173             String directory,
174             String fromDirectory,
175             String toDirectory,
176             int nFiles,
177             boolean maintainState,
178             String workload) {
179         if (inodeopBinary == null) return "";
180 
181         StringBuilder sb = new StringBuilder();
182         sb.append(inodeopBinary);
183         if (directory != null) {
184             sb.append(" -d ");
185             sb.append(directory);
186         }
187         if (fromDirectory != null) {
188             sb.append(" -f ");
189             sb.append(fromDirectory);
190         }
191         if (toDirectory != null) {
192             sb.append(" -t ");
193             sb.append(toDirectory);
194         }
195         if (maintainState) sb.append(" -s");
196         sb.append(" -n ");
197         sb.append(nFiles);
198         // inodeop requires the workload type to come last.
199         if (workload != null) {
200             sb.append(" -w ");
201             sb.append(workload);
202         }
203 
204         return sb.toString();
205     }
206 
207     @Override
setDevice(ITestDevice device)208     public void setDevice(ITestDevice device) {
209         mDevice = device;
210     }
211 
212     @Override
getDevice()213     public ITestDevice getDevice() {
214         return mDevice;
215     }
216 }
217