1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.performance;
17 
18 import com.android.tradefed.config.Option;
19 import com.android.tradefed.config.Option.Importance;
20 import com.android.tradefed.device.DeviceNotAvailableException;
21 import com.android.tradefed.device.ITestDevice;
22 import com.android.tradefed.invoker.TestInformation;
23 import com.android.tradefed.log.LogUtil.CLog;
24 import com.android.tradefed.result.ITestInvocationListener;
25 import com.android.tradefed.testtype.IDeviceTest;
26 import com.android.tradefed.testtype.IRemoteTest;
27 import com.android.tradefed.util.SimpleStats;
28 import java.util.ArrayList;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 
33 /** dd benchmark runner. */
34 public class DDBenchmarkTest implements IRemoteTest, IDeviceTest {
35     @Option(name = "dd-bin", description = "Path to dd binary.", mandatory = true)
36     private String ddBinary = "dd";
37 
38     @Option(
39             name = "name",
40             description = "ASCII name of the benchmark.",
41             mandatory = true,
42             importance = Importance.ALWAYS)
43     private String benchmarkName = "dd_benchmark";
44 
45     @Option(
46             name = "iter",
47             description = "Number of times the dd benchmark is executed.",
48             mandatory = true,
49             importance = Importance.ALWAYS)
50     private int iterations = 1;
51 
52     @Option(name = "if", description = "Read from this file instead of stdin.")
53     private String inputFile = null;
54 
55     @Option(name = "of", description = "Write to this file instead of stdout.")
56     private String outputFile = null;
57 
58     @Option(name = "ibs", description = "Input block size.")
59     private String inputBlockSize = null;
60 
61     @Option(name = "obs", description = "Ouput block size.")
62     private String outputBlockSize = null;
63 
64     @Option(name = "bs", description = "Read and write N bytes at a time.")
65     private String ddBlockSize = null;
66 
67     @Option(name = "count", description = "Copy only N input blocks.")
68     private String count = null;
69 
70     @Option(name = "iflag", description = "Set input flags")
71     private String inputFlags = null;
72 
73     @Option(name = "oflag", description = "Set output flags")
74     private String outputFlags = null;
75 
76     @Option(name = "conv", description = "Convert the file as per the comma separated symbol list")
77     private String conv = null;
78 
79     @Option(name = "create-if", description = "Fill if with count input blocks before running dd.")
80     private boolean createInputFile = false;
81 
82     @Option(name = "clean-if", description = "Delete if after the benchmark ends.")
83     private boolean deleteInputFile = false;
84 
85     @Option(name = "clean-of", description = "Delete of after the benchmark ends.")
86     private boolean deleteOutputFile = false;
87 
88     @Option(name = "reboot-between-runs", description = "Reboot the device before each run.")
89     private boolean rebootBetweenRuns = false;
90 
91     private static final int I_BANDWIDTH = 2;
92     private Map<String, String> metrics = new HashMap<>();
93     private SimpleStats bandwidthStats = new SimpleStats();
94     private boolean hasCollectedMetrics = false;
95     private ITestDevice mDevice;
96 
97     @Override
run(TestInformation testInfo, ITestInvocationListener listener)98     public void run(TestInformation testInfo, ITestInvocationListener listener)
99             throws DeviceNotAvailableException {
100         if (createInputFile) setupInputFile();
101         List<String> results = runDDBenchmark();
102         for (String result : results)
103             hasCollectedMetrics = parseDDOutput(result) || hasCollectedMetrics;
104         reportDDMetrics(listener);
105         cleanup();
106     }
107 
setupInputFile()108     private void setupInputFile() throws DeviceNotAvailableException {
109         // We use the specified inputBlockSize for ibs, obs, bs to make sure we are creating a file
110         // of the correct size.
111         String fillCommand =
112                 buildDDCommand(
113                         ddBinary,
114                         "/dev/zero" /*inputFile*/,
115                         inputFile /*outputFile*/,
116                         inputBlockSize /*ibs*/,
117                         inputBlockSize /*obs*/,
118                         inputBlockSize /*bs*/,
119                         count,
120                         inputFlags,
121                         null /*oflag*/,
122                         "fsync" /*conv*/);
123         getDevice().executeShellCommand(fillCommand);
124     }
125 
runDDBenchmark()126     private List<String> runDDBenchmark() throws DeviceNotAvailableException {
127         List<String> results = new ArrayList<String>();
128         String ddCommand =
129                 buildDDCommand(
130                         ddBinary,
131                         inputFile,
132                         outputFile,
133                         inputBlockSize,
134                         outputBlockSize,
135                         ddBlockSize,
136                         count,
137                         inputFlags,
138                         outputFlags,
139                         conv);
140 
141         for (int i = 0; i < iterations; i++) {
142             dropState();
143             results.add(getDevice().executeShellCommand(ddCommand));
144         }
145 
146         return results;
147     }
148 
getDDVersion()149     private String getDDVersion() throws DeviceNotAvailableException {
150         String ddVersionCommand = String.format("%s --version", ddBinary);
151         return getDevice().executeShellCommand(ddVersionCommand).trim();
152     }
153 
reportDDMetrics(ITestInvocationListener listener)154     private void reportDDMetrics(ITestInvocationListener listener)
155             throws DeviceNotAvailableException {
156         listener.testRunStarted(benchmarkName, 0);
157         if (!hasCollectedMetrics) {
158             String errorMessage = "Failed to collect dd benchmark metrics";
159             CLog.i(errorMessage);
160             listener.testRunFailed(errorMessage);
161             return;
162         }
163         String ddVersion = getDDVersion();
164         String meanMetricName = String.format("%s-%s", ddVersion, "bandwidth_avg_MiB_s");
165         String stdevMetricName = String.format("%s-%s", ddVersion, "bandwidth_stdev_MiB_s");
166         metrics.put(meanMetricName, String.format("%.4f", bandwidthStats.mean()));
167         metrics.put(stdevMetricName, String.format("%.4f", bandwidthStats.stdev()));
168         listener.testRunEnded(0, metrics);
169     }
170 
cleanup()171     private void cleanup() throws DeviceNotAvailableException {
172         if (deleteInputFile) {
173             String rmCommand = String.format("rm %s", inputFile);
174             getDevice().executeShellCommand(rmCommand);
175         }
176         if (deleteOutputFile) {
177             String rmCommand = String.format("rm %s", outputFile);
178             getDevice().executeShellCommand(rmCommand);
179         }
180     }
181 
dropState()182     private void dropState() throws DeviceNotAvailableException {
183         if (rebootBetweenRuns) getDevice().reboot();
184         else getDevice().executeShellCommand("sync; echo 3 > /proc/sys/vm/drop_caches");
185     }
186 
187     /** Parse dd output assuming `toybox 0.8.3-android` version. */
parseDDOutput(String output)188     private boolean parseDDOutput(String output) {
189         /* Output format in case of success:
190          * Line | Content                                 | Notes
191          * -----+-----------------------------------------+-------------------
192          * 1:   | "count=" COUNT_FLAGS                    | Missing if count=0
193          * 2:   | x+y "records in"                        |
194          * 3:   | x+y "records out"                       |
195          * 4:   | x "bytes" (y METRIC_PREFIX) "copied", \ |
196          * 4:   | t TIME_UNIT, y BANDWIDTH_UNIT           |
197          *
198          * Output format in case of failure:
199          * Line | Content
200          * -----+--------------------
201          * 1:   | "dd:" ERROR_MESSAGE
202          */
203         String[] lines = output.split("\n");
204         if (lines.length < 3) return false;
205 
206         String bandwidthLine = lines[lines.length - 1];
207         String[] bandwidthWithUnit = bandwidthLine.split(",")[I_BANDWIDTH].trim().split(" ");
208         String bandwidthS = bandwidthWithUnit[0];
209         String unit = bandwidthWithUnit[1];
210         try {
211             double bandwidth = bandwidthInMiB(bandwidthS, unit);
212             bandwidthStats.add(bandwidth);
213         } catch (IllegalArgumentException e) {
214             CLog.i(String.format("Unknown unit %s while parsing dd output", unit));
215             return false;
216         }
217 
218         return true;
219     }
220 
221     /**
222      * Convert dd output bandwidth to MiB/s.
223      *
224      * <p>dd output bandwidth can have any of the suffixes reported by `dd --help`. This function
225      * uses the values documented for the `toybox 0.8.3-android` version to return a consistent
226      * bandwidth unit (MiB/s).
227      */
bandwidthInMiB(String bandwidth, String unit)228     public static double bandwidthInMiB(String bandwidth, String unit)
229             throws IllegalArgumentException {
230         double multiplier = 1;
231 
232         switch (unit) {
233             case "c/s":
234                 multiplier = 1;
235                 break;
236             case "w/s":
237                 multiplier = 2;
238                 break;
239             case "b/s":
240                 multiplier = 512;
241                 break;
242             case "kD/s":
243                 multiplier = 1000;
244                 break;
245             case "k/s":
246                 multiplier = 1024;
247                 break;
248             case "MD/s":
249                 multiplier = 1000 * 1000;
250                 break;
251             case "M/s":
252                 multiplier = 1024 * 1024;
253                 break;
254             case "GD/s":
255                 multiplier = 1000 * 1000 * 1000;
256                 break;
257             case "G/s":
258                 multiplier = 1024 * 1024 * 1024;
259                 break;
260             default:
261                 throw new IllegalArgumentException(String.format("Unknown unit %s", unit));
262         }
263 
264         double bandwidthInB = Double.parseDouble(bandwidth) * multiplier;
265         return bandwidthInB / (1024 * 1024);
266     }
267 
buildDDCommand( String ddBinary, String inputFile, String outputFile, String inputBlockSize, String outputBlockSize, String ddBlockSize, String count, String inputFlags, String outputFlags, String conv)268     public static String buildDDCommand(
269             String ddBinary,
270             String inputFile,
271             String outputFile,
272             String inputBlockSize,
273             String outputBlockSize,
274             String ddBlockSize,
275             String count,
276             String inputFlags,
277             String outputFlags,
278             String conv) {
279         if (ddBinary == null) return "";
280 
281         StringBuilder sb = new StringBuilder();
282         sb.append(ddBinary);
283         if (inputFile != null) {
284             sb.append(" if=");
285             sb.append(inputFile);
286         }
287         if (outputFile != null) {
288             sb.append(" of=");
289             sb.append(outputFile);
290         }
291         if (inputBlockSize != null) {
292             sb.append(" ibs=");
293             sb.append(inputBlockSize);
294         }
295         if (outputBlockSize != null) {
296             sb.append(" obs=");
297             sb.append(outputBlockSize);
298         }
299         if (ddBlockSize != null) {
300             sb.append(" bs=");
301             sb.append(ddBlockSize);
302         }
303         if (count != null) {
304             sb.append(" count=");
305             sb.append(count);
306         }
307         if (inputFlags != null) {
308             sb.append(" iflag=");
309             sb.append(inputFlags);
310         }
311         if (outputFlags != null) {
312             sb.append(" oflag=");
313             sb.append(outputFlags);
314         }
315         if (conv != null) {
316             sb.append(" conv=");
317             sb.append(conv);
318         }
319 
320         return sb.toString();
321     }
322 
323     @Override
setDevice(ITestDevice device)324     public void setDevice(ITestDevice device) {
325         mDevice = device;
326     }
327 
328     @Override
getDevice()329     public ITestDevice getDevice() {
330         return mDevice;
331     }
332 }
333