1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.device.metric; 17 18 import com.android.tradefed.config.Option; 19 import com.android.tradefed.device.DeviceNotAvailableException; 20 import com.android.tradefed.device.ITestDevice; 21 import com.android.tradefed.device.TestDeviceState; 22 import com.android.tradefed.invoker.logger.CurrentInvocation; 23 import com.android.tradefed.log.LogUtil.CLog; 24 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 25 import com.android.tradefed.util.FileUtil; 26 import com.android.tradefed.util.ZipUtil; 27 import com.android.tradefed.util.proto.TfMetricProtoUtil; 28 29 import java.io.File; 30 import java.io.IOException; 31 import java.util.AbstractMap.SimpleEntry; 32 import java.util.Arrays; 33 import java.util.HashMap; 34 import java.util.HashSet; 35 import java.util.Map; 36 import java.util.Map.Entry; 37 import java.util.Set; 38 import java.util.regex.Pattern; 39 40 /** 41 * A {@link BaseDeviceMetricCollector} that listen for metrics key coming from the device and pull 42 * them as a file from the device. Can be extended for extra-processing of the file. 43 */ 44 public abstract class FilePullerDeviceMetricCollector extends BaseDeviceMetricCollector { 45 46 @Option( 47 name = "pull-pattern-keys", 48 description = 49 "The pattern key name to be pull from the device as a file. Can be repeated.") 50 private Set<String> mKeys = new HashSet<>(); 51 52 @Option( 53 name = "directory-keys", 54 description = "Path to the directory on the device that contains the metrics.") 55 protected Set<String> mDirectoryKeys = new HashSet<>(); 56 57 @Option(name = "compress-directories", 58 description = "Compress multiple files in the matching directory into zip file") 59 private boolean mCompressDirectory = false; 60 61 @Option( 62 name = "clean-up", 63 description = "Whether to delete the file from the device after pulling it or not." 64 ) 65 private boolean mCleanUp = true; 66 67 @Option( 68 name = "collect-on-run-ended-only", 69 description = 70 "Attempt to collect the files on test run end only instead of on both test cases " 71 + "and test run ended. This is safer since test case level collection isn't" 72 + " synchronous." 73 ) 74 private boolean mCollectOnRunEndedOnly = true; 75 public Map<String, String> mTestCaseMetrics = new HashMap<String, String>(); 76 77 @Override onTestEnd(DeviceMetricData testData, Map<String, Metric> currentTestCaseMetrics)78 public void onTestEnd(DeviceMetricData testData, Map<String, Metric> currentTestCaseMetrics) 79 throws DeviceNotAvailableException { 80 if (mCollectOnRunEndedOnly) { 81 // Track test cases metrics in case we don't process here. 82 mTestCaseMetrics.putAll(TfMetricProtoUtil.compatibleConvert(currentTestCaseMetrics)); 83 return; 84 } 85 processMetricRequest(testData, currentTestCaseMetrics); 86 } 87 88 @Override onTestRunEnd(DeviceMetricData runData, final Map<String, Metric> currentRunMetrics)89 public void onTestRunEnd(DeviceMetricData runData, final Map<String, Metric> currentRunMetrics) 90 throws DeviceNotAvailableException { 91 processMetricRequest(runData, currentRunMetrics); 92 mTestCaseMetrics = new HashMap<>(); 93 } 94 95 /** Adds additional pattern keys to the pull from the device. */ addKeys(String... keys)96 protected void addKeys(String... keys) { 97 mKeys.addAll(Arrays.asList(keys)); 98 } 99 100 /** 101 * Implementation of the method should allow to log the file, parse it for metrics to be put in 102 * {@link DeviceMetricData}. 103 * 104 * @param key the option key associated to the file that was pulled. 105 * @param metricFile the {@link File} pulled from the device matching the option key. 106 * @param data the {@link DeviceMetricData} where metrics can be stored. 107 */ processMetricFile(String key, File metricFile, DeviceMetricData data)108 public abstract void processMetricFile(String key, File metricFile, DeviceMetricData data); 109 110 /** 111 * Implementation of the method should allow to log the directory, parse it for metrics to be 112 * put in {@link DeviceMetricData}. 113 * 114 * @param key the option key associated to the directory that was pulled. 115 * @param metricDirectory the {@link File} pulled from the device matching the option key. 116 * @param data the {@link DeviceMetricData} where metrics can be stored. 117 */ processMetricDirectory( String key, File metricDirectory, DeviceMetricData data)118 public abstract void processMetricDirectory( 119 String key, File metricDirectory, DeviceMetricData data); 120 121 /** 122 * Process the file associated with the matching key or directory name and update the data with 123 * any additional metrics. 124 * 125 * @param data where the final metrics will be stored. 126 * @param metrics where the key or directory name will be matched to the keys. 127 */ processMetricRequest(DeviceMetricData data, Map<String, Metric> metrics)128 private void processMetricRequest(DeviceMetricData data, Map<String, Metric> metrics) 129 throws DeviceNotAvailableException { 130 Map<String, String> currentMetrics = TfMetricProtoUtil 131 .compatibleConvert(metrics); 132 currentMetrics.putAll(mTestCaseMetrics); 133 if (mKeys.isEmpty() && mDirectoryKeys.isEmpty()) { 134 return; 135 } 136 Map<ITestDevice, Integer> deviceUsers = new HashMap<>(); 137 if (!mKeys.isEmpty()) { 138 for (ITestDevice device : getRealDevices()) { 139 if (!TestDeviceState.ONLINE.equals(device.getDeviceState())) { 140 CLog.d( 141 "Device '%s' is in state '%s' skipping file puller", 142 device.getSerialNumber(), device.getDeviceState()); 143 return; 144 } 145 deviceUsers.put(device, device.getCurrentUser()); 146 } 147 } 148 for (String key : mKeys) { 149 Map<String, File> pulledMetrics = pullMetricFile(key, currentMetrics, deviceUsers); 150 151 // Process all the metric files that matched the key pattern. 152 for (Map.Entry<String, File> entry : pulledMetrics.entrySet()) { 153 processMetricFile(entry.getKey(), entry.getValue(), data); 154 } 155 } 156 157 for (String key : mDirectoryKeys) { 158 Entry<String, File> pulledMetrics = pullMetricDirectory(key); 159 if (pulledMetrics != null) { 160 if (mCompressDirectory) { 161 File pulledDirectory = pulledMetrics.getValue(); 162 if (pulledDirectory.isDirectory()) { 163 try { 164 File compressedFile = ZipUtil.createZip(pulledDirectory, 165 getFileName(key)); 166 processMetricFile(key, compressedFile, data); 167 } catch (IOException e) { 168 CLog.e("Unable to compress the directory."); 169 } 170 FileUtil.recursiveDelete(pulledDirectory); 171 } 172 continue; 173 } 174 processMetricDirectory(pulledMetrics.getKey(), pulledMetrics.getValue(), data); 175 } 176 } 177 } 178 179 /** 180 * Return the last folder name from the path the in the device where the 181 * directory is pulled. 182 */ getFileName(String key)183 private String getFileName(String key) { 184 return key.substring(key.lastIndexOf("/")+1); 185 } 186 pullMetricFile( String pattern, final Map<String, String> currentMetrics, Map<ITestDevice, Integer> deviceUsers)187 private Map<String, File> pullMetricFile( 188 String pattern, 189 final Map<String, String> currentMetrics, 190 Map<ITestDevice, Integer> deviceUsers) 191 throws DeviceNotAvailableException { 192 Map<String, File> matchedFiles = new HashMap<>(); 193 Pattern p = Pattern.compile(pattern); 194 195 for (Entry<String, String> entry : currentMetrics.entrySet()) { 196 if (p.matcher(entry.getKey()).find()) { 197 for (ITestDevice device : getRealDevices()) { 198 if (!shouldCollect(device)) { 199 continue; 200 } 201 try { 202 File attemptPull = 203 retrieveFile(device, entry.getValue(), deviceUsers.get(device)); 204 if (attemptPull != null) { 205 if (mCleanUp) { 206 device.deleteFile(entry.getValue()); 207 } 208 // Store all the keys that matches the pattern and the corresponding 209 // files pulled from the device. 210 matchedFiles.put(entry.getKey(), attemptPull); 211 } 212 } catch (RuntimeException e) { 213 CLog.e( 214 "Exception when pulling metric file '%s' from %s", 215 entry.getValue(), device.getSerialNumber()); 216 CLog.e(e); 217 } 218 } 219 } 220 } 221 222 if (matchedFiles.isEmpty()) { 223 // Not a hard failure, just nice to know 224 CLog.d("Could not find a device file associated to pattern '%s'.", pattern); 225 226 } 227 return matchedFiles; 228 } 229 230 /** 231 * Pull the file from the specified path in the device. 232 * 233 * @param device which has the file. 234 * @param remoteFilePath location in the device. 235 * @param userId the user id to pull from 236 * @return File retrieved from the given path in the device. 237 * @throws DeviceNotAvailableException 238 */ retrieveFile(ITestDevice device, String remoteFilePath, int userId)239 protected File retrieveFile(ITestDevice device, String remoteFilePath, int userId) 240 throws DeviceNotAvailableException { 241 return device.pullFile(remoteFilePath, userId); 242 } 243 244 /** 245 * Pulls the directory and all its content from the device and save it in the host under the 246 * metric_tmp folder. 247 * 248 * @param keyDirectory path to the source directory in the device. 249 * @return Key,value pair of the directory name and path to the directory in the local host. 250 */ pullMetricDirectory(String keyDirectory)251 private Entry<String, File> pullMetricDirectory(String keyDirectory) 252 throws DeviceNotAvailableException { 253 try { 254 File tmpDestDir = 255 FileUtil.createTempDir("metric_tmp", CurrentInvocation.getWorkFolder()); 256 for (ITestDevice device : getRealDevices()) { 257 if (!shouldCollect(device)) { 258 continue; 259 } 260 try { 261 if (device.pullDir(keyDirectory, tmpDestDir)) { 262 if (mCleanUp) { 263 device.deleteFile(keyDirectory); 264 } 265 return new SimpleEntry<String, File>(keyDirectory, tmpDestDir); 266 } 267 } catch (RuntimeException e) { 268 CLog.e( 269 "Exception when pulling directory '%s' from %s", 270 keyDirectory, device.getSerialNumber()); 271 CLog.e(e); 272 } 273 } 274 } catch (IOException ioe) { 275 CLog.e("Exception while creating the local directory"); 276 CLog.e(ioe); 277 } 278 CLog.e("Could not find a device directory associated to path '%s'.", keyDirectory); 279 return null; 280 } 281 shouldCollect(ITestDevice device)282 private boolean shouldCollect(ITestDevice device) { 283 TestDeviceState state = device.getDeviceState(); 284 if (!TestDeviceState.ONLINE.equals(state)) { 285 CLog.d("Skip %s device is in state '%s'", this, state); 286 return false; 287 } 288 return true; 289 } 290 } 291