1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.clearcut; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.asuite.clearcut.Clientanalytics.ClientInfo; 20 import com.android.asuite.clearcut.Clientanalytics.LogEvent; 21 import com.android.asuite.clearcut.Clientanalytics.LogRequest; 22 import com.android.asuite.clearcut.Clientanalytics.LogResponse; 23 import com.android.asuite.clearcut.Common.UserType; 24 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 25 import com.android.tradefed.log.LogUtil.CLog; 26 import com.android.tradefed.util.CommandResult; 27 import com.android.tradefed.util.CommandStatus; 28 import com.android.tradefed.util.FileUtil; 29 import com.android.tradefed.util.RunUtil; 30 import com.android.tradefed.util.StreamUtil; 31 import com.android.tradefed.util.net.HttpHelper; 32 33 import com.google.common.base.Strings; 34 import com.google.protobuf.util.JsonFormat; 35 36 import java.io.File; 37 import java.io.IOException; 38 import java.io.InputStream; 39 import java.io.OutputStream; 40 import java.io.OutputStreamWriter; 41 import java.net.HttpURLConnection; 42 import java.net.InetAddress; 43 import java.net.URL; 44 import java.net.UnknownHostException; 45 import java.time.Duration; 46 import java.util.ArrayList; 47 import java.util.List; 48 import java.util.UUID; 49 import java.util.concurrent.CompletableFuture; 50 import java.util.concurrent.ExecutionException; 51 import java.util.concurrent.Executors; 52 import java.util.concurrent.ScheduledThreadPoolExecutor; 53 import java.util.concurrent.ThreadFactory; 54 import java.util.concurrent.TimeUnit; 55 56 /** Client that allows reporting usage metrics to clearcut. */ 57 public class ClearcutClient { 58 59 public static final String DISABLE_CLEARCUT_KEY = "DISABLE_CLEARCUT"; 60 private static final String CLEARCUT_SUB_TOOL_NAME = "CLEARCUT_SUB_TOOL_NAME"; 61 62 private static final String CLEARCUT_PROD_URL = "https://play.googleapis.com/log"; 63 private static final int CLIENT_TYPE = 1; 64 private static final int INTERNAL_LOG_SOURCE = 971; 65 private static final int EXTERNAL_LOG_SOURCE = 934; 66 67 private static final long SCHEDULER_INITIAL_DELAY_MILLISECONDS = 1000; 68 private static final long SCHEDULER_PERDIOC_MILLISECONDS = 250; 69 70 private static final String GOOGLE_EMAIL = "@google.com"; 71 private static final String GOOGLE_HOSTNAME = ".google.com"; 72 73 private File mCachedUuidFile = new File(System.getProperty("user.home"), ".tradefed"); 74 private String mRunId; 75 private long mSessionStartTime = 0L; 76 77 private final int mLogSource; 78 private final String mUrl; 79 private final UserType mUserType; 80 private final String mSubToolName; 81 82 // Consider synchronized list 83 private List<LogRequest> mExternalEventQueue; 84 // The pool executor to actually post the metrics 85 private ScheduledThreadPoolExecutor mExecutor; 86 // Whether the clearcut client should be inop 87 private boolean mDisabled = false; 88 ClearcutClient(String subToolName)89 public ClearcutClient(String subToolName) { 90 this(null, subToolName); 91 } 92 93 /** 94 * Create Client with customized posting URL and forcing whether it's internal or external user. 95 */ 96 @VisibleForTesting ClearcutClient(String url, String subToolName)97 protected ClearcutClient(String url, String subToolName) { 98 mDisabled = isClearcutDisabled(); 99 100 // We still have to set the 'final' variable so go through the assignments before returning 101 if (!mDisabled && isGoogleUser()) { 102 mLogSource = INTERNAL_LOG_SOURCE; 103 mUserType = UserType.GOOGLE; 104 } else { 105 mLogSource = EXTERNAL_LOG_SOURCE; 106 mUserType = UserType.EXTERNAL; 107 } 108 if (url == null) { 109 mUrl = CLEARCUT_PROD_URL; 110 } else { 111 mUrl = url; 112 } 113 mRunId = UUID.randomUUID().toString(); 114 mExternalEventQueue = new ArrayList<>(); 115 if (Strings.isNullOrEmpty(subToolName) && System.getenv(CLEARCUT_SUB_TOOL_NAME) != null) { 116 mSubToolName = System.getenv(CLEARCUT_SUB_TOOL_NAME); 117 } else { 118 mSubToolName = subToolName; 119 } 120 121 if (mDisabled) { 122 return; 123 } 124 125 // Print the notice 126 System.out.println(NoticeMessageUtil.getNoticeMessage(mUserType)); 127 128 // Executor to actually send the events. 129 mExecutor = 130 new ScheduledThreadPoolExecutor( 131 1, 132 new ThreadFactory() { 133 @Override 134 public Thread newThread(Runnable r) { 135 Thread t = Executors.defaultThreadFactory().newThread(r); 136 t.setDaemon(true); 137 t.setName("clearcut-client-thread"); 138 return t; 139 } 140 }); 141 Runnable command = 142 new Runnable() { 143 @Override 144 public void run() { 145 flushEvents(); 146 } 147 }; 148 mExecutor.scheduleAtFixedRate( 149 command, 150 SCHEDULER_INITIAL_DELAY_MILLISECONDS, 151 SCHEDULER_PERDIOC_MILLISECONDS, 152 TimeUnit.MILLISECONDS); 153 } 154 155 /** Send the first event to notify that Tradefed was started. */ notifyTradefedStartEvent()156 public void notifyTradefedStartEvent() { 157 if (mDisabled) { 158 return; 159 } 160 mSessionStartTime = System.nanoTime(); 161 long eventTimeMs = System.currentTimeMillis(); 162 CompletableFuture.supplyAsync(() -> createStartEvent(eventTimeMs)); 163 } 164 createStartEvent(long eventTimeMs)165 private boolean createStartEvent(long eventTimeMs) { 166 LogRequest.Builder request = createBaseLogRequest(); 167 LogEvent.Builder logEvent = LogEvent.newBuilder(); 168 logEvent.setEventTimeMs(eventTimeMs); 169 logEvent.setSourceExtension( 170 ClearcutEventHelper.createStartEvent( 171 getGroupingKey(), mRunId, mUserType, mSubToolName)); 172 request.addLogEvent(logEvent); 173 queueEvent(request.build()); 174 return true; 175 } 176 177 /** Send the last event to notify that Tradefed is done. */ notifyTradefedFinishedEvent()178 public void notifyTradefedFinishedEvent() { 179 if (mDisabled) { 180 return; 181 } 182 Duration duration = java.time.Duration.ofNanos(System.nanoTime() - mSessionStartTime); 183 LogRequest.Builder request = createBaseLogRequest(); 184 LogEvent.Builder logEvent = LogEvent.newBuilder(); 185 logEvent.setEventTimeMs(System.currentTimeMillis()); 186 logEvent.setSourceExtension( 187 ClearcutEventHelper.createFinishedEvent( 188 getGroupingKey(), mRunId, mUserType, mSubToolName, duration)); 189 request.addLogEvent(logEvent); 190 queueEvent(request.build()); 191 } 192 193 /** Send the event to notify that a Tradefed invocation was started. */ notifyTradefedInvocationStartEvent()194 public void notifyTradefedInvocationStartEvent() { 195 if (mDisabled) { 196 return; 197 } 198 LogRequest.Builder request = createBaseLogRequest(); 199 LogEvent.Builder logEvent = LogEvent.newBuilder(); 200 logEvent.setEventTimeMs(System.currentTimeMillis()); 201 logEvent.setSourceExtension( 202 ClearcutEventHelper.createRunStartEvent( 203 getGroupingKey(), mRunId, mUserType, mSubToolName)); 204 request.addLogEvent(logEvent); 205 queueEvent(request.build()); 206 } 207 208 /** Send the event to notify that a test run finished. */ notifyTestRunFinished(long startTimeNano)209 public void notifyTestRunFinished(long startTimeNano) { 210 if (mDisabled) { 211 return; 212 } 213 Duration duration = java.time.Duration.ofNanos(System.nanoTime() - startTimeNano); 214 LogRequest.Builder request = createBaseLogRequest(); 215 LogEvent.Builder logEvent = LogEvent.newBuilder(); 216 logEvent.setEventTimeMs(System.currentTimeMillis()); 217 logEvent.setSourceExtension( 218 ClearcutEventHelper.creatRunTestFinished( 219 getGroupingKey(), mRunId, mUserType, mSubToolName, duration)); 220 request.addLogEvent(logEvent); 221 queueEvent(request.build()); 222 } 223 224 /** Stop the periodic sending of clearcut events */ stop()225 public void stop() { 226 if (mExecutor != null) { 227 mExecutor.setRemoveOnCancelPolicy(true); 228 mExecutor.shutdown(); 229 mExecutor = null; 230 } 231 // Send all remaining events 232 flushEvents(); 233 } 234 235 /** Add an event to the queue of events that needs to be send. */ queueEvent(LogRequest event)236 public void queueEvent(LogRequest event) { 237 synchronized (mExternalEventQueue) { 238 mExternalEventQueue.add(event); 239 } 240 } 241 242 /** Returns the current queue size. */ getQueueSize()243 public final int getQueueSize() { 244 synchronized (mExternalEventQueue) { 245 return mExternalEventQueue.size(); 246 } 247 } 248 249 /** Allows to override the default cached uuid file. */ setCachedUuidFile(File uuidFile)250 public void setCachedUuidFile(File uuidFile) { 251 mCachedUuidFile = uuidFile; 252 } 253 254 /** Get a new or the cached uuid for the user. */ 255 @VisibleForTesting getGroupingKey()256 String getGroupingKey() { 257 String uuid = null; 258 if (mCachedUuidFile.exists()) { 259 try { 260 uuid = FileUtil.readStringFromFile(mCachedUuidFile); 261 } catch (IOException e) { 262 CLog.e(e); 263 } 264 } 265 if (uuid == null || uuid.isEmpty()) { 266 uuid = UUID.randomUUID().toString(); 267 try { 268 FileUtil.writeToFile(uuid, mCachedUuidFile); 269 } catch (IOException e) { 270 CLog.e(e); 271 } 272 } 273 return uuid; 274 } 275 276 /** Returns True if clearcut is disabled, False otherwise. */ 277 @VisibleForTesting isClearcutDisabled()278 public boolean isClearcutDisabled() { 279 return "1".equals(System.getenv(DISABLE_CLEARCUT_KEY)); 280 } 281 282 /** Returns True if the user is a Googler, False otherwise. */ 283 @VisibleForTesting isGoogleUser()284 boolean isGoogleUser() { 285 try { 286 String hostname = InetAddress.getLocalHost().getHostName(); 287 if (hostname.contains(GOOGLE_HOSTNAME)) { 288 return true; 289 } 290 } catch (UnknownHostException e) { 291 // Ignore 292 } 293 CommandResult gitRes = 294 RunUtil.getDefault() 295 .runTimedCmdSilently(60000L, "git", "config", "--get", "user.email"); 296 if (CommandStatus.SUCCESS.equals(gitRes.getStatus())) { 297 String stdout = gitRes.getStdout(); 298 if (stdout != null && stdout.trim().endsWith(GOOGLE_EMAIL)) { 299 return true; 300 } 301 } 302 303 return false; 304 } 305 createBaseLogRequest()306 private LogRequest.Builder createBaseLogRequest() { 307 LogRequest.Builder request = LogRequest.newBuilder(); 308 request.setLogSource(mLogSource); 309 request.setClientInfo(ClientInfo.newBuilder().setClientType(CLIENT_TYPE)); 310 return request; 311 } 312 flushEvents()313 private void flushEvents() { 314 List<LogRequest> copy = new ArrayList<>(); 315 synchronized (mExternalEventQueue) { 316 copy.addAll(mExternalEventQueue); 317 mExternalEventQueue.clear(); 318 } 319 List<CompletableFuture<Boolean>> futures = new ArrayList<>(); 320 while (!copy.isEmpty()) { 321 LogRequest event = copy.remove(0); 322 futures.add(CompletableFuture.supplyAsync(() -> sendToClearcut(event))); 323 } 324 325 for (CompletableFuture<Boolean> future : futures) { 326 try { 327 future.get(); 328 } catch (InterruptedException | ExecutionException e) { 329 CLog.e(e); 330 } 331 } 332 } 333 334 /** Send one event to the configured server. */ sendToClearcut(LogRequest event)335 private boolean sendToClearcut(LogRequest event) { 336 HttpHelper helper = new HttpHelper(); 337 338 InputStream inputStream = null; 339 InputStream errorStream = null; 340 OutputStream outputStream = null; 341 OutputStreamWriter outputStreamWriter = null; 342 try (CloseableTraceScope ignored = new CloseableTraceScope("sendToClearcut")) { 343 HttpURLConnection connection = helper.createConnection(new URL(mUrl), "POST", "text"); 344 outputStream = connection.getOutputStream(); 345 outputStreamWriter = new OutputStreamWriter(outputStream); 346 347 String jsonObject = JsonFormat.printer().preservingProtoFieldNames().print(event); 348 outputStreamWriter.write(jsonObject.toString()); 349 outputStreamWriter.flush(); 350 351 inputStream = connection.getInputStream(); 352 LogResponse response = LogResponse.parseFrom(inputStream); 353 354 errorStream = connection.getErrorStream(); 355 if (errorStream != null) { 356 String message = StreamUtil.getStringFromStream(errorStream); 357 CLog.e("Error posting clearcut event: '%s'. LogResponse: '%s'", message, response); 358 } 359 } catch (IOException e) { 360 CLog.e(e); 361 } finally { 362 StreamUtil.close(outputStream); 363 StreamUtil.close(inputStream); 364 StreamUtil.close(outputStreamWriter); 365 StreamUtil.close(errorStream); 366 } 367 return true; 368 } 369 } 370