1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker.logger; 17 18 import com.android.tradefed.log.LogUtil.CLog; 19 20 import java.util.Collections; 21 import java.util.HashMap; 22 import java.util.Map; 23 24 /** A utility class for an invocation to log some metrics. */ 25 public class InvocationMetricLogger { 26 27 /** Some special named key that we will always populate for the invocation. */ 28 public enum InvocationMetricKey { 29 WIFI_AP_NAME("wifi_ap_name", false), 30 WIFI_CONNECT_TIME("wifi_connect_time", true), 31 WIFI_CONNECT_COUNT("wifi_connect_count", true), 32 WIFI_CONNECT_RETRY_COUNT("wifi_connect_retry_count", true), 33 WIFI_HELPER_V2("wifi_helper_v2", false), 34 // Bugreport time and count 35 BUGREPORT_TIME("bugreport_time", true), 36 BUGREPORT_COUNT("bugreport_count", true), 37 ANR_TIME("anr_time", true), 38 ANR_COUNT("anr_count", true), 39 // Logcat dump time and count 40 LOGCAT_DUMP_TIME("logcat_dump_time", true), 41 LOGCAT_DUMP_COUNT("logcat_dump_count", true), 42 CLEARED_RUN_ERROR("cleared_run_error", true), 43 FETCH_BUILD("fetch_build_time_ms", true), 44 SETUP("setup_time_ms", true), 45 SHARDING_DEVICE_SETUP_TIME("remote_device_sharding_setup_ms", true), 46 AUTO_RETRY_TIME("auto_retry_time_ms", true), 47 BACKFILL_BUILD_INFO("backfill_build_info", false), 48 STAGE_TESTS_TIME("stage_tests_time_ms", true), 49 STAGE_REMOTE_TIME("stage_remote_time_ms", true), 50 STAGE_TESTS_BYTES("stage_tests_bytes", true), 51 STAGE_TESTS_INDIVIDUAL_DOWNLOADS("stage_tests_individual_downloads", true), 52 STAGE_UNDEFINED_DEPENDENCY("stage_undefined_dependency", true), 53 SERVER_REFERENCE("server_reference", false), 54 INSTRUMENTATION_RERUN_FROM_FILE("instrumentation_rerun_from_file", true), 55 INSTRUMENTATION_RERUN_SERIAL("instrumentation_rerun_serial", true), 56 DOWNLOAD_RETRY_COUNT("download_retry_count", true), 57 METADATA_RETRY_COUNT("metadata_retry_count", true), 58 XTS_STAGE_TESTS_TIME("xts_stage_tests_time_ms", true), 59 XTS_STAGE_TESTS_BYTES("xts_stage_tests_bytes", true), 60 XTS_PARTIAL_DOWNLOAD_SUCCESS_COUNT("xts_partial_download_success_count", true), 61 XTS_PARTIAL_DOWNLOAD_UNSUPPORTED_FILTER_FALLBACK_COUNT( 62 "xts_partial_download_unsupported_filter_fallback_count", true), 63 XTS_PARTIAL_DOWNLOAD_FALLBACK_COUNT("xts_partial_download_fallback_count", true), 64 XTS_PARTIAL_DOWNLOAD_UNFOUND_MODULES("xts_partial_download_unfound_modules", true), 65 XTS_PARTIAL_DOWNLOAD_TOTAL_COUNT("xts_partial_download_total_count", true), 66 SANDBOX_JAR_STAGING_PARTIAL_DOWNLOAD_FEATURE_COUNT( 67 "sandbox_jar_staging_partial_download_FEATURE_count", true), 68 SANDBOX_JAR_STAGING_PARTIAL_DOWNLOAD_SUCCESS_COUNT( 69 "sandbox_jar_staging_partial_download_SUCCESS_count", true), 70 TEST_DISCOVERY_MODULE_COUNT("test_discovery_module_count", true), 71 // -- Disk memory usage -- 72 // Approximate peak disk space usage of the invocation 73 // Represent files that would usually live for the full invocation (min usage) 74 TEAR_DOWN_DISK_USAGE("teardown_disk_usage_bytes", false), 75 // Recovery Mode 76 AUTO_RECOVERY_MODE_COUNT("recovery_mode_count", true), 77 ATTEMPT_RECOVERY_LOG_COUNT("attempt_pull_recovery_log", true), 78 // Represents the time we spend attempting to recover a device. 79 RECOVERY_TIME("recovery_time", true), 80 // Represents how often we enter the recover device routine. 81 RECOVERY_ROUTINE_COUNT("recovery_routine_count", true), 82 // Represents the time we spend attempting to "adb root" a device. 83 ADB_ROOT_TIME("adb_root_time", true), 84 // Represents how often we enter the "adb root" device routine. 85 ADB_ROOT_ROUTINE_COUNT("adb_root_routine_count", true), 86 // Represents the time we spend attempting to reboot a device. 87 ADB_REBOOT_TIME("adb_reboot_time", true), 88 // Represents how often we attempt to reboot the device. 89 ADB_REBOOT_ROUTINE_COUNT("adb_reboot_routine_count", true), 90 // Represents the time attempting to reboot a device into bootloader 91 BOOTLOADER_REBOOT_TIME("bootloader_reboot_time", true), 92 // Represents how often we attempt to reboot the device into bootloader 93 BOOTLOADER_REBOOT_COUNT("bootloader_reboot_count", true), 94 // Represents the time attempting to reboot a device into fastbootd 95 FASTBOOTD_REBOOT_TIME("fastbootd_reboot_time", true), 96 // Represents how often we attempt to reboot the device into fastbootd 97 FASTBOOTD_REBOOT_COUNT("fastbootd_reboot_count", true), 98 // Represents how often we reboot a device already in bootloader 99 BOOTLOADER_SAME_STATE_REBOOT("bootloader_same_state_reboot", true), 100 // Represents the time we spend during postboot setup 101 POSTBOOT_SETUP_TIME("postboot_setup_time", true), 102 // Represents how often we go through postboot setup 103 POSTBOOT_SETUP_COUNT("postboot_setup_count", true), 104 // Represents the time we spend during postboot wifi setup 105 POSTBOOT_WIFI_SETUP_TIME("postboot_wifi_setup_time", true), 106 // Represents how often we go through postboot wifi setup 107 POSTBOOT_WIFI_SETUP_COUNT("postboot_wifi_setup_count", true), 108 // Represents the time we spend during md5 calculation 109 MD5_CALCULATION_TIME("md5_calculation_time", true), 110 // Represents how often we go through md5 calculation 111 MD5_CALCULATION_COUNT("md5_calculation_count", true), 112 113 // Represents the time we spend pulling file from device. 114 PULL_FILE_TIME("pull_file_time_ms", true), 115 // Represents how many times we pulled file from the device. 116 PULL_FILE_COUNT("pull_file_count", true), 117 // Represents the time we spend pulling dir from device. 118 PULL_DIR_TIME("pull_dir_time_ms", true), 119 // Represents how many times we pulled dir from the device. 120 PULL_DIR_COUNT("pull_dir_count", true), 121 // Represents the time we spend pushing file from device. 122 PUSH_FILE_TIME("push_file_time_ms", true), 123 // Represents how many times we pushed file from the device. 124 PUSH_FILE_COUNT("push_file_count", true), 125 // Represents the time we spend pushing dir from device. 126 PUSH_DIR_TIME("push_dir_time_ms", true), 127 // Represents how many times we pushing dir from the device. 128 PUSH_DIR_COUNT("push_dir_count", true), 129 // Represents the time we spent deleting file on device 130 DELETE_DEVICE_FILE_TIME("delete_device_file_time_ms", true), 131 // Represents how many times we call the delete file method 132 DELETE_DEVICE_FILE_COUNT("delete_device_file_count", true), 133 DOES_FILE_EXISTS_TIME("does_file_exists_time_ms", true), 134 DOES_FILE_EXISTS_COUNT("does_file_exists_count", true), 135 // Represents the time and count for installing packages 136 PACKAGE_INSTALL_TIME("package_install_time_ms", true), 137 PACKAGE_INSTALL_COUNT("package_install_count", true), 138 // Capture the time spent isolating a retry with reset 139 RESET_RETRY_ISOLATION_PAIR("reset_isolation_timestamp_pair", true), 140 // Capture the time spent isolating a retry with reboot 141 REBOOT_RETRY_ISOLATION_PAIR("reboot_isolation_timestamp_pair", true), 142 // Track metrics for skipped retries 143 RETRY_MODULE_SKIPPED_COUNT("retry_module_skipped_count", true), 144 RETRY_TEST_SKIPPED_COUNT("retry_test_skipped_count", true), 145 RETRY_SKIPPED_ALL_FILTERED_COUNT("retry_skipped_all_filtered_count", true), 146 147 // Track dynamic sharding total request latency 148 DYNAMIC_SHARDING_REQUEST_LATENCY("dynamic-sharding-request-latency", true), 149 // Track dynamic sharding total request count 150 DYNAMIC_SHARDING_REQUEST_COUNT("dynamic-sharding-request-count", true), 151 152 // The time spent inside metric collectors 153 COLLECTOR_TIME("collector_time_ms", true), 154 // Track if soft restart is occurring after test module 155 SOFT_RESTART_AFTER_MODULE("soft_restart_after_module", true), 156 CLOUD_DEVICE_PROJECT("cloud_device_project", false), 157 CLOUD_DEVICE_MACHINE_TYPE("cloud_device_machine_type", false), 158 CLOUD_DEVICE_ZONE("cloud_device_zone", false), 159 CLOUD_DEVICE_STABLE_HOST_IMAGE("stable_host_image_name", false), 160 CLOUD_DEVICE_STABLE_HOST_IMAGE_PROJECT("stable_host_image_project", false), 161 162 SHUTDOWN_BEFORE_TEST("shutdown_before_test", false), 163 SHUTDOWN_AFTER_TEST("shutdown_after_test", false), 164 SHUTDOWN_LATENCY("shutdown_latency_ms", false), 165 SHUTDOWN_HARD_LATENCY("shutdown_hard_latency_ms", false), 166 DEVICE_COUNT("device_count", false), 167 DEVICE_DONE_TIMESTAMP("device_done_timestamp", false), 168 DEVICE_RELEASE_STATE("device_release_state", false), 169 DEVICE_LOST_DETECTED("device_lost_detected", false), 170 VIRTUAL_DEVICE_LOST_DETECTED("virtual_device_lost_detected", false), 171 // Count the number of time device recovery like usb reset are successful. 172 DEVICE_RECOVERY("device_recovery", true), 173 DEVICE_RECOVERY_FROM_RECOVERY("device_recovery_from_recovery", true), 174 DEVICE_RECOVERED_FROM_SSH_TUNNEL("device_recovered_from_ssh_tunnel", true), 175 DEVICE_RECOVERED_FROM_DEVICE_RESET("device_recovered_from_device_reset", true), 176 DEVICE_RECOVERY_FAIL("device_recovery_fail", true), 177 SANDBOX_EXIT_CODE("sandbox_exit_code", false), 178 CF_FETCH_ARTIFACT_TIME("cf_fetch_artifact_time_ms", false), 179 CF_GCE_CREATE_TIME("cf_gce_create_time_ms", false), 180 CF_LAUNCH_CVD_TIME("cf_launch_cvd_time_ms", false), 181 CF_INSTANCE_COUNT("cf_instance_count", false), 182 CF_LOG_SIZE("cf_log_size_bytes", true), 183 CF_OXYGEN_SERVER_URL("cf_oxygen_server_url", false), 184 CF_OXYGEN_SESSION_ID("cf_oxygen_session_id", false), 185 CF_OXYGEN_VERSION("cf_oxygen_version", false), 186 CRASH_FAILURES("crash_failures", true), 187 UNCAUGHT_CRASH_FAILURES("uncaught_crash_failures", true), 188 TEST_CRASH_FAILURES("test_crash_failures", true), 189 UNCAUGHT_TEST_CRASH_FAILURES("uncaught_test_crash_failures", true), 190 DEVICE_RESET_COUNT("device_reset_count", true), 191 DEVICE_RESET_MODULES("device_reset_modules", true), 192 DEVICE_POWERWASH_DURATIONS("device_powerwash_durations", true), 193 DEVICE_RESET_MODULES_FOR_TARGET_PREPARER("device_reset_modules_for_target_preparer", true), 194 DEVICE_SNAPSHOT_SUCCESS_COUNT("device_snapshot_success_count", true), 195 DEVICE_SNAPSHOT_FAILURE_COUNT("device_snapshot_failure_count", true), 196 DEVICE_SNAPSHOT_DURATIONS("device_snapshot_durations", true), 197 DEVICE_SNAPSHOT_RESTORE_SUCCESS_COUNT("device_snapshot_restore_success_count", true), 198 DEVICE_SNAPSHOT_RESTORE_FAILURE_COUNT("device_snapshot_restore_failure_count", true), 199 DEVICE_SNAPSHOT_RESTORE_DURATIONS("device_snapshot_restore_durations", true), 200 DEVICE_STOP_SUCCESS_COUNT("device_stop_success_count", true), 201 DEVICE_STOP_FAILURE_COUNT("device_stop_failure_count", true), 202 DEVICE_STOP_DURATIONS("device_stop_durations", true), 203 NONPERSISTENT_DEVICE_PROPERTIES("nonpersistent_device_properties", true), 204 PERSISTENT_DEVICE_PROPERTIES("persistent_device_properties", true), 205 INVOCATION_START("tf_invocation_start_timestamp", false), 206 LOAD_TEST_CONFIGS_TIME("load_test_configs_time_ms", true), 207 OXYGEN_DEVICE_DIRECT_LEASE_COUNT("oxygen_device_direct_lease_count", true), 208 OXYGEN_DEVICE_DIRECT_RELEASE_COUNT("oxygen_device_direct_release_count", true), 209 OXYGEN_DEVICE_RELEASE_FAILURE_COUNT("oxygen_device_release_failure_count", true), 210 OXYGEN_DEVICE_RELEASE_FAILURE_MESSAGE("oxygen_device_release_failure_message", true), 211 212 DYNAMIC_FILE_RESOLVER_PAIR("tf_dynamic_resolver_pair_timestamp", true), 213 ARTIFACTS_DOWNLOAD_SIZE("tf_artifacts_download_size_bytes", true), 214 ARTIFACTS_UPLOAD_SIZE("tf_artifacts_upload_size_bytes", true), 215 LOG_SAVING_TIME("log_saving_time", true), 216 LOG_SAVING_COUNT("log_saving_count", true), 217 // TODO: Delete start/end timestamp in favor of pair. 218 FETCH_BUILD_START("tf_fetch_build_start_timestamp", false), 219 FETCH_BUILD_END("tf_fetch_build_end_timestamp", false), 220 FETCH_BUILD_PAIR("tf_fetch_build_pair_timestamp", true), 221 // TODO: Delete start/end timestamp in favor of pair. 222 SETUP_START("tf_setup_start_timestamp", false), 223 SETUP_END("tf_setup_end_timestamp", false), 224 SETUP_PAIR("tf_setup_pair_timestamp", true), 225 TEST_SETUP_PAIR("tf_test_setup_pair_timestamp", true), 226 FLASHING_FROM_FASTBOOTD("flashing_from_fastbootd", true), 227 FLASHING_TIME("flashing_time_ms", true), 228 FLASHING_PERMIT_LATENCY("flashing_permit_latency_ms", true), 229 FLASHING_METHOD("flashing_method", false), 230 FLASHSTATION_DOWNLOAD_SIZE("flashstation_download_size_bytes", true), 231 DOWNLOAD_PERMIT_LATENCY("download_permit_latency_ms", true), 232 // Unzipping metrics 233 UNZIP_TESTS_DIR_TIME("unzip_tests_dir_time_ms", true), 234 UNZIP_TESTS_DIR_COUNT("unzip_tests_dir_count", true), 235 // Don't aggregate test pair, latest report wins because it's the closest to 236 // the execution like in a subprocess. 237 TEST_PAIR("tf_test_pair_timestamp", false), 238 // TODO: Delete start/end timestamp in favor of pair. 239 TEARDOWN_START("tf_teardown_start_timestamp", false), 240 TEARDOWN_END("tf_teardown_end_timestamp", false), 241 TEARDOWN_PAIR("tf_teardown_pair_timestamp", true), 242 TEST_TEARDOWN_PAIR("tf_test_teardown_pair_timestamp", true), 243 244 INVOCATION_END("tf_invocation_end_timestamp", false), 245 246 MODULE_SETUP_PAIR("tf_module_setup_pair_timestamp", true), 247 MODULE_TEARDOWN_PAIR("tf_module_teardown_pair_timestamp", true), 248 STATUS_CHECKER_PAIR("status_checker_pair", true), 249 250 LAB_PREPARER_NOT_ILAB("lab_preparer_not_ilab", true), 251 TARGET_PREPARER_IS_ILAB("target_preparer_is_ilab", true), 252 253 ART_RUN_TEST_CHECKER_COMMAND_TIME_MS("art_run_test_checker_command_time_ms", true), 254 255 // CAS downloader metrics 256 // Name of files downloaded by CAS downloader. 257 CAS_DOWNLOAD_FILES("cas_download_files", true), 258 CAS_DOWNLOAD_FILE_SUCCESS_COUNT("cas_download_file_success_count", true), 259 CAS_DOWNLOAD_FILE_FAIL_COUNT("cas_download_file_fail_count", true), 260 CAS_DOWNLOAD_TIME("cas_download_time_ms", true), 261 // Records the wait time caused by CAS downloader concurrency limitation. 262 CAS_DOWNLOAD_WAIT_TIME("cas_download_wait_time_ms", true), 263 CAS_LOCK_TIMEOUTS("cas_lock_timeout", true), 264 CAS_CACHE_FALLBACK_COUNT("cas_cache_fallback_count", true), 265 CAS_TIMEOUT_COUNT("cas_timeout_count", true), 266 // Records cache hit metrics 267 CAS_DOWNLOAD_HOT_BYTES("cas_download_hot_bytes", true), 268 CAS_DOWNLOAD_COLD_BYTES("cas_download_cold_bytes", true), 269 CAS_DOWNLOAD_HOT_FILES_COUNT("cas_download_hot_files_count", true), 270 CAS_DOWNLOAD_COLD_FILES_COUNT("cas_download_cold_files_count", true), 271 // Records local cache metrics 272 // CAS downloader local cache type. Can be either NFS or local disk. 273 CAS_DOWNLOAD_LOCAL_CACHE_TYPE("cas_download_local_cache_type", false), 274 // Number of ATE instances that sharing a same NFS backed local cache 275 CAS_DOWNLOAD_NFS_LOCAL_CACHE_CONCURRENCY("cas_download_nfs_local_cache_concurrency", false), 276 CAS_DOWNLOAD_ACQUIRE_LOCAL_CACHE_LOCK_TIME( 277 "cas_download_acquire_local_cache_lock_time_ms", true), 278 279 // Download Cache 280 CACHE_HIT_COUNT("cache_hit_count", true), 281 CACHE_WAIT_FOR_LOCK("cache_wait_for_lock", true), 282 283 // CF Cache metrics 284 CF_CACHE_WAIT_TIME("cf_cache_wait_time_sec", false), 285 CF_ARTIFACTS_FETCH_SOURCE("cf_artifacts_fetch_source", false), 286 287 // Ab downloader metrics 288 AB_BUILD_GET_API_TIME("ab_build_get_api_time", true), 289 AB_DOWNLOAD_SIZE_ELAPSED_TIME("ab_download_size_elapsed_time", true), 290 ZIP_PARTIAL_DOWNLOAD_CACHE_HIT("zip_partial_download_cache_hit", true), 291 DOWNLOAD_BOTH_ZIPS_AND_TS("download_both_zips_and_ts", true), 292 AB_LIST_API_TIME_PAIR("ab_list_api_time_pair", true), 293 AB_TEST_ZIP_NAME("ab_test_zip_name", true), 294 AB_MODULE_IN_ZIP("ab_module_in_zip", true), 295 296 // Ab log saver metrics 297 AB_LOG_SAVER_STAGING_TIME("ab_log_saver_staging_time", true), 298 AB_LOG_SAVER_UPLOAD_TIME("ab_log_saver_upload_time", true), 299 // Ants metrics 300 ANTS_INVOCATION_START_TIME("ants_invocation_start_time", true), 301 ANTS_INVOCATION_END_TIME("ants_invocation_end_time", true), 302 ANTS_MODULE_START_TIME("ants_module_start_time", true), 303 ANTS_MODULE_END_TIME("ants_module_end_time", true), 304 ANTS_RUN_START_TIME("ants_run_start_time", true), 305 ANTS_RUN_END_TIME("ants_run_end_time", true), 306 307 DUPLICATE_MAPPING_DIFFERENT_OPTIONS("duplicate_mapping_different_options", true), 308 309 HAS_ANY_RUN_FAILURES("has_any_run_failures", false), 310 TOTAL_TEST_COUNT("total_test_count", true), 311 312 // Metrics to store Device failure signatures 313 DEVICE_ERROR_SIGNATURES("device_failure_signatures", false), 314 315 DEVICE_IMAGE_NOT_CHANGED("device_image_not_changed", false), 316 IMAGE_CHANGES_IN_KEY_FILE("image_changes_in_key_file", true), 317 DEVICE_IMAGE_FILE_CHANGES("device_image_file_changes", true), 318 DEVICE_IMAGE_USED_HEURISTIC("device_image_used_heuristic", true), 319 TEST_ARTIFACT_NOT_CHANGED("test_artifact_not_changed", true), 320 PURE_DEVICE_IMAGE_UNCHANGED("pure_device_image_unchanged", true), 321 TEST_ARTIFACT_CHANGE_ONLY("test_artifact_change_only", true), 322 WORKDIR_DIFFS_IN_COMMON("workdir_diffs_in_common", true), 323 WOKRDIR_MODULE_WITH_DIFFS("workdir_module_with_diffs", true), 324 WORKDIR_UNCHANGED_MODULES("workdir_unchanged_modules", true), 325 ABORT_CONTENT_ANALYSIS("abort_content_analysis", true), 326 ABORT_CONTENT_ANALYSIS_REASON("abort_content_analysis_reason", true), 327 XTS_DIFFS_IN_COMMON("xts_diffs_in_common", true), 328 XTS_MODULE_WITH_DIFFS("xts_module_with_diffs", true), 329 XTS_UNCHANGED_MODULES("xts_unchanged_modules", true), 330 BUILD_KEY_WITH_DIFFS("build_key_with_diffs", true), 331 FILE_WITH_DIFFS("file_with_diffs", true), 332 UNCHANGED_FILE("unchanged_file", true), 333 MULTI_DEVICES_CONTENT_ANALYSIS("multi_devices_content_analysis", true), 334 DEVICELESS_CONTENT_ANALYSIS("deviceless_content_analysis", true), 335 336 POWERWASH_TIME("powerwash_time_ms", true), 337 POWERWASH_SUCCESS_COUNT("powerwash_success_count", true), 338 POWERWASH_FAILURE_COUNT("powerwash_failure_count", true), 339 LEASE_RETRY_COUNT_SUCCESS("lease_retry_count_success", true), 340 LEASE_RETRY_COUNT_FAILURE("lease_retry_count_failure", true), 341 342 TRACE_INTERNAL_ERROR("trace_internal_error", true), 343 344 INCREMENTAL_FLASHING_TIME("incremental_flashing_time", true), 345 INCREMENTAL_FLASHING_WAIT_PARALLEL_SETUP("incremental_flashing_wait_parallel_setup", true), 346 INCREMENTAL_FLASHING_ATTEMPT_COUNT("incremental_flashing_attempt_count", true), 347 INCREMENTAL_ACROSS_RELEASE_COUNT("incremental_across_release_count", true), 348 INCREMENTAL_FLASHING_TEARDOWN_FAILURE("incremental_flashing_teardown_failure", true), 349 INCREMENTAL_FLASHING_UPDATE_FAILURE("incremental_flashing_update_failure", true), 350 INCREMENTAL_SNAPUSERD_WRITE_TIME("incremental_snapuserd_write_time", true), 351 INCREMENTAL_SNAPUSERD_WRITE_BLOCKING_TIME( 352 "incremental_snapuserd_write_blocking_time", true), 353 INCREMENTAL_FALLBACK_REASON("incremental_fallback_reason", true), 354 DEVICE_IMAGE_CACHE_MISMATCH("device_image_cache_mismatch", true), 355 DEVICE_IMAGE_CACHE_ORIGIN("device_image_cache_origin", true), 356 357 CONTENT_BASED_ANALYSIS_ATTEMPT("content_based_analysis_attempt", true), 358 SKIP_NO_TESTS_DISCOVERED("skip_no_tests_discovered", true), 359 SKIP_NO_CHANGES("skip_no_changes", true), 360 NO_CHANGES_POSTSUBMIT("no_changes_postsubmit", true), 361 SILENT_INVOCATION_SKIP_COUNT("silent_invocation_skip_count", true), 362 DEMOTION_FILTERS_RECEIVED_COUNT("demotion_filters_received_count", true), 363 DEMOTION_ERROR_RESPONSE("demotion_error_response", true), 364 365 // Following are trace events also reporting as metrics 366 invocation_warm_up("invocation_warm_up", true), 367 dynamic_download("dynamic_download", true), 368 fetch_artifact("fetch_artifact", true), 369 start_logcat("start_logcat", true), 370 pre_sharding_required_setup("pre_sharding_required_setup", true), 371 sharding("sharding", true), 372 pre_multi_preparer("pre_multi_preparer", true), 373 lab_setup("lab_setup", true), 374 test_setup("test_setup", true), 375 test_execution("test_execution", true), 376 check_device_availability("check_device_availability", true), 377 bugreport("bugreport", true), 378 host_sleep("host_sleep", true), 379 test_teardown("test_teardown", true), 380 test_cleanup("test_cleanup", true), 381 log_and_release_device("log_and_release_device", true), 382 invocation_events_processing("invocation_events_processing", true), 383 stage_suite_test_artifacts("stage_suite_test_artifacts", true), 384 wait_for_results_update("wait_for_results_update", true), 385 instru_collect_tests("instru_collect_tests", true), 386 TestContentAnalyzer("TestContentAnalyzer", true), 387 screen_on_setup("screen_on_setup", true), 388 389 // Test caching metrics 390 CACHED_MODULE_RESULTS_COUNT("cached_module_results_count", true), 391 ; 392 393 private final String mKeyName; 394 // Whether or not to add the value when the key is added again. 395 private final boolean mAdditive; 396 InvocationMetricKey(String key, boolean additive)397 private InvocationMetricKey(String key, boolean additive) { 398 mKeyName = key; 399 mAdditive = additive; 400 } 401 402 @Override toString()403 public String toString() { 404 return mKeyName; 405 } 406 shouldAdd()407 public boolean shouldAdd() { 408 return mAdditive; 409 } 410 } 411 412 /** Grouping allows to log several groups under a same key. */ 413 public enum InvocationGroupMetricKey { 414 TEST_TYPE_COUNT("test-type-count", true), 415 TARGET_PREPARER_SETUP_LATENCY("target-preparer-setup-latency", true), 416 TARGET_PREPARER_TEARDOWN_LATENCY("target-preparer-teardown-latency", true), 417 LAB_PREPARER_SETUP_LATENCY("lab-preparer-setup-latency", true), 418 LAB_PREPARER_TEARDOWN_LATENCY("lab-preparer-teardown-latency", true), 419 MULTI_TARGET_PREPARER_TEARDOWN_LATENCY("multi-target-preparer-teardown-latency", true), 420 421 INCREMENTAL_FLASHING_PATCHES_SIZE("incremental-flashing-patches-size", true), 422 INCREMENTAL_FLASHING_TARGET_SIZE("incremental-flashing-target-size", true); 423 424 private final String mGroupName; 425 // Whether or not to add the value when the key is added again. 426 private final boolean mAdditive; 427 InvocationGroupMetricKey(String groupName, boolean additive)428 private InvocationGroupMetricKey(String groupName, boolean additive) { 429 mGroupName = groupName; 430 mAdditive = additive; 431 } 432 433 @Override toString()434 public String toString() { 435 return mGroupName; 436 } 437 shouldAdd()438 public boolean shouldAdd() { 439 return mAdditive; 440 } 441 } 442 InvocationMetricLogger()443 private InvocationMetricLogger() {} 444 445 private static ThreadLocal<ThreadGroup> sLocal = new ThreadLocal<>(); 446 447 /** Tracks a localized context when using the properties inside the gRPC server */ setLocalGroup(ThreadGroup tg)448 public static void setLocalGroup(ThreadGroup tg) { 449 sLocal.set(tg); 450 } 451 452 /** Resets the localized context. */ resetLocalGroup()453 public static void resetLocalGroup() { 454 sLocal.remove(); 455 } 456 457 /** 458 * Track metrics per ThreadGroup as a proxy to invocation since an invocation run within one 459 * threadgroup. 460 */ 461 private static final Map<ThreadGroup, Map<String, String>> mPerGroupMetrics = 462 Collections.synchronizedMap(new HashMap<ThreadGroup, Map<String, String>>()); 463 464 /** 465 * Add one key-value to be tracked at the invocation level. 466 * 467 * @param key The key under which the invocation metric will be tracked. 468 * @param value The value of the invocation metric. 469 */ addInvocationMetrics(InvocationMetricKey key, long value)470 public static void addInvocationMetrics(InvocationMetricKey key, long value) { 471 if (key.shouldAdd()) { 472 String existingVal = getInvocationMetrics().get(key.toString()); 473 long existingLong = 0L; 474 if (existingVal != null) { 475 try { 476 existingLong = Long.parseLong(existingVal); 477 } catch (NumberFormatException e) { 478 CLog.e( 479 "%s is expected to contain a number, instead found: %s", 480 key.toString(), existingVal); 481 } 482 } 483 value += existingLong; 484 } 485 addInvocationMetrics(key.toString(), Long.toString(value)); 486 } 487 488 /** 489 * Add one key-value for a given group 490 * 491 * @param groupKey The key of the group 492 * @param group The group name associated with the key 493 * @param value The value for the group 494 */ addInvocationMetrics( InvocationGroupMetricKey groupKey, String group, String value)495 public static void addInvocationMetrics( 496 InvocationGroupMetricKey groupKey, String group, String value) { 497 String key = groupKey.toString() + ":" + group; 498 if (groupKey.shouldAdd()) { 499 String existingVal = getInvocationMetrics().get(key.toString()); 500 if (existingVal != null) { 501 value = String.format("%s,%s", existingVal, value); 502 } 503 } 504 addInvocationMetrics(key, value); 505 } 506 507 /** 508 * Add one key-value to be tracked at the invocation level. Don't expose the String key yet to 509 * avoid abuse, stick to the official {@link InvocationMetricKey} to start with. 510 * 511 * @param key The key under which the invocation metric will be tracked. 512 * @param value The value of the invocation metric. 513 */ addInvocationMetrics(String key, String value)514 private static void addInvocationMetrics(String key, String value) { 515 ThreadGroup group = Thread.currentThread().getThreadGroup(); 516 synchronized (mPerGroupMetrics) { 517 if (sLocal.get() != null) { 518 group = sLocal.get(); 519 } 520 if (mPerGroupMetrics.get(group) == null) { 521 mPerGroupMetrics.put(group, new HashMap<>()); 522 } 523 mPerGroupMetrics.get(group).put(key, value); 524 } 525 } 526 527 /** 528 * Add one key-value to be tracked at the invocation level for a given group. 529 * 530 * @param groupKey The key of the group 531 * @param group The group name associated with the key 532 * @param value The value for the group 533 */ addInvocationMetrics( InvocationGroupMetricKey groupKey, String group, long value)534 public static void addInvocationMetrics( 535 InvocationGroupMetricKey groupKey, String group, long value) { 536 String key = groupKey.toString() + ":" + group; 537 if (groupKey.shouldAdd()) { 538 String existingVal = getInvocationMetrics().get(key); 539 long existingLong = 0L; 540 if (existingVal != null) { 541 try { 542 existingLong = Long.parseLong(existingVal); 543 } catch (NumberFormatException e) { 544 CLog.e( 545 "%s is expected to contain a number, instead found: %s", 546 key.toString(), existingVal); 547 } 548 } 549 value += existingLong; 550 } 551 addInvocationMetrics(key, Long.toString(value)); 552 } 553 554 /** 555 * Add one key-value to be tracked at the invocation level. 556 * 557 * @param key The key under which the invocation metric will be tracked. 558 * @param value The value of the invocation metric. 559 */ addInvocationMetrics(InvocationMetricKey key, String value)560 public static void addInvocationMetrics(InvocationMetricKey key, String value) { 561 if (key.shouldAdd()) { 562 String existingVal = getInvocationMetrics().get(key.toString()); 563 if (existingVal != null) { 564 value = String.format("%s,%s", existingVal, value); 565 } 566 } 567 addInvocationMetrics(key.toString(), value); 568 } 569 570 /** 571 * Add a pair of value associated with the same key. Usually used for timestamp start and end. 572 * 573 * @param key The key under which the invocation metric will be tracked. 574 * @param start The start value of the invocation metric. 575 * @param end The end value of the invocation metric. 576 */ addInvocationPairMetrics(InvocationMetricKey key, long start, long end)577 public static void addInvocationPairMetrics(InvocationMetricKey key, long start, long end) { 578 String value = start + ":" + end; 579 if (key.shouldAdd()) { 580 String existingVal = getInvocationMetrics().get(key.toString()); 581 if (existingVal != null) { 582 value = String.format("%s,%s", existingVal, value); 583 } 584 } 585 addInvocationMetrics(key.toString(), value); 586 } 587 588 /** Returns the Map of invocation metrics for the invocation in progress. */ getInvocationMetrics()589 public static Map<String, String> getInvocationMetrics() { 590 ThreadGroup group = Thread.currentThread().getThreadGroup(); 591 synchronized (mPerGroupMetrics) { 592 if (sLocal.get() != null) { 593 group = sLocal.get(); 594 } 595 if (mPerGroupMetrics.get(group) == null) { 596 mPerGroupMetrics.put(group, new HashMap<>()); 597 } 598 return new HashMap<>(mPerGroupMetrics.get(group)); 599 } 600 } 601 602 /** Clear the invocation metrics for an invocation. */ clearInvocationMetrics()603 public static void clearInvocationMetrics() { 604 ThreadGroup group = Thread.currentThread().getThreadGroup(); 605 synchronized (mPerGroupMetrics) { 606 mPerGroupMetrics.remove(group); 607 } 608 } 609 } 610