1#!/usr/bin/env python3 2# 3# Copyright (C) 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18from __future__ import annotations 19import argparse 20import collections 21from concurrent.futures import Future, ThreadPoolExecutor 22from dataclasses import dataclass 23import datetime 24import json 25import logging 26import os 27from pathlib import Path 28import sys 29from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Union 30 31from simpleperf_report_lib import GetReportLib, SymbolStruct 32from simpleperf_utils import ( 33 Addr2Nearestline, AddrRange, BaseArgumentParser, BinaryFinder, Disassembly, get_script_dir, 34 log_exit, Objdump, open_report_in_browser, ReadElf, ReportLibOptions, SourceFileSearcher) 35 36MAX_CALLSTACK_LENGTH = 750 37 38 39class HtmlWriter(object): 40 41 def __init__(self, output_path: Union[Path, str]): 42 self.fh = open(output_path, 'w') 43 self.tag_stack = [] 44 45 def close(self): 46 self.fh.close() 47 48 def open_tag(self, tag: str, **attrs: Dict[str, str]) -> HtmlWriter: 49 attr_str = '' 50 for key in attrs: 51 attr_str += ' %s="%s"' % (key, attrs[key]) 52 self.fh.write('<%s%s>' % (tag, attr_str)) 53 self.tag_stack.append(tag) 54 return self 55 56 def close_tag(self, tag: Optional[str] = None): 57 if tag: 58 assert tag == self.tag_stack[-1] 59 self.fh.write('</%s>\n' % self.tag_stack.pop()) 60 61 def add(self, text: str) -> HtmlWriter: 62 self.fh.write(text) 63 return self 64 65 def add_file(self, file_path: Union[Path, str]) -> HtmlWriter: 66 file_path = os.path.join(get_script_dir(), file_path) 67 with open(file_path, 'r') as f: 68 self.add(f.read()) 69 return self 70 71 72def modify_text_for_html(text: str) -> str: 73 return text.replace('>', '>').replace('<', '<') 74 75 76def hex_address_for_json(addr: int) -> str: 77 """ To handle big addrs (nears uint64_max) in Javascript, store addrs as hex strings in Json. 78 """ 79 return '0x%x' % addr 80 81 82class EventScope(object): 83 84 def __init__(self, name: str): 85 self.name = name 86 self.processes: Dict[int, ProcessScope] = {} # map from pid to ProcessScope 87 self.sample_count = 0 88 self.event_count = 0 89 90 def get_process(self, pid: int) -> ProcessScope: 91 process = self.processes.get(pid) 92 if not process: 93 process = self.processes[pid] = ProcessScope(pid) 94 return process 95 96 def get_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 97 result = {} 98 result['eventName'] = self.name 99 result['eventCount'] = self.event_count 100 processes = sorted(self.processes.values(), key=lambda a: a.event_count, reverse=True) 101 result['processes'] = [process.get_sample_info(gen_addr_hit_map) 102 for process in processes] 103 return result 104 105 @property 106 def threads(self) -> Iterator[ThreadScope]: 107 for process in self.processes.values(): 108 for thread in process.threads.values(): 109 yield thread 110 111 @property 112 def libraries(self) -> Iterator[LibScope]: 113 for process in self.processes.values(): 114 for thread in process.threads.values(): 115 for lib in thread.libs.values(): 116 yield lib 117 118 119class ProcessScope(object): 120 121 def __init__(self, pid: int): 122 self.pid = pid 123 self.name = '' 124 self.event_count = 0 125 self.threads: Dict[int, ThreadScope] = {} # map from tid to ThreadScope 126 127 def get_thread(self, tid: int, thread_name: str) -> ThreadScope: 128 thread = self.threads.get(tid) 129 if not thread: 130 thread = self.threads[tid] = ThreadScope(tid) 131 thread.name = thread_name 132 if self.pid == tid: 133 self.name = thread_name 134 return thread 135 136 def get_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 137 result = {} 138 result['pid'] = self.pid 139 result['eventCount'] = self.event_count 140 # Sorting threads by sample count is better for profiles recorded with --trace-offcpu. 141 threads = sorted(self.threads.values(), key=lambda a: a.sample_count, reverse=True) 142 result['threads'] = [thread.get_sample_info(gen_addr_hit_map) 143 for thread in threads] 144 return result 145 146 def merge_by_thread_name(self, process: ProcessScope): 147 self.event_count += process.event_count 148 thread_list: List[ThreadScope] = list( 149 self.threads.values()) + list(process.threads.values()) 150 new_threads: Dict[str, ThreadScope] = {} # map from thread name to ThreadScope 151 for thread in thread_list: 152 cur_thread = new_threads.get(thread.name) 153 if cur_thread is None: 154 new_threads[thread.name] = thread 155 else: 156 cur_thread.merge(thread) 157 self.threads = {} 158 for thread in new_threads.values(): 159 self.threads[thread.tid] = thread 160 161 162class ThreadScope(object): 163 164 def __init__(self, tid: int): 165 self.tid = tid 166 self.name = '' 167 self.event_count = 0 168 self.sample_count = 0 169 self.libs: Dict[int, LibScope] = {} # map from lib_id to LibScope 170 self.call_graph = CallNode(-1) 171 self.reverse_call_graph = CallNode(-1) 172 173 def add_callstack( 174 self, event_count: int, callstack: List[Tuple[int, int, int]], 175 build_addr_hit_map: bool): 176 """ callstack is a list of tuple (lib_id, func_id, addr). 177 For each i > 0, callstack[i] calls callstack[i-1].""" 178 hit_func_ids: Set[int] = set() 179 for i, (lib_id, func_id, addr) in enumerate(callstack): 180 # When a callstack contains recursive function, only add for each function once. 181 if func_id in hit_func_ids: 182 continue 183 hit_func_ids.add(func_id) 184 185 lib = self.libs.get(lib_id) 186 if not lib: 187 lib = self.libs[lib_id] = LibScope(lib_id) 188 function = lib.get_function(func_id) 189 function.subtree_event_count += event_count 190 if i == 0: 191 lib.event_count += event_count 192 function.event_count += event_count 193 function.sample_count += 1 194 if build_addr_hit_map: 195 function.build_addr_hit_map(addr, event_count if i == 0 else 0, event_count) 196 197 # build call graph and reverse call graph 198 node = self.call_graph 199 for item in reversed(callstack): 200 node = node.get_child(item[1]) 201 node.event_count += event_count 202 node = self.reverse_call_graph 203 for item in callstack: 204 node = node.get_child(item[1]) 205 node.event_count += event_count 206 207 def update_subtree_event_count(self): 208 self.call_graph.update_subtree_event_count() 209 self.reverse_call_graph.update_subtree_event_count() 210 211 def limit_percents(self, min_func_limit: float, min_callchain_percent: float, 212 hit_func_ids: Set[int]): 213 for lib in self.libs.values(): 214 to_del_funcs = [] 215 for function in lib.functions.values(): 216 if function.subtree_event_count < min_func_limit: 217 to_del_funcs.append(function.func_id) 218 else: 219 hit_func_ids.add(function.func_id) 220 for func_id in to_del_funcs: 221 del lib.functions[func_id] 222 min_limit = min_callchain_percent * 0.01 * self.call_graph.subtree_event_count 223 self.call_graph.cut_edge(min_limit, hit_func_ids) 224 self.reverse_call_graph.cut_edge(min_limit, hit_func_ids) 225 226 def get_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 227 result = {} 228 result['tid'] = self.tid 229 result['eventCount'] = self.event_count 230 result['sampleCount'] = self.sample_count 231 result['libs'] = [lib.gen_sample_info(gen_addr_hit_map) 232 for lib in self.libs.values()] 233 result['g'] = self.call_graph.gen_sample_info() 234 result['rg'] = self.reverse_call_graph.gen_sample_info() 235 return result 236 237 def merge(self, thread: ThreadScope): 238 self.event_count += thread.event_count 239 self.sample_count += thread.sample_count 240 for lib_id, lib in thread.libs.items(): 241 cur_lib = self.libs.get(lib_id) 242 if cur_lib is None: 243 self.libs[lib_id] = lib 244 else: 245 cur_lib.merge(lib) 246 self.call_graph.merge(thread.call_graph) 247 self.reverse_call_graph.merge(thread.reverse_call_graph) 248 249 def sort_call_graph_by_function_name(self, get_func_name: Callable[[int], str]) -> None: 250 self.call_graph.sort_by_function_name(get_func_name) 251 self.reverse_call_graph.sort_by_function_name(get_func_name) 252 253 254class LibScope(object): 255 256 def __init__(self, lib_id: int): 257 self.lib_id = lib_id 258 self.event_count = 0 259 self.functions: Dict[int, FunctionScope] = {} # map from func_id to FunctionScope. 260 261 def get_function(self, func_id: int) -> FunctionScope: 262 function = self.functions.get(func_id) 263 if not function: 264 function = self.functions[func_id] = FunctionScope(func_id) 265 return function 266 267 def gen_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 268 result = {} 269 result['libId'] = self.lib_id 270 result['eventCount'] = self.event_count 271 result['functions'] = [func.gen_sample_info(gen_addr_hit_map) 272 for func in self.functions.values()] 273 return result 274 275 def merge(self, lib: LibScope): 276 self.event_count += lib.event_count 277 for func_id, function in lib.functions.items(): 278 cur_function = self.functions.get(func_id) 279 if cur_function is None: 280 self.functions[func_id] = function 281 else: 282 cur_function.merge(function) 283 284 285class FunctionScope(object): 286 287 def __init__(self, func_id: int): 288 self.func_id = func_id 289 self.sample_count = 0 290 self.event_count = 0 291 self.subtree_event_count = 0 292 self.addr_hit_map = None # map from addr to [event_count, subtree_event_count]. 293 # map from (source_file_id, line) to [event_count, subtree_event_count]. 294 self.line_hit_map = None 295 296 def build_addr_hit_map(self, addr: int, event_count: int, subtree_event_count: int): 297 if self.addr_hit_map is None: 298 self.addr_hit_map = {} 299 count_info = self.addr_hit_map.get(addr) 300 if count_info is None: 301 self.addr_hit_map[addr] = [event_count, subtree_event_count] 302 else: 303 count_info[0] += event_count 304 count_info[1] += subtree_event_count 305 306 def build_line_hit_map(self, source_file_id: int, line: int, event_count: int, 307 subtree_event_count: int): 308 if self.line_hit_map is None: 309 self.line_hit_map = {} 310 key = (source_file_id, line) 311 count_info = self.line_hit_map.get(key) 312 if count_info is None: 313 self.line_hit_map[key] = [event_count, subtree_event_count] 314 else: 315 count_info[0] += event_count 316 count_info[1] += subtree_event_count 317 318 def gen_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 319 result = {} 320 result['f'] = self.func_id 321 result['c'] = [self.sample_count, self.event_count, self.subtree_event_count] 322 if self.line_hit_map: 323 items = [] 324 for key in self.line_hit_map: 325 count_info = self.line_hit_map[key] 326 item = {'f': key[0], 'l': key[1], 'e': count_info[0], 's': count_info[1]} 327 items.append(item) 328 result['s'] = items 329 if gen_addr_hit_map and self.addr_hit_map: 330 items = [] 331 for addr in sorted(self.addr_hit_map): 332 count_info = self.addr_hit_map[addr] 333 items.append( 334 {'a': hex_address_for_json(addr), 335 'e': count_info[0], 336 's': count_info[1]}) 337 result['a'] = items 338 return result 339 340 def merge(self, function: FunctionScope): 341 self.sample_count += function.sample_count 342 self.event_count += function.event_count 343 self.subtree_event_count += function.subtree_event_count 344 self.addr_hit_map = self.__merge_hit_map(self.addr_hit_map, function.addr_hit_map) 345 self.line_hit_map = self.__merge_hit_map(self.line_hit_map, function.line_hit_map) 346 347 @staticmethod 348 def __merge_hit_map(map1: Optional[Dict[int, List[int]]], 349 map2: Optional[Dict[int, List[int]]]) -> Optional[Dict[int, List[int]]]: 350 if not map1: 351 return map2 352 if not map2: 353 return map1 354 for key, value2 in map2.items(): 355 value1 = map1.get(key) 356 if value1 is None: 357 map1[key] = value2 358 else: 359 value1[0] += value2[0] 360 value1[1] += value2[1] 361 return map1 362 363 364class CallNode(object): 365 366 def __init__(self, func_id: int): 367 self.event_count = 0 368 self.subtree_event_count = 0 369 self.func_id = func_id 370 # map from func_id to CallNode 371 self.children: Dict[int, CallNode] = collections.OrderedDict() 372 373 def get_child(self, func_id: int) -> CallNode: 374 child = self.children.get(func_id) 375 if not child: 376 child = self.children[func_id] = CallNode(func_id) 377 return child 378 379 def update_subtree_event_count(self): 380 self.subtree_event_count = self.event_count 381 for child in self.children.values(): 382 self.subtree_event_count += child.update_subtree_event_count() 383 return self.subtree_event_count 384 385 def cut_edge(self, min_limit: float, hit_func_ids: Set[int]): 386 hit_func_ids.add(self.func_id) 387 to_del_children = [] 388 for key in self.children: 389 child = self.children[key] 390 if child.subtree_event_count < min_limit: 391 to_del_children.append(key) 392 else: 393 child.cut_edge(min_limit, hit_func_ids) 394 for key in to_del_children: 395 del self.children[key] 396 397 def gen_sample_info(self) -> Dict[str, Any]: 398 result = {} 399 result['e'] = self.event_count 400 result['s'] = self.subtree_event_count 401 result['f'] = self.func_id 402 result['c'] = [child.gen_sample_info() for child in self.children.values()] 403 return result 404 405 def merge(self, node: CallNode): 406 self.event_count += node.event_count 407 self.subtree_event_count += node.subtree_event_count 408 for key, child in node.children.items(): 409 cur_child = self.children.get(key) 410 if cur_child is None: 411 self.children[key] = child 412 else: 413 cur_child.merge(child) 414 415 def sort_by_function_name(self, get_func_name: Callable[[int], str]) -> None: 416 if self.children: 417 child_func_ids = list(self.children.keys()) 418 child_func_ids.sort(key=get_func_name) 419 new_children = collections.OrderedDict() 420 for func_id in child_func_ids: 421 new_children[func_id] = self.children[func_id] 422 self.children = new_children 423 for child in self.children.values(): 424 child.sort_by_function_name(get_func_name) 425 426 427@dataclass 428class LibInfo: 429 name: str 430 build_id: str 431 432 433class LibSet(object): 434 """ Collection of shared libraries used in perf.data. """ 435 436 def __init__(self): 437 self.lib_name_to_id: Dict[str, int] = {} 438 self.libs: List[LibInfo] = [] 439 440 def get_lib_id(self, lib_name: str) -> Optional[int]: 441 return self.lib_name_to_id.get(lib_name) 442 443 def add_lib(self, lib_name: str, build_id: str) -> int: 444 """ Return lib_id of the newly added lib. """ 445 lib_id = len(self.libs) 446 self.libs.append(LibInfo(lib_name, build_id)) 447 self.lib_name_to_id[lib_name] = lib_id 448 return lib_id 449 450 def get_lib(self, lib_id: int) -> LibInfo: 451 return self.libs[lib_id] 452 453 454class Function(object): 455 """ Represent a function in a shared library. """ 456 457 def __init__(self, lib_id: int, func_name: str, func_id: int, start_addr: int, addr_len: int): 458 self.lib_id = lib_id 459 self.func_name = func_name 460 self.func_id = func_id 461 self.start_addr = start_addr 462 self.addr_len = addr_len 463 self.source_info = None 464 self.disassembly = None 465 466 467class FunctionSet(object): 468 """ Collection of functions used in perf.data. """ 469 470 def __init__(self): 471 self.name_to_func: Dict[Tuple[int, str], Function] = {} 472 self.id_to_func: Dict[int, Function] = {} 473 474 def get_func_id(self, lib_id: int, symbol: SymbolStruct) -> int: 475 key = (lib_id, symbol.symbol_name) 476 function = self.name_to_func.get(key) 477 if function is None: 478 func_id = len(self.id_to_func) 479 function = Function(lib_id, symbol.symbol_name, func_id, symbol.symbol_addr, 480 symbol.symbol_len) 481 self.name_to_func[key] = function 482 self.id_to_func[func_id] = function 483 return function.func_id 484 485 def get_func_name(self, func_id: int) -> str: 486 return self.id_to_func[func_id].func_name 487 488 def trim_functions(self, left_func_ids: Set[int]): 489 """ Remove functions excepts those in left_func_ids. """ 490 for function in self.name_to_func.values(): 491 if function.func_id not in left_func_ids: 492 del self.id_to_func[function.func_id] 493 # name_to_func will not be used. 494 self.name_to_func = None 495 496 497class SourceFile(object): 498 """ A source file containing source code hit by samples. """ 499 500 def __init__(self, file_id: int, abstract_path: str): 501 self.file_id = file_id 502 self.abstract_path = abstract_path # path reported by addr2line 503 self.real_path: Optional[str] = None # file path in the file system 504 self.requested_lines: Optional[Set[int]] = set() 505 self.line_to_code: Dict[int, str] = {} # map from line to code in that line. 506 507 def request_lines(self, start_line: int, end_line: int): 508 self.requested_lines |= set(range(start_line, end_line + 1)) 509 510 def add_source_code(self, real_path: str): 511 self.real_path = real_path 512 with open(real_path, 'r') as f: 513 source_code = f.readlines() 514 max_line = len(source_code) 515 for line in self.requested_lines: 516 if line > 0 and line <= max_line: 517 self.line_to_code[line] = source_code[line - 1] 518 # requested_lines is no longer used. 519 self.requested_lines = None 520 521 522class SourceFileSet(object): 523 """ Collection of source files. """ 524 525 def __init__(self): 526 self.path_to_source_files: Dict[str, SourceFile] = {} # map from file path to SourceFile. 527 528 def get_source_file(self, file_path: str) -> SourceFile: 529 source_file = self.path_to_source_files.get(file_path) 530 if not source_file: 531 source_file = SourceFile(len(self.path_to_source_files), file_path) 532 self.path_to_source_files[file_path] = source_file 533 return source_file 534 535 def load_source_code(self, source_dirs: List[str]): 536 file_searcher = SourceFileSearcher(source_dirs) 537 for source_file in self.path_to_source_files.values(): 538 real_path = file_searcher.get_real_path(source_file.abstract_path) 539 if real_path: 540 source_file.add_source_code(real_path) 541 542 543class RecordData(object): 544 545 """RecordData reads perf.data, and generates data used by report_html.js in json format. 546 All generated items are listed as below: 547 1. recordTime: string 548 2. machineType: string 549 3. androidVersion: string 550 4. recordCmdline: string 551 5. totalSamples: int 552 6. processNames: map from pid to processName. 553 7. threadNames: map from tid to threadName. 554 8. libList: an array of libNames, indexed by libId. 555 9. functionMap: map from functionId to funcData. 556 funcData = { 557 l: libId 558 f: functionName 559 s: [sourceFileId, startLine, endLine] [optional] 560 d: [(disassembly, addr)] [optional] 561 } 562 563 10. sampleInfo = [eventInfo] 564 eventInfo = { 565 eventName 566 eventCount 567 processes: [processInfo] 568 } 569 processInfo = { 570 pid 571 eventCount 572 threads: [threadInfo] 573 } 574 threadInfo = { 575 tid 576 eventCount 577 sampleCount 578 libs: [libInfo], 579 g: callGraph, 580 rg: reverseCallgraph 581 } 582 libInfo = { 583 libId, 584 eventCount, 585 functions: [funcInfo] 586 } 587 funcInfo = { 588 f: functionId 589 c: [sampleCount, eventCount, subTreeEventCount] 590 s: [sourceCodeInfo] [optional] 591 a: [addrInfo] (sorted by addrInfo.addr) [optional] 592 } 593 callGraph and reverseCallGraph are both of type CallNode. 594 callGraph shows how a function calls other functions. 595 reverseCallGraph shows how a function is called by other functions. 596 CallNode { 597 e: selfEventCount 598 s: subTreeEventCount 599 f: functionId 600 c: [CallNode] # children 601 } 602 603 sourceCodeInfo { 604 f: sourceFileId 605 l: line 606 e: eventCount 607 s: subtreeEventCount 608 } 609 610 addrInfo { 611 a: addr 612 e: eventCount 613 s: subtreeEventCount 614 } 615 616 11. sourceFiles: an array of sourceFile, indexed by sourceFileId. 617 sourceFile { 618 path 619 code: # a map from line to code for that line. 620 } 621 """ 622 623 def __init__( 624 self, binary_cache_path: Optional[str], 625 ndk_path: Optional[str], 626 build_addr_hit_map: bool): 627 self.binary_cache_path = binary_cache_path 628 self.ndk_path = ndk_path 629 self.build_addr_hit_map = build_addr_hit_map 630 self.meta_info: Optional[Dict[str, str]] = None 631 self.cmdline: Optional[str] = None 632 self.arch: Optional[str] = None 633 self.events: Dict[str, EventScope] = {} 634 self.libs = LibSet() 635 self.functions = FunctionSet() 636 self.total_samples = 0 637 self.source_files = SourceFileSet() 638 self.gen_addr_hit_map_in_record_info = False 639 self.binary_finder = BinaryFinder(binary_cache_path, ReadElf(ndk_path)) 640 641 def load_record_file(self, record_file: str, report_lib_options: ReportLibOptions): 642 lib = GetReportLib(record_file) 643 # If not showing ip for unknown symbols, the percent of the unknown symbol may be 644 # accumulated to very big, and ranks first in the sample table. 645 lib.ShowIpForUnknownSymbol() 646 if self.binary_cache_path: 647 lib.SetSymfs(self.binary_cache_path) 648 lib.SetReportOptions(report_lib_options) 649 self.meta_info = lib.MetaInfo() 650 self.cmdline = lib.GetRecordCmd() 651 self.arch = lib.GetArch() 652 while True: 653 raw_sample = lib.GetNextSample() 654 if not raw_sample: 655 lib.Close() 656 break 657 raw_event = lib.GetEventOfCurrentSample() 658 symbol = lib.GetSymbolOfCurrentSample() 659 callchain = lib.GetCallChainOfCurrentSample() 660 event = self._get_event(raw_event.name) 661 self.total_samples += 1 662 event.sample_count += 1 663 event.event_count += raw_sample.period 664 process = event.get_process(raw_sample.pid) 665 process.event_count += raw_sample.period 666 thread = process.get_thread(raw_sample.tid, raw_sample.thread_comm) 667 thread.event_count += raw_sample.period 668 thread.sample_count += 1 669 670 lib_id = self.libs.get_lib_id(symbol.dso_name) 671 if lib_id is None: 672 lib_id = self.libs.add_lib(symbol.dso_name, lib.GetBuildIdForPath(symbol.dso_name)) 673 func_id = self.functions.get_func_id(lib_id, symbol) 674 callstack = [(lib_id, func_id, symbol.vaddr_in_file)] 675 for i in range(callchain.nr): 676 symbol = callchain.entries[i].symbol 677 lib_id = self.libs.get_lib_id(symbol.dso_name) 678 if lib_id is None: 679 lib_id = self.libs.add_lib( 680 symbol.dso_name, lib.GetBuildIdForPath(symbol.dso_name)) 681 func_id = self.functions.get_func_id(lib_id, symbol) 682 callstack.append((lib_id, func_id, symbol.vaddr_in_file)) 683 if len(callstack) > MAX_CALLSTACK_LENGTH: 684 callstack = callstack[:MAX_CALLSTACK_LENGTH] 685 thread.add_callstack(raw_sample.period, callstack, self.build_addr_hit_map) 686 687 for event in self.events.values(): 688 for thread in event.threads: 689 thread.update_subtree_event_count() 690 691 def aggregate_by_thread_name(self): 692 for event in self.events.values(): 693 new_processes = {} # from process name to ProcessScope 694 for process in event.processes.values(): 695 cur_process = new_processes.get(process.name) 696 if cur_process is None: 697 new_processes[process.name] = process 698 else: 699 cur_process.merge_by_thread_name(process) 700 event.processes = {} 701 for process in new_processes.values(): 702 event.processes[process.pid] = process 703 704 def limit_percents(self, min_func_percent: float, min_callchain_percent: float): 705 hit_func_ids: Set[int] = set() 706 for event in self.events.values(): 707 min_limit = event.event_count * min_func_percent * 0.01 708 to_del_processes = [] 709 for process in event.processes.values(): 710 to_del_threads = [] 711 for thread in process.threads.values(): 712 if thread.call_graph.subtree_event_count < min_limit: 713 to_del_threads.append(thread.tid) 714 else: 715 thread.limit_percents(min_limit, min_callchain_percent, hit_func_ids) 716 for thread in to_del_threads: 717 del process.threads[thread] 718 if not process.threads: 719 to_del_processes.append(process.pid) 720 for process in to_del_processes: 721 del event.processes[process] 722 self.functions.trim_functions(hit_func_ids) 723 724 def sort_call_graph_by_function_name(self) -> None: 725 for event in self.events.values(): 726 for process in event.processes.values(): 727 for thread in process.threads.values(): 728 thread.sort_call_graph_by_function_name(self.functions.get_func_name) 729 730 def _get_event(self, event_name: str) -> EventScope: 731 if event_name not in self.events: 732 self.events[event_name] = EventScope(event_name) 733 return self.events[event_name] 734 735 def add_source_code(self, source_dirs: List[str], filter_lib: Callable[[str], bool], jobs: int): 736 """ Collect source code information: 737 1. Find line ranges for each function in FunctionSet. 738 2. Find line for each addr in FunctionScope.addr_hit_map. 739 3. Collect needed source code in SourceFileSet. 740 """ 741 addr2line = Addr2Nearestline(self.ndk_path, self.binary_finder, False) 742 # Request line range for each function. 743 for function in self.functions.id_to_func.values(): 744 if function.func_name == 'unknown': 745 continue 746 lib_info = self.libs.get_lib(function.lib_id) 747 if filter_lib(lib_info.name): 748 addr2line.add_addr(lib_info.name, lib_info.build_id, 749 function.start_addr, function.start_addr) 750 addr2line.add_addr(lib_info.name, lib_info.build_id, function.start_addr, 751 function.start_addr + function.addr_len - 1) 752 # Request line for each addr in FunctionScope.addr_hit_map. 753 for event in self.events.values(): 754 for lib in event.libraries: 755 lib_info = self.libs.get_lib(lib.lib_id) 756 if filter_lib(lib_info.name): 757 for function in lib.functions.values(): 758 func_addr = self.functions.id_to_func[function.func_id].start_addr 759 for addr in function.addr_hit_map: 760 addr2line.add_addr(lib_info.name, lib_info.build_id, func_addr, addr) 761 addr2line.convert_addrs_to_lines(jobs) 762 763 # Set line range for each function. 764 for function in self.functions.id_to_func.values(): 765 if function.func_name == 'unknown': 766 continue 767 dso = addr2line.get_dso(self.libs.get_lib(function.lib_id).name) 768 if not dso: 769 continue 770 start_source = addr2line.get_addr_source(dso, function.start_addr) 771 end_source = addr2line.get_addr_source(dso, function.start_addr + function.addr_len - 1) 772 if not start_source or not end_source: 773 continue 774 start_file_path, start_line = start_source[-1] 775 end_file_path, end_line = end_source[-1] 776 if start_file_path != end_file_path or start_line > end_line: 777 continue 778 source_file = self.source_files.get_source_file(start_file_path) 779 source_file.request_lines(start_line, end_line) 780 function.source_info = (source_file.file_id, start_line, end_line) 781 782 # Build FunctionScope.line_hit_map. 783 for event in self.events.values(): 784 for lib in event.libraries: 785 dso = addr2line.get_dso(self.libs.get_lib(lib.lib_id).name) 786 if not dso: 787 continue 788 for function in lib.functions.values(): 789 for addr in function.addr_hit_map: 790 source = addr2line.get_addr_source(dso, addr) 791 if not source: 792 continue 793 for file_path, line in source: 794 source_file = self.source_files.get_source_file(file_path) 795 # Show [line - 5, line + 5] of the line hit by a sample. 796 source_file.request_lines(line - 5, line + 5) 797 count_info = function.addr_hit_map[addr] 798 function.build_line_hit_map(source_file.file_id, line, count_info[0], 799 count_info[1]) 800 801 # Collect needed source code in SourceFileSet. 802 self.source_files.load_source_code(source_dirs) 803 804 def add_disassembly(self, filter_lib: Callable[[str], bool], 805 jobs: int, disassemble_job_size: int): 806 """ Collect disassembly information: 807 1. Use objdump to collect disassembly for each function in FunctionSet. 808 2. Set flag to dump addr_hit_map when generating record info. 809 """ 810 objdump = Objdump(self.ndk_path, self.binary_finder) 811 lib_functions: Dict[int, List[Function]] = collections.defaultdict(list) 812 813 for function in self.functions.id_to_func.values(): 814 if function.func_name == 'unknown': 815 continue 816 lib_functions[function.lib_id].append(function) 817 818 with ThreadPoolExecutor(jobs) as executor: 819 futures: List[Future] = [] 820 all_tasks = [] 821 for lib_id, functions in lib_functions.items(): 822 lib = self.libs.get_lib(lib_id) 823 if not filter_lib(lib.name): 824 continue 825 dso_info = objdump.get_dso_info(lib.name, lib.build_id) 826 if not dso_info: 827 continue 828 829 tasks = self.split_disassembly_jobs(functions, disassemble_job_size) 830 logging.debug('create %d jobs to disassemble %d functions in %s', 831 len(tasks), len(functions), lib.name) 832 for task in tasks: 833 futures.append(executor.submit( 834 self._disassemble_functions, objdump, dso_info, task)) 835 all_tasks.append(task) 836 837 for task, future in zip(all_tasks, futures): 838 result = future.result() 839 if result and len(result) == len(task): 840 for function, disassembly in zip(task, result): 841 function.disassembly = disassembly.lines 842 843 logging.debug('finished all disassemble jobs') 844 self.gen_addr_hit_map_in_record_info = True 845 846 def split_disassembly_jobs(self, functions: List[Function], 847 disassemble_job_size: int) -> List[List[Function]]: 848 """ Decide how to split the task of dissassembly functions in one library. """ 849 if not functions: 850 return [] 851 functions.sort(key=lambda f: f.start_addr) 852 result = [] 853 job_start_addr = None 854 for function in functions: 855 if (job_start_addr is None or 856 function.start_addr - job_start_addr > disassemble_job_size): 857 job_start_addr = function.start_addr 858 result.append([function]) 859 else: 860 result[-1].append(function) 861 return result 862 863 def _disassemble_functions(self, objdump: Objdump, dso_info, 864 functions: List[Function]) -> Optional[List[Disassembly]]: 865 addr_ranges = [AddrRange(f.start_addr, f.addr_len) for f in functions] 866 return objdump.disassemble_functions(dso_info, addr_ranges) 867 868 def gen_record_info(self) -> Dict[str, Any]: 869 """ Return json data which will be used by report_html.js. """ 870 record_info = {} 871 timestamp = self.meta_info.get('timestamp') 872 if timestamp: 873 t = datetime.datetime.fromtimestamp(int(timestamp)) 874 else: 875 t = datetime.datetime.now() 876 record_info['recordTime'] = t.strftime('%Y-%m-%d (%A) %H:%M:%S') 877 878 product_props = self.meta_info.get('product_props') 879 machine_type = self.arch 880 if product_props: 881 manufacturer, model, name = product_props.split(':') 882 machine_type = '%s (%s) by %s, arch %s' % (model, name, manufacturer, self.arch) 883 record_info['machineType'] = machine_type 884 record_info['androidVersion'] = self.meta_info.get('android_version', '') 885 record_info['androidBuildFingerprint'] = self.meta_info.get('android_build_fingerprint', '') 886 record_info['kernelVersion'] = self.meta_info.get('kernel_version', '') 887 record_info['recordCmdline'] = self.cmdline 888 record_info['totalSamples'] = self.total_samples 889 record_info['processNames'] = self._gen_process_names() 890 record_info['threadNames'] = self._gen_thread_names() 891 record_info['libList'] = self._gen_lib_list() 892 record_info['functionMap'] = self._gen_function_map() 893 record_info['sampleInfo'] = self._gen_sample_info() 894 record_info['sourceFiles'] = self._gen_source_files() 895 return record_info 896 897 def _gen_process_names(self) -> Dict[int, str]: 898 process_names: Dict[int, str] = {} 899 for event in self.events.values(): 900 for process in event.processes.values(): 901 process_names[process.pid] = process.name 902 return process_names 903 904 def _gen_thread_names(self) -> Dict[int, str]: 905 thread_names: Dict[int, str] = {} 906 for event in self.events.values(): 907 for process in event.processes.values(): 908 for thread in process.threads.values(): 909 thread_names[thread.tid] = thread.name 910 return thread_names 911 912 def _gen_lib_list(self) -> List[str]: 913 return [modify_text_for_html(lib.name) for lib in self.libs.libs] 914 915 def _gen_function_map(self) -> Dict[int, Any]: 916 func_map: Dict[int, Any] = {} 917 for func_id in sorted(self.functions.id_to_func): 918 function = self.functions.id_to_func[func_id] 919 func_data = {} 920 func_data['l'] = function.lib_id 921 func_data['f'] = modify_text_for_html(function.func_name) 922 if function.source_info: 923 func_data['s'] = function.source_info 924 if function.disassembly: 925 disassembly_list = [] 926 for code, addr in function.disassembly: 927 disassembly_list.append( 928 [modify_text_for_html(code), 929 hex_address_for_json(addr)]) 930 func_data['d'] = disassembly_list 931 func_map[func_id] = func_data 932 return func_map 933 934 def _gen_sample_info(self) -> List[Dict[str, Any]]: 935 return [event.get_sample_info(self.gen_addr_hit_map_in_record_info) 936 for event in self.events.values()] 937 938 def _gen_source_files(self) -> List[Dict[str, Any]]: 939 source_files = sorted(self.source_files.path_to_source_files.values(), 940 key=lambda x: x.file_id) 941 file_list = [] 942 for source_file in source_files: 943 file_data = {} 944 if not source_file.real_path: 945 file_data['path'] = '' 946 file_data['code'] = {} 947 else: 948 file_data['path'] = source_file.real_path 949 code_map = {} 950 for line in source_file.line_to_code: 951 code_map[line] = modify_text_for_html(source_file.line_to_code[line]) 952 file_data['code'] = code_map 953 file_list.append(file_data) 954 return file_list 955 956 957URLS = { 958 'jquery': 'https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js', 959 'bootstrap4-css': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.2/css/bootstrap.min.css', 960 'bootstrap4-popper': 961 'https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js', 962 'bootstrap4': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.2/js/bootstrap.min.js', 963 'dataTable': 'https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js', 964 'dataTable-bootstrap4': 'https://cdn.datatables.net/1.10.19/js/dataTables.bootstrap4.min.js', 965 'dataTable-css': 'https://cdn.datatables.net/1.10.19/css/dataTables.bootstrap4.min.css', 966 'gstatic-charts': 'https://www.gstatic.com/charts/loader.js', 967} 968 969 970class ReportGenerator(object): 971 972 def __init__(self, html_path: Union[Path, str]): 973 self.hw = HtmlWriter(html_path) 974 self.hw.open_tag('html') 975 self.hw.open_tag('head') 976 for css in ['bootstrap4-css', 'dataTable-css']: 977 self.hw.open_tag('link', rel='stylesheet', type='text/css', href=URLS[css]).close_tag() 978 for js in ['jquery', 'bootstrap4-popper', 'bootstrap4', 'dataTable', 'dataTable-bootstrap4', 979 'gstatic-charts']: 980 self.hw.open_tag('script', src=URLS[js]).close_tag() 981 982 self.hw.open_tag('script').add( 983 "google.charts.load('current', {'packages': ['corechart', 'table']});").close_tag() 984 self.hw.open_tag('style', type='text/css').add(""" 985 .colForLine { width: 50px; text-align: right; } 986 .colForCount { width: 100px; text-align: right; } 987 .tableCell { font-size: 17px; } 988 .boldTableCell { font-weight: bold; font-size: 17px; } 989 .textRight { text-align: right; } 990 """).close_tag() 991 self.hw.close_tag('head') 992 self.hw.open_tag('body') 993 994 def write_content_div(self): 995 self.hw.open_tag('div', id='report_content').close_tag() 996 997 def write_record_data(self, record_data: Dict[str, Any]): 998 self.hw.open_tag('script', id='record_data', type='application/json') 999 self.hw.add(json.dumps(record_data)) 1000 self.hw.close_tag() 1001 1002 def write_script(self): 1003 self.hw.open_tag('script').add_file('report_html.js').close_tag() 1004 1005 def finish(self): 1006 self.hw.close_tag('body') 1007 self.hw.close_tag('html') 1008 self.hw.close() 1009 1010 1011def get_args() -> argparse.Namespace: 1012 parser = BaseArgumentParser(description='report profiling data') 1013 parser.add_argument('-i', '--record_file', nargs='+', default=['perf.data'], help=""" 1014 Set profiling data file to report.""") 1015 parser.add_argument('-o', '--report_path', default='report.html', help='Set output html file') 1016 parser.add_argument('--min_func_percent', default=0.01, type=float, help=""" 1017 Set min percentage of functions shown in the report. 1018 For example, when set to 0.01, only functions taking >= 0.01%% of total 1019 event count are collected in the report.""") 1020 parser.add_argument('--min_callchain_percent', default=0.01, type=float, help=""" 1021 Set min percentage of callchains shown in the report. 1022 It is used to limit nodes shown in the function flamegraph. For example, 1023 when set to 0.01, only callchains taking >= 0.01%% of the event count of 1024 the starting function are collected in the report.""") 1025 parser.add_argument('--add_source_code', action='store_true', help='Add source code.') 1026 parser.add_argument('--source_dirs', nargs='+', help='Source code directories.') 1027 parser.add_argument('--add_disassembly', action='store_true', help='Add disassembled code.') 1028 parser.add_argument('--disassemble-job-size', type=int, default=1024*1024, 1029 help='address range for one disassemble job') 1030 parser.add_argument('--binary_filter', nargs='+', help="""Annotate source code and disassembly 1031 only for selected binaries, whose recorded paths contains [BINARY_FILTER] as 1032 a substring. Example: to select binaries belonging to an app with package 1033 name 'com.example.myapp', use `--binary_filter com.example.myapp`. 1034 """) 1035 parser.add_argument( 1036 '-j', '--jobs', type=int, default=os.cpu_count(), 1037 help='Use multithreading to speed up disassembly and source code annotation.') 1038 parser.add_argument('--ndk_path', nargs=1, help='Find tools in the ndk path.') 1039 parser.add_argument('--no_browser', action='store_true', help="Don't open report in browser.") 1040 parser.add_argument('--aggregate-by-thread-name', action='store_true', help="""aggregate 1041 samples by thread name instead of thread id. This is useful for 1042 showing multiple perf.data generated for the same app.""") 1043 parser.add_report_lib_options() 1044 return parser.parse_args() 1045 1046 1047def main(): 1048 sys.setrecursionlimit(MAX_CALLSTACK_LENGTH * 2 + 50) 1049 args = get_args() 1050 1051 # 1. Process args. 1052 binary_cache_path = 'binary_cache' 1053 if not os.path.isdir(binary_cache_path): 1054 if args.add_source_code or args.add_disassembly: 1055 log_exit("""binary_cache/ doesn't exist. Can't add source code or disassembled code 1056 without collected binaries. Please run binary_cache_builder.py to 1057 collect binaries for current profiling data, or run app_profiler.py 1058 without -nb option.""") 1059 binary_cache_path = None 1060 1061 if args.add_source_code and not args.source_dirs: 1062 log_exit('--source_dirs is needed to add source code.') 1063 build_addr_hit_map = args.add_source_code or args.add_disassembly 1064 ndk_path = None if not args.ndk_path else args.ndk_path[0] 1065 if args.jobs < 1: 1066 log_exit('Invalid --jobs option.') 1067 1068 # 2. Produce record data. 1069 record_data = RecordData(binary_cache_path, ndk_path, build_addr_hit_map) 1070 for record_file in args.record_file: 1071 record_data.load_record_file(record_file, args.report_lib_options) 1072 if args.aggregate_by_thread_name: 1073 record_data.aggregate_by_thread_name() 1074 record_data.limit_percents(args.min_func_percent, args.min_callchain_percent) 1075 record_data.sort_call_graph_by_function_name() 1076 1077 def filter_lib(lib_name: str) -> bool: 1078 if not args.binary_filter: 1079 return True 1080 for binary in args.binary_filter: 1081 if binary in lib_name: 1082 return True 1083 return False 1084 if args.add_source_code: 1085 record_data.add_source_code(args.source_dirs, filter_lib, args.jobs) 1086 if args.add_disassembly: 1087 record_data.add_disassembly(filter_lib, args.jobs, args.disassemble_job_size) 1088 1089 # 3. Generate report html. 1090 report_generator = ReportGenerator(args.report_path) 1091 report_generator.write_script() 1092 report_generator.write_content_div() 1093 report_generator.write_record_data(record_data.gen_record_info()) 1094 report_generator.finish() 1095 1096 if not args.no_browser: 1097 open_report_in_browser(args.report_path) 1098 logging.info("Report generated at '%s'." % args.report_path) 1099 1100 1101if __name__ == '__main__': 1102 main() 1103