1#!/usr/bin/env python3
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18"""app_profiler.py: Record cpu profiling data of an android app or native program.
19
20    It downloads simpleperf on device, uses it to collect profiling data on the selected app,
21    and pulls profiling data and related binaries on host.
22"""
23
24import logging
25import os
26import os.path
27import re
28import subprocess
29import sys
30import time
31from typing import Optional
32
33from simpleperf_utils import (
34    AdbHelper, BaseArgumentParser, bytes_to_str, extant_dir, get_script_dir, get_target_binary_path,
35    log_exit, ReadElf, remove, str_to_bytes)
36
37NATIVE_LIBS_DIR_ON_DEVICE = '/data/local/tmp/native_libs/'
38
39SHELL_PS_UID_PATTERN = re.compile(r'USER.*\nu(\d+)_.*')
40
41
42class HostElfEntry(object):
43    """Represent a native lib on host in NativeLibDownloader."""
44
45    def __init__(self, path, name, score):
46        self.path = path
47        self.name = name
48        self.score = score
49
50    def __repr__(self):
51        return self.__str__()
52
53    def __str__(self):
54        return '[path: %s, name %s, score %s]' % (self.path, self.name, self.score)
55
56
57class NativeLibDownloader(object):
58    """Download native libs on device.
59
60    1. Collect info of all native libs in the native_lib_dir on host.
61    2. Check the available native libs in /data/local/tmp/native_libs on device.
62    3. Sync native libs on device.
63    """
64
65    def __init__(self, ndk_path, device_arch, adb):
66        self.adb = adb
67        self.readelf = ReadElf(ndk_path)
68        self.device_arch = device_arch
69        self.need_archs = self._get_need_archs()
70        self.host_build_id_map = {}  # Map from build_id to HostElfEntry.
71        self.device_build_id_map = {}  # Map from build_id to relative_path on device.
72        # Map from filename to HostElfEntry for elf files without build id.
73        self.no_build_id_file_map = {}
74        self.name_count_map = {}  # Used to give a unique name for each library.
75        self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
76        self.build_id_list_file = 'build_id_list'
77
78    def _get_need_archs(self):
79        """Return the archs of binaries needed on device."""
80        if self.device_arch == 'arm64':
81            return ['arm', 'arm64']
82        if self.device_arch == 'arm':
83            return ['arm']
84        if self.device_arch == 'x86_64':
85            return ['x86', 'x86_64']
86        if self.device_arch == 'x86':
87            return ['x86']
88        return []
89
90    def collect_native_libs_on_host(self, native_lib_dir):
91        self.host_build_id_map.clear()
92        for root, _, files in os.walk(native_lib_dir):
93            for name in files:
94                if not name.endswith('.so'):
95                    continue
96                self.add_native_lib_on_host(os.path.join(root, name), name)
97
98    def add_native_lib_on_host(self, path, name):
99        arch = self.readelf.get_arch(path)
100        if arch not in self.need_archs:
101            return
102        sections = self.readelf.get_sections(path)
103        score = 0
104        if '.debug_info' in sections:
105            score = 3
106        elif '.gnu_debugdata' in sections:
107            score = 2
108        elif '.symtab' in sections:
109            score = 1
110        build_id = self.readelf.get_build_id(path)
111        if build_id:
112            entry = self.host_build_id_map.get(build_id)
113            if entry:
114                if entry.score < score:
115                    entry.path = path
116                    entry.score = score
117            else:
118                repeat_count = self.name_count_map.get(name, 0)
119                self.name_count_map[name] = repeat_count + 1
120                unique_name = name if repeat_count == 0 else name + '_' + str(repeat_count)
121                self.host_build_id_map[build_id] = HostElfEntry(path, unique_name, score)
122        else:
123            entry = self.no_build_id_file_map.get(name)
124            if entry:
125                if entry.score < score:
126                    entry.path = path
127                    entry.score = score
128            else:
129                self.no_build_id_file_map[name] = HostElfEntry(path, name, score)
130
131    def collect_native_libs_on_device(self):
132        self.device_build_id_map.clear()
133        self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
134        if os.path.exists(self.build_id_list_file):
135            os.remove(self.build_id_list_file)
136        result, output = self.adb.run_and_return_output(['shell', 'ls', self.dir_on_device])
137        if not result:
138            return
139        file_set = set(output.strip().split())
140        if self.build_id_list_file not in file_set:
141            return
142        self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
143        if os.path.exists(self.build_id_list_file):
144            with open(self.build_id_list_file, 'rb') as fh:
145                for line in fh.readlines():
146                    line = bytes_to_str(line).strip()
147                    items = line.split('=')
148                    if len(items) == 2:
149                        build_id, filename = items
150                        if filename in file_set:
151                            self.device_build_id_map[build_id] = filename
152            remove(self.build_id_list_file)
153
154    def sync_native_libs_on_device(self):
155        # Push missing native libs on device.
156        for build_id in self.host_build_id_map:
157            if build_id not in self.device_build_id_map:
158                entry = self.host_build_id_map[build_id]
159                self.adb.check_run(['push', entry.path, self.dir_on_device + entry.name])
160        # Remove native libs not exist on host.
161        for build_id in self.device_build_id_map:
162            if build_id not in self.host_build_id_map:
163                name = self.device_build_id_map[build_id]
164                self.adb.run(['shell', 'rm', self.dir_on_device + name])
165        # Push new build_id_list on device.
166        with open(self.build_id_list_file, 'wb') as fh:
167            for build_id in self.host_build_id_map:
168                s = str_to_bytes('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name))
169                fh.write(s)
170        self.adb.check_run(['push', self.build_id_list_file,
171                            self.dir_on_device + self.build_id_list_file])
172        os.remove(self.build_id_list_file)
173
174        # Push elf files without build id on device.
175        for entry in self.no_build_id_file_map.values():
176            target = self.dir_on_device + entry.name
177
178            # Skip download if we have a file with the same name and size on device.
179            result, output = self.adb.run_and_return_output(['shell', 'ls', '-l', target])
180            if result:
181                items = output.split()
182                if len(items) > 5:
183                    try:
184                        file_size = int(items[4])
185                    except ValueError:
186                        file_size = 0
187                    if file_size == os.path.getsize(entry.path):
188                        continue
189            self.adb.check_run(['push', entry.path, target])
190
191
192class ProfilerBase(object):
193    """Base class of all Profilers."""
194
195    def __init__(self, args):
196        self.args = args
197        self.adb = AdbHelper(enable_switch_to_root=not args.disable_adb_root)
198        if not self.adb.is_device_available():
199            log_exit('No Android device is connected via ADB.')
200        self.is_root_device = self.adb.switch_to_root()
201        self.android_version = self.adb.get_android_version()
202        if self.android_version < 7:
203            log_exit("""app_profiler.py isn't supported on Android < N, please switch to use
204                        simpleperf binary directly.""")
205        self.device_arch = self.adb.get_device_arch()
206        self.record_subproc = None
207
208    def profile(self):
209        logging.info('prepare profiling')
210        self.prepare()
211        logging.info('start profiling')
212        self.start()
213        self.wait_profiling()
214        logging.info('collect profiling data')
215        self.collect_profiling_data()
216        logging.info('profiling is finished.')
217
218    def prepare(self):
219        """Prepare recording. """
220        self.download_simpleperf()
221        if self.args.native_lib_dir:
222            self.download_libs()
223
224    def download_simpleperf(self):
225        simpleperf_binary = get_target_binary_path(self.device_arch, 'simpleperf')
226        self.adb.check_run(['push', simpleperf_binary, '/data/local/tmp'])
227        self.adb.check_run(['shell', 'chmod', 'a+x', '/data/local/tmp/simpleperf'])
228
229    def download_libs(self):
230        downloader = NativeLibDownloader(self.args.ndk_path, self.device_arch, self.adb)
231        downloader.collect_native_libs_on_host(self.args.native_lib_dir)
232        downloader.collect_native_libs_on_device()
233        downloader.sync_native_libs_on_device()
234
235    def start(self):
236        raise NotImplementedError
237
238    def start_profiling(self, target_args):
239        """Start simpleperf record process on device."""
240        args = ['/data/local/tmp/simpleperf', 'record', '-o', '/data/local/tmp/perf.data',
241                self.args.record_options]
242        if self.adb.run(['shell', 'ls', NATIVE_LIBS_DIR_ON_DEVICE]):
243            args += ['--symfs', NATIVE_LIBS_DIR_ON_DEVICE]
244        args += ['--log', self.args.log]
245        args += target_args
246        adb_args = [self.adb.adb_path, 'shell'] + args
247        logging.info('run adb cmd: %s' % adb_args)
248        self.record_subproc = subprocess.Popen(adb_args)
249
250    def wait_profiling(self):
251        """Wait until profiling finishes, or stop profiling when user presses Ctrl-C."""
252        returncode = None
253        try:
254            returncode = self.record_subproc.wait()
255        except KeyboardInterrupt:
256            self.stop_profiling()
257            self.record_subproc = None
258            # Don't check return value of record_subproc. Because record_subproc also
259            # receives Ctrl-C, and always returns non-zero.
260            returncode = 0
261        logging.debug('profiling result [%s]' % (returncode == 0))
262        if returncode != 0:
263            log_exit('Failed to record profiling data.')
264
265    def stop_profiling(self):
266        """Stop profiling by sending SIGINT to simpleperf, and wait until it exits
267           to make sure perf.data is completely generated."""
268        has_killed = False
269        while True:
270            (result, _) = self.adb.run_and_return_output(['shell', 'pidof', 'simpleperf'])
271            if not result:
272                break
273            if not has_killed:
274                has_killed = True
275                self.adb.run_and_return_output(['shell', 'pkill', '-l', '2', 'simpleperf'])
276            time.sleep(1)
277
278    def collect_profiling_data(self):
279        self.adb.check_run_and_return_output(['pull', '/data/local/tmp/perf.data',
280                                              self.args.perf_data_path])
281        if not self.args.skip_collect_binaries:
282            binary_cache_args = [sys.executable,
283                                 os.path.join(get_script_dir(), 'binary_cache_builder.py')]
284            binary_cache_args += ['-i', self.args.perf_data_path, '--log', self.args.log]
285            if self.args.native_lib_dir:
286                binary_cache_args += ['-lib', self.args.native_lib_dir]
287            if self.args.disable_adb_root:
288                binary_cache_args += ['--disable_adb_root']
289            if self.args.ndk_path:
290                binary_cache_args += ['--ndk_path', self.args.ndk_path]
291            subprocess.check_call(binary_cache_args)
292
293
294class AppProfiler(ProfilerBase):
295    """Profile an Android app."""
296
297    def prepare(self):
298        super(AppProfiler, self).prepare()
299        self.app_versioncode = self.get_app_versioncode()
300        if self.args.compile_java_code:
301            self.compile_java_code()
302
303    def get_app_versioncode(self) -> Optional[str]:
304        result, output = self.adb.run_and_return_output(
305            ['shell', 'pm', 'list', 'packages', '--show-versioncode'])
306        if not result:
307            return None
308        prefix = f'package:{self.args.app} '
309        for line in output.splitlines():
310            if line.startswith(prefix):
311                pos = line.find('versionCode:')
312                if pos != -1:
313                    return line[pos + len('versionCode:'):].strip()
314        return None
315
316    def compile_java_code(self):
317        self.kill_app_process()
318        # Fully compile Java code on Android >= N.
319        self.adb.set_property('debug.generate-debug-info', 'true')
320        self.adb.check_run(['shell', 'cmd', 'package', 'compile', '-f', '-m', 'speed',
321                            self.args.app])
322
323    def kill_app_process(self):
324        if self.find_app_process():
325            self.adb.check_run(['shell', 'am', 'force-stop', self.args.app])
326            count = 0
327            while True:
328                time.sleep(1)
329                pid = self.find_app_process()
330                if not pid:
331                    break
332                count += 1
333                if count >= 5:
334                    logging.info('unable to kill %s, skipping...' % self.args.app)
335                    break
336                # When testing on Android N, `am force-stop` sometimes can't kill
337                # com.example.simpleperf.simpleperfexampleofkotlin. So use kill when this happens.
338                if count >= 3:
339                    self.run_in_app_dir(['kill', '-9', str(pid)])
340
341    def find_app_process(self):
342        result, pidof_output = self.adb.run_and_return_output(
343            ['shell', 'pidof', self.args.app])
344        if not result:
345            return None
346        result, current_user = self.adb.run_and_return_output(
347            ['shell', 'am', 'get-current-user'])
348        if not result:
349            return None
350        pids = pidof_output.split()
351        for pid in pids:
352            result, ps_output = self.adb.run_and_return_output(
353                ['shell', 'ps', '-p', pid, '-o', 'USER'])
354            if not result:
355                return None
356            uid = SHELL_PS_UID_PATTERN.search(ps_output).group(1)
357            if uid == current_user.strip():
358                return int(pid)
359        return None
360
361    def run_in_app_dir(self, args):
362        if self.is_root_device:
363            adb_args = ['shell', 'cd /data/data/' + self.args.app + ' && ' + (' '.join(args))]
364        else:
365            adb_args = ['shell', 'run-as', self.args.app] + args
366        return self.adb.run_and_return_output(adb_args)
367
368    def start(self):
369        if self.args.launch or self.args.activity or self.args.test:
370            self.kill_app_process()
371        args = ['--app', self.args.app]
372        if self.app_versioncode:
373            args += ['--add-meta-info', f'app_versioncode={self.app_versioncode}']
374        self.start_profiling(args)
375        if self.args.launch:
376            self.start_app()
377        if self.args.activity:
378            self.start_activity()
379        elif self.args.test:
380            self.start_test()
381        # else: no need to start an activity or test.
382
383    def start_app(self):
384        result = self.adb.run(['shell', 'monkey', '-p', self.args.app, '1'])
385        if not result:
386            self.record_subproc.terminate()
387            log_exit(f"Can't start {self.args.app}")
388
389    def start_activity(self):
390        activity = self.args.app + '/' + self.args.activity
391        result = self.adb.run(['shell', 'am', 'start', '-n', activity])
392        if not result:
393            self.record_subproc.terminate()
394            log_exit("Can't start activity %s" % activity)
395
396    def start_test(self):
397        runner = self.args.app + '/androidx.test.runner.AndroidJUnitRunner'
398        result = self.adb.run(['shell', 'am', 'instrument', '-e', 'class',
399                               self.args.test, runner])
400        if not result:
401            self.record_subproc.terminate()
402            log_exit("Can't start instrumentation test  %s" % self.args.test)
403
404
405class NativeProgramProfiler(ProfilerBase):
406    """Profile a native program."""
407
408    def start(self):
409        logging.info('Waiting for native process %s' % self.args.native_program)
410        while True:
411            (result, pid) = self.adb.run_and_return_output(['shell', 'pidof',
412                                                            self.args.native_program])
413            if not result:
414                # Wait for 1 millisecond.
415                time.sleep(0.001)
416            else:
417                self.start_profiling(['-p', str(int(pid))])
418                break
419
420
421class NativeCommandProfiler(ProfilerBase):
422    """Profile running a native command."""
423
424    def start(self):
425        self.start_profiling([self.args.cmd])
426
427
428class NativeProcessProfiler(ProfilerBase):
429    """Profile processes given their pids."""
430
431    def start(self):
432        self.start_profiling(['-p', ','.join(self.args.pid)])
433
434
435class NativeThreadProfiler(ProfilerBase):
436    """Profile threads given their tids."""
437
438    def start(self):
439        self.start_profiling(['-t', ','.join(self.args.tid)])
440
441
442class SystemWideProfiler(ProfilerBase):
443    """Profile system wide."""
444
445    def start(self):
446        self.start_profiling(['-a'])
447
448
449def main():
450    parser = BaseArgumentParser(description=__doc__)
451
452    target_group = parser.add_argument_group(title='Select profiling target'
453                                             ).add_mutually_exclusive_group(required=True)
454    target_group.add_argument('-p', '--app', help="""Profile an Android app, given the package name.
455                              Like `-p com.example.android.myapp`.""")
456
457    target_group.add_argument('-np', '--native_program', help="""Profile a native program running on
458                              the Android device. Like `-np surfaceflinger`.""")
459
460    target_group.add_argument('-cmd', help="""Profile running a command on the Android device.
461                              Like `-cmd "pm -l"`.""")
462
463    target_group.add_argument('--pid', nargs='+', help="""Profile native processes running on device
464                              given their process ids.""")
465
466    target_group.add_argument('--tid', nargs='+', help="""Profile native threads running on device
467                              given their thread ids.""")
468
469    target_group.add_argument('--system_wide', action='store_true', help="""Profile system wide.""")
470
471    app_target_group = parser.add_argument_group(title='Extra options for profiling an app')
472    app_target_group.add_argument('--compile_java_code', action='store_true', help="""Used with -p.
473                                  On Android N and Android O, we need to compile Java code into
474                                  native instructions to profile Java code. Android O also needs
475                                  wrap.sh in the apk to use the native instructions.""")
476
477    app_start_group = app_target_group.add_mutually_exclusive_group()
478    app_start_group.add_argument('--launch', action='store_true', help="""Used with -p. Profile the
479                                 launch time of an Android app. The app will be started or
480                                 restarted.""")
481    app_start_group.add_argument('-a', '--activity', help="""Used with -p. Profile the launch time
482                                 of an activity in an Android app. The app will be started or
483                                 restarted to run the activity. Like `-a .MainActivity`.""")
484
485    app_start_group.add_argument('-t', '--test', help="""Used with -p. Profile the launch time of an
486                                 instrumentation test in an Android app. The app will be started or
487                                 restarted to run the instrumentation test. Like
488                                 `-t test_class_name`.""")
489
490    record_group = parser.add_argument_group('Select recording options')
491    record_group.add_argument('-r', '--record_options',
492                              default='-e task-clock:u -f 1000 -g --duration 10', help="""Set
493                              recording options for `simpleperf record` command. Use
494                              `run_simpleperf_on_device.py record -h` to see all accepted options.
495                              Default is "-e task-clock:u -f 1000 -g --duration 10".""")
496
497    record_group.add_argument('-lib', '--native_lib_dir', type=extant_dir,
498                              help="""When profiling an Android app containing native libraries,
499                                      the native libraries are usually stripped and lake of symbols
500                                      and debug information to provide good profiling result. By
501                                      using -lib, you tell app_profiler.py the path storing
502                                      unstripped native libraries, and app_profiler.py will search
503                                      all shared libraries with suffix .so in the directory. Then
504                                      the native libraries will be downloaded on device and
505                                      collected in build_cache.""")
506
507    record_group.add_argument('-o', '--perf_data_path', default='perf.data',
508                              help='The path to store profiling data. Default is perf.data.')
509
510    record_group.add_argument('-nb', '--skip_collect_binaries', action='store_true',
511                              help="""By default we collect binaries used in profiling data from
512                                      device to binary_cache directory. It can be used to annotate
513                                      source code and disassembly. This option skips it.""")
514
515    other_group = parser.add_argument_group('Other options')
516    other_group.add_argument('--ndk_path', type=extant_dir,
517                             help="""Set the path of a ndk release. app_profiler.py needs some
518                                     tools in ndk, like readelf.""")
519
520    other_group.add_argument('--disable_adb_root', action='store_true',
521                             help="""Force adb to run in non root mode. By default, app_profiler.py
522                                     will try to switch to root mode to be able to profile released
523                                     Android apps.""")
524
525    def check_args(args):
526        if (not args.app) and (args.compile_java_code or args.activity or args.test):
527            log_exit('--compile_java_code, -a, -t can only be used when profiling an Android app.')
528
529    args = parser.parse_args()
530    check_args(args)
531    if args.app:
532        profiler = AppProfiler(args)
533    elif args.native_program:
534        profiler = NativeProgramProfiler(args)
535    elif args.cmd:
536        profiler = NativeCommandProfiler(args)
537    elif args.pid:
538        profiler = NativeProcessProfiler(args)
539    elif args.tid:
540        profiler = NativeThreadProfiler(args)
541    elif args.system_wide:
542        profiler = SystemWideProfiler(args)
543    profiler.profile()
544
545
546if __name__ == '__main__':
547    main()
548