1#!/usr/bin/env python3
2
3import collections
4import os
5import re
6import shutil
7import subprocess
8import sys
9import tempfile
10import collections
11
12
13SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
14
15try:
16    AOSP_DIR = os.environ['ANDROID_BUILD_TOP']
17except KeyError:
18    print('error: ANDROID_BUILD_TOP environment variable is not set.',
19          file=sys.stderr)
20    sys.exit(1)
21
22BUILTIN_HEADERS_DIR = (
23    os.path.join(AOSP_DIR, 'bionic', 'libc', 'include'),
24    os.path.join(AOSP_DIR, 'external', 'libcxx', 'include'),
25    os.path.join(AOSP_DIR, 'prebuilts', 'clang-tools', 'linux-x86',
26                 'clang-headers'),
27)
28
29SO_EXT = '.so'
30SOURCE_ABI_DUMP_EXT_END = '.lsdump'
31SOURCE_ABI_DUMP_EXT = SO_EXT + SOURCE_ABI_DUMP_EXT_END
32KNOWN_ABI_DUMP_EXTS = {
33    SOURCE_ABI_DUMP_EXT,
34    SO_EXT + '.apex' + SOURCE_ABI_DUMP_EXT_END,
35    SO_EXT + '.llndk' + SOURCE_ABI_DUMP_EXT_END,
36}
37
38DEFAULT_CPPFLAGS = ['-x', 'c++', '-std=c++11']
39DEFAULT_CFLAGS = ['-std=gnu99']
40DEFAULT_HEADER_FLAGS = ["-dump-function-declarations"]
41DEFAULT_FORMAT = 'ProtobufTextFormat'
42
43BuildTarget = collections.namedtuple(
44    'BuildTarget', ['product', 'release', 'variant'])
45
46
47class Arch(object):
48    """A CPU architecture of a build target."""
49    def __init__(self, is_2nd, build_target):
50        extra = '_2ND' if is_2nd else ''
51        build_vars_to_fetch = ['TARGET_ARCH',
52                               'TARGET{}_ARCH'.format(extra),
53                               'TARGET{}_ARCH_VARIANT'.format(extra),
54                               'TARGET{}_CPU_VARIANT'.format(extra)]
55        build_vars = get_build_vars(build_vars_to_fetch, build_target)
56        self.primary_arch = build_vars[0]
57        assert self.primary_arch != ''
58        self.arch = build_vars[1]
59        self.arch_variant = build_vars[2]
60        self.cpu_variant = build_vars[3]
61
62    def get_arch_str(self):
63        """Return a string that represents the architecture and the primary
64        architecture.
65        """
66        if not self.arch or self.arch == self.primary_arch:
67            return self.primary_arch
68        return self.arch + '_' + self.primary_arch
69
70    def get_arch_cpu_str(self):
71        """Return a string that represents the architecture, the architecture
72        variant, and the CPU variant.
73
74        If TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant
75        empty. This is the case for aosp_x86_64.
76        """
77        if not self.arch_variant or self.arch_variant == self.arch:
78            arch_variant = ''
79        else:
80            arch_variant = '_' + self.arch_variant
81
82        if not self.cpu_variant or self.cpu_variant == 'generic':
83            cpu_variant = ''
84        else:
85            cpu_variant = '_' + self.cpu_variant
86
87        return self.arch + arch_variant + cpu_variant
88
89
90def _strip_dump_name_ext(filename):
91    """Remove .so*.lsdump from a file name."""
92    for ext in KNOWN_ABI_DUMP_EXTS:
93        if filename.endswith(ext) and len(filename) > len(ext):
94            return filename[:-len(ext)]
95    raise ValueError(f'{filename} has an unknown file name extension.')
96
97
98def _validate_dump_content(dump_path):
99    """Make sure that the dump contains relative source paths."""
100    with open(dump_path, 'r') as f:
101        for line_number, line in enumerate(f, 1):
102            start = 0
103            while True:
104                start = line.find(AOSP_DIR, start)
105                if start < 0:
106                    break
107                # The substring is not preceded by a common path character.
108                if start == 0 or not (line[start - 1].isalnum() or
109                                      line[start - 1] in '.-_/'):
110                    raise ValueError(f'{dump_path} contains absolute path to '
111                                     f'$ANDROID_BUILD_TOP at line '
112                                     f'{line_number}:\n{line}')
113                start += len(AOSP_DIR)
114
115
116def copy_reference_dump(lib_path, reference_dump_dir):
117    _validate_dump_content(lib_path)
118    ref_dump_name = (_strip_dump_name_ext(os.path.basename(lib_path)) +
119                     SOURCE_ABI_DUMP_EXT)
120    ref_dump_path = os.path.join(reference_dump_dir, ref_dump_name)
121    os.makedirs(reference_dump_dir, exist_ok=True)
122    shutil.copyfile(lib_path, ref_dump_path)
123    print(f'Created abi dump at {ref_dump_path}')
124    return ref_dump_path
125
126
127def run_header_abi_dumper(input_path, output_path, cflags=tuple(),
128                          export_include_dirs=tuple(), flags=tuple()):
129    """Run header-abi-dumper to dump ABI from `input_path` and the output is
130    written to `output_path`."""
131    input_ext = os.path.splitext(input_path)[1]
132    cmd = ['header-abi-dumper', '-o', output_path, input_path]
133    for dir in export_include_dirs:
134        cmd += ['-I', dir]
135    cmd += flags
136    if '-output-format' not in flags:
137        cmd += ['-output-format', DEFAULT_FORMAT]
138    if input_ext == ".h":
139        cmd += DEFAULT_HEADER_FLAGS
140    cmd += ['--']
141    cmd += cflags
142    if input_ext in ('.cpp', '.cc', '.h'):
143        cmd += DEFAULT_CPPFLAGS
144    else:
145        cmd += DEFAULT_CFLAGS
146
147    for dir in BUILTIN_HEADERS_DIR:
148        cmd += ['-isystem', dir]
149    # The export include dirs imply local include dirs.
150    for dir in export_include_dirs:
151        cmd += ['-I', dir]
152    subprocess.check_call(cmd, cwd=AOSP_DIR)
153    _validate_dump_content(output_path)
154
155
156def run_header_abi_linker(inputs, output_path, version_script, api, arch_str,
157                          flags=tuple()):
158    """Link inputs, taking version_script into account"""
159    cmd = ['header-abi-linker', '-o', output_path, '-v', version_script,
160           '-api', api, '-arch', arch_str]
161    cmd += flags
162    if '-input-format' not in flags:
163        cmd += ['-input-format', DEFAULT_FORMAT]
164    if '-output-format' not in flags:
165        cmd += ['-output-format', DEFAULT_FORMAT]
166    cmd += inputs
167    subprocess.check_call(cmd, cwd=AOSP_DIR)
168    _validate_dump_content(output_path)
169
170
171def make_targets(build_target, args):
172    make_cmd = ['build/soong/soong_ui.bash', '--make-mode', '-j',
173                'TARGET_PRODUCT=' + build_target.product,
174                'TARGET_BUILD_VARIANT=' + build_target.variant]
175    if build_target.release:
176        make_cmd.append('TARGET_RELEASE=' + build_target.release)
177    make_cmd += args
178    subprocess.check_call(make_cmd, cwd=AOSP_DIR)
179
180
181def make_libraries(build_target, arches, libs, lsdump_filter):
182    """Build lsdump files for specific libs."""
183    lsdump_paths = read_lsdump_paths(build_target, arches, lsdump_filter,
184                                     build=True)
185    make_target_paths = []
186    for name in libs:
187        if not (name in lsdump_paths and lsdump_paths[name]):
188            raise KeyError('Cannot find lsdump for %s.' % name)
189        for tag_path_dict in lsdump_paths[name].values():
190            make_target_paths.extend(tag_path_dict.values())
191    make_targets(build_target, make_target_paths)
192
193
194def get_lsdump_paths_file_path(build_target):
195    """Get the path to lsdump_paths.txt."""
196    product_out = get_build_vars(['PRODUCT_OUT'], build_target)[0]
197    return os.path.join(product_out, 'lsdump_paths.txt')
198
199
200def _get_module_variant_sort_key(suffix):
201    for variant in suffix.split('_'):
202        match = re.match(r'apex(\d+)$', variant)
203        if match:
204            return (int(match.group(1)), suffix)
205    return (-1, suffix)
206
207
208def _get_module_variant_dir_name(tag, arch_cpu_str):
209    """Return the module variant directory name.
210
211    For example, android_x86_shared, android_vendor.R_arm_armv7-a-neon_shared.
212    """
213    if tag in ('LLNDK', 'NDK', 'PLATFORM', 'APEX'):
214        return f'android_{arch_cpu_str}_shared'
215    if tag == 'VENDOR':
216        return f'android_vendor_{arch_cpu_str}_shared'
217    if tag == 'PRODUCT':
218        return f'android_product_{arch_cpu_str}_shared'
219    raise ValueError(tag + ' is not a known tag.')
220
221
222def _read_lsdump_paths(lsdump_paths_file_path, arches, lsdump_filter):
223    """Read lsdump paths from lsdump_paths.txt for each libname and variant.
224
225    This function returns a dictionary, {lib_name: {arch_cpu: {tag: path}}}.
226    For example,
227    {
228      "libc": {
229        "x86_x86_64": {
230          "NDK": "path/to/libc.so.lsdump"
231        }
232      }
233    }
234    """
235    lsdump_paths = collections.defaultdict(
236        lambda: collections.defaultdict(dict))
237    suffixes = collections.defaultdict(
238        lambda: collections.defaultdict(dict))
239
240    with open(lsdump_paths_file_path, 'r') as lsdump_paths_file:
241        for line in lsdump_paths_file:
242            if not line.strip():
243                continue
244            tag, path = (x.strip() for x in line.split(':', 1))
245            dir_path, filename = os.path.split(path)
246            libname = _strip_dump_name_ext(filename)
247            if not lsdump_filter(tag, libname):
248                continue
249            # dir_path may contain soong config hash.
250            # For example, the following dir_paths are valid.
251            # android_x86_x86_64_shared/012abc/libc.so.lsdump
252            # android_x86_x86_64_shared/libc.so.lsdump
253            dirnames = []
254            dir_path, dirname = os.path.split(dir_path)
255            dirnames.append(dirname)
256            dirname = os.path.basename(dir_path)
257            dirnames.append(dirname)
258            for arch in arches:
259                arch_cpu = arch.get_arch_cpu_str()
260                prefix = _get_module_variant_dir_name(tag, arch_cpu)
261                variant = next((d for d in dirnames if d.startswith(prefix)),
262                               None)
263                if not variant:
264                    continue
265                new_suffix = variant[len(prefix):]
266                old_suffix = suffixes[libname][arch_cpu].get(tag)
267                if (not old_suffix or
268                        _get_module_variant_sort_key(new_suffix) >
269                        _get_module_variant_sort_key(old_suffix)):
270                    lsdump_paths[libname][arch_cpu][tag] = path
271                    suffixes[libname][arch_cpu][tag] = new_suffix
272    return lsdump_paths
273
274
275def read_lsdump_paths(build_target, arches, lsdump_filter, build):
276    """Build lsdump_paths.txt and read the paths."""
277    lsdump_paths_file_path = get_lsdump_paths_file_path(build_target)
278    lsdump_paths_file_abspath = os.path.join(AOSP_DIR, lsdump_paths_file_path)
279    if build:
280        if os.path.lexists(lsdump_paths_file_abspath):
281            os.unlink(lsdump_paths_file_abspath)
282        make_targets(build_target, [lsdump_paths_file_path])
283    return _read_lsdump_paths(lsdump_paths_file_abspath, arches, lsdump_filter)
284
285
286def find_lib_lsdumps(lsdump_paths, libs, arch):
287    """Find the lsdump corresponding to libs for the given architecture.
288
289    This function returns a list of (tag, absolute_path).
290    For example,
291    [
292      (
293        "NDK",
294        "/path/to/libc.so.lsdump"
295      )
296    ]
297    """
298    arch_cpu = arch.get_arch_cpu_str()
299    result = []
300    if libs:
301        for lib_name in libs:
302            if not (lib_name in lsdump_paths and
303                    arch_cpu in lsdump_paths[lib_name]):
304                raise KeyError('Cannot find lsdump for %s, %s.' %
305                               (lib_name, arch_cpu))
306            result.extend(lsdump_paths[lib_name][arch_cpu].items())
307    else:
308        for arch_tag_path_dict in lsdump_paths.values():
309            result.extend(arch_tag_path_dict[arch_cpu].items())
310    return [(tag, os.path.join(AOSP_DIR, path)) for tag, path in result]
311
312
313def run_abi_diff(old_dump_path, new_dump_path, output_path, arch_str, lib_name,
314                 flags):
315    abi_diff_cmd = ['header-abi-diff', '-new', new_dump_path, '-old',
316                    old_dump_path, '-arch', arch_str, '-lib', lib_name,
317                    '-o', output_path]
318    abi_diff_cmd += flags
319    if '-input-format-old' not in flags:
320        abi_diff_cmd += ['-input-format-old', DEFAULT_FORMAT]
321    if '-input-format-new' not in flags:
322        abi_diff_cmd += ['-input-format-new', DEFAULT_FORMAT]
323    return subprocess.run(abi_diff_cmd).returncode
324
325
326def run_and_read_abi_diff(old_dump_path, new_dump_path, arch_str, lib_name,
327                          flags=tuple()):
328    with tempfile.TemporaryDirectory() as tmp:
329        output_name = os.path.join(tmp, lib_name) + '.abidiff'
330        result = run_abi_diff(old_dump_path, new_dump_path, output_name,
331                              arch_str, lib_name, flags)
332        with open(output_name, 'r') as output_file:
333            return result, output_file.read()
334
335
336def get_build_vars(names, build_target):
337    """ Get build system variable for the launched target."""
338    env = os.environ.copy()
339    env['TARGET_PRODUCT'] = build_target.product
340    env['TARGET_BUILD_VARIANT'] = build_target.variant
341    if build_target.release:
342        env['TARGET_RELEASE'] = build_target.release
343    cmd = [
344        os.path.join('build', 'soong', 'soong_ui.bash'),
345        '--dumpvars-mode', '-vars', ' '.join(names),
346    ]
347
348    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
349                            stderr=subprocess.PIPE, cwd=AOSP_DIR, env=env)
350    out, err = proc.communicate()
351
352    if proc.returncode != 0:
353        print("error: %s" % err.decode('utf-8'), file=sys.stderr)
354        return None
355
356    build_vars = out.decode('utf-8').strip().splitlines()
357
358    build_vars_list = []
359    for build_var in build_vars:
360        value = build_var.partition('=')[2]
361        build_vars_list.append(value.replace('\'', ''))
362    return build_vars_list
363