1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import json
19import logging
20import os
21import shutil
22import sys
23import tempfile
24import unittest
25
26from vts.testcases.vndk import utils
27from vts.testcases.vndk.golden import vndk_data
28from vts.utils.python.library import elf_parser
29from vts.utils.python.library.vtable import vtable_dumper
30from vts.utils.python.vndk import vndk_utils
31
32
33class VtsVndkAbiTest(unittest.TestCase):
34    """A test module to verify ABI compliance of vendor libraries.
35
36    Attributes:
37        _dut: the AndroidDevice under test.
38        _temp_dir: The temporary directory for libraries copied from device.
39    """
40
41    def setUp(self):
42        """Initializes data file path, device, and temporary directory."""
43        serial_number = os.environ.get("ANDROID_SERIAL", "")
44        self.assertTrue(serial_number, "$ANDROID_SERIAL is empty")
45        self._dut = utils.AndroidDevice(serial_number)
46        self._temp_dir = tempfile.mkdtemp()
47
48    def tearDown(self):
49        """Deletes the temporary directory."""
50        logging.info("Delete %s", self._temp_dir)
51        shutil.rmtree(self._temp_dir)
52
53    def _PullOrCreateDir(self, target_dir, host_dir):
54        """Copies a directory from device. Creates an empty one if not exist.
55
56        Args:
57            target_dir: The directory to copy from device.
58            host_dir: The directory to copy to host.
59        """
60        if not self._dut.IsDirectory(target_dir):
61            logging.info("%s doesn't exist. Create %s.", target_dir, host_dir)
62            os.makedirs(host_dir)
63            return
64        parent_dir = os.path.dirname(host_dir)
65        if parent_dir and not os.path.isdir(parent_dir):
66            os.makedirs(parent_dir)
67        logging.info("adb pull %s %s", target_dir, host_dir)
68        self._dut.AdbPull(target_dir, host_dir)
69
70    def _ToHostPath(self, target_path):
71        """Maps target path to host path in self._temp_dir."""
72        return os.path.join(self._temp_dir, *target_path.strip("/").split("/"))
73
74    @staticmethod
75    def _LoadGlobalSymbolsFromDump(dump_obj):
76        """Loads global symbols from a dump object.
77
78        Args:
79            dump_obj: A dict, the dump in JSON format.
80
81        Returns:
82            A set of strings, the symbol names.
83        """
84        symbols = set()
85        for key in ("elf_functions", "elf_objects"):
86            symbols.update(
87                symbol.get("name", "") for symbol in dump_obj.get(key, []) if
88                symbol.get("binding", "global") == "global")
89        return symbols
90
91    def _DiffElfSymbols(self, dump_obj, parser):
92        """Checks if a library includes all symbols in a dump.
93
94        Args:
95            dump_obj: A dict, the dump in JSON format.
96            parser: An elf_parser.ElfParser that loads the library.
97
98        Returns:
99            A list of strings, the global symbols that are in the dump but not
100            in the library.
101
102        Raises:
103            elf_parser.ElfError if fails to load the library.
104        """
105        dump_symbols = self._LoadGlobalSymbolsFromDump(dump_obj)
106        lib_symbols = parser.ListGlobalDynamicSymbols(include_weak=True)
107        return sorted(dump_symbols.difference(lib_symbols))
108
109    @staticmethod
110    def _DiffVtableComponent(offset, expected_symbol, vtable):
111        """Checks if a symbol is in a vtable entry.
112
113        Args:
114            offset: An integer, the offset of the expected symbol.
115            expected_symbol: A string, the name of the expected symbol.
116            vtable: A dict of {offset: [entry]} where offset is an integer and
117                    entry is an instance of vtable_dumper.VtableEntry.
118
119        Returns:
120            A list of strings, the actual possible symbols if expected_symbol
121            does not match the vtable entry.
122            None if expected_symbol matches the entry.
123        """
124        if offset not in vtable:
125            return []
126
127        entry = vtable[offset]
128        if not entry.names:
129            return [hex(entry.value).rstrip('L')]
130
131        if expected_symbol not in entry.names:
132            return entry.names
133
134    def _DiffVtableComponents(self, dump_obj, dumper):
135        """Checks if a library includes all vtable entries in a dump.
136
137        Args:
138            dump_obj: A dict, the dump in JSON format.
139            dumper: An vtable_dumper.VtableDumper that loads the library.
140            bitness: 32 or 64, the size of the vtable entries.
141
142        Returns:
143            A list of tuples (VTABLE, OFFSET, EXPECTED_SYMBOL, ACTUAL).
144            ACTUAL can be "missing", a list of symbol names, or an ELF virtual
145            address.
146
147        Raises:
148            vtable_dumper.VtableError if fails to dump vtable from the library.
149        """
150        function_kinds = [
151            "function_pointer",
152            "complete_dtor_pointer",
153            "deleting_dtor_pointer"
154        ]
155        non_function_kinds = [
156            "vcall_offset",
157            "vbase_offset",
158            "offset_to_top",
159            "rtti",
160            "unused_function_pointer"
161        ]
162        default_vtable_component_kind = "function_pointer"
163
164        global_symbols = self._LoadGlobalSymbolsFromDump(dump_obj)
165
166        lib_vtables = {vtable.name: vtable
167                       for vtable in dumper.DumpVtables()}
168        logging.debug("\n\n".join(str(vtable)
169                                  for _, vtable in lib_vtables.items()))
170
171        vtables_diff = []
172        for record_type in dump_obj.get("record_types", []):
173            # Since Android R, unique_id has been replaced with linker_set_key.
174            # unique_id starts with "_ZTI"; linker_set_key starts with "_ZTS".
175            type_name_symbol = record_type.get("unique_id", "")
176            if type_name_symbol:
177                vtable_symbol = type_name_symbol.replace("_ZTS", "_ZTV", 1)
178            else:
179                type_name_symbol = record_type.get("linker_set_key", "")
180                vtable_symbol = type_name_symbol.replace("_ZTI", "_ZTV", 1)
181
182            # Skip if the vtable symbol isn't global.
183            if vtable_symbol not in global_symbols:
184                continue
185
186            # Collect vtable entries from library dump.
187            if vtable_symbol in lib_vtables:
188                lib_vtable = {entry.offset: entry
189                              for entry in lib_vtables[vtable_symbol].entries}
190            else:
191                lib_vtable = dict()
192
193            for index, entry in enumerate(record_type.get("vtable_components",
194                                                          [])):
195                entry_offset = index * dumper.bitness // 8
196                entry_kind = entry.get("kind", default_vtable_component_kind)
197                entry_symbol = entry.get("mangled_component_name", "")
198                entry_is_pure = entry.get("is_pure", False)
199
200                if entry_kind in non_function_kinds:
201                    continue
202
203                if entry_kind not in function_kinds:
204                    logging.warning("%s: Unexpected vtable entry kind %s",
205                                    vtable_symbol, entry_kind)
206
207                if entry_symbol not in global_symbols:
208                    # Itanium cxx abi doesn't specify pure virtual vtable
209                    # entry's behaviour. However we can still do some checks
210                    # based on compiler behaviour.
211                    # Even though we don't check weak symbols, we can still
212                    # issue a warning when a pure virtual function pointer
213                    # is missing.
214                    if entry_is_pure and entry_offset not in lib_vtable:
215                        logging.warning("%s: Expected pure virtual function"
216                                        "in %s offset %s",
217                                        vtable_symbol, vtable_symbol,
218                                        entry_offset)
219                    continue
220
221                diff_symbols = self._DiffVtableComponent(
222                    entry_offset, entry_symbol, lib_vtable)
223                if diff_symbols is None:
224                    continue
225
226                vtables_diff.append(
227                    (vtable_symbol, str(entry_offset), entry_symbol,
228                     (",".join(diff_symbols) if diff_symbols else "missing")))
229
230        return vtables_diff
231
232    def _ScanLibDirs(self, dump_zip, dump_paths, lib_dirs, dump_version):
233        """Compares dump files with libraries copied from device.
234
235        Args:
236            dump_zip: A zip_file.ZipFile object containing the dumps.
237            dump_paths: A dict of {library name: dump resource path}.
238            lib_dirs: The list of directories containing libraries.
239            dump_version: The VNDK version of the dump files. If the device has
240                          no VNDK version or has extension in vendor partition,
241                          this method compares the unversioned VNDK directories
242                          with the dump directories of the given version.
243
244        Returns:
245            A list of strings, the incompatible libraries.
246        """
247        error_list = []
248        lib_paths = dict()
249        for lib_dir in lib_dirs:
250            for parent_dir, dir_names, lib_names in os.walk(lib_dir):
251                for lib_name in lib_names:
252                    if lib_name not in lib_paths:
253                        lib_paths[lib_name] = os.path.join(parent_dir,
254                                                           lib_name)
255        for lib_name, dump_path in dump_paths.items():
256            if lib_name not in lib_paths:
257                logging.info("%s: Not found on target", lib_name)
258                continue
259            lib_path = lib_paths[lib_name]
260            rel_path = os.path.relpath(lib_path, self._temp_dir)
261
262            has_exception = False
263            missing_symbols = []
264            vtable_diff = []
265
266            try:
267                with dump_zip.open(dump_path, "r") as dump_file:
268                    dump_obj = json.load(dump_file)
269                with vtable_dumper.VtableDumper(lib_path) as dumper:
270                    missing_symbols = self._DiffElfSymbols(
271                        dump_obj, dumper)
272                    vtable_diff = self._DiffVtableComponents(
273                        dump_obj, dumper)
274            except (IOError,
275                    elf_parser.ElfError,
276                    vtable_dumper.VtableError) as e:
277                logging.exception("%s: Cannot diff ABI", rel_path)
278                has_exception = True
279
280            if missing_symbols:
281                logging.error("%s: Missing Symbols:\n%s",
282                              rel_path, "\n".join(missing_symbols))
283            if vtable_diff:
284                logging.error("%s: Vtable Difference:\n"
285                              "vtable offset expected actual\n%s",
286                              rel_path,
287                              "\n".join(" ".join(e) for e in vtable_diff))
288            if (has_exception or missing_symbols or vtable_diff):
289                error_list.append(rel_path)
290            else:
291                logging.info("%s: Pass", rel_path)
292        return error_list
293
294    @staticmethod
295    def _GetLinkerSearchIndex(target_path):
296        """Returns the key for sorting linker search paths."""
297        index = 0
298        for prefix in ("/odm", "/vendor", "/apex"):
299            if target_path.startswith(prefix):
300                return index
301            index += 1
302        return index
303
304    def _TestAbiCompatibility(self, bitness):
305        """Checks ABI compliance of VNDK libraries.
306
307        Args:
308            bitness: 32 or 64, the bitness of the tested libraries.
309        """
310        if not self._dut.GetCpuAbiList(bitness):
311            logging.info("Skip the test as the device doesn't support %d-bit "
312                         "ABI.", bitness)
313            return
314        if not vndk_utils.IsVndkRequired(self._dut):
315            logging.info("Skip the test as the device does not require VNDK.")
316            return
317        self.assertTrue(self._dut.IsRoot(), "This test requires adb root.")
318        primary_abi = self._dut.GetCpuAbiList()[0]
319        binder_bitness = self._dut.GetBinderBitness()
320        self.assertTrue(binder_bitness, "Cannot determine binder bitness.")
321        dump_version = self._dut.GetVndkVersion()
322        self.assertTrue(dump_version, "Cannot determine VNDK version.")
323
324        if vndk_utils.IsVndkInstalledInVendor(self._dut):
325            logging.info("Skip the test as VNDK should be installed in vendor "
326                         "partition. version: %s ABI: %s bitness: %d",
327                         dump_version, primary_abi, bitness)
328            return
329
330        dump_paths = vndk_data.GetAbiDumpPathsFromResources(
331            dump_version,
332            binder_bitness,
333            primary_abi,
334            bitness)
335        self.assertTrue(
336            dump_paths,
337            "No dump files. version: %s ABI: %s bitness: %d" % (
338                dump_version, primary_abi, bitness))
339
340        target_dirs = vndk_utils.GetVndkExtDirectories(bitness)
341        target_dirs += vndk_utils.GetVndkSpExtDirectories(bitness)
342        target_dirs += [vndk_utils.GetVndkDirectory(bitness, dump_version)]
343        target_dirs.sort(key=self._GetLinkerSearchIndex)
344
345        host_dirs = [self._ToHostPath(x) for x in target_dirs]
346        for target_dir, host_dir in zip(target_dirs, host_dirs):
347            self._PullOrCreateDir(target_dir, host_dir)
348
349        with vndk_data.AbiDumpResource() as dump_resource:
350            assert_lines = self._ScanLibDirs(dump_resource.zip_file,
351                                             dump_paths, host_dirs,
352                                             dump_version)
353
354        if assert_lines:
355            error_count = len(assert_lines)
356            if error_count > 20:
357                assert_lines = assert_lines[:20] + ["..."]
358            assert_lines.append("Total number of errors: " + str(error_count))
359            self.fail("\n".join(assert_lines))
360
361    def testAbiCompatibility32(self):
362        """Checks ABI compliance of 32-bit VNDK libraries."""
363        self._TestAbiCompatibility(32)
364
365    def testAbiCompatibility64(self):
366        """Checks ABI compliance of 64-bit VNDK libraries."""
367        self._TestAbiCompatibility(64)
368
369
370if __name__ == "__main__":
371    # The logs are written to stdout so that TradeFed test runner can parse the
372    # results from stderr.
373    logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
374    # Setting verbosity is required to generate output that the TradeFed test
375    # runner can parse.
376    unittest.main(verbosity=3)
377