1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import collections
19import logging
20import os
21import re
22import sys
23import unittest
24
25from vts.testcases.vndk import utils
26from vts.testcases.vndk.golden import vndk_data
27from vts.utils.python.library import elf_parser
28from vts.utils.python.vndk import vndk_utils
29
30
31class VtsVndkDependencyTest(unittest.TestCase):
32    """A test case to verify vendor library dependency.
33
34    Attributes:
35        _dut: The AndroidDevice under test.
36        _vndk_version: The VNDK version of the device.
37        _ll_ndk: Set of strings. The names of low-level NDK libraries in
38                 /system/lib[64].
39        _sp_hal: List of patterns. The names of the same-process HAL libraries
40                 expected to be in /vendor/lib[64].
41        _vndk: Set of strings. The names of VNDK-core libraries.
42        _vndk_sp: Set of strings. The names of VNDK-SP libraries.
43        _VENDOR_DIRS: The directories of vendor partitions.
44        _SP_HAL_LINK_PATHS: Format strings of same-process HAL's default link
45                            paths.
46        _VENDOR_LINK_PATHS: Format strings of vendor processes' default link
47                            paths.
48        _VENDOR_PERMITTED_PATHS: Same-process HAL and vendor processes'
49                                 permitted link paths.
50        _VENDOR_APP_DIRS: The app directories in vendor partitions.
51    """
52    _VENDOR_DIRS = [
53        "/odm", "/vendor"
54    ]
55    _SP_HAL_LINK_PATHS = [
56        "/odm/{LIB}/egl", "/odm/{LIB}/hw", "/odm/{LIB}",
57        "/vendor/{LIB}/egl", "/vendor/{LIB}/hw", "/vendor/{LIB}"
58    ]
59    _VENDOR_LINK_PATHS = [
60        "/odm/{LIB}/hw", "/odm/{LIB}/egl", "/odm/{LIB}",
61        "/vendor/{LIB}/hw", "/vendor/{LIB}/egl", "/vendor/{LIB}"
62    ]
63    _VENDOR_PERMITTED_PATHS = [
64        "/odm", "/vendor"
65    ]
66    _VENDOR_APP_DIRS = [
67        "/vendor/app", "/vendor/priv-app", "/odm/app", "/odm/priv-app"
68    ]
69    _DEFAULT_PROGRAM_INTERPRETERS = [
70        "/system/bin/linker", "/system/bin/linker64"
71    ]
72
73    class ElfObject(object):
74        """Contains dependencies of an ELF file on target device.
75
76        Attributes:
77            target_path: String. The path to the ELF file on target.
78            name: String. File name of the ELF.
79            target_dir: String. The directory containing the ELF file on
80                        target.
81            bitness: Integer. Bitness of the ELF.
82            deps: List of strings. The names of the depended libraries.
83            runpaths: List of strings. The library search paths.
84            custom_link_paths: List of strings. The library search paths.
85        """
86
87        def __init__(self, target_path, bitness, deps, runpaths,
88                     custom_link_paths):
89            self.target_path = target_path
90            self.name = os.path.basename(target_path)
91            self.target_dir = os.path.dirname(target_path)
92            self.bitness = bitness
93            self.deps = deps
94            # Format runpaths
95            self.runpaths = []
96            lib_dir_name = "lib64" if bitness == 64 else "lib"
97            for runpath in runpaths:
98                path = runpath.replace("${LIB}", lib_dir_name)
99                path = path.replace("$LIB", lib_dir_name)
100                path = path.replace("${ORIGIN}", self.target_dir)
101                path = path.replace("$ORIGIN", self.target_dir)
102                self.runpaths.append(path)
103            self.custom_link_paths = custom_link_paths
104
105    def setUp(self):
106        """Initializes VNDK lists."""
107        self._dut = utils.AndroidDevice()
108        self.assertTrue(self._dut.IsRoot(), "This test requires adb root.")
109
110        self._vndk_version = self._dut.GetVndkVersion()
111        vndk_lists = vndk_data.LoadVndkLibraryListsFromResources(
112            self._vndk_version,
113            vndk_data.SP_HAL,
114            vndk_data.LL_NDK,
115            vndk_data.VNDK,
116            vndk_data.VNDK_SP)
117        self.assertTrue(vndk_lists, "Cannot load VNDK library lists.")
118
119        sp_hal_strings = vndk_lists[0]
120        self._sp_hal = [re.compile(x) for x in sp_hal_strings]
121        if vndk_utils.IsVndkRequired(self._dut):
122            self._ll_ndk = vndk_lists[1]
123            if vndk_utils.IsVndkInstalledInVendor(self._dut):
124                (self._vndk, self._vndk_sp) = ([], [])
125            else:
126                (self._vndk, self._vndk_sp) = vndk_lists[2:]
127        else:
128            self._ll_ndk = self._dut.GetLlndkList()
129            (self._vndk, self._vndk_sp) = ([], [])
130
131        logging.debug("LL_NDK: %s", self._ll_ndk)
132        logging.debug("SP_HAL: %s", sp_hal_strings)
133        logging.debug("VNDK: %s", self._vndk)
134        logging.debug("VNDK_SP: %s", self._vndk_sp)
135
136    def _IsElfObjectForAp(self, elf, target_path, abi_list):
137        """Checks whether an ELF object is for application processor.
138
139        Args:
140            elf: The object of elf_parser.ElfParser.
141            target_path: The path to the ELF file on target.
142            abi_list: A list of strings, the ABIs of the application processor.
143
144        Returns:
145            A boolean, whether the ELF object is for application processor.
146        """
147        if not any(elf.MatchCpuAbi(x) for x in abi_list):
148            logging.debug("%s does not match the ABI", target_path)
149            return False
150
151        # b/115567177 Skip an ELF file if it meets the following 3 conditions:
152        # The ELF type is executable.
153        if not elf.IsExecutable():
154            return True
155
156        # It requires special program interpreter.
157        interp = elf.GetProgramInterpreter()
158        if not interp or interp in self._DEFAULT_PROGRAM_INTERPRETERS:
159            return True
160
161        # It does not have execute permission in the file system.
162        if self._dut.IsExecutable(target_path):
163            return True
164
165        return False
166
167    def _IsElfObjectBuiltForAndroid(self, elf, target_path):
168        """Checks whether an ELF object is built for Android.
169
170        Some ELF objects in vendor partition require special program
171        interpreters. Such executable files have .interp sections, but shared
172        libraries don't. As there is no reliable way to identify those
173        libraries. This method checks .note.android.ident section which is
174        created by Android build system.
175
176        Args:
177            elf: The object of elf_parser.ElfParser.
178            target_path: The path to the ELF file on target.
179
180        Returns:
181            A boolean, whether the ELF object is built for Android.
182        """
183        # b/133399940 Skip an ELF file if it does not have .note.android.ident
184        # section and meets one of the following conditions:
185        if elf.HasAndroidIdent():
186            return True
187
188        # It's in the specific directory and is a shared library.
189        if (target_path.startswith("/vendor/arib/lib/") and
190                ".so" in target_path and
191                elf.IsSharedObject()):
192            return False
193
194        # It's in the specific directory, requires special program interpreter,
195        # and is executable.
196        if target_path.startswith("/vendor/arib/bin/"):
197            interp = elf.GetProgramInterpreter()
198            if interp and interp not in self._DEFAULT_PROGRAM_INTERPRETERS:
199                if elf.IsExecutable() or self._dut.IsExecutable(target_path):
200                    return False
201
202        return True
203
204    @staticmethod
205    def _IterateFiles(target_dir):
206        """Iterates files in a directory.
207
208        Args:
209            target_dir: The directory.
210
211        Yields:
212            The file paths under the directory.
213        """
214        for root_dir, dir_names, file_names in os.walk(target_dir):
215            for file_name in file_names:
216                yield os.path.join(root_dir, file_name)
217
218    def _LoadElfObjects(self, target_dir, abi_list, elf_error_handler):
219        """Scans a directory recursively and loads all ELF files in it.
220
221        Args:
222            target_dir: The directory to scan.
223            abi_list: A list of strings, the ABIs of the ELF files to load.
224            elf_error_handler: A function that takes 2 arguments
225                               (target_path, exception). It is called when
226                               the parser fails to read an ELF file.
227
228        Returns:
229            List of ElfObject.
230        """
231        objs = []
232        for target_path in self._IterateFiles(target_dir):
233            try:
234                elf = elf_parser.ElfParser(target_path)
235            except elf_parser.ElfError:
236                logging.debug("%s is not an ELF file", target_path)
237                continue
238            try:
239                if not self._IsElfObjectForAp(elf, target_path, abi_list):
240                    logging.info("%s is not for application processor",
241                                 target_path)
242                    continue
243                if not self._IsElfObjectBuiltForAndroid(elf, target_path):
244                    logging.warning("%s is not built for Android, which is no "
245                                    "longer exempted.", target_path)
246
247                deps, runpaths = elf.ListDependencies()
248            except elf_parser.ElfError as e:
249                elf_error_handler(target_path, e)
250                continue
251            finally:
252                elf.Close()
253
254            logging.info("%s depends on: %s", target_path, ", ".join(deps))
255            if runpaths:
256                logging.info("%s has runpaths: %s",
257                             target_path, ":".join(runpaths))
258
259            # b/123216664 App libraries depend on those in the same directory.
260            custom_link_paths = []
261            if any(target_path.startswith(app_dir + os.path.sep) for
262                    app_dir in self._VENDOR_APP_DIRS):
263                custom_link_paths.append(os.path.dirname(target_path))
264
265            objs.append(self.ElfObject(target_path, elf.bitness, deps,
266                                       runpaths, custom_link_paths))
267        return objs
268
269    def _FindLibsInLinkPaths(self, bitness, link_paths, objs):
270        """Finds libraries in link paths.
271
272        Args:
273            bitness: 32 or 64, the bitness of the returned libraries.
274            link_paths: List of strings, the default link paths.
275            objs: List of ElfObject, the libraries/executables to be filtered
276                  by bitness and path.
277
278        Returns:
279            A defaultdict, {dir: {name: obj}} where obj is an ElfObject, dir
280            is obj.target_dir, and name is obj.name.
281        """
282        namespace = collections.defaultdict(dict)
283        for obj in objs:
284            if (obj.bitness == bitness and
285                    any(obj.target_path.startswith(link_path + os.path.sep)
286                        for link_path in link_paths)):
287                namespace[obj.target_dir][obj.name] = obj
288        return namespace
289
290    def _DfsDependencies(self, lib, searched, namespace, link_paths):
291        """Depth-first-search for library dependencies.
292
293        Args:
294            lib: ElfObject, the library to search for dependencies.
295            searched: The set of searched libraries.
296            namespace: Defaultdict, {dir: {name: obj}} containing all
297                       searchable libraries.
298            link_paths: List of strings, the default link paths.
299        """
300        if lib in searched:
301            return
302        searched.add(lib)
303        for dep_name in lib.deps:
304            for link_path in lib.custom_link_paths + lib.runpaths + link_paths:
305                if dep_name in namespace[link_path]:
306                    self._DfsDependencies(namespace[link_path][dep_name],
307                                          searched, namespace, link_paths)
308                    break
309
310    def _FindDisallowedDependencies(self, objs, namespace, link_paths,
311                                    *vndk_lists):
312        """Tests if libraries/executables have disallowed dependencies.
313
314        Args:
315            objs: Collection of ElfObject, the libraries/executables under
316                  test.
317            namespace: Defaultdict, {dir: {name: obj}} containing all libraries
318                       in the linker namespace.
319            link_paths: List of strings, the default link paths.
320            vndk_lists: Collections of library names in VNDK, VNDK-SP, etc.
321
322        Returns:
323            List of tuples (path, disallowed_dependencies).
324        """
325        dep_errors = []
326        for obj in objs:
327            disallowed_libs = []
328            for dep_name in obj.deps:
329                if any((dep_name in vndk_list) for vndk_list in vndk_lists):
330                    continue
331                if any((dep_name in namespace[link_path]) for link_path in
332                       obj.custom_link_paths + obj.runpaths + link_paths):
333                    continue
334                disallowed_libs.append(dep_name)
335
336            if disallowed_libs:
337                dep_errors.append((obj.target_path, disallowed_libs))
338        return dep_errors
339
340    def _TestElfDependency(self, bitness, objs):
341        """Tests vendor libraries/executables and SP-HAL dependencies.
342
343        Args:
344            bitness: 32 or 64, the bitness of the vendor libraries.
345            objs: List of ElfObject. The libraries/executables in odm and
346                  vendor partitions.
347
348        Returns:
349            List of tuples (path, disallowed_dependencies).
350        """
351        vndk_sp_ext_dirs = vndk_utils.GetVndkSpExtDirectories(bitness)
352
353        vendor_link_paths = [vndk_utils.FormatVndkPath(x, bitness) for
354                             x in self._VENDOR_LINK_PATHS]
355        vendor_namespace = self._FindLibsInLinkPaths(
356            bitness, self._VENDOR_PERMITTED_PATHS, objs)
357        # Exclude VNDK and VNDK-SP extensions from vendor libraries.
358        for vndk_ext_dir in (vndk_utils.GetVndkExtDirectories(bitness) +
359                             vndk_utils.GetVndkSpExtDirectories(bitness)):
360            vendor_namespace.pop(vndk_ext_dir, None)
361        logging.info("%d-bit odm, vendor, and SP-HAL libraries:", bitness)
362        for dir_path, libs in vendor_namespace.items():
363            logging.info("%s: %s", dir_path, ",".join(libs.keys()))
364
365        sp_hal_link_paths = [vndk_utils.FormatVndkPath(x, bitness) for
366                             x in self._SP_HAL_LINK_PATHS]
367        sp_hal_namespace = self._FindLibsInLinkPaths(
368            bitness, self._VENDOR_PERMITTED_PATHS, objs)
369
370        if vndk_utils.IsVndkInstalledInVendor(self._dut):
371            vndk_in_vendor = [
372                os.path.basename(lib_path) for lib_path in
373                self._dut.FindFiles(
374                    vndk_utils.GetVndkDirectory(bitness, self._vndk_version),
375                    "*", "!", "-type", "d")]
376            logging.info("%d-bit VNDK libraries installed in vendor: %s",
377                         bitness, vndk_in_vendor)
378        else:
379            vndk_in_vendor = []
380
381        # Find same-process HAL and dependencies
382        sp_hal_libs = set()
383        for link_path in sp_hal_link_paths:
384            for obj in sp_hal_namespace[link_path].values():
385                if any(x.match(obj.target_path) for x in self._sp_hal):
386                    self._DfsDependencies(obj, sp_hal_libs, sp_hal_namespace,
387                                          sp_hal_link_paths)
388        logging.info("%d-bit SP-HAL libraries: %s",
389                     bitness, ", ".join(x.name for x in sp_hal_libs))
390
391        # Find VNDK-SP extension libraries and their dependencies.
392        vndk_sp_ext_libs = set(obj for obj in objs if
393                               obj.bitness == bitness and
394                               obj.target_dir in vndk_sp_ext_dirs)
395        vndk_sp_ext_deps = set()
396        for lib in vndk_sp_ext_libs:
397            self._DfsDependencies(lib, vndk_sp_ext_deps, sp_hal_namespace,
398                                  sp_hal_link_paths)
399        logging.info("%d-bit VNDK-SP extension libraries and dependencies: %s",
400                     bitness, ", ".join(x.name for x in vndk_sp_ext_deps))
401
402        # A vendor library/executable is allowed to depend on
403        # LL-NDK
404        # VNDK
405        # VNDK-SP
406        # Other libraries in vendor link paths
407        vendor_objs = {obj for obj in objs if
408                       obj.bitness == bitness and
409                       obj not in sp_hal_libs and
410                       obj not in vndk_sp_ext_deps}
411        dep_errors = self._FindDisallowedDependencies(
412            vendor_objs, vendor_namespace, vendor_link_paths,
413            self._ll_ndk, self._vndk, self._vndk_sp, vndk_in_vendor)
414
415        # A VNDK-SP extension library/dependency is allowed to depend on
416        # LL-NDK
417        # VNDK-SP
418        # Libraries in vendor link paths
419        # Other VNDK-SP extension libraries, which is a subset of VNDK-SP
420        #
421        # However, it is not allowed to indirectly depend on VNDK. i.e., the
422        # depended vendor libraries must not depend on VNDK.
423        #
424        # vndk_sp_ext_deps and sp_hal_libs may overlap. Their dependency
425        # restrictions are the same.
426        dep_errors.extend(self._FindDisallowedDependencies(
427            vndk_sp_ext_deps - sp_hal_libs, vendor_namespace,
428            vendor_link_paths, self._ll_ndk, self._vndk_sp, vndk_in_vendor))
429
430        if not vndk_utils.IsVndkRuntimeEnforced(self._dut):
431            logging.warning("Ignore dependency errors: %s", dep_errors)
432            dep_errors = []
433
434        # A same-process HAL library is allowed to depend on
435        # LL-NDK
436        # VNDK-SP
437        # Other same-process HAL libraries and dependencies
438        dep_errors.extend(self._FindDisallowedDependencies(
439            sp_hal_libs, sp_hal_namespace, sp_hal_link_paths,
440            self._ll_ndk, self._vndk_sp, vndk_in_vendor))
441        return dep_errors
442
443    def testElfDependency(self):
444        """Tests vendor libraries/executables and SP-HAL dependencies."""
445        read_errors = []
446        abi_list = self._dut.GetCpuAbiList()
447        objs = []
448        for target_dir in self._VENDOR_DIRS:
449            objs += self._LoadElfObjects(
450                target_dir, abi_list,
451                lambda p, e: read_errors.append((p, str(e))))
452
453        dep_errors = []
454        if self._dut.GetCpuAbiList(32):
455            dep_errors.extend(self._TestElfDependency(32, objs))
456        if self._dut.GetCpuAbiList(64):
457            dep_errors.extend(self._TestElfDependency(64, objs))
458
459        assert_lines = []
460        if read_errors:
461            error_lines = ["%s: %s" % (x[0], x[1]) for x in read_errors]
462            logging.error("%d read errors:\n%s",
463                          len(read_errors), "\n".join(error_lines))
464            assert_lines.extend(error_lines[:20])
465
466        if dep_errors:
467            error_lines = ["%s: %s" % (x[0], ", ".join(x[1]))
468                           for x in dep_errors]
469            logging.error("%d disallowed dependencies:\n%s",
470                          len(dep_errors), "\n".join(error_lines))
471            assert_lines.extend(error_lines[:20])
472
473        error_count = len(read_errors) + len(dep_errors)
474        if error_count:
475            if error_count > len(assert_lines):
476                assert_lines.append("...")
477            assert_lines.append("Total number of errors: " + str(error_count))
478            self.fail("\n".join(assert_lines))
479
480
481if __name__ == "__main__":
482    # Write logs to stdout and results to stderr.
483    logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
484    unittest.main(verbosity=3)
485