1#!/usr/bin/env python3
2
3"""Tool to find static libraries that maybe should be shared libraries and shared libraries that maybe should be static libraries.
4
5This tool only looks at the module-info.json for the current target.
6
7Example of "class" types for each of the modules in module-info.json
8  "EXECUTABLES": 2307,
9  "ETC": 9094,
10  "NATIVE_TESTS": 10461,
11  "APPS": 2885,
12  "JAVA_LIBRARIES": 5205,
13  "EXECUTABLES/JAVA_LIBRARIES": 119,
14  "FAKE": 553,
15  "SHARED_LIBRARIES/STATIC_LIBRARIES": 7591,
16  "STATIC_LIBRARIES": 11535,
17  "SHARED_LIBRARIES": 10852,
18  "HEADER_LIBRARIES": 1897,
19  "DYLIB_LIBRARIES": 1262,
20  "RLIB_LIBRARIES": 3413,
21  "ROBOLECTRIC": 39,
22  "PACKAGING": 5,
23  "PROC_MACRO_LIBRARIES": 36,
24  "RENDERSCRIPT_BITCODE": 17,
25  "DYLIB_LIBRARIES/RLIB_LIBRARIES": 8,
26  "ETC/FAKE": 1
27
28None of the "SHARED_LIBRARIES/STATIC_LIBRARIES" are double counted in the
29modules with one class
30RLIB/
31
32All of these classes have shared_libs and/or static_libs
33    "EXECUTABLES",
34    "SHARED_LIBRARIES",
35    "STATIC_LIBRARIES",
36    "SHARED_LIBRARIES/STATIC_LIBRARIES", # cc_library
37    "HEADER_LIBRARIES",
38    "NATIVE_TESTS", # test modules
39    "DYLIB_LIBRARIES", # rust
40    "RLIB_LIBRARIES", # rust
41    "ETC", # rust_bindgen
42"""
43
44from collections import defaultdict
45
46import json, os, argparse
47
48ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT")
49# If a shared library is used less than MAX_SHARED_INCLUSIONS times in a target,
50# then it will likely save memory by changing it to a static library
51# This move will also use less storage
52MAX_SHARED_INCLUSIONS = 2
53# If a static library is used more than MAX_STATIC_INCLUSIONS times in a target,
54# then it will likely save memory by changing it to a shared library
55# This move will also likely use less storage
56MIN_STATIC_INCLUSIONS = 3
57
58
59def parse_args():
60  parser = argparse.ArgumentParser(
61      description=(
62          "Parse module-info.jso and display information about static and"
63          " shared library dependencies."
64      )
65  )
66  parser.add_argument(
67      "--module", dest="module", help="Print the info for the module."
68  )
69  parser.add_argument(
70      "--shared",
71      dest="print_shared",
72      action=argparse.BooleanOptionalAction,
73      help=(
74          "Print the list of libraries that are shared_libs for fewer than {}"
75          " modules.".format(MAX_SHARED_INCLUSIONS)
76      ),
77  )
78  parser.add_argument(
79      "--static",
80      dest="print_static",
81      action=argparse.BooleanOptionalAction,
82      help=(
83          "Print the list of libraries that are static_libs for more than {}"
84          " modules.".format(MIN_STATIC_INCLUSIONS)
85      ),
86  )
87  parser.add_argument(
88      "--recursive",
89      dest="recursive",
90      action=argparse.BooleanOptionalAction,
91      default=True,
92      help=(
93          "Gather all dependencies of EXECUTABLES recursvily before calculating"
94          " the stats. This eliminates duplicates from multiple libraries"
95          " including the same dependencies in a single binary."
96      ),
97  )
98  parser.add_argument(
99      "--both",
100      dest="both",
101      action=argparse.BooleanOptionalAction,
102      default=False,
103      help=(
104          "Print a list of libraries that are including libraries as both"
105          " static and shared"
106      ),
107  )
108  return parser.parse_args()
109
110
111class TransitiveHelper:
112
113  def __init__(self):
114    # keep a list of already expanded libraries so we don't end up in a cycle
115    self.visited = defaultdict(lambda: defaultdict(set))
116
117  # module is an object from the module-info dictionary
118  # module_info is the dictionary from module-info.json
119  # modify the module's shared_libs and static_libs with all of the transient
120  # dependencies required from all of the explicit dependencies
121  def flattenDeps(self, module, module_info):
122    libs_snapshot = dict(shared_libs = set(module.get("shared_libs",{})), static_libs = set(module.get("static_libs",{})))
123
124    for lib_class in ["shared_libs", "static_libs"]:
125      for lib in libs_snapshot[lib_class]:
126        if not lib or lib not in module_info or lib_class not in module:
127          continue
128        if lib in self.visited:
129          module[lib_class].update(self.visited[lib][lib_class])
130        else:
131          res = self.flattenDeps(module_info[lib], module_info)
132          module[lib_class].update(res.get(lib_class, {}))
133          self.visited[lib][lib_class].update(res.get(lib_class, {}))
134
135    return module
136
137def main():
138  module_info = json.load(open(ANDROID_PRODUCT_OUT + "/module-info.json"))
139
140  args = parse_args()
141
142  if args.module:
143    if args.module not in module_info:
144      print("Module {} does not exist".format(args.module))
145      exit(1)
146
147  # turn all of the static_libs and shared_libs lists into sets to make them
148  # easier to update
149  for _, module in module_info.items():
150    module["shared_libs"] = set(module.get("shared_libs", {}))
151    module["static_libs"] = set(module.get("static_libs", {}))
152
153  includedStatically = defaultdict(set)
154  includedSharedly = defaultdict(set)
155  includedBothly = defaultdict(set)
156  transitive = TransitiveHelper()
157  for name, module in module_info.items():
158    if args.recursive:
159      # in this recursive mode we only want to see what is included by the executables
160      if "EXECUTABLES" not in module["class"]:
161        continue
162      module = transitive.flattenDeps(module, module_info)
163      # filter out fuzzers by their dependency on clang
164      if "static_libs" in module:
165        if "libclang_rt.fuzzer" in module["static_libs"]:
166          continue
167    else:
168      if "NATIVE_TESTS" in module["class"]:
169        # We don't care about how tests are including libraries
170        continue
171
172    # count all of the shared and static libs included in this module
173    if "shared_libs" in module:
174      for lib in module["shared_libs"]:
175        includedSharedly[lib].add(name)
176    if "static_libs" in module:
177      for lib in module["static_libs"]:
178        includedStatically[lib].add(name)
179
180    if "shared_libs" in module and  "static_libs" in module:
181      intersection = set(module["shared_libs"]).intersection(
182          module["static_libs"]
183      )
184      if intersection:
185        includedBothly[name] = intersection
186
187  if args.print_shared:
188    print(
189        "Shared libraries that are included by fewer than {} modules on a"
190        " device:".format(MAX_SHARED_INCLUSIONS)
191    )
192    for name, libs in includedSharedly.items():
193      if len(libs) < MAX_SHARED_INCLUSIONS:
194        print("{}: {} included by: {}".format(name, len(libs), libs))
195
196  if args.print_static:
197    print(
198        "Libraries that are included statically by more than {} modules on a"
199        " device:".format(MIN_STATIC_INCLUSIONS)
200    )
201    for name, libs in includedStatically.items():
202      if len(libs) > MIN_STATIC_INCLUSIONS:
203        print("{}: {} included by: {}".format(name, len(libs), libs))
204
205  if args.both:
206    allIncludedBothly = set()
207    for name, libs in includedBothly.items():
208      allIncludedBothly.update(libs)
209
210    print(
211        "List of libraries used both statically and shared in the same"
212        " processes:\n {}\n\n".format("\n".join(sorted(allIncludedBothly)))
213    )
214    print(
215        "List of libraries used both statically and shared in any processes:\n {}".format("\n".join(sorted(includedStatically.keys() & includedSharedly.keys()))))
216
217  if args.module:
218    print(json.dumps(module_info[args.module], default=list, indent=2))
219    print(
220        "{} is included in shared_libs {} times by these modules: {}".format(
221            args.module, len(includedSharedly[args.module]),
222            includedSharedly[args.module]
223        )
224    )
225    print(
226        "{} is included in static_libs {} times by these modules: {}".format(
227            args.module, len(includedStatically[args.module]),
228            includedStatically[args.module]
229        )
230    )
231    print("Shared libs included by this module that are used in fewer than {} processes:\n{}".format(
232        MAX_SHARED_INCLUSIONS, [x for x in module_info[args.module]["shared_libs"] if len(includedSharedly[x]) < MAX_SHARED_INCLUSIONS]))
233
234
235
236if __name__ == "__main__":
237  main()
238