1#!/usr/bin/env python3
2
3import argparse
4import fnmatch
5import html
6import io
7import json
8import os
9import pathlib
10import subprocess
11import types
12import sys
13
14
15class Graph:
16    def __init__(self, modules):
17        def get_or_make_node(dictionary, id, module):
18            node = dictionary.get(id)
19            if node:
20                if module and not node.module:
21                    node.module = module
22                return node
23            node = Node(id, module)
24            dictionary[id] = node
25            return node
26        self.nodes = dict()
27        for module in modules.values():
28            node = get_or_make_node(self.nodes, module.id, module)
29            for d in module.deps:
30                dep = get_or_make_node(self.nodes, d.id, None)
31                node.deps.add(dep)
32                dep.rdeps.add(node)
33                node.dep_tags.setdefault(dep, list()).append(d)
34
35    def find_paths(self, id1, id2):
36        # Throws KeyError if one of the names isn't found
37        def recurse(node1, node2, visited):
38            result = set()
39            for dep in node1.rdeps:
40                if dep == node2:
41                    result.add(node2)
42                if dep not in visited:
43                    visited.add(dep)
44                    found = recurse(dep, node2, visited)
45                    if found:
46                        result |= found
47                        result.add(dep)
48            return result
49        node1 = self.nodes[id1]
50        node2 = self.nodes[id2]
51        # Take either direction
52        p = recurse(node1, node2, set())
53        if p:
54            p.add(node1)
55            return p
56        p = recurse(node2, node1, set())
57        p.add(node2)
58        return p
59
60
61class Node:
62    def __init__(self, id, module):
63        self.id = id
64        self.module = module
65        self.deps = set()
66        self.rdeps = set()
67        self.dep_tags = {}
68
69
70PROVIDERS = [
71    "android/soong/java.JarJarProviderData",
72    "android/soong/java.BaseJarJarProviderData",
73]
74
75
76def format_dep_label(node, dep):
77    tags = node.dep_tags.get(dep)
78    labels = []
79    if tags:
80        labels = [tag.tag_type.split("/")[-1] for tag in tags]
81        labels = sorted(set(labels))
82    if labels:
83        result = "<<table border=\"0\" cellborder=\"0\" cellspacing=\"0\" cellpadding=\"0\">"
84        for label in labels:
85            result += f"<tr><td>{label}</td></tr>"
86        result += "</table>>"
87        return result
88
89
90def format_node_label(node, module_formatter):
91    result = "<<table border=\"0\" cellborder=\"0\" cellspacing=\"0\" cellpadding=\"0\">"
92
93    # node name
94    result += f"<tr><td><b>{node.module.name if node.module else node.id}</b></td></tr>"
95
96    if node.module:
97        # node_type
98        result += f"<tr><td>{node.module.type}</td></tr>"
99
100        # module_formatter will return a list of rows
101        for row in module_formatter(node.module):
102            row = html.escape(row)
103            result += f"<tr><td><font color=\"#666666\">{row}</font></td></tr>"
104
105    result += "</table>>"
106    return result
107
108
109def format_source_pos(file, lineno):
110    result = file
111    if lineno:
112        result += f":{lineno}"
113    return result
114
115
116STRIP_TYPE_PREFIXES = [
117    "android/soong/",
118    "github.com/google/",
119]
120
121
122def format_provider(provider):
123    result = ""
124    for prefix in STRIP_TYPE_PREFIXES:
125        if provider.type.startswith(prefix):
126            result = provider.type[len(prefix):]
127            break
128    if not result:
129        result = provider.type
130    if True and provider.debug:
131        result += " (" + provider.debug + ")"
132    return result
133
134
135def load_soong_debug():
136    # Read the json
137    try:
138        with open(SOONG_DEBUG_DATA_FILENAME) as f:
139            info = json.load(f, object_hook=lambda d: types.SimpleNamespace(**d))
140    except IOError:
141        sys.stderr.write(f"error: Unable to open {SOONG_DEBUG_DATA_FILENAME}. Make sure you have"
142                         + " built with GENERATE_SOONG_DEBUG.\n")
143        sys.exit(1)
144
145    # Construct IDs, which are name + variant if the
146    name_counts = dict()
147    for m in info.modules:
148        name_counts[m.name] = name_counts.get(m.name, 0) + 1
149    def get_id(m):
150        result = m.name
151        if name_counts[m.name] > 1 and m.variant:
152            result += "@@" + m.variant
153        return result
154    for m in info.modules:
155        m.id = get_id(m)
156        for dep in m.deps:
157            dep.id = get_id(dep)
158
159    return info
160
161
162def load_modules():
163    info = load_soong_debug()
164
165    # Filter out unnamed modules
166    modules = dict()
167    for m in info.modules:
168        if not m.name:
169            continue
170        modules[m.id] = m
171
172    return modules
173
174
175def load_graph():
176    modules=load_modules()
177    return Graph(modules)
178
179
180def module_selection_args(parser):
181    parser.add_argument("modules", nargs="*",
182                        help="Modules to match. Can be glob-style wildcards.")
183    parser.add_argument("--provider", nargs="+",
184                        help="Match the given providers.")
185    parser.add_argument("--dep", nargs="+",
186                        help="Match the given providers.")
187
188
189def load_and_filter_modules(args):
190    # Which modules are printed
191    matchers = []
192    if args.modules:
193        matchers.append(lambda m: [True for pattern in args.modules
194                                   if fnmatch.fnmatchcase(m.name, pattern)])
195    if args.provider:
196        matchers.append(lambda m: [True for pattern in args.provider
197                                   if [True for p in m.providers if p.type.endswith(pattern)]])
198    if args.dep:
199        matchers.append(lambda m: [True for pattern in args.dep
200                                   if [True for d in m.deps if d.id == pattern]])
201
202    if not matchers:
203        sys.stderr.write("error: At least one module matcher must be supplied\n")
204        sys.exit(1)
205
206    info = load_soong_debug()
207    for m in sorted(info.modules, key=lambda m: (m.name, m.variant)):
208        if len([matcher for matcher in matchers if matcher(m)]) == len(matchers):
209            yield m
210
211
212def print_args(parser):
213    parser.add_argument("--label", action="append", metavar="JQ_FILTER",
214                        help="jq query for each module metadata")
215    parser.add_argument("--deptags", action="store_true",
216                        help="show dependency tags (makes the graph much more complex)")
217
218    group = parser.add_argument_group("output formats",
219                                      "If no format is provided, a dot file will be written to"
220                                      + " stdout.")
221    output = group.add_mutually_exclusive_group()
222    output.add_argument("--dot", type=str, metavar="FILENAME",
223                        help="Write the graph to this file as dot (graphviz format)")
224    output.add_argument("--svg", type=str, metavar="FILENAME",
225                        help="Write the graph to this file as svg")
226
227
228def print_nodes(args, nodes, module_formatter):
229    # Generate the graphviz
230    dep_tag_id = 0
231    dot = io.StringIO()
232    dot.write("digraph {\n")
233    dot.write("node [shape=box];")
234
235    for node in nodes:
236        dot.write(f"\"{node.id}\" [label={format_node_label(node, module_formatter)}];\n")
237        for dep in node.deps:
238            if dep in nodes:
239                if args.deptags:
240                    dot.write(f"\"{node.id}\" -> \"__dep_tag_{dep_tag_id}\" [ arrowhead=none ];\n")
241                    dot.write(f"\"__dep_tag_{dep_tag_id}\" -> \"{dep.id}\";\n")
242                    dot.write(f"\"__dep_tag_{dep_tag_id}\""
243                                  + f"[label={format_dep_label(node, dep)} shape=ellipse"
244                                  + " color=\"#666666\" fontcolor=\"#666666\"];\n")
245                else:
246                    dot.write(f"\"{node.id}\" -> \"{dep.id}\";\n")
247                dep_tag_id += 1
248    dot.write("}\n")
249    text = dot.getvalue()
250
251    # Write it somewhere
252    if args.dot:
253        with open(args.dot, "w") as f:
254            f.write(text)
255    elif args.svg:
256        subprocess.run(["dot", "-Tsvg", "-o", args.svg],
257                              input=text, text=True, check=True)
258    else:
259        sys.stdout.write(text)
260
261
262def get_deps(nodes, root, maxdepth, reverse):
263    if root in nodes:
264        return
265    nodes.add(root)
266    if maxdepth != 0:
267        for dep in (root.rdeps if reverse else root.deps):
268            get_deps(nodes, dep, maxdepth-1, reverse)
269
270
271def new_module_formatter(args):
272    def module_formatter(module):
273        if not args.label:
274            return []
275        result = []
276        text = json.dumps(module, default=lambda o: o.__dict__)
277        for jq_filter in args.label:
278            proc = subprocess.run(["jq", jq_filter],
279                                  input=text, text=True, check=True, stdout=subprocess.PIPE)
280            if proc.stdout:
281                o = json.loads(proc.stdout)
282                if type(o) == list:
283                    for row in o:
284                        if row:
285                            result.append(row)
286                elif type(o) == dict:
287                    result.append(str(proc.stdout).strip())
288                else:
289                    if o:
290                        result.append(str(o).strip())
291        return result
292    return module_formatter
293
294
295class BetweenCommand:
296    help = "Print the module graph between two nodes."
297
298    def args(self, parser):
299        parser.add_argument("module", nargs=2,
300                            help="the two modules")
301        print_args(parser)
302
303    def run(self, args):
304        graph = load_graph()
305        print_nodes(args, graph.find_paths(args.module[0], args.module[1]),
306                    new_module_formatter(args))
307
308
309class DepsCommand:
310    help = "Print the module graph of dependencies of one or more modules"
311
312    def args(self, parser):
313        parser.add_argument("module", nargs="+",
314                            help="Module to print dependencies of")
315        parser.add_argument("--reverse", action="store_true",
316                            help="traverse reverse dependencies")
317        parser.add_argument("--depth", type=int, default=-1,
318                            help="max depth of dependencies (can keep the graph size reasonable)")
319        print_args(parser)
320
321    def run(self, args):
322        graph = load_graph()
323        nodes = set()
324        err = False
325        for id in args.module:
326            root = graph.nodes.get(id)
327            if not root:
328                sys.stderr.write(f"error: Can't find root: {id}\n")
329                err = True
330                continue
331            get_deps(nodes, root, args.depth, args.reverse)
332        if err:
333            sys.exit(1)
334        print_nodes(args, nodes, new_module_formatter(args))
335
336
337class IdCommand:
338    help = "Print the id (name + variant) of matching modules"
339
340    def args(self, parser):
341        module_selection_args(parser)
342
343    def run(self, args):
344        for m in load_and_filter_modules(args):
345            print(m.id)
346
347
348class JsonCommand:
349    help = "Print metadata about modules in json format"
350
351    def args(self, parser):
352        module_selection_args(parser)
353        parser.add_argument("--list", action="store_true",
354                            help="Print the results in a json list. If not set and multiple"
355                            + " modules are matched, the output won't be valid json.")
356
357    def run(self, args):
358        modules = load_and_filter_modules(args)
359        if args.list:
360            json.dump([m for m in modules], sys.stdout, indent=4, default=lambda o: o.__dict__)
361        else:
362            for m in modules:
363                json.dump(m, sys.stdout, indent=4, default=lambda o: o.__dict__)
364                print()
365
366
367class QueryCommand:
368    help = "Query details about modules"
369
370    def args(self, parser):
371        module_selection_args(parser)
372
373    def run(self, args):
374        for m in load_and_filter_modules(args):
375            print(m.id)
376            print(f"    type:     {m.type}")
377            print(f"    location: {format_source_pos(m.source_file, m.source_line)}")
378            for p in m.providers:
379                print(f"    provider: {format_provider(p)}")
380            for d in m.deps:
381                print(f"    dep:      {d.id}")
382
383
384COMMANDS = {
385    "between": BetweenCommand(),
386    "deps": DepsCommand(),
387    "id": IdCommand(),
388    "json": JsonCommand(),
389    "query": QueryCommand(),
390}
391
392
393def assert_env(name):
394    val = os.getenv(name)
395    if not val:
396        sys.stderr.write(f"{name} not set. please make sure you've run lunch.")
397    return val
398
399ANDROID_BUILD_TOP = assert_env("ANDROID_BUILD_TOP")
400
401TARGET_PRODUCT = assert_env("TARGET_PRODUCT")
402OUT_DIR = os.getenv("OUT_DIR")
403if not OUT_DIR:
404    OUT_DIR = "out"
405if OUT_DIR[0] != "/":
406    OUT_DIR = pathlib.Path(ANDROID_BUILD_TOP).joinpath(OUT_DIR)
407SOONG_DEBUG_DATA_FILENAME = pathlib.Path(OUT_DIR).joinpath("soong/soong-debug-info.json")
408
409
410def main():
411    parser = argparse.ArgumentParser()
412    subparsers = parser.add_subparsers(required=True, dest="command")
413    for name in sorted(COMMANDS.keys()):
414        command = COMMANDS[name]
415        subparser = subparsers.add_parser(name, help=command.help)
416        command.args(subparser)
417    args = parser.parse_args()
418    COMMANDS[args.command].run(args)
419    sys.exit(0)
420
421
422if __name__ == "__main__":
423    main()
424
425