1#!/usr/bin/env python3
2#
3# Copyright (C) 2022 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""A tool to print human-readable metrics information regarding the last build.
17
18By default, the consumed files will be located in $ANDROID_BUILD_TOP/out/. You
19may pass in a different directory instead using the metrics_files_dir flag.
20"""
21
22import argparse
23import json
24import os
25import shutil
26import subprocess
27import sys
28import tarfile
29
30from bazel_metrics_proto.bazel_metrics_pb2 import BazelMetrics
31from bp2build_metrics_proto.bp2build_metrics_pb2 import Bp2BuildMetrics, UnconvertedReasonType
32from google.protobuf import json_format
33from metrics_proto.metrics_pb2 import MetricsBase, SoongBuildMetrics
34
35
36class Event(object):
37  """Contains nested event data.
38
39  Fields:
40    name: The short name of this event e.g. the 'b' in an event called a.b.
41    start_time_relative_ns: Time since the epoch that the event started
42    duration_ns: Duration of this event, including time spent in children.
43  """
44
45  def __init__(self, name, start_time_relative_ns, duration_ns):
46    self.name = name
47    self.start_time_relative_ns = start_time_relative_ns
48    self.duration_ns = duration_ns
49
50
51def _get_output_file(output_dir, filename):
52  file_base = os.path.splitext(filename)[0]
53  return os.path.join(output_dir, file_base + ".json")
54
55
56def _get_default_out_dir(metrics_dir):
57  return os.path.join(metrics_dir, "analyze_build_output")
58
59
60def _get_default_metrics_dir():
61  """Returns the filepath for the build output."""
62  out_dir = os.getenv("OUT_DIR")
63  if out_dir:
64    return out_dir
65  build_top = os.getenv("ANDROID_BUILD_TOP")
66  if not build_top:
67    raise Exception(
68        "$ANDROID_BUILD_TOP not found in environment. Have you run lunch?"
69    )
70  return os.path.join(build_top, "out")
71
72
73def _write_event(out, event):
74  """Writes an event. See _write_events for args."""
75  out.write(
76      "%(start)9s  %(duration)9s  %(name)s\n"
77      % {
78          "start": _format_ns(event.start_time_relative_ns),
79          "duration": _format_ns(event.duration_ns),
80          "name": event.name,
81      }
82  )
83
84
85def _print_metrics_event_times(description, metrics):
86  # Bail if there are no events
87  raw_events = metrics.events
88  if not raw_events:
89    print("%s: No events to display" % description)
90    return
91  print("-- %s events --" % description)
92
93  # Update the start times to be based on the first event
94  first_time_ns = min([event.start_time for event in raw_events])
95  events = [
96      Event(
97          getattr(e, "description", e.name),
98          e.start_time - first_time_ns,
99          e.real_time,
100      )
101      for e in raw_events
102  ]
103
104  # Sort by start time so the nesting also is sorted by time
105  events.sort(key=lambda x: x.start_time_relative_ns)
106
107  # Output the results
108  print("    start   duration")
109
110  for event in events:
111    _write_event(sys.stdout, event)
112  print()
113
114
115def _format_ns(duration_ns):
116  "Pretty print duration in nanoseconds"
117  return "%.02fs" % (duration_ns / 1_000_000_000)
118
119
120def _read_data(filepath, proto):
121  with open(filepath, "rb") as f:
122    proto.ParseFromString(f.read())
123    f.close()
124
125
126def _maybe_save_data(proto, filename, args):
127  if args.skip_metrics:
128    return
129  json_out = json_format.MessageToJson(proto)
130  output_filepath = _get_output_file(args.output_dir, filename)
131  _save_file(json_out, output_filepath)
132
133
134def _save_file(data, file):
135  with open(file, "w") as f:
136    f.write(data)
137    f.close()
138
139
140def _handle_missing_metrics(args, filename):
141  """Handles cleanup for a metrics file that doesn't exist.
142
143  This will delete any output files under the tool's output directory that
144  would have been generated as a result of a metrics file from a previous
145  build. This prevents stale analysis files from polluting the output dir.
146  """
147  if args.skip_metrics:
148    # If skip_metrics is enabled, then don't write or delete any data.
149    return
150  output_filepath = _get_output_file(args.output_dir, filename)
151  if os.path.exists(output_filepath):
152    os.remove(output_filepath)
153
154
155def process_timing_mode(args):
156  metrics_files_dir = args.metrics_files_dir
157  if not args.skip_metrics:
158    os.makedirs(args.output_dir, exist_ok=True)
159    print("Writing build analysis files to " + args.output_dir, file=sys.stderr)
160
161  bp2build_file = os.path.join(metrics_files_dir, "bp2build_metrics.pb")
162  if os.path.exists(bp2build_file):
163    bp2build_metrics = Bp2BuildMetrics()
164    _read_data(bp2build_file, bp2build_metrics)
165    _print_metrics_event_times("bp2build", bp2build_metrics)
166    _maybe_save_data(bp2build_metrics, "bp2build_metrics.pb", args)
167  else:
168    _handle_missing_metrics(args, "bp2build_metrics.pb")
169
170  soong_build_file = os.path.join(metrics_files_dir, "soong_build_metrics.pb")
171  if os.path.exists(soong_build_file):
172    soong_build_metrics = SoongBuildMetrics()
173    _read_data(soong_build_file, soong_build_metrics)
174    _print_metrics_event_times("soong_build", soong_build_metrics)
175    _maybe_save_data(soong_build_metrics, "soong_build_metrics.pb", args)
176  else:
177    _handle_missing_metrics(args, "soong_build_metrics.pb")
178
179  soong_metrics_file = os.path.join(metrics_files_dir, "soong_metrics")
180  if os.path.exists(soong_metrics_file):
181    metrics_base = MetricsBase()
182    _read_data(soong_metrics_file, metrics_base)
183    _maybe_save_data(metrics_base, "soong_metrics", args)
184  else:
185    _handle_missing_metrics(args, "soong_metrics")
186
187  bazel_metrics_file = os.path.join(metrics_files_dir, "bazel_metrics.pb")
188  if os.path.exists(bazel_metrics_file):
189    bazel_metrics = BazelMetrics()
190    _read_data(bazel_metrics_file, bazel_metrics)
191    _maybe_save_data(bazel_metrics, "bazel_metrics.pb", args)
192  else:
193    _handle_missing_metrics(args, "bazel_metrics.pb")
194
195
196def process_build_files_mode(args):
197  if args.skip_metrics:
198    raise Exception("build_files mode incompatible with --skip-metrics")
199  os.makedirs(args.output_dir, exist_ok=True)
200  tar_out = os.path.join(args.output_dir, "build_files.tar.gz")
201
202  os.chdir(args.metrics_files_dir)
203
204  if os.path.exists(tar_out):
205    os.remove(tar_out)
206  print("adding build files to", tar_out, "...", file=sys.stderr)
207
208  with tarfile.open(tar_out, "w:gz", dereference=True) as tar:
209    for root, dirs, files in os.walk("."):
210      for file in files:
211        if (
212            file.endswith(".bzl")
213            or file.endswith("BUILD")
214            or file.endswith("BUILD.bazel")
215        ):
216          tar.add(os.path.join(root, file), arcname=os.path.join(root, file))
217
218
219def process_bp2build_mode(args):
220  metrics_files_dir = args.metrics_files_dir
221  if not args.skip_metrics:
222    os.makedirs(args.output_dir, exist_ok=True)
223    print("Writing build analysis files to " + args.output_dir, file=sys.stderr)
224
225  bp2build_file = os.path.join(metrics_files_dir, "bp2build_metrics.pb")
226  if not os.path.exists(bp2build_file):
227    raise Exception("bp2build mode requires that the last build ran bp2build")
228
229  bp2build_metrics = Bp2BuildMetrics()
230  _read_data(bp2build_file, bp2build_metrics)
231  _maybe_save_data(bp2build_metrics, "bp2build_metrics.pb", args)
232  converted_modules = {}
233  for module in bp2build_metrics.convertedModules:
234    converted_modules[module] = True
235
236  if len(args.module_names) > 0:
237    modules_to_report = args.module_names
238  else:
239    all_modules = {}
240    for m in converted_modules:
241      all_modules[m] = True
242    for m in bp2build_metrics.unconvertedModules:
243      all_modules[m] = True
244    modules_to_report = sorted(all_modules)
245
246  for name in modules_to_report:
247    if name in converted_modules:
248      print(name, "converted successfully.")
249    elif name in bp2build_metrics.unconvertedModules:
250      unconverted_summary = name + " not converted: "
251      t = bp2build_metrics.unconvertedModules[name].type
252      if t > -1 and t < len(UnconvertedReasonType.keys()):
253        unconverted_summary += UnconvertedReasonType.keys()[t]
254      else:
255        unconverted_summary += "UNKNOWN_TYPE"
256      if len(bp2build_metrics.unconvertedModules[name].detail) > 0:
257        unconverted_summary += (
258            " detail: " + bp2build_metrics.unconvertedModules[name].detail
259        )
260      print(unconverted_summary)
261    else:
262      print(name, "does not exist.")
263
264
265def _define_global_flags(parser, suppress_default=False):
266  """Adds global flags to the given parser object.
267
268  Global flags should be added to both the global args parser and subcommand
269  parsers. This allows global flags to be specified before or after the
270  subcommand.
271
272  Subcommand parser binding should pass suppress_default=True. This uses the
273  default value specified in the global parser.
274  """
275  parser.add_argument(
276      "--metrics_files_dir",
277      default=(
278          argparse.SUPPRESS if suppress_default else _get_default_metrics_dir()
279      ),
280      help="The directory contained metrics files to analyze."
281      + " Defaults to $OUT_DIR if set, $ANDROID_BUILD_TOP/out otherwise.",
282  )
283  parser.add_argument(
284      "--skip-metrics",
285      action="store_true",
286      default=(argparse.SUPPRESS if suppress_default else None),
287      help="If set, do not save the output of printproto commands.",
288  )
289  parser.add_argument(
290      "--output_dir",
291      default=(argparse.SUPPRESS if suppress_default else None),
292      help="The directory to save analyzed proto output to. "
293      + "If unspecified, will default to the directory specified with"
294      " --metrics_files_dir + '/analyze_build_output/'",
295  )
296
297
298def main():
299  # Parse args
300  parser = argparse.ArgumentParser(
301      description=(
302          "Analyzes build artifacts from the user's most recent build. Prints"
303          " and/or saves data in a user-friendly format. See"
304          " subcommand-specific help for analysis options."
305      ),
306      prog="analyze_build",
307  )
308  _define_global_flags(parser)
309  subparsers = parser.add_subparsers(
310      title="subcommands",
311      help='types of analysis to run, "timing" by default.',
312      dest="mode",
313  )
314  timing_parser = subparsers.add_parser(
315      "timing", help="print per-phase build timing information"
316  )
317  _define_global_flags(timing_parser, True)
318  build_files_parser = subparsers.add_parser(
319      "build_files",
320      help="create a tar containing all bazel-related build files",
321  )
322  _define_global_flags(build_files_parser, True)
323  bp2build_parser = subparsers.add_parser(
324      "bp2build",
325      help="print whether a module was generated by bp2build",
326  )
327  _define_global_flags(bp2build_parser, True)
328  bp2build_parser.add_argument(
329      "module_names",
330      metavar="module_name",
331      nargs="*",
332      help="print conversion info about these modules",
333  )
334
335  args = parser.parse_args()
336
337  # Use `timing` as the default build mode.
338  if not args.mode:
339    args.mode = "timing"
340  # Check the metrics dir.
341  if not os.path.exists(args.metrics_files_dir):
342    raise Exception(
343        "Directory "
344        + arg.metrics_files_dir
345        + " not found. Did you run a build?"
346    )
347
348  args.output_dir = args.output_dir or _get_default_out_dir(
349      args.metrics_files_dir
350  )
351
352  if args.mode == "timing":
353    process_timing_mode(args)
354  elif args.mode == "build_files":
355    process_build_files_mode(args)
356  elif args.mode == "bp2build":
357    process_bp2build_mode(args)
358
359
360if __name__ == "__main__":
361  main()
362