1# Copyright (C) 2022 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14import csv
15import dataclasses
16import datetime
17import enum
18import functools
19import json
20import logging
21import os
22import re
23import subprocess
24import sys
25from datetime import date
26from pathlib import Path
27import textwrap
28from typing import Callable
29from typing import Final
30from typing import Generator
31from typing import TypeVar
32
33INDICATOR_FILE: Final[str] = "build/soong/soong_ui.bash"
34# metrics.csv is written to but not read by this tool.
35# It's supposed to be viewed as a spreadsheet that compiles data from multiple
36# builds to be analyzed by other external tools.
37METRICS_TABLE: Final[str] = "metrics.csv"
38RUN_DIR_PREFIX: Final[str] = "run"
39BUILD_INFO_JSON: Final[str] = "build_info.json"
40
41
42@functools.cache
43def _is_important(column) -> bool:
44    patterns = {
45        "actions",
46        r"build_ninja_(?:hash|size)",
47        "build_type",
48        "cquery_out_size",
49        "description",
50        "log",
51        r"mixed\.enabled",
52        "targets",
53        # the following are time-based values
54        "bp2build",
55        r"kati/kati (?:build|package)",
56        "ninja/ninja",
57        "soong/soong",
58        r"soong_build/\*(?:\.bazel)?",
59        "symlink_forest",
60        "time",
61    }
62    for pattern in patterns:
63        if re.fullmatch(pattern, column):
64            return True
65    return False
66
67
68class BuildResult(enum.Enum):
69    SUCCESS = enum.auto()
70    FAILED = enum.auto()
71    TEST_FAILURE = enum.auto()
72
73
74class BuildType(enum.Enum):
75    # see https://docs.python.org/3/library/enum.html#enum.Enum._ignore_
76    _ignore_ = "_soong_cmd"
77    # _soong_cmd_ will not be listed as an enum constant because of `_ignore_`
78    _soong_cmd = ["build/soong/soong_ui.bash", "--make-mode", "--skip-soong-tests"]
79
80    SOONG_ONLY = [*_soong_cmd, "BUILD_BROKEN_DISABLE_BAZEL=true"]
81    MIXED_PROD = [*_soong_cmd, "--bazel-mode"]
82    MIXED_STAGING = [*_soong_cmd, "--bazel-mode-staging"]
83    B_BUILD = ["build/bazel/bin/b", "build"]
84    B_ANDROID = [*B_BUILD, "--config=android"]
85
86    @staticmethod
87    def from_flag(s: str) -> list["BuildType"]:
88        chosen: list[BuildType] = []
89        for e in BuildType:
90            if s.lower() in e.name.lower():
91                chosen.append(e)
92        if len(chosen) == 0:
93            raise RuntimeError(f"no such build type: {s}")
94        return chosen
95
96    def to_flag(self):
97        return self.name.lower()
98
99
100CURRENT_BUILD_TYPE: BuildType
101"""global state capturing what the current build type is"""
102
103
104@dataclasses.dataclass
105class BuildInfo:
106    actions: int
107    bp_size_total: int
108    build_ninja_hash: str  # hash
109    build_ninja_size: int
110    build_result: BuildResult
111    build_type: BuildType
112    bz_size_total: int
113    cquery_out_size: int
114    description: str
115    product: str
116    targets: tuple[str, ...]
117    time: datetime.timedelta
118    tag: str = None
119    rebuild: bool = False
120    warmup: bool = False
121
122
123class CustomEncoder(json.JSONEncoder):
124    def default(self, obj):
125        if isinstance(obj, BuildInfo):
126            return self.default(dataclasses.asdict(obj))
127        if isinstance(obj, dict):
128            return {k: v for k, v in obj.items() if v is not None}
129        if isinstance(obj, datetime.timedelta):
130            return hhmmss(obj, decimal_precision=True)
131        if isinstance(obj, enum.Enum):
132            return obj.name
133        return json.JSONEncoder.default(self, obj)
134
135
136def get_csv_columns_cmd(d: Path) -> str:
137    """
138    :param d: the log directory
139    :return: a quick shell command to view columns in metrics.csv
140    """
141    csv_file = d.joinpath(METRICS_TABLE).resolve()
142    return f'head -n 1 "{csv_file}" | sed "s/,/\\n/g" | less -N'
143
144
145def get_cmd_to_display_tabulated_metrics(d: Path, ci_mode: bool) -> str:
146    """
147    :param d: the log directory
148    :param ci_mode: if true all top-level events are displayed
149    :return: a quick shell command to view some collected metrics
150    """
151    csv_file = d.joinpath(METRICS_TABLE)
152    headers: list[str] = []
153    if csv_file.exists():
154        with open(csv_file) as r:
155            reader = csv.DictReader(r)
156            headers = reader.fieldnames or []
157
158    cols: list[int] = [i + 1 for i, h in enumerate(headers) if _is_important(h)]
159    if ci_mode:
160        # ci mode contains all information about the top level events
161        for i, h in enumerate(headers):
162            if re.match(r"^\w+/[^.]+$", h) and i not in cols:
163                cols.append(i)
164
165    if len(cols) == 0:
166        # syntactically correct command even if the file doesn't exist
167        cols.append(1)
168
169    f = ",".join(str(i) for i in cols)
170    # the sed invocations are to account for
171    # https://man7.org/linux/man-pages/man1/column.1.html#BUGS
172    # example: if a row were `,,,hi,,,,`
173    # the successive sed conversions would be
174    #    `,,,hi,,,,` =>
175    #    `,--,,hi,--,,--,` =>
176    #    `,--,--,hi,--,--,--,` =>
177    #    `--,--,--,hi,--,--,--,` =>
178    #    `--,--,--,hi,--,--,--,--`
179    # Note sed doesn't support lookahead or lookbehinds
180    return textwrap.dedent(
181        f"""\
182        grep -v "WARMUP\\|rebuild-" "{csv_file}" | \\
183          sed "s/,,/,--,/g" | \\
184          sed "s/,,/,--,/g" | \\
185          sed "s/^,/--,/" | \\
186          sed "s/,$/,--/" | \\
187          cut -d, -f{f} | column -t -s,"""
188    )
189
190
191@functools.cache
192def get_top_dir(d: Path = Path(".").resolve()) -> Path:
193    """Get the path to the root of the Android source tree"""
194    top_dir = os.environ.get("ANDROID_BUILD_TOP")
195    if top_dir:
196        logging.info("ANDROID BUILD TOP = %s", d)
197        return Path(top_dir).resolve()
198    logging.debug("Checking if Android source tree root is %s", d)
199    if d.parent == d:
200        sys.exit(
201            "Unable to find ROOT source directory, specifically,"
202            f"{INDICATOR_FILE} not found anywhere. "
203            "Try `m nothing` and `repo sync`"
204        )
205    if d.joinpath(INDICATOR_FILE).is_file():
206        logging.info("ANDROID BUILD TOP assumed to be %s", d)
207        return d
208    return get_top_dir(d.parent)
209
210
211@functools.cache
212def get_out_dir() -> Path:
213    out_dir = os.environ.get("OUT_DIR")
214    return Path(out_dir).resolve() if out_dir else get_top_dir().joinpath("out")
215
216
217@functools.cache
218def get_default_log_dir() -> Path:
219    return get_top_dir().parent.joinpath(f'timing-{date.today().strftime("%b%d")}')
220
221
222def is_interactive_shell() -> bool:
223    return (
224        sys.__stdin__.isatty() and sys.__stdout__.isatty() and sys.__stderr__.isatty()
225    )
226
227
228# see test_next_path_helper() for examples
229def _next_path_helper(basename: str) -> str:
230    def padded_suffix(i: int) -> str:
231        return f"{i:03d}"
232
233    # find the sequence digits following "-" and preceding the file extension
234    name = re.sub(
235        r"(?<=-)(?P<suffix>\d+)$",
236        lambda d: padded_suffix(int(d.group("suffix")) + 1),
237        basename,
238    )
239
240    if name == basename:
241        # basename didn't have any numeric suffix
242        name = f"{name}-{padded_suffix(1)}"
243    return name
244
245
246def next_path(path: Path) -> Generator[Path, None, None]:
247    """
248    Generator for indexed paths
249    :returns a new Path with an increasing number suffix to the name
250    e.g. _to_file('a.txt') = a-5.txt (if a-4.txt already exists)
251    """
252    while True:
253        name = _next_path_helper(path.name)
254        path = path.parent.joinpath(name)
255        if not path.exists():
256            yield path
257
258
259def has_uncommitted_changes() -> bool:
260    """
261    effectively a quick 'repo status' that fails fast
262    if any project has uncommitted changes
263    """
264    for cmd in ["diff", "diff --staged"]:
265        diff = subprocess.run(
266            f"repo forall -c git {cmd} --quiet --exit-code",
267            cwd=get_top_dir(),
268            shell=True,
269            text=True,
270            capture_output=True,
271        )
272        if diff.returncode != 0:
273            logging.error(diff.stderr)
274            return True
275    return False
276
277
278def hhmmss(t: datetime.timedelta, decimal_precision: bool = False) -> str:
279    """pretty prints time periods, prefers mm:ss.sss and resorts to hh:mm:ss.sss
280    only if t >= 1 hour.
281    Examples(non_decimal_precision): 02:12, 1:12:13
282    Examples(decimal_precision): 02:12.231, 00:00.512, 00:01:11.321, 1:12:13.121
283    See unit test for more examples."""
284    h, f = divmod(t.seconds, 60 * 60)
285    m, f = divmod(f, 60)
286    s = f + t.microseconds / 1000_000
287    if decimal_precision:
288        return f"{h}:{m:02d}:{s:06.3f}" if h else f"{m:02d}:{s:06.3f}"
289    else:
290        return f"{h}:{m:02}:{s:02.0f}" if h else f"{m:02}:{s:02.0f}"
291
292
293def period_to_seconds(s: str) -> float:
294    """converts a time period into seconds. The input is expected to be in the
295    format used by hhmmss().
296    Example: 02:04 -> 125
297    See unit test for more examples."""
298    if s == "":
299        return 0.0
300    acc = 0.0
301    while True:
302        [left, *right] = s.split(":", 1)
303        acc = acc * 60 + float(left)
304        if right:
305            s = right[0]
306        else:
307            return acc
308
309
310R = TypeVar("R")
311
312
313def groupby(xs: list[R], key: Callable[[R], str]) -> dict[str, list[R]]:
314    grouped: dict[str, list[R]] = {}
315    for x in xs:
316        grouped.setdefault(key(x), []).append(x)
317    return grouped
318