1# Copyright (C) 2022 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import functools
16import io
17import logging
18import shutil
19import tempfile
20import uuid
21from pathlib import Path
22from typing import Final, Iterable, Optional
23
24import clone
25import cuj
26import finder
27import util
28import cuj_regex_based
29from cuj import CujGroup
30from cuj import CujStep
31from cuj import InWorkspace
32from cuj import Verifier
33from cuj import de_src
34from cuj import src
35from util import BuildType
36
37"""
38Provides some representative CUJs. If you wanted to manually run something but
39would like the metrics to be collated in the metrics.csv file, use
40`perf_metrics.py` as a stand-alone after your build.
41"""
42
43
44class Clean(CujGroup):
45    def __init__(self):
46        super().__init__("clean")
47
48    def get_steps(self) -> Iterable[CujStep]:
49        def clean():
50            if util.get_out_dir().exists():
51                shutil.rmtree(util.get_out_dir())
52
53        return [CujStep("", clean)]
54
55
56class NoChange(CujGroup):
57    def __init__(self):
58        super().__init__("no change")
59
60    def get_steps(self) -> Iterable[CujStep]:
61        return [CujStep("", lambda: None)]
62
63
64Warmup: Final[CujGroup] = NoChange()
65Warmup._desc = "WARMUP"
66
67
68class Modify(CujGroup):
69    """
70    A pair of CujSteps, where the first modifies the file and the
71    second reverts the modification
72    Arguments:
73        file: the file to be modified and reverted
74        text: the text to be appended to the file to modify it
75    """
76
77    def __init__(self, file: Path, text: Optional[str] = None):
78        super().__init__(f"modify {de_src(file)}")
79        if not file.exists():
80            raise RuntimeError(f"{file} does not exist")
81        self.file = file
82        self.text = text
83
84    def get_steps(self) -> Iterable[CujStep]:
85        if self.text is None:
86            self.text = f"//BOGUS {uuid.uuid4()}\n"
87
88        def add_line():
89            with open(self.file, mode="a") as f:
90                f.write(self.text)
91
92        def revert():
93            with open(self.file, mode="rb+") as f:
94                # assume UTF-8
95                f.seek(-len(self.text), io.SEEK_END)
96                f.truncate()
97
98        return [CujStep("", add_line), CujStep("revert", revert)]
99
100
101class Create(CujGroup):
102    """
103    A pair of CujSteps, where the fist creates the file and the
104    second deletes it
105    Attributes:
106        file: the file to be created and deleted
107        ws: the expectation for the counterpart file in symlink
108            forest (aka the synthetic bazel workspace) when its created
109        text: the content of the file
110    """
111
112    def __init__(self, file: Path, ws: InWorkspace, text: Optional[str] = None):
113        super().__init__(f"create {de_src(file)}")
114        if file.exists():
115            raise RuntimeError(
116                f"File {file} already exists. Interrupted an earlier run?\n"
117                "TIP: `repo status` and revert changes!!!"
118            )
119        self.file = file
120        self.ws = ws
121        self.text = text
122        if self.text is None:
123            self.text = f"//Test File: safe to delete {uuid.uuid4()}\n"
124
125    def get_steps(self) -> Iterable[CujStep]:
126        missing_dirs = [f for f in self.file.parents if not f.exists()]
127        shallowest_missing_dir = missing_dirs[-1] if len(missing_dirs) else None
128
129        def create():
130            self.file.parent.mkdir(parents=True, exist_ok=True)
131            self.file.touch(exist_ok=False)
132            with open(self.file, mode="w") as f:
133                f.write(self.text)
134
135        def delete():
136            if shallowest_missing_dir:
137                shutil.rmtree(shallowest_missing_dir)
138            else:
139                self.file.unlink(missing_ok=False)
140
141        return [
142            CujStep("", create, self.ws.verifier(self.file)),
143            CujStep("revert", delete, InWorkspace.OMISSION.verifier(self.file)),
144        ]
145
146
147class CreateBp(Create):
148    """
149    This is basically the same as "Create" but with canned content for
150    an Android.bp file.
151    """
152
153    def __init__(self, bp_file: Path):
154        super().__init__(
155            bp_file,
156            InWorkspace.SYMLINK,
157            'filegroup { name: "test-bogus-filegroup", srcs: ["**/*.md"] }',
158        )
159
160
161class Delete(CujGroup):
162    """
163    A pair of CujSteps, where the first deletes a file and the second
164    restores it
165    Attributes:
166        original: The file to be deleted then restored
167        ws: When restored, expectation for the file's counterpart in the
168            symlink forest (aka synthetic bazel workspace)
169    """
170
171    def __init__(self, original: Path, ws: InWorkspace):
172        super().__init__(f"delete {de_src(original)}")
173        self.original = original
174        self.ws = ws
175
176    def get_steps(self) -> Iterable[CujStep]:
177        tempdir = Path(tempfile.gettempdir())
178        if tempdir.is_relative_to(util.get_top_dir()):
179            raise SystemExit(f"Temp dir {tempdir} is under source tree")
180        if tempdir.is_relative_to(util.get_out_dir()):
181            raise SystemExit(
182                f"Temp dir {tempdir} is under " f"OUT dir {util.get_out_dir()}"
183            )
184        copied = tempdir.joinpath(f"{self.original.name}-{uuid.uuid4()}.bak")
185
186        def move_to_tempdir_to_mimic_deletion():
187            logging.warning("MOVING %s TO %s", de_src(self.original), copied)
188            self.original.rename(copied)
189
190        return [
191            CujStep(
192                "",
193                move_to_tempdir_to_mimic_deletion,
194                InWorkspace.OMISSION.verifier(self.original),
195            ),
196            CujStep(
197                "revert",
198                lambda: copied.rename(self.original),
199                self.ws.verifier(self.original),
200            ),
201        ]
202
203
204class ReplaceFileWithDir(CujGroup):
205    """Replace a file with a non-empty directory"""
206
207    def __init__(self, p: Path):
208        super().__init__(f"replace {de_src(p)} with dir")
209        self.p = p
210
211    def get_steps(self) -> Iterable[CujStep]:
212        # an Android.bp is always a symlink in the workspace and thus its parent
213        # will be a directory in the workspace
214        create_dir: CujStep
215        delete_dir: CujStep
216        create_dir, delete_dir, *tail = CreateBp(
217            self.p.joinpath("Android.bp")
218        ).get_steps()
219        assert len(tail) == 0
220
221        original_text: str
222
223        def replace_it():
224            nonlocal original_text
225            original_text = self.p.read_text()
226            self.p.unlink()
227            create_dir.apply_change()
228
229        def revert():
230            delete_dir.apply_change()
231            self.p.write_text(original_text)
232
233        return [
234            CujStep(f"", replace_it, create_dir.verify),
235            CujStep(f"revert", revert, InWorkspace.SYMLINK.verifier(self.p)),
236        ]
237
238
239def content_verfiers(ws_build_file: Path, content: str) -> tuple[Verifier, Verifier]:
240    def search() -> bool:
241        with open(ws_build_file, "r") as f:
242            for line in f:
243                if line == content:
244                    return True
245        return False
246
247    @cuj.skip_for(BuildType.SOONG_ONLY)
248    def contains():
249        if not search():
250            raise AssertionError(
251                f"{de_src(ws_build_file)} expected to contain {content}"
252            )
253        logging.info(f"VERIFIED {de_src(ws_build_file)} contains {content}")
254
255    @cuj.skip_for(BuildType.SOONG_ONLY)
256    def does_not_contain():
257        if search():
258            raise AssertionError(
259                f"{de_src(ws_build_file)} not expected to contain {content}"
260            )
261        logging.info(f"VERIFIED {de_src(ws_build_file)} does not contain {content}")
262
263    return contains, does_not_contain
264
265
266class ModifyKeptBuildFile(CujGroup):
267    def __init__(self, build_file: Path):
268        super().__init__(f"modify kept {de_src(build_file)}")
269        self.build_file = build_file
270
271    def get_steps(self) -> Iterable[CujStep]:
272        content = f"//BOGUS {uuid.uuid4()}\n"
273        step1, step2, *tail = Modify(self.build_file, content).get_steps()
274        assert len(tail) == 0
275        ws_build_file = InWorkspace.ws_counterpart(self.build_file).with_name(
276            "BUILD.bazel"
277        )
278        merge_prover, merge_disprover = content_verfiers(ws_build_file, content)
279        return [
280            CujStep(
281                step1.verb,
282                step1.apply_change,
283                cuj.sequence(step1.verify, merge_prover),
284            ),
285            CujStep(
286                step2.verb,
287                step2.apply_change,
288                cuj.sequence(step2.verify, merge_disprover),
289            ),
290        ]
291
292
293class CreateKeptBuildFile(CujGroup):
294    def __init__(self, build_file: Path):
295        super().__init__(f"create kept {de_src(build_file)}")
296        self.build_file = build_file
297        if self.build_file.name == "BUILD.bazel":
298            self.ws = InWorkspace.NOT_UNDER_SYMLINK
299        elif self.build_file.name == "BUILD":
300            self.ws = InWorkspace.SYMLINK
301        else:
302            raise RuntimeError(f"Illegal name for a build file {self.build_file}")
303
304    def get_steps(self):
305        content = f"//BOGUS {uuid.uuid4()}\n"
306        ws_build_file = InWorkspace.ws_counterpart(self.build_file).with_name(
307            "BUILD.bazel"
308        )
309        merge_prover, merge_disprover = content_verfiers(ws_build_file, content)
310
311        step1: CujStep
312        step2: CujStep
313        step1, step2, *tail = Create(self.build_file, self.ws, content).get_steps()
314        assert len(tail) == 0
315        return [
316            CujStep(
317                step1.verb,
318                step1.apply_change,
319                cuj.sequence(step1.verify, merge_prover),
320            ),
321            CujStep(
322                step2.verb,
323                step2.apply_change,
324                cuj.sequence(step2.verify, merge_disprover),
325            ),
326        ]
327
328
329class CreateUnkeptBuildFile(CujGroup):
330    def __init__(self, build_file: Path):
331        super().__init__(f"create unkept {de_src(build_file)}")
332        self.build_file = build_file
333
334    def get_steps(self):
335        content = f"//BOGUS {uuid.uuid4()}\n"
336        ws_build_file = InWorkspace.ws_counterpart(self.build_file).with_name(
337            "BUILD.bazel"
338        )
339        step1: CujStep
340        step2: CujStep
341        step1, step2, *tail = Create(
342            self.build_file, InWorkspace.OMISSION, content
343        ).get_steps()
344        assert len(tail) == 0
345        _, merge_disprover = content_verfiers(ws_build_file, content)
346        return [
347            CujStep(step1.verb, step1.apply_change, merge_disprover),
348            CujStep(step2.verb, step2.apply_change, merge_disprover),
349        ]
350
351
352def _mixed_build_launch_cujs() -> tuple[CujGroup, ...]:
353    core_settings = src("frameworks/base/core/java/android/provider/Settings.java")
354    ams = src(
355        "frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java"
356    )
357    resource = src("frameworks/base/core/res/res/values/config.xml")
358    return (
359        Modify(src("bionic/libc/tzcode/asctime.c")),
360        Modify(src("bionic/libc/stdio/stdio.cpp")),
361        Modify(src("packages/modules/adb/daemon/main.cpp")),
362        Modify(src("frameworks/base/core/java/android/view/View.java")),
363        Modify(core_settings),
364        cuj_regex_based.modify_private_method(core_settings),
365        cuj_regex_based.add_private_field(core_settings),
366        cuj_regex_based.add_public_api(core_settings),
367        cuj_regex_based.modify_private_method(ams),
368        cuj_regex_based.add_private_field(ams),
369        cuj_regex_based.add_public_api(ams),
370        cuj_regex_based.modify_resource(resource),
371        cuj_regex_based.add_resource(resource),
372    )
373
374
375def _cloning_cujs() -> tuple[CujGroup, ...]:
376    cc_ = (
377        lambda t, name: t.startswith("cc_")
378        and "test" not in t
379        and not name.startswith("libcrypto")  # has some unique hash
380    )
381    libNN = lambda t, name: t == "cc_library_shared" and name == "libneuralnetworks"
382    return (
383        clone.Clone("clone genrules", {src("."): clone.type_in("genrule")}),
384        clone.Clone("clone cc_", {src("."): cc_}),
385        clone.Clone(
386            "clone adbd",
387            {src("packages/modules/adb/Android.bp"): clone.name_in("adbd")},
388        ),
389        clone.Clone(
390            "clone libNN",
391            {src("packages/modules/NeuralNetworks/runtime/Android.bp"): libNN},
392        ),
393        clone.Clone(
394            "clone adbd&libNN",
395            {
396                src("packages/modules/adb/Android.bp"): clone.name_in("adbd"),
397                src("packages/modules/NeuralNetworks/runtime/Android.bp"): libNN,
398            },
399        ),
400    )
401
402
403@functools.cache
404def get_cujgroups() -> tuple[CujGroup, ...]:
405    # we are choosing "package" directories that have Android.bp but
406    # not BUILD nor BUILD.bazel because
407    # we can't tell if ShouldKeepExistingBuildFile would be True or not
408    non_empty_dir = "*/*"
409    pkg = src("art")
410    finder.confirm(pkg, non_empty_dir, "Android.bp", "!BUILD*")
411    pkg_free = src("bionic/docs")
412    finder.confirm(pkg_free, non_empty_dir, "!**/Android.bp", "!**/BUILD*")
413    ancestor = src("bionic")
414    finder.confirm(ancestor, "**/Android.bp", "!Android.bp", "!BUILD*")
415    leaf_pkg_free = src("bionic/build")
416    finder.confirm(leaf_pkg_free, f"!{non_empty_dir}", "!**/Android.bp", "!**/BUILD*")
417
418    android_bp_cujs = (
419        Modify(src("Android.bp")),
420        *(
421            CreateBp(d.joinpath("Android.bp"))
422            for d in [ancestor, pkg_free, leaf_pkg_free]
423        ),
424    )
425    unreferenced_file_cujs = (
426        *(
427            Create(d.joinpath("unreferenced.txt"), InWorkspace.SYMLINK)
428            for d in [ancestor, pkg]
429        ),
430        *(
431            Create(d.joinpath("unreferenced.txt"), InWorkspace.UNDER_SYMLINK)
432            for d in [pkg_free, leaf_pkg_free]
433        ),
434    )
435
436    return (
437        Clean(),
438        NoChange(),
439        *_cloning_cujs(),
440        Create(src("bionic/libc/tzcode/globbed.c"), InWorkspace.UNDER_SYMLINK),
441        # TODO (usta): find targets that should be affected
442        *(
443            Delete(f, InWorkspace.SYMLINK)
444            for f in [
445                src("bionic/libc/version_script.txt"),
446                src("external/cbor-java/AndroidManifest.xml"),
447            ]
448        ),
449        *unreferenced_file_cujs,
450        *_mixed_build_launch_cujs(),
451        *android_bp_cujs,
452        ReplaceFileWithDir(src("bionic/README.txt")),
453        # TODO(usta): add a dangling symlink
454    )
455