1#!/usr/bin/env python3 2# Copyright (C) 2023 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import sys 17if __name__ == "__main__": 18 sys.dont_write_bytecode = True 19 20import argparse 21import dataclasses 22import datetime 23import json 24import os 25import pathlib 26import random 27import re 28import shutil 29import subprocess 30import time 31import uuid 32from typing import Optional 33 34import pretty 35import utils 36 37 38class FatalError(Exception): 39 def __init__(self): 40 pass 41 42 43class OptionsError(Exception): 44 def __init__(self, message): 45 self.message = message 46 47 48@dataclasses.dataclass(frozen=True) 49class Lunch: 50 "Lunch combination" 51 52 target_product: str 53 "TARGET_PRODUCT" 54 55 target_release: str 56 "TARGET_RELEASE" 57 58 target_build_variant: str 59 "TARGET_BUILD_VARIANT" 60 61 def ToDict(self): 62 return { 63 "TARGET_PRODUCT": self.target_product, 64 "TARGET_RELEASE": self.target_release, 65 "TARGET_BUILD_VARIANT": self.target_build_variant, 66 } 67 68 def Combine(self): 69 return f"{self.target_product}-{self.target_release}-{self.target_build_variant}" 70 71 72@dataclasses.dataclass(frozen=True) 73class Change: 74 "A change that we make to the tree, and how to undo it" 75 label: str 76 "String to print in the log when the change is made" 77 78 change: callable 79 "Function to change the source tree" 80 81 undo: callable 82 "Function to revert the source tree to its previous condition in the most minimal way possible." 83 84_DUMPVARS_VARS=[ 85 "COMMON_LUNCH_CHOICES", 86 "HOST_PREBUILT_TAG", 87 "print", 88 "PRODUCT_OUT", 89 "report_config", 90 "TARGET_ARCH", 91 "TARGET_BUILD_VARIANT", 92 "TARGET_DEVICE", 93 "TARGET_PRODUCT", 94] 95 96_DUMPVARS_ABS_VARS =[ 97 "ANDROID_CLANG_PREBUILTS", 98 "ANDROID_JAVA_HOME", 99 "ANDROID_JAVA_TOOLCHAIN", 100 "ANDROID_PREBUILTS", 101 "HOST_OUT", 102 "HOST_OUT_EXECUTABLES", 103 "HOST_OUT_TESTCASES", 104 "OUT_DIR", 105 "print", 106 "PRODUCT_OUT", 107 "SOONG_HOST_OUT", 108 "SOONG_HOST_OUT_EXECUTABLES", 109 "TARGET_OUT_TESTCASES", 110] 111 112@dataclasses.dataclass(frozen=True) 113class Benchmark: 114 "Something we measure" 115 116 id: str 117 "Short ID for the benchmark, for the command line" 118 119 title: str 120 "Title for reports" 121 122 change: Change 123 "Source tree modification for the benchmark that will be measured" 124 125 dumpvars: Optional[bool] = False 126 "If specified, soong will run in dumpvars mode rather than build-mode." 127 128 modules: Optional[list[str]] = None 129 "Build modules to build on soong command line" 130 131 preroll: Optional[int] = 0 132 "Number of times to run the build command to stabilize" 133 134 postroll: Optional[int] = 3 135 "Number of times to run the build command after reverting the action to stabilize" 136 137 def build_description(self): 138 "Short description of the benchmark's Soong invocation." 139 if self.dumpvars: 140 return "dumpvars" 141 elif self.modules: 142 return " ".join(self.modules) 143 return "" 144 145 146 def soong_command(self, root): 147 "Command line args to soong_ui for this benchmark." 148 if self.dumpvars: 149 return [ 150 "--dumpvars-mode", 151 f"--vars=\"{' '.join(_DUMPVARS_VARS)}\"", 152 f"--abs-vars=\"{' '.join(_DUMPVARS_ABS_VARS)}\"", 153 "--var-prefix=var_cache_", 154 "--abs-var-prefix=abs_var_cache_", 155 ] 156 elif self.modules: 157 return [ 158 "--build-mode", 159 "--all-modules", 160 f"--dir={root}", 161 "--skip-metrics-upload", 162 ] + self.modules 163 else: 164 raise Exception("Benchmark must specify dumpvars or modules") 165 166 167@dataclasses.dataclass(frozen=True) 168class FileSnapshot: 169 "Snapshot of a file's contents." 170 171 filename: str 172 "The file that was snapshottened" 173 174 contents: str 175 "The contents of the file" 176 177 def write(self): 178 "Write the contents back to the file" 179 with open(self.filename, "w") as f: 180 f.write(self.contents) 181 182 183def Snapshot(filename): 184 """Return a FileSnapshot with the file's current contents.""" 185 with open(filename) as f: 186 contents = f.read() 187 return FileSnapshot(filename, contents) 188 189 190def Clean(): 191 """Remove the out directory.""" 192 def remove_out(): 193 out_dir = utils.get_out_dir() 194 #only remove actual contents, in case out is a symlink (as is the case for cog) 195 if os.path.exists(out_dir): 196 for filename in os.listdir(out_dir): 197 p = os.path.join(out_dir, filename) 198 if os.path.isfile(p) or os.path.islink(p): 199 os.remove(p) 200 elif os.path.isdir(p): 201 shutil.rmtree(p) 202 return Change(label="Remove out", change=remove_out, undo=lambda: None) 203 204 205def NoChange(): 206 """No change to the source tree.""" 207 return Change(label="No change", change=lambda: None, undo=lambda: None) 208 209 210def Create(filename): 211 "Create an action to create `filename`. The parent directory must exist." 212 def create(): 213 with open(filename, "w") as f: 214 pass 215 def delete(): 216 os.remove(filename) 217 return Change( 218 label=f"Create {filename}", 219 change=create, 220 undo=delete, 221 ) 222 223 224def Modify(filename, contents, before=None): 225 """Create an action to modify `filename` by appending the result of `contents` 226 before the last instances of `before` in the file. 227 228 Raises an error if `before` doesn't appear in the file. 229 """ 230 orig = Snapshot(filename) 231 if before: 232 index = orig.contents.rfind(before) 233 if index < 0: 234 report_error(f"{filename}: Unable to find string '{before}' for modify operation.") 235 raise FatalError() 236 else: 237 index = len(orig.contents) 238 modified = FileSnapshot(filename, orig.contents[:index] + contents() + orig.contents[index:]) 239 if False: 240 print(f"Modify: {filename}") 241 x = orig.contents.replace("\n", "\n ORIG") 242 print(f" ORIG {x}") 243 x = modified.contents.replace("\n", "\n MODIFIED") 244 print(f" MODIFIED {x}") 245 246 return Change( 247 label="Modify " + filename, 248 change=lambda: modified.write(), 249 undo=lambda: orig.write() 250 ) 251 252def ChangePublicApi(): 253 change = AddJavaField("frameworks/base/core/java/android/provider/Settings.java", 254 "@android.annotation.SuppressLint(\"UnflaggedApi\") public") 255 orig_current_text = Snapshot("frameworks/base/core/api/current.txt") 256 257 def undo(): 258 change.undo() 259 orig_current_text.write() 260 261 return Change( 262 label=change.label, 263 change=change.change, 264 undo=lambda: undo() 265 ) 266 267def AddJavaField(filename, prefix): 268 return Modify(filename, 269 lambda: f"{prefix} static final int BENCHMARK = {random.randint(0, 1000000)};\n", 270 before="}") 271 272 273def Comment(prefix, suffix=""): 274 return lambda: prefix + " " + str(uuid.uuid4()) + suffix 275 276 277class BenchmarkReport(): 278 "Information about a run of the benchmark" 279 280 lunch: Lunch 281 "lunch combo" 282 283 benchmark: Benchmark 284 "The benchmark object." 285 286 iteration: int 287 "Which iteration of the benchmark" 288 289 log_dir: str 290 "Path the the log directory, relative to the root of the reports directory" 291 292 preroll_duration_ns: [int] 293 "Durations of the in nanoseconds." 294 295 duration_ns: int 296 "Duration of the measured portion of the benchmark in nanoseconds." 297 298 postroll_duration_ns: [int] 299 "Durations of the postrolls in nanoseconds." 300 301 complete: bool 302 "Whether the benchmark made it all the way through the postrolls." 303 304 def __init__(self, lunch, benchmark, iteration, log_dir): 305 self.lunch = lunch 306 self.benchmark = benchmark 307 self.iteration = iteration 308 self.log_dir = log_dir 309 self.preroll_duration_ns = [] 310 self.duration_ns = -1 311 self.postroll_duration_ns = [] 312 self.complete = False 313 314 def ToDict(self): 315 return { 316 "lunch": self.lunch.ToDict(), 317 "id": self.benchmark.id, 318 "title": self.benchmark.title, 319 "modules": self.benchmark.modules, 320 "dumpvars": self.benchmark.dumpvars, 321 "change": self.benchmark.change.label, 322 "iteration": self.iteration, 323 "log_dir": self.log_dir, 324 "preroll_duration_ns": self.preroll_duration_ns, 325 "duration_ns": self.duration_ns, 326 "postroll_duration_ns": self.postroll_duration_ns, 327 "complete": self.complete, 328 } 329 330class Runner(): 331 """Runs the benchmarks.""" 332 333 def __init__(self, options): 334 self._options = options 335 self._reports = [] 336 self._complete = False 337 338 def Run(self): 339 """Run all of the user-selected benchmarks.""" 340 # Clean out the log dir or create it if necessary 341 prepare_log_dir(self._options.LogDir()) 342 343 try: 344 for lunch in self._options.Lunches(): 345 print(lunch) 346 for benchmark in self._options.Benchmarks(): 347 for iteration in range(self._options.Iterations()): 348 self._run_benchmark(lunch, benchmark, iteration) 349 self._complete = True 350 finally: 351 self._write_summary() 352 353 354 def _run_benchmark(self, lunch, benchmark, iteration): 355 """Run a single benchmark.""" 356 benchmark_log_subdir = self._benchmark_log_dir(lunch, benchmark, iteration) 357 benchmark_log_dir = self._options.LogDir().joinpath(benchmark_log_subdir) 358 359 sys.stderr.write(f"STARTING BENCHMARK: {benchmark.id}\n") 360 sys.stderr.write(f" lunch: {lunch.Combine()}\n") 361 sys.stderr.write(f" iteration: {iteration}\n") 362 sys.stderr.write(f" benchmark_log_dir: {benchmark_log_dir}\n") 363 364 report = BenchmarkReport(lunch, benchmark, iteration, benchmark_log_subdir) 365 self._reports.append(report) 366 367 # Preroll builds 368 for i in range(benchmark.preroll): 369 ns = self._run_build(lunch, benchmark_log_dir.joinpath(f"pre_{i}"), benchmark) 370 report.preroll_duration_ns.append(ns) 371 372 sys.stderr.write(f"PERFORMING CHANGE: {benchmark.change.label}\n") 373 if not self._options.DryRun(): 374 benchmark.change.change() 375 try: 376 377 # Measured build 378 ns = self._run_build(lunch, benchmark_log_dir.joinpath("measured"), benchmark) 379 report.duration_ns = ns 380 381 dist_one = self._options.DistOne() 382 if dist_one: 383 # If we're disting just one benchmark, save the logs and we can stop here. 384 self._dist(utils.get_dist_dir(), benchmark.dumpvars) 385 else: 386 self._dist(benchmark_log_dir, benchmark.dumpvars, store_metrics_only=True) 387 # Postroll builds 388 for i in range(benchmark.postroll): 389 ns = self._run_build(lunch, benchmark_log_dir.joinpath(f"post_{i}"), 390 benchmark) 391 report.postroll_duration_ns.append(ns) 392 393 finally: 394 # Always undo, even if we crashed or the build failed and we stopped. 395 sys.stderr.write(f"UNDOING CHANGE: {benchmark.change.label}\n") 396 if not self._options.DryRun(): 397 benchmark.change.undo() 398 399 self._write_summary() 400 sys.stderr.write(f"FINISHED BENCHMARK: {benchmark.id}\n") 401 402 def _benchmark_log_dir(self, lunch, benchmark, iteration): 403 """Construct the log directory fir a benchmark run.""" 404 path = f"{lunch.Combine()}/{benchmark.id}" 405 # Zero pad to the correct length for correct alpha sorting 406 path += ("/%0" + str(len(str(self._options.Iterations()))) + "d") % iteration 407 return path 408 409 def _run_build(self, lunch, build_log_dir, benchmark): 410 """Builds the modules. Saves interesting log files to log_dir. Raises FatalError 411 if the build fails. 412 """ 413 sys.stderr.write(f"STARTING BUILD {benchmark.build_description()}\n") 414 415 before_ns = time.perf_counter_ns() 416 if not self._options.DryRun(): 417 cmd = [ 418 "build/soong/soong_ui.bash", 419 ] + benchmark.soong_command(self._options.root) 420 env = dict(os.environ) 421 env["TARGET_PRODUCT"] = lunch.target_product 422 env["TARGET_RELEASE"] = lunch.target_release 423 env["TARGET_BUILD_VARIANT"] = lunch.target_build_variant 424 returncode = subprocess.call(cmd, env=env) 425 if returncode != 0: 426 report_error(f"Build failed: {' '.join(cmd)}") 427 raise FatalError() 428 429 after_ns = time.perf_counter_ns() 430 431 # TODO: Copy some log files. 432 433 sys.stderr.write(f"FINISHED BUILD {benchmark.build_description()}\n") 434 435 return after_ns - before_ns 436 437 def _dist(self, dist_dir, dumpvars, store_metrics_only=False): 438 out_dir = utils.get_out_dir() 439 dest_dir = dist_dir.joinpath("logs") 440 os.makedirs(dest_dir, exist_ok=True) 441 basenames = [ 442 "soong_build_metrics.pb", 443 "soong_metrics", 444 ] 445 if not store_metrics_only: 446 basenames.extend([ 447 "build.trace.gz", 448 "soong.log", 449 ]) 450 if dumpvars: 451 basenames = ['dumpvars-'+b for b in basenames] 452 for base in basenames: 453 src = out_dir.joinpath(base) 454 if src.exists(): 455 sys.stderr.write(f"DIST: copied {src} to {dest_dir}\n") 456 shutil.copy(src, dest_dir) 457 458 def _write_summary(self): 459 # Write the results, even if the build failed or we crashed, including 460 # whether we finished all of the benchmarks. 461 data = { 462 "start_time": self._options.Timestamp().isoformat(), 463 "branch": self._options.Branch(), 464 "tag": self._options.Tag(), 465 "benchmarks": [report.ToDict() for report in self._reports], 466 "complete": self._complete, 467 } 468 with open(self._options.LogDir().joinpath("summary.json"), "w", encoding="utf-8") as f: 469 json.dump(data, f, indent=2, sort_keys=True) 470 471 472def benchmark_table(benchmarks): 473 rows = [("ID", "DESCRIPTION", "REBUILD"),] 474 rows += [(benchmark.id, benchmark.title, benchmark.build_description()) for benchmark in 475 benchmarks] 476 return rows 477 478 479def prepare_log_dir(directory): 480 if os.path.exists(directory): 481 # If it exists and isn't a directory, fail. 482 if not os.path.isdir(directory): 483 report_error(f"Log directory already exists but isn't a directory: {directory}") 484 raise FatalError() 485 # Make sure the directory is empty. Do this rather than deleting it to handle 486 # symlinks cleanly. 487 for filename in os.listdir(directory): 488 entry = os.path.join(directory, filename) 489 if os.path.isdir(entry): 490 shutil.rmtree(entry) 491 else: 492 os.unlink(entry) 493 else: 494 # Create it 495 os.makedirs(directory) 496 497 498class Options(): 499 def __init__(self): 500 self._had_error = False 501 502 # Wall time clock when we started 503 self._timestamp = datetime.datetime.now(datetime.timezone.utc) 504 505 # Move to the root of the tree right away. Everything must happen from there. 506 self.root = utils.get_root() 507 if not self.root: 508 report_error("Unable to find root of tree from cwd.") 509 raise FatalError() 510 os.chdir(self.root) 511 512 # Initialize the Benchmarks. Note that this pre-loads all of the files, etc. 513 # Doing all that here forces us to fail fast if one of them can't load a required 514 # file, at the cost of a small startup speed. Don't make this do something slow 515 # like scan the whole tree. 516 self._init_benchmarks() 517 518 # Argument parsing 519 epilog = f""" 520benchmarks: 521{pretty.FormatTable(benchmark_table(self._benchmarks), prefix=" ")} 522""" 523 524 parser = argparse.ArgumentParser( 525 prog="benchmarks", 526 allow_abbrev=False, # Don't let people write unsupportable scripts. 527 formatter_class=argparse.RawDescriptionHelpFormatter, 528 epilog=epilog, 529 description="Run build system performance benchmarks.") 530 self.parser = parser 531 532 parser.add_argument("--log-dir", 533 help="Directory for logs. Default is $TOP/../benchmarks/.") 534 parser.add_argument("--dated-logs", action="store_true", 535 help="Append timestamp to log dir.") 536 parser.add_argument("-n", action="store_true", dest="dry_run", 537 help="Dry run. Don't run the build commands but do everything else.") 538 parser.add_argument("--tag", 539 help="Variant of the run, for when there are multiple perf runs.") 540 parser.add_argument("--lunch", nargs="*", 541 help="Lunch combos to test") 542 parser.add_argument("--iterations", type=int, default=1, 543 help="Number of iterations of each test to run.") 544 parser.add_argument("--branch", type=str, 545 help="Specify branch. Otherwise a guess will be made based on repo.") 546 parser.add_argument("--benchmark", nargs="*", default=[b.id for b in self._benchmarks], 547 metavar="BENCHMARKS", 548 help="Benchmarks to run. Default suite will be run if omitted.") 549 parser.add_argument("--dist-one", action="store_true", 550 help="Copy logs and metrics to the given dist dir. Requires that only" 551 + " one benchmark be supplied. Postroll steps will be skipped.") 552 553 self._args = parser.parse_args() 554 555 self._branch = self._branch() 556 self._log_dir = self._log_dir() 557 self._lunches = self._lunches() 558 559 # Validate the benchmark ids 560 all_ids = [benchmark.id for benchmark in self._benchmarks] 561 bad_ids = [id for id in self._args.benchmark if id not in all_ids] 562 if bad_ids: 563 for id in bad_ids: 564 self._error(f"Invalid benchmark: {id}") 565 566 # --dist-one requires that only one benchmark be supplied 567 if self._args.dist_one and len(self.Benchmarks()) != 1: 568 self._error("--dist-one requires that exactly one --benchmark.") 569 570 if self._had_error: 571 raise FatalError() 572 573 def Timestamp(self): 574 return self._timestamp 575 576 def _branch(self): 577 """Return the branch, either from the command line or by guessing from repo.""" 578 if self._args.branch: 579 return self._args.branch 580 try: 581 branch = subprocess.check_output(f"cd {self.root}/.repo/manifests" 582 + " && git rev-parse --abbrev-ref --symbolic-full-name @{u}", 583 shell=True, encoding="utf-8") 584 return branch.strip().split("/")[-1] 585 except subprocess.CalledProcessError as ex: 586 report_error("Can't get branch from .repo dir. Specify --branch argument") 587 report_error(str(ex)) 588 raise FatalError() 589 590 def Branch(self): 591 return self._branch 592 593 def _log_dir(self): 594 "The log directory to use, based on the current options" 595 if self._args.log_dir: 596 d = pathlib.Path(self._args.log_dir).resolve().absolute() 597 else: 598 d = self.root.joinpath("..", utils.DEFAULT_REPORT_DIR) 599 if self._args.dated_logs: 600 d = d.joinpath(self._timestamp.strftime('%Y-%m-%d')) 601 d = d.joinpath(self._branch) 602 if self._args.tag: 603 d = d.joinpath(self._args.tag) 604 return d.resolve().absolute() 605 606 def LogDir(self): 607 return self._log_dir 608 609 def Benchmarks(self): 610 return [b for b in self._benchmarks if b.id in self._args.benchmark] 611 612 def Tag(self): 613 return self._args.tag 614 615 def DryRun(self): 616 return self._args.dry_run 617 618 def _lunches(self): 619 def parse_lunch(lunch): 620 parts = lunch.split("-") 621 if len(parts) != 3: 622 raise OptionsError(f"Invalid lunch combo: {lunch}") 623 return Lunch(parts[0], parts[1], parts[2]) 624 # If they gave lunch targets on the command line use that 625 if self._args.lunch: 626 result = [] 627 # Split into Lunch objects 628 for lunch in self._args.lunch: 629 try: 630 result.append(parse_lunch(lunch)) 631 except OptionsError as ex: 632 self._error(ex.message) 633 return result 634 # Use whats in the environment 635 product = os.getenv("TARGET_PRODUCT") 636 release = os.getenv("TARGET_RELEASE") 637 variant = os.getenv("TARGET_BUILD_VARIANT") 638 if (not product) or (not release) or (not variant): 639 # If they didn't give us anything, fail rather than guessing. There's no good 640 # default for AOSP. 641 self._error("No lunch combo specified. Either pass --lunch argument or run lunch.") 642 return [] 643 return [Lunch(product, release, variant),] 644 645 def Lunches(self): 646 return self._lunches 647 648 def Iterations(self): 649 return self._args.iterations 650 651 def DistOne(self): 652 return self._args.dist_one 653 654 def _init_benchmarks(self): 655 """Initialize the list of benchmarks.""" 656 # Assumes that we've already chdired to the root of the tree. 657 self._benchmarks = [ 658 Benchmark( 659 id="full_lunch", 660 title="Lunch from clean out", 661 change=Clean(), 662 dumpvars=True, 663 preroll=0, 664 postroll=0, 665 ), 666 Benchmark( 667 id="noop_lunch", 668 title="Lunch with no change", 669 change=NoChange(), 670 dumpvars=True, 671 preroll=1, 672 postroll=0, 673 ), 674 Benchmark(id="full", 675 title="Full build", 676 change=Clean(), 677 modules=["droid"], 678 preroll=0, 679 postroll=3, 680 ), 681 Benchmark(id="nochange", 682 title="No change", 683 change=NoChange(), 684 modules=["droid"], 685 preroll=2, 686 postroll=3, 687 ), 688 Benchmark(id="unreferenced", 689 title="Create unreferenced file", 690 change=Create("bionic/unreferenced.txt"), 691 modules=["droid"], 692 preroll=1, 693 postroll=2, 694 ), 695 Benchmark(id="modify_bp", 696 title="Modify Android.bp", 697 change=Modify("bionic/libc/Android.bp", Comment("//")), 698 modules=["droid"], 699 preroll=1, 700 postroll=3, 701 ), 702 Benchmark(id="modify_stdio", 703 title="Modify stdio.cpp", 704 change=Modify("bionic/libc/stdio/stdio.cpp", Comment("//")), 705 modules=["libc"], 706 preroll=1, 707 postroll=2, 708 ), 709 Benchmark(id="modify_adbd", 710 title="Modify adbd", 711 change=Modify("packages/modules/adb/daemon/main.cpp", Comment("//")), 712 modules=["adbd"], 713 preroll=1, 714 postroll=2, 715 ), 716 Benchmark(id="services_private_field", 717 title="Add private field to ActivityManagerService.java", 718 change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java", 719 "private"), 720 modules=["services"], 721 preroll=1, 722 postroll=2, 723 ), 724 Benchmark(id="services_public_field", 725 title="Add public field to ActivityManagerService.java", 726 change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java", 727 "/** @hide */ public"), 728 modules=["services"], 729 preroll=1, 730 postroll=2, 731 ), 732 Benchmark(id="services_api", 733 title="Add API to ActivityManagerService.javaa", 734 change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java", 735 "@android.annotation.SuppressLint(\"UnflaggedApi\") public"), 736 modules=["services"], 737 preroll=1, 738 postroll=2, 739 ), 740 Benchmark(id="framework_private_field", 741 title="Add private field to Settings.java", 742 change=AddJavaField("frameworks/base/core/java/android/provider/Settings.java", 743 "private"), 744 modules=["framework-minus-apex"], 745 preroll=1, 746 postroll=2, 747 ), 748 Benchmark(id="framework_public_field", 749 title="Add public field to Settings.java", 750 change=AddJavaField("frameworks/base/core/java/android/provider/Settings.java", 751 "/** @hide */ public"), 752 modules=["framework-minus-apex"], 753 preroll=1, 754 postroll=2, 755 ), 756 Benchmark(id="framework_api", 757 title="Add API to Settings.java", 758 change=ChangePublicApi(), 759 modules=["api-stubs-docs-non-updatable-update-current-api", "framework-minus-apex"], 760 preroll=1, 761 postroll=2, 762 ), 763 Benchmark(id="modify_framework_resource", 764 title="Modify framework resource", 765 change=Modify("frameworks/base/core/res/res/values/config.xml", 766 lambda: str(uuid.uuid4()), 767 before="</string>"), 768 modules=["framework-minus-apex"], 769 preroll=1, 770 postroll=2, 771 ), 772 Benchmark(id="add_framework_resource", 773 title="Add framework resource", 774 change=Modify("frameworks/base/core/res/res/values/config.xml", 775 lambda: f"<string name=\"BENCHMARK\">{uuid.uuid4()}</string>", 776 before="</resources>"), 777 modules=["framework-minus-apex"], 778 preroll=1, 779 postroll=2, 780 ), 781 Benchmark(id="add_systemui_field", 782 title="Add SystemUI field", 783 change=AddJavaField("frameworks/base/packages/SystemUI/src/com/android/systemui/wmshell/WMShell.java", 784 "public"), 785 modules=["SystemUI"], 786 preroll=1, 787 postroll=2, 788 ), 789 ] 790 791 def _error(self, message): 792 report_error(message) 793 self._had_error = True 794 795 796def report_error(message): 797 sys.stderr.write(f"error: {message}\n") 798 799 800def main(argv): 801 try: 802 options = Options() 803 runner = Runner(options) 804 runner.Run() 805 except FatalError: 806 sys.stderr.write(f"FAILED\n") 807 sys.exit(1) 808 809 810if __name__ == "__main__": 811 main(sys.argv) 812