1# Copyright (C) 2023 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""A script to rebuild QEMU from scratch on Linux."""
16
17import argparse
18from enum import Enum
19import os
20from pathlib import Path
21import shlex
22import shutil
23import subprocess
24import sys
25from typing import Any
26from typing import Callable
27from typing import Dict
28from typing import List
29from typing import Sequence
30
31
32class Architecture(Enum):
33  AARCH64 = "aarch64"
34  X86_64 = "x86_64"
35
36
37def copy_file(src: Path, dst: Path) -> None:
38  log("  COPY_FILE %s --> %s" % (src, dst))
39  os.makedirs(dst.parent, exist_ok=True)
40  shutil.copy2(src, dst)
41
42
43def log(msg: str) -> None:
44  print(msg)
45
46
47def create_dev_environment(
48    build_dir: Path,
49    install_dir: Path,
50    prebuilts_dir: Path,
51    clang_dir: Path,
52    target_arch: Architecture,
53) -> Dict[str, str]:
54  sysroot = str(build_dir / "sysroot")
55  binprefix = "%s/" % (clang_dir / "bin")
56  env = os.environ.copy()
57  path = env["PATH"]
58  ld_library_path = env.get("LD_LIBRARY_PATH", "")
59
60  if target_arch == Architecture.AARCH64:
61    cargo_path = Path(env["HOME"]) / ".cargo" / "bin"
62    env.update({
63        "CC": f"clang",
64        "CXX": f"clang++ -stdlib=libc++",
65        # FIXME: this file does not exist.
66        "LD": f"llvm-ld",
67        "AR": f"llvm-ar",
68        "NM": f"llvm-nm",
69        "PKG_CONFIG_PATH": ":".join([
70            f"{install_dir}/usr/lib/aarch64-linux-gnu/pkgconfig",
71            f"{install_dir}/usr/lib/pkgconfig",
72        ]),
73        "PATH": f"{cargo_path}:{install_dir}/usr/bin:{path}",
74        "LD_LIBRARY_PATH": ":".join([
75            f"{install_dir}/usr/lib/aarch64-linux-gnu",
76            f"{install_dir}/usr/lib",
77            f"{ld_library_path}",
78        ]),
79        # Required to ensure that configure scripts that do not rely
80        # on pkg-config find their dependencies properly.
81        "CFLAGS": f"-I{install_dir}/usr/include",
82        "CXXFLAGS": f"-I{install_dir}/usr/include",
83        "LDFLAGS": f"-Wl,-L{install_dir}/usr/lib",
84    })
85  else:
86    env.update({
87        "CC": f"{binprefix}clang --sysroot={sysroot}",
88        "CXX": f"{binprefix}clang++ --sysroot={sysroot} -stdlib=libc++",
89        # FIXME: this file does not exist.
90        "LD": f"{binprefix}llvm-ld --sysroot={sysroot}",
91        "AR": f"{binprefix}llvm-ar",
92        "NM": f"{binprefix}llvm-nm",
93        "PKG_CONFIG_PATH": ":".join([
94            f"{install_dir}/usr/lib/x86_64-linux-gnu/pkgconfig",
95            f"{install_dir}/usr/lib/pkgconfig",
96            f"{install_dir}/usr/lib64/pkgconfig",
97            f"{sysroot}/usr/lib/pkgconfig",
98        ]),
99        "PATH": f"{install_dir}/usr/bin:{path}",
100        "LD_LIBRARY_PATH": ":".join([
101            f"{install_dir}/usr/lib/x86_64-linux-gnu",
102            f"{install_dir}/usr/lib",
103            f"{clang_dir}/lib:{ld_library_path}",
104        ]),
105        # Required to ensure that configure scripts that do not rely
106        # on pkg-config find their dependencies properly.
107        "CFLAGS": f"-I{install_dir}/usr/include",
108        "CXXFLAGS": f"-I{install_dir}/usr/include",
109        "LDFLAGS": f"-Wl,-L{install_dir}/usr/lib",
110    })
111  return env
112
113
114_CLANG_VERSION = "r487747"
115
116
117def generate_shell_command(
118    cmd_args: Sequence[str],
119    build_dir: Path | None,
120    env: Dict[str, str] | None = None,
121    base_env: Dict[str, str] | None = None,
122) -> str:
123  """Generate a shell command that can be printed or written to a script.
124
125  Arguments:
126    cmd_args: A list of strings for the command arguments.
127    build_dir: An optional path to a build directory. None if the command must
128      run in the current one.
129    env: An optional dictionary of environment variable definitions
130    base_env: An optional base environment. Values in env will be compared to
131      it, and only differences will appear in the result. If None then
132      os.environ will be used.
133
134  Returns:
135    A single string that can be printed or written to a script.
136    All command arguments and variable values will be properly quoted.
137  """
138  if base_env is None:
139    base_env = dict(os.environ)
140  environ = []
141  if env:
142    environ = [
143        "%s=%s" % (k, shlex.quote(v))
144        for k, v in sorted(env.items())
145        if k not in base_env or base_env[k] != v
146    ]
147  result = ""
148  result_wrap = False
149  if build_dir:
150    result += f"cd {build_dir} && \\\n"
151    result_wrap = True
152  if environ:
153    result += " \\\n".join(environ) + " \\\n"
154
155  result += " ".join(shlex.quote(c) for c in cmd_args)
156  if result_wrap:
157    result = f"({result})"
158  return result
159
160
161def run_command(
162    cmd_args: Sequence[Any],
163    build_dir: Path | None = None,
164    env: Dict[str, str] | None = None,
165) -> None:
166  # Convert arguments to strings, to support Path items directly.
167  cmd_args = [str(c) for c in cmd_args]
168
169  # Log a copy pastable command to help with iteration.
170  log(generate_shell_command(cmd_args, build_dir, env) + "\n")
171  subprocess.run(cmd_args, cwd=build_dir, env=env).check_returncode()
172
173
174##########################################################################
175##########################################################################
176#####
177#####  B U I L D   C O N F I G
178#####
179##########################################################################
180##########################################################################
181
182
183class BuildConfig(object):
184  """Global build configuration object that is passed to all functions
185
186  that implement a specific task build instructions below.
187
188  This provides readonly directory paths, a default development
189  environment used to launch all action commands, and ways to augment it
190  with custom modifications (e.g. to add specific compiler flags).
191
192  Usage is the following:
193
194     1) Create instance, passing the path of the build directory, and
195        the path to the read-only top directory project.
196
197     2) Later, the instance is passed to each build function, which will
198        be able to call its various methods to perform the operations
199        required for its task, e.g. unpacking archives, applying patches
200        or running commands.
201  """
202
203  def __init__(self, build_dir: Path, top_dir: Path, target_arch: Architecture):
204    self._target_arch = target_arch
205    self._build_dir = build_dir
206    self._sysroot_dir = build_dir / "sysroot"
207    self._install_dir = build_dir / "dest-install"
208    self._prebuilts_dir = top_dir / "prebuilts"
209    self._clang_dir = self._prebuilts_dir / "clang" / f"clang-{_CLANG_VERSION}"
210    self._third_party_dir = top_dir / "third_party"
211    self._env = create_dev_environment(
212        self._build_dir,
213        self._install_dir,
214        self._prebuilts_dir,
215        self._clang_dir,
216        self._target_arch,
217    )
218
219    # By default, run commands directly. Subclasses can override
220    # this value to record the commands instead, for example
221    # to write them to a script or into a Makefile or Ninja build plan.
222    self._runner = run_command
223
224  def enable_ccache(self) -> None:
225    for varname in ("CC", "CXX"):
226      self._env[varname] = "ccache " + self._env[varname]
227
228  @property
229  def target_arch(self) -> Architecture:
230    return self._target_arch
231
232  @property
233  def build_dir(self) -> Path:
234    return self._build_dir
235
236  @property
237  def install_dir(self) -> Path:
238    return self._install_dir
239
240  @property
241  def prebuilts_dir(self) -> Path:
242    return self._prebuilts_dir
243
244  @property
245  def clang_dir(self) -> Path:
246    return self._clang_dir
247
248  @property
249  def sysroot_dir(self) -> Path:
250    return self._sysroot_dir
251
252  @property
253  def third_party_dir(self) -> Path:
254    return self._third_party_dir
255
256  def env_copy(self):
257    """Return a copy of the current environment dictionary."""
258    return self._env.copy()
259
260  def env_copy_with(self, new_values: Dict[str, str]) -> Dict[str, str]:
261    """Return a copy of the current environment, updated with new variables."""
262    env = self._env.copy()
263    env.update(new_values)
264    return env
265
266  def env_with_DESTDIR(self, dest_dir: Path | None = None) -> Dict[str, str]:
267    """Return a copy of the current environment, with DESTDIR set to
268
269    the installation directory.
270    """
271    return self.env_copy_with({"DESTDIR": str(dest_dir or self._install_dir)})
272
273  def make_subdir(self, subdir: Path) -> Path:
274    path = self.build_dir / subdir
275    self._runner(["rm", "-rf", path], self.build_dir, None)
276    self._runner(["mkdir", "-p", path], self.build_dir, None)
277    return path
278
279  def run(
280      self,
281      args: Sequence[Path | str],
282      sub_build_dir: Path | None = None,
283      env: Dict[str, str] | None = None,
284  ) -> None:
285    """Run a command in |sub_build_dir|, with optional |env|."""
286    cur_dir = self.build_dir
287    if sub_build_dir:
288      cur_dir = cur_dir / sub_build_dir
289    if env is None:
290      env = self.env_copy()
291    self._runner(args, cur_dir, env)
292
293  def run_make_build(
294      self, sub_build_dir: Path | None = None, extra_args: List[str] = []
295  ) -> None:
296    """Run `make -j<numcpus>` in |sub_build_dir|."""
297    self.run(["make", f"-j{os.cpu_count()}"] + extra_args, sub_build_dir)
298
299  def run_make_install(
300      self,
301      sub_build_dir: Path | None = None,
302      use_DESTDIR: bool = False,
303      dest_dir: Path | None = None,
304  ) -> None:
305    """Run `make install` in |sub_build_dir|.
306
307    If use_DESTDIR is True, set DESTDIR env variable.
308    """
309    env = None
310    if use_DESTDIR:
311      env = self.env_with_DESTDIR(dest_dir=dest_dir)
312    self.run(["make", "install"], sub_build_dir, env)
313
314  def copy_file(self, src_path: Path, dst_path: Path):
315    if dst_path.is_dir():
316      raise ValueError(
317          f"Misuse: dst_path ({dst_path}) points at an existing directory."
318      )
319    self._runner(["mkdir", "-p", dst_path.parent], None, None)
320    self._runner(["cp", "-f", src_path, dst_path], None, None)
321
322  def copy_dir(self, src_dir: Path, dst_dir: Path):
323    self._runner(["mkdir", "-p", dst_dir.parent], None, None)
324    self._runner(
325        ["cp", "-rfL", "--no-target-directory", src_dir, dst_dir], None, None
326    )
327
328
329##########################################################################
330##########################################################################
331#####
332#####  B U I L D   S E Q U E N C  E R
333#####
334#####  A |Project| can register build tasks and their dependencies.
335#####  Then it can return a build plan to be executed in sequence.
336#####
337##########################################################################
338##########################################################################
339
340BuildTaskFn = Callable[[BuildConfig], None]
341
342
343class Project:
344  # Type of a build task function that takes a single |BuildConfig| argument.
345
346  def __init__(self):
347    self.tasks = {}
348
349  def task(self, deps: List[BuildTaskFn]):
350    """Decorator that registers a |BuildTaskFn| and its dependencies."""
351
352    def decorator(fn: BuildTaskFn) -> BuildTaskFn:
353      for dep in deps:
354        if dep not in self.tasks:
355          raise ValueError(
356              f"Task {fn} depends on {dep}, but {dep} is was not yet defined."
357              " Did you forgot to annotate it?"
358          )
359      if fn in self.tasks:
360        raise ValueError(f"Task {fn} already defined.")
361      self.tasks[fn] = deps
362      return fn
363
364    return decorator
365
366  def get_build_task_list(
367      self, task_function: BuildTaskFn
368  ) -> List[BuildTaskFn]:
369    """Returns the transitive dependency list of the current task."""
370    # Rely on the fact that:
371    # a - function are registered in topological order
372    # b - python dictionaries are iterated in insertion order.
373    task_list = list(self.tasks.keys())
374    return task_list[: task_list.index(task_function) + 1]
375
376
377project = Project()
378
379##########################################################################
380##########################################################################
381#####
382#####  I N D I V I D U A L   T A S K S
383#####
384#####  Each build_task_for_xxx() function below should only access a single
385#####  BuildConfig argument, be decorated with `project.task` and enumerate
386#####  the tasks it depends on.
387#####
388#####  These functions should also only use the BuildConfig methods
389#####  to do their work, i.e. they shall not directly modify the
390#####  filesystem or environment in any way.
391#####
392##########################################################################
393##########################################################################
394
395
396@project.task([])
397def build_task_for_sysroot(build: BuildConfig):
398  if build.target_arch == Architecture.AARCH64:
399    # We don't build the sysroot for AARCH64 or its dependencies.
400    return
401
402  # populate_sysroot(build.build_dir / 'sysroot', build.prebuilts_dir),
403  dst_sysroot = build.build_dir / "sysroot"
404  dst_sysroot_lib_dir = dst_sysroot / "usr" / "lib"
405
406  # Copy the content of the sysroot first.
407  src_sysroot_dir = build.prebuilts_dir / "gcc/sysroot"
408  build.copy_dir(src_sysroot_dir / "usr", dst_sysroot / "usr")
409
410  # Add the static gcc runtime libraries and C runtime objects.
411  static_libgcc_dir = build.prebuilts_dir / "gcc/lib/gcc/x86_64-linux/4.8.3"
412  for lib in [
413      "libgcc.a",
414      "libgcc_eh.a",
415      "crtbegin.o",
416      "crtbeginS.o",
417      "crtbeginT.o",
418      "crtend.o",
419      "crtendS.o",
420  ]:
421    build.copy_file(static_libgcc_dir / lib, dst_sysroot_lib_dir / lib)
422
423  # Add the shared gcc runtime libraries.
424  # Do we need libatomic.so and others?
425  shared_libgcc_dir = build.prebuilts_dir / "gcc/x86_64-linux/lib64"
426  for lib in ["libgcc_s.so", "libgcc_s.so.1", "libstdc++.a", "libstdc++.so"]:
427    build.copy_file(shared_libgcc_dir / lib, dst_sysroot_lib_dir / lib)
428
429
430@project.task([build_task_for_sysroot])
431def build_task_for_ninja(build: BuildConfig):
432  if build.target_arch == Architecture.AARCH64:
433    # We use apt to install ninja.
434    return
435
436  build.copy_file(
437      build.prebuilts_dir / "ninja" / "ninja",
438      build.install_dir / "usr" / "bin" / "ninja",
439  )
440
441
442@project.task([])
443def build_task_for_python(build: BuildConfig):
444  if build.target_arch == Architecture.AARCH64:
445    # We use apt to install python3.10.
446    return
447
448  src_python_dir = build.third_party_dir / "python"
449  dst_python_dir = build.install_dir / "usr"
450  for d in ("bin", "lib", "share"):
451    build.copy_dir(src_python_dir / d, dst_python_dir / d)
452
453
454@project.task(
455    [build_task_for_sysroot, build_task_for_ninja, build_task_for_python]
456)
457def build_task_for_meson(build: BuildConfig):
458  meson_packager = (
459      build.third_party_dir / "meson" / "packaging" / "create_zipapp.py"
460  )
461  usr_bin = build.install_dir / "usr" / "bin"
462  build.run(["mkdir", "-p", usr_bin], env=None)
463  build.run([
464      "python3",
465      "-S",
466      meson_packager,
467      "--outfile=%s" % (usr_bin / "meson"),
468      "--interpreter",
469      "/usr/bin/env python3",
470      build.third_party_dir / "meson",
471  ])
472
473
474@project.task([])
475def build_task_for_rust(build: BuildConfig):
476  if build.target_arch == Architecture.AARCH64:
477    # We use rustup to install rust, but we need the following libraries
478    # for the aarch64 portable build.
479    dst_dir = build.install_dir / "usr" / "lib64"
480    for lib in ["libc++.so.1", "libc++abi.so.1", "libunwind.so.1"]:
481      build.copy_file(Path(f"/lib/aarch64-linux-gnu/{lib}"), dst_dir / lib)
482    return
483
484  src_rust_dir = build.prebuilts_dir / "rust" / "linux-x86" / "1.73.0"
485  dst_rust_dir = build.install_dir / "usr"
486  for d in ("bin", "lib", "lib64", "share"):
487    src_dir = src_rust_dir / d
488    dst_dir = dst_rust_dir / d
489    build.copy_dir(src_dir, dst_dir)
490  build.run(
491      ["ln", "-sf", "libc++.so", "libc++.so.1"],
492      sub_build_dir=dst_rust_dir / "lib64",
493      env={},
494  )
495
496
497@project.task([build_task_for_sysroot])
498def build_task_for_make(build: BuildConfig) -> None:
499  if build.target_arch == Architecture.AARCH64:
500    # We use apt to install make.
501    return
502
503  build.copy_file(
504      build.prebuilts_dir / "build-tools" / "linux-x86" / "bin" / "make",
505      build.install_dir / "usr" / "bin" / "make",
506  )
507
508
509@project.task([])
510def build_task_for_cmake(build: BuildConfig):
511  if build.target_arch == Architecture.AARCH64:
512    # We use apt to install cmake.
513    return
514
515  build.copy_file(
516      build.prebuilts_dir / "cmake" / "bin" / "cmake",
517      build.install_dir / "usr" / "bin" / "cmake",
518  )
519  build.copy_dir(
520      build.prebuilts_dir / "cmake" / "share",
521      build.install_dir / "usr" / "share",
522  )
523
524
525@project.task([build_task_for_make])
526def build_task_for_bzip2(build: BuildConfig):
527  build_dir = build.make_subdir(Path("bzip2"))
528  build.copy_dir(build.third_party_dir / "bzip2", build_dir)
529  env = build.env_copy()
530  build.run(
531      [
532          "make",
533          f"-j{os.cpu_count()}",
534          "CC=%s" % env["CC"],
535          "AR=%s" % env["AR"],
536          "CFLAGS=%s -O2 -D_FILE_OFFSET_BITS=64" % env["CFLAGS"],
537          "LDFLAGS=%s" % env["LDFLAGS"],
538      ],
539      build_dir,
540  )
541  build.run(
542      [
543          "make",
544          "install",
545          f"PREFIX={build.install_dir}/usr",
546      ],
547      build_dir,
548  )
549
550
551@project.task([build_task_for_make])
552def build_task_for_pkg_config(build: BuildConfig):
553  build_dir = build.make_subdir(Path("pkg-config"))
554  build.copy_dir(build.third_party_dir / "pkg-config", build_dir)
555  build.run(
556      [
557          "sed",
558          "-i",
559          "s/m4_copy(/m4_copy_force(/g",
560          "glib/m4macros/glib-gettext.m4",
561      ],
562      build_dir,
563  )
564  # Run configure separately so that we can pass "--with-internal-glib".
565  build.run(["./autogen.sh", "--no-configure"], build_dir)
566
567  cmd_env = build.env_copy()
568  cmd_env["CFLAGS"] += " -Wno-int-conversion"
569  cmd_args = [
570      "./configure",
571      "--prefix=%s/usr" % build.install_dir,
572      "--disable-shared",
573      "--with-internal-glib",
574  ]
575  build.run(cmd_args, build_dir, cmd_env)
576  build.run_make_build(build_dir)
577  build.run_make_install(build_dir)
578
579
580@project.task([build_task_for_make])
581def build_task_for_patchelf(build: BuildConfig):
582  build_dir = build.make_subdir(Path("patchelf"))
583  build.copy_dir(build.third_party_dir / "patchelf", build_dir)
584  # Run configure separately so that we can pass "--with-internal-glib".
585  build.run(["./bootstrap.sh"], build_dir)
586  build.run(
587      [
588          "./configure",
589          "--prefix=%s/usr" % build.install_dir,
590      ],
591      build_dir,
592  )
593  build.run_make_build(build_dir)
594  build.run_make_install(build_dir)
595
596
597@project.task([build_task_for_make, build_task_for_cmake])
598def build_task_for_zlib(build: BuildConfig):
599  lib_name = "zlib"
600  src_dir = build.third_party_dir / lib_name
601  build_dir = build.make_subdir(Path(lib_name))
602
603  # `--undefined-version` workaround the pickiness of lld.
604  # Some symbols of the link script are not found which
605  # is an error for lld and not for ld.
606
607  # `-Wunused-command-line-argument` remove annoying warnings
608  # introduces by adding a linker flag to all the clang
609  # invocations.
610
611  # `--no-deprecated-non-prototype` removes warning due
612  # to the use of deprecated C features.
613  env = build.env_copy()
614  env["CC"] += " -Wl,--undefined-version -Wno-unused-command-line-argument"
615  env["CC"] += " -Wno-deprecated-non-prototype"
616
617  cmd_args = [
618      "cmake",
619      f"-DCMAKE_INSTALL_PREFIX={build.install_dir}/usr",
620      src_dir,
621  ]
622  build.run(cmd_args, build_dir, env=env)
623  build.run_make_build(build_dir)
624  build.run_make_install(build_dir, use_DESTDIR=False)
625
626
627@project.task([build_task_for_make, build_task_for_bzip2])
628def build_task_for_libpcre2(build: BuildConfig):
629  build_dir = build.make_subdir(Path("pcre"))
630  build.copy_dir(build.third_party_dir / "pcre", build_dir)
631
632  cmd_args = [
633      "./configure",
634      "--prefix=/usr",
635      "--disable-shared",
636  ]
637  build.run(cmd_args, build_dir)
638  build.run_make_build(build_dir)
639  build.run_make_install(build_dir, use_DESTDIR=True)
640
641
642@project.task([build_task_for_make])
643def build_task_for_libffi(build: BuildConfig):
644  build_dir = build.make_subdir(Path("libffi"))
645  build.copy_dir(build.third_party_dir / "libffi", build_dir)
646
647  build.run(["./autogen.sh"], build_dir)
648
649  cmd_args = [
650      "./configure",
651      "--prefix=/usr",
652      "--disable-shared",
653  ]
654  build.run(cmd_args, build_dir)
655  build.run_make_build(build_dir)
656  build.run_make_install(build_dir, use_DESTDIR=True)
657
658
659@project.task([
660    build_task_for_make,
661    build_task_for_meson,
662    build_task_for_libffi,
663    build_task_for_libpcre2,
664    build_task_for_zlib,
665    build_task_for_pkg_config,
666])
667def build_task_for_glib(build: BuildConfig):
668  src_dir = build.third_party_dir / "glib"
669  build_dir = build.make_subdir(Path("glib"))
670
671  # --prefix=$DESTDIR is required to ensure the pkg-config .pc files contain
672  #     the right absolute path.
673  #
674  # --includedir=$DESTDIR/include is required to avoid installs to
675  #     /out/dest-install/out/dest-install/usr/include!
676  #
677  build.run(
678      [
679          "meson",
680          "setup",
681          "--default-library=static",
682          "--prefix=%s/usr" % build.install_dir,
683          "--includedir=%s/usr/include" % build.install_dir,
684          "--libdir=%s/usr/lib" % build.install_dir,
685          "--buildtype=release",
686          "--wrap-mode=nofallback",
687          build_dir,
688          src_dir,
689      ],
690  )
691
692  build.run(["ninja", "install"], build_dir)
693
694
695@project.task([
696    build_task_for_make,
697    build_task_for_meson,
698    build_task_for_pkg_config,
699])
700def build_task_for_pixman(build: BuildConfig):
701  src_dir = build.third_party_dir / "pixman"
702  build_dir = build.make_subdir(Path("pixman"))
703  cmd_args = [
704      "meson",
705      "setup",
706      "--prefix=%s/usr" % build.install_dir,
707      "--includedir=%s/usr/include" % build.install_dir,
708      "--libdir=%s/usr/lib" % build.install_dir,
709      "--default-library=static",
710      "-Dtests=disabled",
711      "--buildtype=release",
712      build_dir,
713      src_dir,
714  ]
715  env = build.env_copy()
716  env["CC"] += " -ldl -Wno-implicit-function-declaration"
717  build.run(cmd_args, env=env)
718  build.run(
719      [
720          "meson",
721          "compile",
722      ],
723      build_dir,
724  )
725  build.run(
726      [
727          "meson",
728          "install",
729      ],
730      build_dir,
731  )
732
733
734@project.task([
735    build_task_for_make,
736    build_task_for_glib,
737])
738def build_task_for_libslirp(build: BuildConfig):
739  src_dir = build.third_party_dir / "libslirp"
740  build_dir = build.make_subdir(Path("libslirp"))
741
742  cmd_args = [
743      "meson",
744      "setup",
745      "--prefix=%s/usr" % build.install_dir,
746      "--includedir=%s/usr/include" % build.install_dir,
747      "--libdir=%s/usr/lib" % build.install_dir,
748      "--default-library=static",
749      "--buildtype=release",
750      build_dir,
751      src_dir,
752  ]
753  build.run(cmd_args, src_dir)
754  build.run(["ninja", "install"], build_dir)
755
756
757@project.task([
758    build_task_for_make,
759    build_task_for_cmake,
760])
761def build_task_for_googletest(build: BuildConfig):
762  dir_name = Path("googletest")
763  build.make_subdir(dir_name)
764  cmd_args = [
765      "cmake",
766      f"-DCMAKE_INSTALL_PREFIX={build.install_dir}/usr",
767      build.third_party_dir / dir_name,
768  ]
769  build.run(cmd_args, dir_name)
770  build.run_make_build(dir_name)
771  build.run_make_install(dir_name, use_DESTDIR=False)
772
773
774@project.task([
775    build_task_for_make,
776    build_task_for_cmake,
777    build_task_for_googletest,
778])
779def build_task_for_aemu_base(build: BuildConfig):
780  dir_name = Path("aemu")
781  build.make_subdir(dir_name)
782  # Options from third_party/aemu/rebuild.sh
783  cmd_args = [
784      "cmake",
785      "-DAEMU_COMMON_GEN_PKGCONFIG=ON",
786      "-DAEMU_COMMON_BUILD_CONFIG=gfxstream",
787      "-DENABLE_VKCEREAL_TESTS=ON",  # `ON` for `aemu-base-testing-support`.
788      f"-DCMAKE_INSTALL_PREFIX={build.install_dir}/usr",
789      build.third_party_dir / dir_name,
790  ]
791  build.run(cmd_args, dir_name)
792  build.run_make_build(dir_name)
793  build.run_make_install(dir_name, use_DESTDIR=False)
794
795
796@project.task([
797    build_task_for_make,
798    build_task_for_cmake,
799])
800def build_task_for_flatbuffers(build: BuildConfig):
801  dir_name = Path("flatbuffers")
802  build.make_subdir(dir_name)
803  cmd_args = [
804      "cmake",
805      f"-DCMAKE_INSTALL_PREFIX={build.install_dir}/usr",
806      build.third_party_dir / dir_name,
807  ]
808  build.run(cmd_args, dir_name)
809  build.run_make_build(dir_name)
810  build.run_make_install(dir_name, use_DESTDIR=False)
811
812
813@project.task([
814    build_task_for_make,
815    build_task_for_meson,
816])
817def build_task_for_libpciaccess(build: BuildConfig):
818  dir_name = Path("libpciaccess")
819  src_dir = build.third_party_dir / dir_name
820  build_dir = build.make_subdir(dir_name)
821
822  build.run(
823      [
824          "meson",
825          "setup",
826          "--prefix=%s/usr" % build.install_dir,
827          build_dir,
828          src_dir,
829      ],
830  )
831  build.run(
832      [
833          "meson",
834          "compile",
835      ],
836      build_dir,
837  )
838  build.run(
839      [
840          "meson",
841          "install",
842      ],
843      build_dir,
844  )
845
846
847@project.task([
848    build_task_for_make,
849    build_task_for_meson,
850    build_task_for_libpciaccess,
851])
852def build_task_for_libdrm(build: BuildConfig):
853  dir_name = Path("libdrm")
854  src_dir = build.third_party_dir / dir_name
855  build_dir = build.make_subdir(dir_name)
856
857  build.run(
858      [
859          "meson",
860          "setup",
861          f"--prefix={build.install_dir}/usr",
862          build_dir,
863          src_dir,
864      ],
865  )
866  build.run(
867      [
868          "meson",
869          "compile",
870      ],
871      build_dir,
872  )
873  build.run(
874      [
875          "meson",
876          "install",
877      ],
878      build_dir,
879  )
880
881
882@project.task([build_task_for_sysroot])
883def build_task_for_egl(build: BuildConfig):
884  if build.target_arch == Architecture.AARCH64:
885    # We use apt to install the EGL headers.
886    return
887
888  build.copy_dir(
889      build.third_party_dir / "egl" / "api" / "KHR",
890      build.sysroot_dir / "usr" / "include" / "KHR",
891  )
892  build.copy_dir(
893      build.third_party_dir / "egl" / "api" / "EGL",
894      build.sysroot_dir / "usr" / "include" / "EGL",
895  )
896
897
898@project.task([
899    build_task_for_meson,
900    build_task_for_aemu_base,
901    build_task_for_flatbuffers,
902    build_task_for_egl,
903    build_task_for_libdrm,
904])
905def build_task_for_gfxstream(build: BuildConfig):
906  dir_name = Path("gfxstream")
907  src_dir = build.third_party_dir / dir_name
908  build_dir = build.make_subdir(dir_name)
909  build.run(
910      [
911          "meson",
912          "setup",
913          f"--prefix={build.install_dir}/usr",
914          build_dir,
915          src_dir,
916      ],
917  )
918  build.run(
919      [
920          "meson",
921          "compile",
922      ],
923      build_dir,
924  )
925  build.run(
926      [
927          "meson",
928          "install",
929      ],
930      build_dir,
931  )
932
933
934@project.task([
935    build_task_for_make,
936    build_task_for_rust,
937    build_task_for_gfxstream,
938])
939def build_task_for_rutabaga(build: BuildConfig):
940  out_dir = build.make_subdir(Path("rutabaga"))
941  cmd_args = [
942      "cargo",
943      "build",
944      "--offline",
945      "--features=gfxstream",
946      "--release",
947  ]
948  if build.target_arch == Architecture.AARCH64:
949    cargo_path = Path(os.environ["HOME"]) / ".cargo" / "bin"
950    env = {
951        "CARGO_TARGET_DIR": str(out_dir),
952        "GFXSTREAM_PATH": str(build.build_dir / "gfxstream" / "host"),
953        "PATH": (
954            f"{cargo_path}:{build.install_dir}/usr/bin:{os.environ['PATH']}"
955        ),
956        "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER": "clang",
957        "RUSTFLAGS": f"-Clink-arg=-Wl,-rpath,$ORIGIN",
958    }
959  else:
960    env = {
961        "CARGO_TARGET_DIR": str(out_dir),
962        "GFXSTREAM_PATH": str(build.build_dir / "gfxstream" / "host"),
963        "PATH": f"{build.install_dir}/usr/bin:{os.environ['PATH']}",
964        "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER": (
965            f"{build.clang_dir}/bin/clang"
966        ),
967        "RUSTFLAGS": (
968            f"-Clink-arg=--sysroot={build.sysroot_dir} -Clink-arg=-Wl,-rpath,$ORIGIN"
969        ),
970    }
971  rutabaga_src_dir = build.third_party_dir / "crosvm" / "rutabaga_gfx" / "ffi"
972  build.run(cmd_args, rutabaga_src_dir, env)
973
974  build.copy_file(
975      out_dir / "release" / "librutabaga_gfx_ffi.so",
976      build.install_dir / "usr" / "lib" / "librutabaga_gfx_ffi.so",
977  )
978  build.run(
979      ["ln", "-sf", "librutabaga_gfx_ffi.so", "librutabaga_gfx_ffi.so.0"],
980      build.install_dir / "usr" / "lib",
981      env={},
982  )
983  build.copy_file(
984      out_dir / "release" / "rutabaga_gfx_ffi.pc",
985      build.install_dir / "usr" / "lib" / "pkgconfig" / "rutabaga_gfx_ffi.pc",
986  )
987  build.copy_file(
988      rutabaga_src_dir / "src" / "include" / "rutabaga_gfx_ffi.h",
989      build.install_dir
990      / "usr"
991      / "include"
992      / "rutabaga_gfx"
993      / "rutabaga_gfx_ffi.h",
994  )
995
996
997@project.task([])
998def build_task_for_libgbm(build: BuildConfig):
999  # gbm is part of mesa which is a large project.
1000  # The dependency is taken fron the system.
1001  build.copy_file(
1002      Path(f"/usr/lib/{build.target_arch.value}-linux-gnu/libgbm.so.1"),
1003      build.install_dir / "usr/lib/libgbm.so.1",
1004  )
1005  build.copy_file(
1006      Path(f"/usr/lib/{build.target_arch.value}-linux-gnu/libgbm.so"),
1007      build.install_dir / "usr/lib/libgbm.so",
1008  )
1009  build.copy_file(
1010      Path(f"/usr/lib/{build.target_arch.value}-linux-gnu/libgbm.so.1.0.0"),
1011      build.install_dir / "usr/lib/libgbm.so.1.0.0",
1012  )
1013  build.copy_file(
1014      Path(f"/usr/lib/{build.target_arch.value}-linux-gnu/pkgconfig/gbm.pc"),
1015      build.install_dir / "usr/lib/pkgconfig/gbm.pc",
1016  )
1017  build.copy_file(
1018      Path("/usr/include/gbm.h"), build.install_dir / "usr/include/gbm.h"
1019  )
1020
1021
1022@project.task([
1023    build_task_for_egl,
1024    build_task_for_libgbm,
1025    build_task_for_meson,
1026    build_task_for_ninja,
1027])
1028def build_task_for_libepoxy(build: BuildConfig):
1029  src_dir = build.third_party_dir / "libepoxy"
1030  build_dir = build.make_subdir(Path("libepoxy"))
1031  build.run(
1032      [
1033          "meson",
1034          "setup",
1035          "--prefix=%s/usr" % build.install_dir,
1036          "--libdir=%s/usr/lib" % build.install_dir,
1037          "-Dtests=false",
1038          build_dir,
1039          src_dir,
1040      ],
1041  )
1042
1043  build.run(["ninja", "install"], build_dir)
1044  # There is a bug in`qemu/third_party/libepoxy/src/meson.build`
1045  # that result in a corrupted line `Requires.private: x11,` in `epoxy.pc`.
1046  # This is not valid and causes the failure:
1047  # `Empty package name in Requires or Conflicts in file '[...]epoxy.pc'`
1048  # This is because 'x11' is found as an implicit dependency and the
1049  # pkgconfig specification in the meson file adds an empty element.
1050  # Until a better solution is found, remove the dependency.
1051  build.run([
1052      "sed",
1053      "-i",
1054      "s/Requires.private: x11, $//g",
1055      build.install_dir / "usr/lib/pkgconfig/epoxy.pc",
1056  ])
1057
1058
1059@project.task([
1060    build_task_for_egl,
1061    build_task_for_libdrm,
1062    build_task_for_libepoxy,
1063    build_task_for_libgbm,
1064    build_task_for_meson,
1065    build_task_for_ninja,
1066])
1067def build_task_for_virglrenderer(build: BuildConfig):
1068  src_dir = build.third_party_dir / "virglrenderer"
1069  build_dir = build.make_subdir(Path("virglrenderer"))
1070  build.run(
1071      [
1072          "meson",
1073          "setup",
1074          "--prefix=%s/usr" % build.install_dir,
1075          "--libdir=%s/usr/lib" % build.install_dir,
1076          "-Dplatforms=egl",
1077          "-Dtests=false",
1078          build_dir,
1079          src_dir,
1080      ],
1081  )
1082
1083  build.run(["ninja", "install"], build_dir)
1084
1085
1086@project.task([
1087    build_task_for_meson,
1088    build_task_for_ninja,
1089])
1090def build_task_for_dtc(build: BuildConfig):
1091  src_dir = build.third_party_dir / "dtc"
1092  build_dir = build.make_subdir(Path("dtc"))
1093  build.run(
1094      [
1095          "meson",
1096          "setup",
1097          "--prefix=%s/usr" % build.install_dir,
1098          "--libdir=%s/usr/lib" % build.install_dir,
1099          # Option taken from qemu/meson.build subproject dtc.
1100          "-Dtools=false",
1101          "-Dyaml=disabled",
1102          "-Dpython=disabled",
1103          "-Ddefault_library=static",
1104          build_dir,
1105          src_dir,
1106      ],
1107  )
1108
1109  build.run(["ninja", "install"], build_dir)
1110
1111
1112@project.task([
1113    build_task_for_make,
1114    build_task_for_libslirp,
1115    build_task_for_glib,
1116    build_task_for_pixman,
1117    build_task_for_zlib,
1118    build_task_for_dtc,
1119    build_task_for_pkg_config,
1120    build_task_for_rutabaga,
1121    build_task_for_gfxstream,
1122    build_task_for_virglrenderer,
1123])
1124def build_task_for_qemu(build: BuildConfig):
1125  target_list = [
1126      "aarch64-softmmu",
1127      "riscv64-softmmu",
1128      "x86_64-softmmu",
1129  ]
1130  src_dir = build.third_party_dir / "qemu"
1131  build_dir = build.make_subdir(Path("qemu"))
1132
1133  # Copy third-pary projects to qemu/subprojects directory so that meson does
1134  # not try to checkout those sources with git.
1135  for name in ("keycodemapdb", "berkeley-softfloat-3", "berkeley-testfloat-3"):
1136    subproject = src_dir / "subprojects" / name
1137    build.run(["rm", "-rf", src_dir / "subprojects" / name], env={})
1138    build.run(
1139        ["cp", "-r", build.third_party_dir / name, src_dir / "subprojects"],
1140        env={},
1141    )
1142    # Add project files to the sources.
1143    packagefiles = src_dir / "subprojects" / "packagefiles" / name
1144    for package_file in packagefiles.glob("*"):
1145      build.run(
1146          ["cp", package_file, subproject],
1147          env={},
1148      )
1149
1150  cmd_args: List[str | Path] = [
1151      src_dir.resolve() / "configure",
1152      "--prefix=/usr",
1153      "--target-list=%s" % ",".join(target_list),
1154      "--disable-plugins",
1155      "--enable-virglrenderer",
1156      # Disable source checkout of missing dependencies, so that
1157      # missing project chan be found early .
1158      "--disable-download",
1159      # Cuttlefish is packaged in host archives that are assembled in
1160      # `$ANDROID_BUILD_TOP/out/host/linux-x86`.
1161      # Binaries are in `./bin` and resources are in `./usr/share` which is
1162      # different from QEMU default expectations. Details in b/296286524.
1163      # Move the binary directory up by one. This path is relative to
1164      # `--prefix` above.
1165      "-Dbindir=../bin",
1166      # Because the canonicalized `bindir` is `/bin` and does not start
1167      # with the `--prefix` the `qemu_firmwarepath` is interpreted differently.
1168      # Hence we have to rewrite it to work as expected.
1169      "-Dqemu_firmwarepath=../usr/share/qemu",
1170      # `gfxstream` is is only capable to output a dynamic library for now
1171      # `libgfxstream_backend.so`
1172      # "--static",
1173      # "--with-git-submodules=ignore",
1174  ]
1175  build.run(cmd_args, build_dir)
1176  build.run_make_build(build_dir)
1177
1178
1179@project.task([
1180    build_task_for_qemu,
1181    build_task_for_patchelf,
1182])
1183def build_task_for_qemu_portable(build: BuildConfig):
1184  package_dir = build.make_subdir(Path("qemu-portable"))
1185  # Install to a new directory rather than to the common taks install dir.
1186  build.run_make_install(
1187      build.build_dir / "qemu", use_DESTDIR=True, dest_dir=package_dir
1188  )
1189  bin_dir = package_dir / "bin"
1190  files = [
1191      "dest-install/usr/lib/libgbm.so.1",
1192      "dest-install/usr/lib/libz.so.1",
1193      "dest-install/usr/lib/libepoxy.so.0",
1194      "dest-install/usr/lib/libvirglrenderer.so.1",
1195      "dest-install/usr/lib/librutabaga_gfx_ffi.so.0",
1196      "dest-install/usr/lib64/libc++.so.1",
1197  ]
1198  if build.target_arch == Architecture.AARCH64:
1199    files += [
1200        "dest-install/usr/lib64/libc++abi.so.1",
1201        "dest-install/usr/lib64/libunwind.so.1",
1202    ]
1203
1204  # Meson install directory depends on the system and differs between podman and
1205  # the developer's workstation. Probe the file system to pick the right location.
1206  either_or = [
1207      f"dest-install/usr/lib/{build.target_arch}-linux-gnu/libgfxstream_backend.so.0",
1208      "dest-install/usr/lib/libgfxstream_backend.so.0",
1209      "dest-install/usr/lib64/libgfxstream_backend.so.0",
1210  ]
1211  try:
1212    files.append(
1213        next(
1214            path for path in either_or if os.path.isfile(build.build_dir / path)
1215        )
1216    )
1217  except StopIteration:
1218    raise FileNotFoundError(f"None of the paths exist: {either_or}")
1219
1220  build.run(["cp", "-t", bin_dir] + files)
1221  build.run(["chmod", "a+rx"] + list(bin_dir.glob("*")))
1222  build.run(["patchelf", "--set-rpath", "$ORIGIN"] + list(bin_dir.glob("*")))
1223  build.run(
1224      [
1225          "tar",
1226          "-czvf",
1227          "qemu-portable.tar.gz",
1228          "--directory",
1229          "qemu-portable",
1230          ".",
1231      ],
1232      build.build_dir,
1233  )
1234
1235
1236@project.task([
1237    build_task_for_qemu_portable,
1238])
1239def build_task_for_qemu_test(build: BuildConfig):
1240  build.run(["make", "test"], build.build_dir / "qemu")
1241
1242
1243##########################################################################
1244##########################################################################
1245#####
1246#####  B U I L D   T A S K S
1247#####
1248##########################################################################
1249##########################################################################
1250
1251
1252def main() -> int:
1253  parser = argparse.ArgumentParser(description=__doc__)
1254  parser.add_argument(
1255      "--arch",
1256      type=Architecture,
1257      choices=list(Architecture),
1258      default=Architecture.X86_64,
1259      help="Target architecture.",
1260  )
1261  parser.add_argument("--build-dir", required=True, help="Build directory.")
1262  parser.add_argument("--ccache", action="store_true", help="Enable ccache.")
1263  parser.add_argument(
1264      "--run-tests",
1265      action="store_true",
1266      help="Run QEMU test suite after the build.",
1267  )
1268  parser.add_argument(
1269      "tasks",
1270      metavar="T",
1271      type=str,
1272      nargs="*",
1273      help="run task by names in the specified order",
1274  )
1275  args = parser.parse_args()
1276
1277  build_dir = Path(args.build_dir)
1278
1279  top_dir = Path(os.path.dirname(__file__)).parent
1280  build_config = BuildConfig(build_dir, top_dir, args.arch)
1281
1282  if args.ccache:
1283    build_config.enable_ccache()
1284
1285  if args.tasks:
1286    for task in args.tasks:
1287      globals()[task](build_config)
1288  else:
1289    if build_dir.exists():
1290      print("Cleaning up build directory...")
1291      for f in os.listdir(build_dir):
1292        path = build_dir / f
1293        if os.path.isfile(path):
1294          os.remove(path)
1295        else:
1296          shutil.rmtree(path)
1297    else:
1298      os.makedirs(build_dir)
1299
1300    # Compute the build plan to get 'qemu'
1301    build_tasks = project.get_build_task_list(
1302        build_task_for_qemu_test
1303        if args.run_tests
1304        else build_task_for_qemu_portable
1305    )
1306
1307    print("BUILD PLAN: %s" % ", ".join([t.__name__ for t in build_tasks]))
1308
1309    for task in build_tasks:
1310      task(build_config)
1311
1312  return 0
1313
1314
1315if __name__ == "__main__":
1316  sys.exit(main())
1317