1# Copyright (C) 2023 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Generates an xml manifest for the `repo` tool.
16
17It enumerates all git submodules recursively, and
18creates a `project` entry in the manifest, pointing
19at the repository's url and hash.
20
21Repository's url are remapped to point either at AOSP
22or at experimental-qemu-build-internal when the AOSP
23repository is not ready.
24
25Some submodules are not used by the build and are not mapped.
26
27Nested repositories are relocated into ./third_party and a
28symlink is created to recreate the structure.
29"""
30
31import argparse
32import dataclasses
33import os
34import subprocess
35import sys
36import textwrap
37import xml.etree.ElementTree as ET
38
39# Host name of AOSP Git on borg repositories.
40# See rpc://android.googlesource.com/{repo name}
41AOSP_HOST = "android.googlesource.com"
42
43# Submodules that are not used by the build.
44REPO_TO_SKIP = {
45    "https://boringssl.googlesource.com/boringssl",
46    "https://chromium.googlesource.com/chromium/tools/depot_tools.git",
47    "https://chromium.googlesource.com/chromiumos/platform/minigbm",
48    "https://chromium.googlesource.com/chromiumos/platform/minijail",
49    "https://chromium.googlesource.com/chromiumos/third_party/tpm2",
50    "https://chromium.googlesource.com/crosvm/perfetto/",
51    "https://github.com/akheron/jansson",
52    "https://github.com/google/brotli",
53    "https://github.com/google/wycheproof",
54    "https://github.com/kkos/oniguruma",
55    "https://github.com/krb5/krb5",
56    "https://github.com/openssl/openssl",
57    "https://github.com/pyca/cryptography.git",
58    "https://github.com/tianocore/edk2-cmocka.git",
59    "https://github.com/ucb-bar/berkeley-softfloat-3.git",
60    "https://gitlab.com/libvirt/libvirt-ci.git",
61    "https://github.com/gost-engine/engine",
62    "https://github.com/provider-corner/libprov.git",
63    "https://github.com/MIPI-Alliance/public-mipi-sys-t.git",
64    "https://github.com/Zeex/subhook.git",
65    "https://github.com/zeux/pugixml.git",
66    "https://github.com/devicetree-org/pylibfdt.git",
67    "https://gitlab.com/qemu-project/edk2.git",
68    "https://gitlab.com/qemu-project/ipxe.git",
69    "https://gitlab.com/qemu-project/libvfio-user.git",
70    "https://gitlab.com/qemu-project/openbios.git",
71    "https://gitlab.com/qemu-project/opensbi.git",
72    "https://gitlab.com/qemu-project/qboot.git",
73    "https://gitlab.com/qemu-project/qemu-palcode.git",
74    "https://gitlab.com/qemu-project/QemuMacDrivers.git",
75    "https://gitlab.com/qemu-project/seabios-hppa.git",
76    "https://gitlab.com/qemu-project/seabios.git/",
77    "https://gitlab.com/qemu-project/skiboot.git",
78    "https://gitlab.com/qemu-project/SLOF.git",
79    "https://gitlab.com/qemu-project/u-boot-sam460ex.git",
80    "https://gitlab.com/qemu-project/vbootrom.git",
81}
82
83# Replaces repositories URLs.
84REPO_MAPPING = {
85    # Submodules remapped to android AOSP.
86    "git://anongit.freedesktop.org/git/pixman.git": (
87        "https://android.googlesource.com/platform/external/pixman"
88    ),
89    "https://boringssl.googlesource.com/boringssl": (
90        "https://android.googlesource.com/platform/external/boringssl"
91    ),
92    "https://chromium.googlesource.com/chromiumos/third_party/virglrenderer": (
93        "https://android.googlesource.com/platform/external/virglrenderer"
94    ),
95    "https://chromium.googlesource.com/crosvm/crosvm": (
96        "https://android.googlesource.com/platform/external/crosvm"
97    ),
98    "https://github.com/google/googletest.git": (
99        "https://android.googlesource.com/platform/external/googletest"
100    ),
101    "https://github.com/mesonbuild/meson.git": (
102        "https://android.googlesource.com/trusty/external/qemu-meson"
103    ),
104    "https://gitlab.com/qemu-project/dtc.git": (
105        "https://android.googlesource.com/platform/external/dtc"
106    ),
107    "https://gitlab.com/qemu-project/meson.git": (
108        "https://android.googlesource.com/trusty/external/qemu-meson"
109    ),
110    "https://gitlab.com/qemu-project/u-boot.git": (  # Can probably be removed.
111        "https://android.googlesource.com/platform/external/u-boot"
112    ),
113    "https://gitlab.freedesktop.org/slirp/libslirp.git": (
114        "https://android.googlesource.com/trusty/external/qemu-libslirp"
115    ),
116    "https://gitlab.gnome.org/GNOME/glib.git": (
117        "https://android.googlesource.com/platform/external/bluetooth/glib"
118    ),
119    "https://github.com/KhronosGroup/EGL-Registry.git": (
120        "https://android.googlesource.com/platform/external/egl-registry"
121    ),
122    "https://gitlab.com/qemu-project/berkeley-softfloat-3.git": "https://android.googlesource.com/platform/external/berkeley-softfloat-3",
123    "https://gitlab.com/qemu-project/berkeley-testfloat-3.git": "https://android.googlesource.com/platform/external/berkeley-testfloat-3",
124    "https://salsa.debian.org/xorg-team/lib/libpciaccess.git": (
125        "https://android.googlesource.com/platform/external/libpciaccess"
126    ),
127    "https://gitlab.com/qemu-project/keycodemapdb.git": (
128        "https://android.googlesource.com/trusty/external/qemu-keycodemapdb"
129    ),
130    "https://gitlab.gnome.org/GNOME/gvdb.git": (
131        "https://android.googlesource.com/platform/external/gvdb"
132    ),
133    "https://github.com/anholt/libepoxy.git": (
134        "https://android.googlesource.com/platform/external/libepoxy"
135    ),
136    "https://gitlab.freedesktop.org/virgl/virglrenderer.git": (
137        "https://android.googlesource.com/platform/external/virglrenderer"
138    ),
139}
140
141
142@dataclasses.dataclass
143class Submodule:
144  """Occurrence of a module in the tree."""
145
146  path: str
147  origin_url: str
148  hash: str
149
150
151@dataclasses.dataclass
152class Project:
153  """Project in a repo manifest."""
154
155  origin_url: str
156  gob_host: str
157  gob_path: str
158  revision: str
159  shallow: bool
160  linkat: list
161
162
163def GetAllSubmodules(path):
164  """Yields a Submodule for each module recursively found in the specified repo."""
165
166  yield Submodule(
167      ".",
168      "https://android.googlesource.com/device/google/cuttlefish_vmm",
169      "main",
170  )
171
172  output = subprocess.check_output([
173      "git",
174      "submodule",
175      "foreach",
176      "--recursive",
177      "-q",
178      "echo ${displaypath} $(git remote get-url origin) ${sha1}",
179  ])
180  for line in output.decode().strip().split("\n"):
181    yield Submodule(*line.split(" "))
182
183
184def MatchProject(url):
185  prefix_to_remote = {
186      "https://android.googlesource.com/": AOSP_HOST,
187      "sso://android/": AOSP_HOST,
188      "persistent-https://android.git.corp.google.com/": AOSP_HOST,
189  }
190  for prefix, remote in prefix_to_remote.items():
191    if url.startswith(prefix):
192      return remote, url.removeprefix(prefix)
193  raise ValueError(
194      f"Url {url} could neither be mapped to AOSP to"
195      " experimental-qemu-build-internal. If this is a new submodule, verify"
196      " if it is used by the build. If it is, we should import it to AOSP. If"
197      " not, add it to the REPO_TO_SKIP."
198  )
199
200
201def main():
202  parser = argparse.ArgumentParser(description=__doc__)
203  parser.add_argument("input", help="git root directory")
204  parser.add_argument(
205      "--repo_manifest",
206      type=argparse.FileType("w"),
207      help="output repo tool manifest",
208  )
209
210  args = parser.parse_args()
211
212  submodules = list(GetAllSubmodules(args.input))
213
214  # Build a list of rep project.
215  project_by_name = {}
216  for module in submodules:
217    repo_url = module.origin_url
218    # Skip unused code repositories.
219    if repo_url in REPO_TO_SKIP:
220      continue
221    # Remap repository to their new location or to None.
222    if repo_url in REPO_MAPPING:
223      repo_url = REPO_MAPPING[repo_url]
224    gob_host, gob_path = MatchProject(repo_url)
225    # Add the repository instance to the list merging by gob_path
226    if gob_path not in project_by_name:
227      project_by_name[gob_path] = Project(
228          origin_url=module.origin_url,
229          gob_host=gob_host,
230          gob_path=gob_path,
231          revision=module.hash,
232          shallow=module.path.startswith("qemu/prebuilts/"),
233          linkat=[],
234      )
235
236    assert project_by_name[gob_path].gob_host == gob_host
237    project_by_name[gob_path].linkat.append(module.path)
238
239  GenerateRepoManifest(project_by_name.values(), args.repo_manifest)
240
241
242def SimpleName(project):
243  return project.gob_path.replace("/", "_")
244
245
246def GenerateRepoManifest(projects, out):
247  # Generate repo manifest xml.
248  manifest = ET.Element("manifest")
249  ET.SubElement(
250      manifest,
251      "remote",
252      name="aosp",
253      fetch="sso://android.googlesource.com",
254      review="sso://android/",
255  )
256  ET.SubElement(
257      manifest,
258      "remote",
259      name="experimental-qemu-build-internal",
260      fetch="sso://experimental-qemu-build-internal",
261  )
262  ET.SubElement(
263      manifest,
264      "default",
265      attrib={"sync-j": "16"},
266      revision="main",
267      remote="aosp",
268  )
269
270  for project in projects:
271    # The project is instantiated only once.
272    elem = ET.SubElement(
273        manifest,
274        "project",
275        path=project.linkat[0],
276        name=project.gob_path,
277        revision=project.revision,
278    )
279    if project.gob_host != AOSP_HOST:
280      elem.attrib["remote"] = project.gob_host
281    else:
282      pass  # Use the default value which is AOSP.
283
284    if project.shallow:
285      elem.attrib["clone-depth"] = "1"
286
287    for linkat in project.linkat[1:]:
288      ET.SubElement(elem, "linkfile", src=".", dest=linkat)
289
290  tree = ET.ElementTree(manifest)
291  tree.getroot().insert(
292      0,
293      ET.Comment(
294          "DO NOT EDIT. This file is generated by " + os.path.basename(__file__)
295      ),
296  )
297  ET.indent(tree)
298  tree.write(out, encoding="unicode")
299
300
301if __name__ == "__main__":
302  sys.exit(main())
303