1#!/usr/bin/python3
2
3import argparse
4import glob
5import json
6import os
7import re
8import shlex
9import shutil
10import subprocess
11import sys
12import tempfile
13import zipfile
14
15from collections import defaultdict
16from pathlib import Path
17
18# See go/fetch_artifact for details on this script.
19FETCH_ARTIFACT = '/google/data/ro/projects/android/fetch_artifact'
20COMPAT_REPO = Path('prebuilts/sdk')
21COMPAT_README = Path('extensions/README.md')
22# This build target is used when fetching from a train build (TXXXXXXXX)
23BUILD_TARGET_TRAIN = 'train_build'
24# This build target is used when fetching from a non-train build (XXXXXXXX)
25BUILD_TARGET_CONTINUOUS = 'mainline_modules_sdks-userdebug'
26BUILD_TARGET_CONTINUOUS_MAIN = 'mainline_modules_sdks-{release_config}-userdebug'
27# The glob of sdk artifacts to fetch from remote build
28ARTIFACT_PATTERN = 'mainline-sdks/for-next-build/current/{module_name}/sdk/*.zip'
29# The glob of sdk artifacts to fetch from local build
30ARTIFACT_LOCAL_PATTERN = 'out/dist/mainline-sdks/for-next-build/current/{module_name}/sdk/*.zip'
31ARTIFACT_MODULES_INFO = 'mainline-modules-info.json'
32ARTIFACT_LOCAL_MODULES_INFO = 'out/dist/mainline-modules-info.json'
33COMMIT_TEMPLATE = """Finalize artifacts for extension SDK %d
34
35Import from build id %s.
36
37Generated with:
38$ %s
39
40Bug: %d
41Test: presubmit"""
42
43def fail(*args, **kwargs):
44    print(*args, file=sys.stderr, **kwargs)
45    sys.exit(1)
46
47def fetch_mainline_modules_info_artifact(target, build_id):
48    tmpdir = Path(tempfile.TemporaryDirectory().name)
49    tmpdir.mkdir()
50    if args.local_mode:
51        artifact_path = ARTIFACT_LOCAL_MODULES_INFO
52        print('Copying %s to %s ...' % (artifact_path, tmpdir))
53        shutil.copy(artifact_path, tmpdir)
54    else:
55        artifact_path = ARTIFACT_MODULES_INFO
56        print('Fetching %s from %s ...' % (artifact_path, target))
57        fetch_cmd = [FETCH_ARTIFACT]
58        fetch_cmd.extend(['--bid', str(build_id)])
59        fetch_cmd.extend(['--target', target])
60        fetch_cmd.append(artifact_path)
61        fetch_cmd.append(str(tmpdir))
62        print("Running: " + ' '.join(fetch_cmd))
63        try:
64            subprocess.check_output(fetch_cmd, stderr=subprocess.STDOUT)
65        except subprocess.CalledProcessError:
66            fail(
67                'FAIL: Unable to retrieve %s artifact for build ID %s for %s target'
68                % (artifact_path, build_id, target)
69            )
70    return os.path.join(tmpdir, ARTIFACT_MODULES_INFO)
71
72def fetch_artifacts(target, build_id, module_name):
73    tmpdir = Path(tempfile.TemporaryDirectory().name)
74    tmpdir.mkdir()
75    if args.local_mode:
76        artifact_path = ARTIFACT_LOCAL_PATTERN.format(module_name='*')
77        print('Copying %s to %s ...' % (artifact_path, tmpdir))
78        for file in glob.glob(artifact_path):
79            shutil.copy(file, tmpdir)
80    else:
81        artifact_path = ARTIFACT_PATTERN.format(module_name=module_name)
82        print('Fetching %s from %s ...' % (artifact_path, target))
83        fetch_cmd = [FETCH_ARTIFACT]
84        fetch_cmd.extend(['--bid', str(build_id)])
85        fetch_cmd.extend(['--target', target])
86        fetch_cmd.append(artifact_path)
87        fetch_cmd.append(str(tmpdir))
88        print("Running: " + ' '.join(fetch_cmd))
89        try:
90            subprocess.check_output(fetch_cmd, stderr=subprocess.STDOUT)
91        except subprocess.CalledProcessError:
92            fail(
93                "FAIL: Unable to retrieve %s artifact for build ID %s"
94                % (artifact_path, build_id)
95            )
96    return tmpdir
97
98def repo_for_sdk(sdk_filename, mainline_modules_info):
99    for module in mainline_modules_info:
100        if mainline_modules_info[module]["sdk_name"] in sdk_filename:
101            project_path = mainline_modules_info[module]["module_sdk_project"]
102            if args.gantry_mode:
103                project_path = "/tmp/" + project_path
104                os.makedirs(project_path , exist_ok = True, mode = 0o777)
105            print(f"module_sdk_path for {module}: {project_path}")
106            return Path(project_path)
107
108    fail('"%s" has no valid mapping to any mainline module.' % sdk_filename)
109
110def dir_for_sdk(filename, version):
111    base = str(version)
112    if 'test-exports' in filename:
113        return os.path.join(base, 'test-exports')
114    if 'host-exports' in filename:
115        return os.path.join(base, 'host-exports')
116    return base
117
118def is_ignored(file):
119    # Conscrypt has some legacy API tracking files that we don't consider for extensions.
120    bad_stem_prefixes = ['conscrypt.module.intra.core.api', 'conscrypt.module.platform.api']
121    return any([file.stem.startswith(p) for p in bad_stem_prefixes])
122
123
124def maybe_tweak_compat_stem(file):
125    # For legacy reasons, art and conscrypt txt file names in the SDKs (*.module.public.api)
126    # do not match their expected filename in prebuilts/sdk (art, conscrypt). So rename them
127    # to match.
128    new_stem = file.stem
129    new_stem = new_stem.replace('art.module.public.api', 'art')
130    new_stem = new_stem.replace('conscrypt.module.public.api', 'conscrypt')
131
132    # The stub jar artifacts from official builds are named '*-stubs.jar', but
133    # the convention for the copies in prebuilts/sdk is just '*.jar'. Fix that.
134    new_stem = new_stem.replace('-stubs', '')
135
136    return file.with_stem(new_stem)
137
138parser = argparse.ArgumentParser(description=('Finalize an extension SDK with prebuilts'))
139parser.add_argument('-f', '--finalize_sdk', type=int, required=True, help='The numbered SDK to finalize.')
140parser.add_argument('-c', '--release_config', type=str, help='The release config to use to finalize.')
141parser.add_argument('-b', '--bug', type=int, required=True, help='The bug number to add to the commit message.')
142parser.add_argument('-r', '--readme', required=True, help='Version history entry to add to %s' % (COMPAT_REPO / COMPAT_README))
143parser.add_argument('-a', '--amend_last_commit', action="store_true", help='Amend current HEAD commits instead of making new commits.')
144parser.add_argument('-m', '--modules', action='append', help='Modules to include. Can be provided multiple times, or not at all for all modules.')
145parser.add_argument('-l', '--local_mode', action="store_true", help='Local mode: use locally built artifacts and don\'t upload the result to Gerrit.')
146parser.add_argument('-g', '--gantry_mode', action="store_true", help='Script executed via Gantry in google3.')
147parser.add_argument('bid', help='Build server build ID')
148args = parser.parse_args()
149
150if not os.path.isdir('build/soong') and not args.gantry_mode:
151    fail("This script must be run from the top of an Android source tree.")
152
153if args.release_config:
154    BUILD_TARGET_CONTINUOUS = BUILD_TARGET_CONTINUOUS_MAIN.format(release_config=args.release_config)
155build_target = BUILD_TARGET_TRAIN if args.bid[0] == 'T' else BUILD_TARGET_CONTINUOUS
156branch_name = 'finalize-%d' % args.finalize_sdk
157cmdline = shlex.join([x for x in sys.argv if x not in ['-a', '--amend_last_commit', '-l', '--local_mode']])
158commit_message = COMMIT_TEMPLATE % (args.finalize_sdk, args.bid, cmdline, args.bug)
159module_names = args.modules or ['*']
160
161if args.gantry_mode:
162    COMPAT_REPO = Path('/tmp/') / COMPAT_REPO
163compat_dir = COMPAT_REPO.joinpath('extensions/%d' % args.finalize_sdk)
164if compat_dir.is_dir():
165    print('Removing existing dir %s' % compat_dir)
166    shutil.rmtree(compat_dir)
167
168created_dirs = defaultdict(set)
169mainline_modules_info_file = fetch_mainline_modules_info_artifact(build_target, args.bid)
170with open(mainline_modules_info_file, "r", encoding="utf8",) as file:
171    mainline_modules_info = json.load(file)
172
173for m in module_names:
174    tmpdir = fetch_artifacts(build_target, args.bid, m)
175    for f in tmpdir.iterdir():
176        repo = repo_for_sdk(f.name, mainline_modules_info)
177        dir = dir_for_sdk(f.name, args.finalize_sdk)
178        target_dir = repo.joinpath(dir)
179        if target_dir.is_dir():
180            print('Removing existing dir %s' % target_dir)
181            shutil.rmtree(target_dir)
182        with zipfile.ZipFile(tmpdir.joinpath(f)) as zipFile:
183            zipFile.extractall(target_dir)
184
185        # Disable the Android.bp, but keep it for reference / potential future use.
186        shutil.move(target_dir.joinpath('Android.bp'), target_dir.joinpath('Android.bp.auto'))
187
188        print('Created %s' % target_dir)
189        created_dirs[repo].add(dir)
190
191        # Copy api txt files to compat tracking dir
192        src_files = [Path(p) for p in glob.glob(os.path.join(target_dir, 'sdk_library/*/*.txt')) + glob.glob(os.path.join(target_dir, 'sdk_library/*/*.jar'))]
193        for src_file in src_files:
194            if is_ignored(src_file):
195                continue
196            api_type = src_file.parts[-2]
197            dest_dir = compat_dir.joinpath(api_type, 'api') if src_file.suffix == '.txt' else compat_dir.joinpath(api_type)
198            dest_file = maybe_tweak_compat_stem(dest_dir.joinpath(src_file.name))
199            os.makedirs(dest_dir, exist_ok = True)
200            shutil.copy(src_file, dest_file)
201            created_dirs[COMPAT_REPO].add(dest_dir.relative_to(COMPAT_REPO))
202
203if args.local_mode:
204    print('Updated prebuilts using locally built artifacts. Don\'t submit or use for anything besides local testing.')
205    sys.exit(0)
206
207# Do not commit any changes when the script is executed via Gantry.
208if args.gantry_mode:
209    sys.exit(0)
210
211subprocess.check_output(['repo', 'start', branch_name] + list(created_dirs.keys()))
212print('Running git commit')
213for repo in created_dirs:
214    git = ['git', '-C', str(repo)]
215    subprocess.check_output(git + ['add'] + list(created_dirs[repo]))
216
217    if repo == COMPAT_REPO:
218        with open(COMPAT_REPO / COMPAT_README, "a") as readme:
219            readme.write(f"- {args.finalize_sdk}: {args.readme}\n")
220        subprocess.check_output(git + ['add', COMPAT_README])
221
222    if args.amend_last_commit:
223        change_id_match = re.search(r'Change-Id: [^\\n]+', str(subprocess.check_output(git + ['log', '-1'])))
224        if change_id_match:
225            change_id = '\n' + change_id_match.group(0)
226        else:
227            fail('FAIL: Unable to find change_id of the last commit.')
228        subprocess.check_output(git + ['commit', '--amend', '-m', commit_message + change_id])
229    else:
230        subprocess.check_output(git + ['commit', '-m', commit_message])
231