1#
2# Copyright (C) 2018 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""A commandline tool to check and update packages in external/
16
17Example usage:
18updater.sh checkall
19updater.sh update kotlinc
20updater.sh update --refresh --keep_date rust/crates/libc
21"""
22
23import argparse
24from collections.abc import Iterable
25import enum
26import json
27import logging
28import os
29import subprocess
30import sys
31import textwrap
32import time
33from typing import Dict, Iterator, List, Union, Tuple, Type
34from pathlib import Path
35
36from base_updater import Updater
37from crates_updater import CratesUpdater
38from git_updater import GitUpdater
39from github_archive_updater import GithubArchiveUpdater
40import fileutils
41import git_utils
42# pylint: disable=import-error
43import metadata_pb2  # type: ignore
44import updater_utils
45
46UPDATERS: List[Type[Updater]] = [
47    CratesUpdater,
48    GithubArchiveUpdater,
49    GitUpdater,
50]
51
52TMP_BRANCH_NAME = 'tmp_auto_upgrade'
53USE_COLOR = sys.stdout.isatty()
54
55
56@enum.unique
57class Color(enum.Enum):
58    """Colors for output to console."""
59    FRESH = '\x1b[32m'
60    STALE = '\x1b[31;1m'
61    ERROR = '\x1b[31m'
62
63
64END_COLOR = '\033[0m'
65
66
67def color_string(string: str, color: Color) -> str:
68    """Changes the color of a string when print to terminal."""
69    if not USE_COLOR:
70        return string
71    return color.value + string + END_COLOR
72
73
74def build_updater(proj_path: Path) -> Tuple[Updater, metadata_pb2.MetaData]:
75    """Build updater for a project specified by proj_path.
76
77    Reads and parses METADATA file. And builds updater based on the information.
78
79    Args:
80      proj_path: Absolute or relative path to the project.
81
82    Returns:
83      The updater object built. None if there's any error.
84    """
85
86    proj_path = fileutils.get_absolute_project_path(proj_path)
87    metadata = fileutils.read_metadata(proj_path)
88    metadata = fileutils.convert_url_to_identifier(metadata)
89    updater = updater_utils.create_updater(metadata, proj_path, UPDATERS)
90    return updater, metadata
91
92
93def commit_message_generator(project_name: str, version: str, path: str, bug: int | None = None) -> str:
94    header = f"Upgrade {project_name} to {version}\n"
95    body = textwrap.dedent(f"""
96    This project was upgraded with external_updater.
97    Usage: tools/external_updater/updater.sh update external/{path}
98    For more info, check https://cs.android.com/android/platform/superproject/+/main:tools/external_updater/README.md\n\n""")
99    if bug is None:
100        footer = "Test: TreeHugger"
101    else:
102        footer = f"Bug: {bug}\nTest: TreeHugger"
103    return header + body + footer
104
105
106def _do_update(args: argparse.Namespace, updater: Updater,
107               metadata: metadata_pb2.MetaData) -> None:
108    full_path = updater.project_path
109
110    if not args.keep_local_changes:
111        git_utils.detach_to_android_head(full_path)
112        if TMP_BRANCH_NAME in git_utils.list_local_branches(full_path):
113            git_utils.delete_branch(full_path, TMP_BRANCH_NAME)
114            git_utils.reset_hard(full_path)
115            git_utils.clean(full_path)
116        git_utils.start_branch(full_path, TMP_BRANCH_NAME)
117
118    try:
119        updater.update()
120
121        updated_metadata = updater.update_metadata(metadata)
122        fileutils.write_metadata(full_path, updated_metadata, args.keep_date)
123        git_utils.add_file(full_path, 'METADATA')
124
125        try:
126            rel_proj_path = str(fileutils.get_relative_project_path(full_path))
127        except ValueError:
128            # Absolute paths to other trees will not be relative to our tree. There are
129            # not portable instructions for upgrading that project, since the path will
130            # differ between machines (or checkouts).
131            rel_proj_path = "<absolute path to project>"
132        commit_message = commit_message_generator(metadata.name, updater.latest_version, rel_proj_path, args.bug)
133        git_utils.remove_gitmodules(full_path)
134        git_utils.add_file(full_path, '*')
135        git_utils.commit(full_path, commit_message, args.no_verify)
136
137        if not args.skip_post_update:
138            updater_utils.run_post_update(full_path, full_path)
139            git_utils.add_file(full_path, '*')
140            git_utils.commit_amend(full_path)
141
142        if args.build:
143            try:
144                updater_utils.build(full_path)
145            except subprocess.CalledProcessError:
146                logging.exception("Build failed, aborting upload")
147                return
148    except Exception as err:
149        if updater.rollback():
150            print('Rolled back.')
151        raise err
152
153    if not args.no_upload:
154        git_utils.push(full_path, args.remote_name, updater.has_errors)
155
156
157def has_new_version(updater: Updater) -> bool:
158    """Checks if a newer version of the project is available."""
159    if updater.current_version != updater.latest_version:
160        return True
161    return False
162
163
164def print_project_status(updater: Updater) -> None:
165    """Prints the current status of the project on console."""
166
167    current_version = updater.current_version
168    latest_version = updater.latest_version
169    alternative_latest_version = updater.alternative_latest_version
170
171    print(f'Current version: {current_version}')
172    print(f'Latest version: {latest_version}')
173    if alternative_latest_version is not None:
174        print(f'Alternative latest version: {alternative_latest_version}')
175    if has_new_version(updater):
176        print(color_string('Out of date!', Color.STALE))
177    else:
178        print(color_string('Up to date.', Color.FRESH))
179
180
181def find_ver_types(current_version: str) -> Tuple[str, str]:
182    if git_utils.is_commit(current_version):
183        alternative_ver_type = 'tag'
184        latest_ver_type = 'sha'
185    else:
186        alternative_ver_type = 'sha'
187        latest_ver_type = 'tag'
188    return latest_ver_type, alternative_ver_type
189
190
191def use_alternative_version(updater: Updater) -> bool:
192    """This function only runs when there is an alternative version available."""
193
194    latest_ver_type, alternative_ver_type = find_ver_types(updater.current_version)
195    latest_version = updater.latest_version
196    alternative_version = updater.alternative_latest_version
197    new_version_available = has_new_version(updater)
198
199    out_of_date_question = f'Would you like to upgrade to {alternative_ver_type} {alternative_version} instead of {latest_ver_type} {latest_version}? (yes/no)\n'
200    up_to_date_question = f'Would you like to upgrade to {alternative_ver_type} {alternative_version}? (yes/no)\n'
201    recom_message = color_string(f'We recommend upgrading to {alternative_ver_type} {alternative_version} instead. ', Color.FRESH)
202    not_recom_message = color_string(f'We DO NOT recommend upgrading to {alternative_ver_type} {alternative_version}. ', Color.STALE)
203
204    # If alternative_version is not None, there are ONLY three possible
205    # scenarios:
206    # Scenario 1, out of date, we recommend switching to tag:
207    # Current version: sha1
208    # Latest version: sha2
209    # Alternative latest version: tag
210
211    # Scenario 2, out of date, we DO NOT recommend switching to sha.
212    # Current version: tag1
213    # Latest version: tag2
214    # Alternative latest version: sha
215
216    # Scenario 3, up to date, we DO NOT recommend switching to sha.
217    # Current version: tag1
218    # Latest version: tag1
219    # Alternative latest version: sha
220
221    if alternative_ver_type == 'tag':
222        warning = out_of_date_question + recom_message
223    else:
224        if not new_version_available:
225            warning = up_to_date_question + not_recom_message
226        else:
227            warning = out_of_date_question + not_recom_message
228
229    answer = input(warning)
230    if "yes".startswith(answer.lower()):
231        return True
232    elif answer.lower().startswith("no"):
233        return False
234    # If user types something that is not "yes" or "no" or something similar, abort.
235    else:
236        raise ValueError(f"Invalid input: {answer}")
237
238
239
240def check_and_update(args: argparse.Namespace,
241                     proj_path: Path,
242                     update_lib=False) -> Union[Updater, str]:
243    """Checks updates for a project.
244
245    Args:
246      args: commandline arguments
247      proj_path: Absolute or relative path to the project.
248      update_lib: If false, will only check for new version, but not update.
249    """
250
251    try:
252        canonical_path = fileutils.canonicalize_project_path(proj_path)
253        print(f'Checking {canonical_path}...')
254        updater, metadata = build_updater(proj_path)
255        updater.check()
256
257        alternative_version = updater.alternative_latest_version
258        new_version_available = has_new_version(updater)
259        print_project_status(updater)
260
261        if update_lib:
262            if args.refresh:
263                print('Refreshing the current version')
264                updater.refresh_without_upgrading()
265
266            answer = False
267            if alternative_version is not None:
268                answer = use_alternative_version(updater)
269                if answer:
270                    updater.set_new_version(alternative_version)
271            if new_version_available or args.force or args.refresh or answer:
272                _do_update(args, updater, metadata)
273        return updater
274    # pylint: disable=broad-except
275    except Exception as err:
276        logging.exception("Failed to check or update %s", proj_path)
277        return str(err)
278
279
280def check_and_update_path(args: argparse.Namespace, paths: Iterable[Path],
281                          update_lib: bool,
282                          delay: int) -> Dict[str, Dict[str, str]]:
283    results = {}
284    for path in paths:
285        res = {}
286        updater = check_and_update(args, path, update_lib)
287        if isinstance(updater, str):
288            res['error'] = updater
289        else:
290            res['current'] = updater.current_version
291            res['latest'] = updater.latest_version
292        results[str(fileutils.canonicalize_project_path(path))] = res
293        time.sleep(delay)
294    return results
295
296
297def _list_all_metadata() -> Iterator[str]:
298    for path, dirs, files in os.walk(fileutils.external_path()):
299        if fileutils.METADATA_FILENAME in files:
300            # Skip sub directories.
301            dirs[:] = []
302            yield path
303        dirs.sort(key=lambda d: d.lower())
304
305
306def write_json(json_file: str, results: Dict[str, Dict[str, str]]) -> None:
307    """Output a JSON report."""
308    with Path(json_file).open('w', encoding='utf-8') as res_file:
309        json.dump(results, res_file, sort_keys=True, indent=4)
310
311
312def validate(args: argparse.Namespace) -> None:
313    """Handler for validate command."""
314    paths = fileutils.resolve_command_line_paths(args.paths)
315    try:
316        canonical_path = fileutils.canonicalize_project_path(paths[0])
317        print(f'Validating {canonical_path}')
318        updater, _ = build_updater(paths[0])
319        print(updater.validate())
320    except Exception:  # pylint: disable=broad-exception-caught
321        logging.exception("Failed to check or update %s", paths)
322
323
324def check(args: argparse.Namespace) -> None:
325    """Handler for check command."""
326    if args.all:
327        paths = [Path(p) for p in _list_all_metadata()]
328    else:
329        paths = fileutils.resolve_command_line_paths(args.paths)
330    results = check_and_update_path(args, paths, False, args.delay)
331
332    if args.json_output is not None:
333        write_json(args.json_output, results)
334
335
336def update(args: argparse.Namespace) -> None:
337    """Handler for update command."""
338    all_paths = fileutils.resolve_command_line_paths(args.paths)
339    # Remove excluded paths.
340    excludes = set() if args.exclude is None else set(args.exclude)
341    filtered_paths = [path for path in all_paths
342                      if not path.name in excludes]
343    # Now we can update each path.
344    results = check_and_update_path(args, filtered_paths, True, 0)
345
346    if args.json_output is not None:
347        write_json(args.json_output, results)
348
349
350def parse_args() -> argparse.Namespace:
351    """Parses commandline arguments."""
352
353    parser = argparse.ArgumentParser(
354        description='Check updates for third party projects in external/.')
355    subparsers = parser.add_subparsers(dest='cmd')
356    subparsers.required = True
357
358    diff_parser = subparsers.add_parser('validate',
359                                        help='Check if aosp version is what it claims to be.')
360    diff_parser.add_argument(
361        'paths',
362        nargs='*',
363        help='Paths of the project. '
364             'Relative paths will be resolved from external/.')
365    diff_parser.set_defaults(func=validate)
366
367    # Creates parser for check command.
368    check_parser = subparsers.add_parser('check',
369                                         help='Check update for one project.')
370    check_parser.add_argument(
371        'paths',
372        nargs='*',
373        help='Paths of the project. '
374        'Relative paths will be resolved from external/.')
375    check_parser.add_argument('--json-output',
376                              help='Path of a json file to write result to.')
377    check_parser.add_argument(
378        '--all',
379        action='store_true',
380        help='If set, check updates for all supported projects.')
381    check_parser.add_argument(
382        '--delay',
383        default=0,
384        type=int,
385        help='Time in seconds to wait between checking two projects.')
386    check_parser.set_defaults(func=check)
387
388    # Creates parser for update command.
389    update_parser = subparsers.add_parser('update', help='Update one project.')
390    update_parser.add_argument(
391        'paths',
392        nargs='*',
393        help='Paths of the project as globs. '
394        'Relative paths will be resolved from external/.')
395    update_parser.add_argument('--json-output',
396                               help='Path of a json file to write result to.')
397    update_parser.add_argument(
398        '--force',
399        help='Run update even if there\'s no new version.',
400        action='store_true')
401    update_parser.add_argument(
402        '--refresh',
403        help='Run update and refresh to the current version.',
404        action='store_true')
405    update_parser.add_argument(
406        '--keep-date',
407        help='Run update and do not change date in METADATA.',
408        action='store_true')
409    update_parser.add_argument('--no-upload',
410                               action='store_true',
411                               help='Does not upload to Gerrit after upgrade')
412    update_parser.add_argument('--keep-local-changes',
413                               action='store_true',
414                               help='Updates the current branch')
415    update_parser.add_argument('--skip-post-update',
416                               action='store_true',
417                               help='Skip post_update script')
418    update_parser.add_argument('--no-build',
419                               action='store_false',
420                               dest='build',
421                               help='Skip building')
422    update_parser.add_argument('--no-verify',
423                               action='store_true',
424                               help='Pass --no-verify to git commit')
425    update_parser.add_argument('--bug',
426                               type=int,
427                               help='Bug number for this update')
428    update_parser.add_argument('--remote-name',
429                               default='aosp',
430                               required=False,
431                               help='Upstream remote name.')
432    update_parser.add_argument('--exclude',
433                               action='append',
434                               help='Names of projects to exclude. '
435                               'These are just the final part of the path '
436                               'with no directories.')
437    update_parser.set_defaults(func=update)
438
439    return parser.parse_args()
440
441
442def main() -> None:
443    """The main entry."""
444
445    args = parse_args()
446    args.func(args)
447
448
449if __name__ == '__main__':
450    main()
451