1#!/usr/bin/python3 -B 2 3# Copyright 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Generates the timezone data files used by Android.""" 18 19import argparse 20import glob 21import os 22import re 23import subprocess 24import sys 25import tarfile 26import tempfile 27 28sys.path.append('%s/external/icu/tools' % os.environ.get('ANDROID_BUILD_TOP')) 29import i18nutil 30import icuutil 31import tzdatautil 32 33# Unix epoch timestamp in seconds up to which transitions in tzdata will be pregenerated. 34pregeneration_upper_bound = 4102444800 # 1 Jan 2100 35 36# Calculate the paths that are referred to by multiple functions. 37android_build_top = i18nutil.GetAndroidRootOrDie() 38timezone_dir = os.path.realpath('%s/system/timezone' % android_build_top) 39i18nutil.CheckDirExists(timezone_dir, 'system/timezone') 40 41android_host_out = i18nutil.GetAndroidHostOutOrDie() 42 43zone_compactor_dir = os.path.realpath('%s/system/timezone/input_tools/android' % android_build_top) 44i18nutil.CheckDirExists(zone_compactor_dir, 'system/timezone/input_tools/android') 45 46timezone_input_tools_dir = os.path.realpath('%s/input_tools' % timezone_dir) 47timezone_input_data_dir = os.path.realpath('%s/input_data' % timezone_dir) 48 49timezone_output_data_dir = '%s/output_data' % timezone_dir 50i18nutil.CheckDirExists(timezone_output_data_dir, 'output_data') 51 52tmp_dir = tempfile.mkdtemp('-tzdata') 53 54def GenerateZicInputFile(extracted_iana_data_dir): 55 # Android APIs assume DST means "summer time" so we follow the rearguard format 56 # introduced in 2018e. 57 zic_input_file_name = 'rearguard.zi' 58 59 # 'NDATA=' is used to remove unnecessary rules files. 60 subprocess.check_call(['make', '-C', extracted_iana_data_dir, 'NDATA=', zic_input_file_name]) 61 62 zic_input_file = '%s/%s' % (extracted_iana_data_dir, zic_input_file_name) 63 if not os.path.exists(zic_input_file): 64 print('Could not find %s' % zic_input_file) 65 sys.exit(1) 66 return zic_input_file 67 68 69def WriteSetupFile(zic_input_file): 70 """Writes the list of zones that ZoneCompactor should process.""" 71 links = [] 72 zones = [] 73 for line in open(zic_input_file): 74 fields = line.split() 75 if fields: 76 line_type = fields[0] 77 if line_type == 'Link': 78 # Each "Link" line requires the creation of a link from an old tz ID to 79 # a new tz ID, and implies the existence of a zone with the old tz ID. 80 # 81 # IANA terminology: 82 # TARGET = the new tz ID, LINK-NAME = the old tz ID 83 target = fields[1] 84 link_name = fields[2] 85 links.append('Link %s %s' % (target, link_name)) 86 zones.append('Zone %s' % link_name) 87 elif line_type == 'Zone': 88 # Each "Zone" line indicates the existence of a tz ID. 89 # 90 # IANA terminology: 91 # NAME is the tz ID, other fields like STDOFF, RULES, FORMAT,[UNTIL] are 92 # ignored. 93 name = fields[1] 94 zones.append('Zone %s' % name) 95 96 zone_compactor_setup_file = '%s/setup' % tmp_dir 97 setup = open(zone_compactor_setup_file, 'w') 98 99 # Ordering requirement from ZoneCompactor: Links must come first. 100 for link in sorted(set(links)): 101 setup.write('%s\n' % link) 102 for zone in sorted(set(zones)): 103 setup.write('%s\n' % zone) 104 setup.close() 105 return zone_compactor_setup_file 106 107 108def BuildIcuData(iana_data_tar_file): 109 icu_build_dir = '%s/icu' % tmp_dir 110 111 icuutil.PrepareIcuBuild(icu_build_dir) 112 icuutil.MakeTzDataFiles(icu_build_dir, iana_data_tar_file) 113 114 # Create ICU system image files. 115 icuutil.MakeAndCopyIcuDataFiles(icu_build_dir) 116 117 # Create the ICU's .res time zone files. 118 icu_overlay_dir = '%s/icu_overlay' % timezone_output_data_dir 119 icuutil.MakeAndCopyIcuTzFiles(icu_build_dir, icu_overlay_dir) 120 121 # There are files in ICU which generation depends on ICU itself, 122 # so multiple builds might be needed. 123 icuutil.GenerateIcuDataFiles() 124 125 # Copy ICU license file(s) 126 icuutil.CopyLicenseFiles(icu_overlay_dir) 127 128 129def GetIanaVersion(iana_tar_file): 130 iana_tar_filename = os.path.basename(iana_tar_file) 131 iana_version = re.search('tz(?:data|code)(.+)\\.tar\\.gz', iana_tar_filename).group(1) 132 return iana_version 133 134 135def ExtractTarFile(tar_file, dir): 136 print('Extracting %s...' % tar_file) 137 if not os.path.exists(dir): 138 os.mkdir(dir) 139 tar = tarfile.open(tar_file, 'r') 140 tar.extractall(dir) 141 142 143def BuildZic(iana_tools_dir): 144 iana_zic_code_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzcode') 145 iana_zic_code_version = GetIanaVersion(iana_zic_code_tar_file) 146 iana_zic_data_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzdata') 147 iana_zic_data_version = GetIanaVersion(iana_zic_data_tar_file) 148 149 print('Found IANA zic release %s/%s in %s/%s ...' \ 150 % (iana_zic_code_version, iana_zic_data_version, iana_zic_code_tar_file, 151 iana_zic_data_tar_file)) 152 153 zic_build_dir = '%s/zic' % tmp_dir 154 ExtractTarFile(iana_zic_code_tar_file, zic_build_dir) 155 ExtractTarFile(iana_zic_data_tar_file, zic_build_dir) 156 157 # zic 158 print('Building zic...') 159 # VERSION_DEPS= is to stop the build process looking for files that might not 160 # be present across different versions. 161 subprocess.check_call(['make', '-C', zic_build_dir, 'zic']) 162 163 zic_binary_file = '%s/zic' % zic_build_dir 164 if not os.path.exists(zic_binary_file): 165 print('Could not find %s' % zic_binary_file) 166 sys.exit(1) 167 return zic_binary_file 168 169 170def BuildTzdata(zic_binary_file, extracted_iana_data_dir, iana_data_version): 171 print('Generating zic input file...') 172 zic_input_file = GenerateZicInputFile(extracted_iana_data_dir) 173 174 print('Calling zic...') 175 zic_output_dir = '%s/data' % tmp_dir 176 os.mkdir(zic_output_dir) 177 # -R specifies upper bound for generated transitions. 178 zic_cmd = [zic_binary_file, '-b', 'slim', '-R', f'@{pregeneration_upper_bound}', '-d', zic_output_dir, zic_input_file] 179 subprocess.check_call(zic_cmd) 180 181 # ZoneCompactor 182 zone_compactor_setup_file = WriteSetupFile(zic_input_file) 183 184 print('Calling ZoneCompactor to update tzdata to %s...' % iana_data_version) 185 186 tzdatautil.InvokeSoong(android_build_top, ['zone_compactor']) 187 188 # Create args for ZoneCompactor 189 header_string = 'tzdata%s' % iana_data_version 190 191 print('Executing ZoneCompactor...') 192 command = '%s/bin/zone_compactor' % android_host_out 193 iana_output_data_dir = '%s/iana' % timezone_output_data_dir 194 subprocess.check_call([command, zone_compactor_setup_file, zic_output_dir, iana_output_data_dir, 195 header_string]) 196 197 198def BuildTzlookupAndTzIds(iana_data_dir): 199 countryzones_source_file = '%s/android/countryzones.txt' % timezone_input_data_dir 200 tzlookup_dest_file = '%s/android/tzlookup.xml' % timezone_output_data_dir 201 tzids_dest_file = '%s/android/tzids.prototxt' % timezone_output_data_dir 202 203 print('Calling TzLookupGenerator to create tzlookup.xml / tzids.prototxt...') 204 tzdatautil.InvokeSoong(android_build_top, ['tzlookup_generator']) 205 206 zone_tab_file = '%s/zone.tab' % iana_data_dir 207 command = '%s/bin/tzlookup_generator' % android_host_out 208 subprocess.check_call([command, countryzones_source_file, zone_tab_file, tzlookup_dest_file, 209 tzids_dest_file]) 210 211 212def BuildTelephonylookup(): 213 telephonylookup_source_file = '%s/android/telephonylookup.txt' % timezone_input_data_dir 214 telephonylookup_dest_file = '%s/android/telephonylookup.xml' % timezone_output_data_dir 215 216 print('Calling TelephonyLookupGenerator to create telephonylookup.xml...') 217 tzdatautil.InvokeSoong(android_build_top, ['telephonylookup_generator']) 218 219 command = '%s/bin/telephonylookup_generator' % android_host_out 220 subprocess.check_call([command, telephonylookup_source_file, telephonylookup_dest_file]) 221 222 223def CreateTzVersion(iana_data_version, android_revision, output_version_file): 224 create_tz_version_script = '%s/input_tools/version/create-tz_version.py' % timezone_dir 225 226 subprocess.check_call([create_tz_version_script, 227 '-iana_version', iana_data_version, 228 '-revision', str(android_revision), 229 '-output_version_file', output_version_file]) 230 231def UpdateTestFiles(): 232 testing_data_dir = '%s/testing/data' % timezone_dir 233 update_test_files_script = '%s/create-test-data.sh' % testing_data_dir 234 subprocess.check_call([update_test_files_script], cwd=testing_data_dir) 235 236 237# Run from any directory, with no special setup required. 238# In the rare case when tzdata has to be updated, but under the same version, 239# pass "-revision" argument. 240# See http://www.iana.org/time-zones/ for more about the source of this data. 241def main(): 242 parser = argparse.ArgumentParser() 243 parser.add_argument('-revision', type=int, default=1, 244 help='Revision of current the IANA version, default = 1') 245 246 args = parser.parse_args() 247 android_revision = args.revision 248 249 print('Source data file structure: %s' % timezone_input_data_dir) 250 print('Source tools file structure: %s' % timezone_input_tools_dir) 251 print('Intermediate / working dir: %s' % tmp_dir) 252 print('Output data file structure: %s' % timezone_output_data_dir) 253 254 iana_input_data_dir = '%s/iana' % timezone_input_data_dir 255 iana_data_tar_file = tzdatautil.GetIanaTarFile(iana_input_data_dir, 'tzdata') 256 iana_data_version = GetIanaVersion(iana_data_tar_file) 257 print('IANA time zone data release %s in %s ...' % (iana_data_version, iana_data_tar_file)) 258 259 icu_dir = icuutil.icuDir() 260 print('Found icu in %s ...' % icu_dir) 261 262 BuildIcuData(iana_data_tar_file) 263 264 iana_tools_dir = '%s/iana' % timezone_input_tools_dir 265 zic_binary_file = BuildZic(iana_tools_dir) 266 267 iana_data_dir = '%s/iana_data' % tmp_dir 268 ExtractTarFile(iana_data_tar_file, iana_data_dir) 269 BuildTzdata(zic_binary_file, iana_data_dir, iana_data_version) 270 271 BuildTzlookupAndTzIds(iana_data_dir) 272 273 BuildTelephonylookup() 274 275 # Create a version file from the output from prior stages. 276 output_version_file = '%s/version/tz_version' % timezone_output_data_dir 277 CreateTzVersion(iana_data_version, android_revision, output_version_file) 278 279 # Update test versions of data files too. 280 UpdateTestFiles() 281 282 print('Look in %s and %s for new files' % (timezone_output_data_dir, icu_dir)) 283 sys.exit(0) 284 285 286if __name__ == '__main__': 287 main() 288