1#!/usr/bin/python3 -B
2
3# Copyright 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Generates the timezone data files used by Android."""
18
19import argparse
20import glob
21import os
22import re
23import subprocess
24import sys
25import tarfile
26import tempfile
27
28sys.path.append('%s/external/icu/tools' % os.environ.get('ANDROID_BUILD_TOP'))
29import i18nutil
30import icuutil
31import tzdatautil
32
33# Unix epoch timestamp in seconds up to which transitions in tzdata will be pregenerated.
34pregeneration_upper_bound = 4102444800 # 1 Jan 2100
35
36# Calculate the paths that are referred to by multiple functions.
37android_build_top = i18nutil.GetAndroidRootOrDie()
38timezone_dir = os.path.realpath('%s/system/timezone' % android_build_top)
39i18nutil.CheckDirExists(timezone_dir, 'system/timezone')
40
41android_host_out = i18nutil.GetAndroidHostOutOrDie()
42
43zone_compactor_dir = os.path.realpath('%s/system/timezone/input_tools/android' % android_build_top)
44i18nutil.CheckDirExists(zone_compactor_dir, 'system/timezone/input_tools/android')
45
46timezone_input_tools_dir = os.path.realpath('%s/input_tools' % timezone_dir)
47timezone_input_data_dir = os.path.realpath('%s/input_data' % timezone_dir)
48
49timezone_output_data_dir = '%s/output_data' % timezone_dir
50i18nutil.CheckDirExists(timezone_output_data_dir, 'output_data')
51
52tmp_dir = tempfile.mkdtemp('-tzdata')
53
54def GenerateZicInputFile(extracted_iana_data_dir):
55  # Android APIs assume DST means "summer time" so we follow the rearguard format
56  # introduced in 2018e.
57  zic_input_file_name = 'rearguard.zi'
58
59  # 'NDATA=' is used to remove unnecessary rules files.
60  subprocess.check_call(['make', '-C', extracted_iana_data_dir, 'NDATA=', zic_input_file_name])
61
62  zic_input_file = '%s/%s' % (extracted_iana_data_dir, zic_input_file_name)
63  if not os.path.exists(zic_input_file):
64    print('Could not find %s' % zic_input_file)
65    sys.exit(1)
66  return zic_input_file
67
68
69def WriteSetupFile(zic_input_file):
70  """Writes the list of zones that ZoneCompactor should process."""
71  links = []
72  zones = []
73  for line in open(zic_input_file):
74    fields = line.split()
75    if fields:
76      line_type = fields[0]
77      if line_type == 'Link':
78        # Each "Link" line requires the creation of a link from an old tz ID to
79        # a new tz ID, and implies the existence of a zone with the old tz ID.
80        #
81        # IANA terminology:
82        # TARGET = the new tz ID, LINK-NAME = the old tz ID
83        target = fields[1]
84        link_name = fields[2]
85        links.append('Link %s %s' % (target, link_name))
86        zones.append('Zone %s' % link_name)
87      elif line_type == 'Zone':
88        # Each "Zone" line indicates the existence of a tz ID.
89        #
90        # IANA terminology:
91        # NAME is the tz ID, other fields like STDOFF, RULES, FORMAT,[UNTIL] are
92        # ignored.
93        name = fields[1]
94        zones.append('Zone %s' % name)
95
96  zone_compactor_setup_file = '%s/setup' % tmp_dir
97  setup = open(zone_compactor_setup_file, 'w')
98
99  # Ordering requirement from ZoneCompactor: Links must come first.
100  for link in sorted(set(links)):
101    setup.write('%s\n' % link)
102  for zone in sorted(set(zones)):
103    setup.write('%s\n' % zone)
104  setup.close()
105  return zone_compactor_setup_file
106
107
108def BuildIcuData(iana_data_tar_file):
109  icu_build_dir = '%s/icu' % tmp_dir
110
111  icuutil.PrepareIcuBuild(icu_build_dir)
112  icuutil.MakeTzDataFiles(icu_build_dir, iana_data_tar_file)
113
114  # Create ICU system image files.
115  icuutil.MakeAndCopyIcuDataFiles(icu_build_dir)
116
117  # Create the ICU's .res time zone files.
118  icu_overlay_dir = '%s/icu_overlay' % timezone_output_data_dir
119  icuutil.MakeAndCopyIcuTzFiles(icu_build_dir, icu_overlay_dir)
120
121  # There are files in ICU which generation depends on ICU itself,
122  # so multiple builds might be needed.
123  icuutil.GenerateIcuDataFiles()
124
125  # Copy ICU license file(s)
126  icuutil.CopyLicenseFiles(icu_overlay_dir)
127
128
129def GetIanaVersion(iana_tar_file):
130  iana_tar_filename = os.path.basename(iana_tar_file)
131  iana_version = re.search('tz(?:data|code)(.+)\\.tar\\.gz', iana_tar_filename).group(1)
132  return iana_version
133
134
135def ExtractTarFile(tar_file, dir):
136  print('Extracting %s...' % tar_file)
137  if not os.path.exists(dir):
138    os.mkdir(dir)
139  tar = tarfile.open(tar_file, 'r')
140  tar.extractall(dir)
141
142
143def BuildZic(iana_tools_dir):
144  iana_zic_code_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzcode')
145  iana_zic_code_version = GetIanaVersion(iana_zic_code_tar_file)
146  iana_zic_data_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzdata')
147  iana_zic_data_version = GetIanaVersion(iana_zic_data_tar_file)
148
149  print('Found IANA zic release %s/%s in %s/%s ...' \
150      % (iana_zic_code_version, iana_zic_data_version, iana_zic_code_tar_file,
151         iana_zic_data_tar_file))
152
153  zic_build_dir = '%s/zic' % tmp_dir
154  ExtractTarFile(iana_zic_code_tar_file, zic_build_dir)
155  ExtractTarFile(iana_zic_data_tar_file, zic_build_dir)
156
157  # zic
158  print('Building zic...')
159  # VERSION_DEPS= is to stop the build process looking for files that might not
160  # be present across different versions.
161  subprocess.check_call(['make', '-C', zic_build_dir, 'zic'])
162
163  zic_binary_file = '%s/zic' % zic_build_dir
164  if not os.path.exists(zic_binary_file):
165    print('Could not find %s' % zic_binary_file)
166    sys.exit(1)
167  return zic_binary_file
168
169
170def BuildTzdata(zic_binary_file, extracted_iana_data_dir, iana_data_version):
171  print('Generating zic input file...')
172  zic_input_file = GenerateZicInputFile(extracted_iana_data_dir)
173
174  print('Calling zic...')
175  zic_output_dir = '%s/data' % tmp_dir
176  os.mkdir(zic_output_dir)
177  # -R specifies upper bound for generated transitions.
178  zic_cmd = [zic_binary_file, '-b', 'slim', '-R', f'@{pregeneration_upper_bound}', '-d', zic_output_dir, zic_input_file]
179  subprocess.check_call(zic_cmd)
180
181  # ZoneCompactor
182  zone_compactor_setup_file = WriteSetupFile(zic_input_file)
183
184  print('Calling ZoneCompactor to update tzdata to %s...' % iana_data_version)
185
186  tzdatautil.InvokeSoong(android_build_top, ['zone_compactor'])
187
188  # Create args for ZoneCompactor
189  header_string = 'tzdata%s' % iana_data_version
190
191  print('Executing ZoneCompactor...')
192  command = '%s/bin/zone_compactor' % android_host_out
193  iana_output_data_dir = '%s/iana' % timezone_output_data_dir
194  subprocess.check_call([command, zone_compactor_setup_file, zic_output_dir, iana_output_data_dir,
195                         header_string])
196
197
198def BuildTzlookupAndTzIds(iana_data_dir):
199  countryzones_source_file = '%s/android/countryzones.txt' % timezone_input_data_dir
200  tzlookup_dest_file = '%s/android/tzlookup.xml' % timezone_output_data_dir
201  tzids_dest_file = '%s/android/tzids.prototxt' % timezone_output_data_dir
202
203  print('Calling TzLookupGenerator to create tzlookup.xml / tzids.prototxt...')
204  tzdatautil.InvokeSoong(android_build_top, ['tzlookup_generator'])
205
206  zone_tab_file = '%s/zone.tab' % iana_data_dir
207  command = '%s/bin/tzlookup_generator' % android_host_out
208  subprocess.check_call([command, countryzones_source_file, zone_tab_file, tzlookup_dest_file,
209                         tzids_dest_file])
210
211
212def BuildTelephonylookup():
213  telephonylookup_source_file = '%s/android/telephonylookup.txt' % timezone_input_data_dir
214  telephonylookup_dest_file = '%s/android/telephonylookup.xml' % timezone_output_data_dir
215
216  print('Calling TelephonyLookupGenerator to create telephonylookup.xml...')
217  tzdatautil.InvokeSoong(android_build_top, ['telephonylookup_generator'])
218
219  command = '%s/bin/telephonylookup_generator' % android_host_out
220  subprocess.check_call([command, telephonylookup_source_file, telephonylookup_dest_file])
221
222
223def CreateTzVersion(iana_data_version, android_revision, output_version_file):
224  create_tz_version_script = '%s/input_tools/version/create-tz_version.py' % timezone_dir
225
226  subprocess.check_call([create_tz_version_script,
227      '-iana_version', iana_data_version,
228      '-revision', str(android_revision),
229      '-output_version_file', output_version_file])
230
231def UpdateTestFiles():
232  testing_data_dir = '%s/testing/data' % timezone_dir
233  update_test_files_script = '%s/create-test-data.sh' % testing_data_dir
234  subprocess.check_call([update_test_files_script], cwd=testing_data_dir)
235
236
237# Run from any directory, with no special setup required.
238# In the rare case when tzdata has to be updated, but under the same version,
239# pass "-revision" argument.
240# See http://www.iana.org/time-zones/ for more about the source of this data.
241def main():
242  parser = argparse.ArgumentParser()
243  parser.add_argument('-revision', type=int, default=1,
244      help='Revision of current the IANA version, default = 1')
245
246  args = parser.parse_args()
247  android_revision = args.revision
248
249  print('Source data file structure: %s' % timezone_input_data_dir)
250  print('Source tools file structure: %s' % timezone_input_tools_dir)
251  print('Intermediate / working dir: %s' % tmp_dir)
252  print('Output data file structure: %s' % timezone_output_data_dir)
253
254  iana_input_data_dir = '%s/iana' % timezone_input_data_dir
255  iana_data_tar_file = tzdatautil.GetIanaTarFile(iana_input_data_dir, 'tzdata')
256  iana_data_version = GetIanaVersion(iana_data_tar_file)
257  print('IANA time zone data release %s in %s ...' % (iana_data_version, iana_data_tar_file))
258
259  icu_dir = icuutil.icuDir()
260  print('Found icu in %s ...' % icu_dir)
261
262  BuildIcuData(iana_data_tar_file)
263
264  iana_tools_dir = '%s/iana' % timezone_input_tools_dir
265  zic_binary_file = BuildZic(iana_tools_dir)
266
267  iana_data_dir = '%s/iana_data' % tmp_dir
268  ExtractTarFile(iana_data_tar_file, iana_data_dir)
269  BuildTzdata(zic_binary_file, iana_data_dir, iana_data_version)
270
271  BuildTzlookupAndTzIds(iana_data_dir)
272
273  BuildTelephonylookup()
274
275  # Create a version file from the output from prior stages.
276  output_version_file = '%s/version/tz_version' % timezone_output_data_dir
277  CreateTzVersion(iana_data_version, android_revision, output_version_file)
278
279  # Update test versions of data files too.
280  UpdateTestFiles()
281
282  print('Look in %s and %s for new files' % (timezone_output_data_dir, icu_dir))
283  sys.exit(0)
284
285
286if __name__ == '__main__':
287  main()
288