1#!/usr/bin/env python
2#
3# Copyright (C) 2022 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may not
6# use this file except in compliance with the License. You may obtain a copy of
7# the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations under
15# the License.
16#
17"""Functions for merging META/* files from partial builds.
18
19Expects items in OPTIONS prepared by merge_target_files.py.
20"""
21
22import logging
23import os
24import re
25import shutil
26
27import build_image
28import common
29import merge_utils
30import sparse_img
31import verity_utils
32from ota_utils import ParseUpdateEngineConfig
33
34from common import ExternalError
35
36logger = logging.getLogger(__name__)
37
38OPTIONS = common.OPTIONS
39
40# In apexkeys.txt or apkcerts.txt, we will find partition tags on each entry in
41# the file. We use these partition tags to filter the entries in those files
42# from the two different target files packages to produce a merged apexkeys.txt
43# or apkcerts.txt file. A partition tag (e.g., for the product partition) looks
44# like this: 'partition="product"'. We use the group syntax grab the value of
45# the tag. We use non-greedy matching in case there are other fields on the
46# same line.
47
48PARTITION_TAG_PATTERN = re.compile(r'partition="(.*?)"')
49
50# The sorting algorithm for apexkeys.txt and apkcerts.txt does not include the
51# ".apex" or ".apk" suffix, so we use the following pattern to extract a key.
52
53MODULE_KEY_PATTERN = re.compile(r'name="(.+)\.(apex|apk)"')
54
55
56def MergeUpdateEngineConfig(framework_meta_dir, vendor_meta_dir,
57                            merged_meta_dir):
58  """Merges META/update_engine_config.txt.
59
60  The output is the configuration for maximum compatibility.
61  """
62  _CONFIG_NAME = 'update_engine_config.txt'
63  framework_config_path = os.path.join(framework_meta_dir, _CONFIG_NAME)
64  vendor_config_path = os.path.join(vendor_meta_dir, _CONFIG_NAME)
65  merged_config_path = os.path.join(merged_meta_dir, _CONFIG_NAME)
66
67  if os.path.exists(framework_config_path):
68    framework_config = ParseUpdateEngineConfig(framework_config_path)
69    vendor_config = ParseUpdateEngineConfig(vendor_config_path)
70    # Copy older config to merged target files for maximum compatibility
71    # update_engine in system partition is from system side, but
72    # update_engine_sideload in recovery is from vendor side.
73    if framework_config < vendor_config:
74      shutil.copy(framework_config_path, merged_config_path)
75    else:
76      shutil.copy(vendor_config_path, merged_config_path)
77  else:
78    if not OPTIONS.allow_partial_ab:
79      raise FileNotFoundError(framework_config_path)
80    shutil.copy(vendor_config_path, merged_config_path)
81
82
83def MergeMetaFiles(temp_dir, merged_dir, framework_partitions):
84  """Merges various files in META/*."""
85
86  framework_meta_dir = os.path.join(temp_dir, 'framework_meta', 'META')
87  merge_utils.CollectTargetFiles(
88      input_zipfile_or_dir=OPTIONS.framework_target_files,
89      output_dir=os.path.dirname(framework_meta_dir),
90      item_list=('META/*',))
91
92  vendor_meta_dir = os.path.join(temp_dir, 'vendor_meta', 'META')
93  merge_utils.CollectTargetFiles(
94      input_zipfile_or_dir=OPTIONS.vendor_target_files,
95      output_dir=os.path.dirname(vendor_meta_dir),
96      item_list=('META/*',))
97
98  merged_meta_dir = os.path.join(merged_dir, 'META')
99
100  # Merge META/misc_info.txt into OPTIONS.merged_misc_info,
101  # but do not write it yet. The following functions may further
102  # modify this dict.
103  OPTIONS.merged_misc_info = MergeMiscInfo(
104      framework_meta_dir=framework_meta_dir,
105      vendor_meta_dir=vendor_meta_dir,
106      merged_meta_dir=merged_meta_dir)
107
108  CopyNamedFileContexts(
109      framework_meta_dir=framework_meta_dir,
110      vendor_meta_dir=vendor_meta_dir,
111      merged_meta_dir=merged_meta_dir)
112
113  if OPTIONS.merged_misc_info.get('use_dynamic_partitions') == 'true':
114    MergeDynamicPartitionsInfo(
115        framework_meta_dir=framework_meta_dir,
116        vendor_meta_dir=vendor_meta_dir,
117        merged_meta_dir=merged_meta_dir)
118
119  if OPTIONS.merged_misc_info.get('ab_update') == 'true':
120    MergeAbPartitions(
121        framework_meta_dir=framework_meta_dir,
122        vendor_meta_dir=vendor_meta_dir,
123        merged_meta_dir=merged_meta_dir,
124        framework_partitions=framework_partitions)
125    UpdateCareMapImageSizeProps(images_dir=os.path.join(merged_dir, 'IMAGES'))
126
127  for file_name in ('apkcerts.txt', 'apexkeys.txt'):
128    MergePackageKeys(
129        framework_meta_dir=framework_meta_dir,
130        vendor_meta_dir=vendor_meta_dir,
131        merged_meta_dir=merged_meta_dir,
132        file_name=file_name)
133
134  if OPTIONS.merged_misc_info.get('ab_update') == 'true':
135    MergeUpdateEngineConfig(
136        framework_meta_dir, vendor_meta_dir, merged_meta_dir)
137
138  # Write the now-finalized OPTIONS.merged_misc_info.
139  merge_utils.WriteSortedData(
140      data=OPTIONS.merged_misc_info,
141      path=os.path.join(merged_meta_dir, 'misc_info.txt'))
142
143
144def MergeAbPartitions(framework_meta_dir, vendor_meta_dir, merged_meta_dir,
145                      framework_partitions):
146  """Merges META/ab_partitions.txt.
147
148  The output contains the union of the partition names.
149  """
150  framework_ab_partitions = []
151  framework_ab_config = os.path.join(framework_meta_dir, 'ab_partitions.txt')
152  if os.path.exists(framework_ab_config):
153    with open(framework_ab_config) as f:
154      # Filter out some partitions here to support the case that the
155      # ab_partitions.txt of framework-target-files has non-framework
156      # partitions. This case happens when we use a complete merged target
157      # files package as the framework-target-files.
158      framework_ab_partitions.extend([
159          partition
160          for partition in f.read().splitlines()
161          if partition in framework_partitions
162      ])
163  else:
164    if not OPTIONS.allow_partial_ab:
165      raise FileNotFoundError(framework_ab_config)
166    logger.info('Use partial AB because framework ab_partitions.txt does not '
167                'exist.')
168
169  with open(os.path.join(vendor_meta_dir, 'ab_partitions.txt')) as f:
170    vendor_ab_partitions = f.read().splitlines()
171
172  merge_utils.WriteSortedData(
173      data=set(framework_ab_partitions + vendor_ab_partitions),
174      path=os.path.join(merged_meta_dir, 'ab_partitions.txt'))
175
176
177def MergeMiscInfo(framework_meta_dir, vendor_meta_dir, merged_meta_dir):
178  """Merges META/misc_info.txt.
179
180  The output contains a combination of key=value pairs from both inputs.
181  Most pairs are taken from the vendor input, while some are taken from
182  the framework input.
183  """
184
185  OPTIONS.framework_misc_info = common.LoadDictionaryFromFile(
186      os.path.join(framework_meta_dir, 'misc_info.txt'))
187  OPTIONS.vendor_misc_info = common.LoadDictionaryFromFile(
188      os.path.join(vendor_meta_dir, 'misc_info.txt'))
189
190  # Merged misc info is a combination of vendor misc info plus certain values
191  # from the framework misc info.
192
193  merged_dict = OPTIONS.vendor_misc_info
194  for key in OPTIONS.framework_misc_info_keys:
195    if key in OPTIONS.framework_misc_info:
196      merged_dict[key] = OPTIONS.framework_misc_info[key]
197
198  # If AVB is enabled then ensure that we build vbmeta.img.
199  # Partial builds with AVB enabled may set PRODUCT_BUILD_VBMETA_IMAGE=false to
200  # skip building an incomplete vbmeta.img.
201  if merged_dict.get('avb_enable') == 'true':
202    merged_dict['avb_building_vbmeta_image'] = 'true'
203
204  return merged_dict
205
206
207def MergeDynamicPartitionsInfo(framework_meta_dir, vendor_meta_dir,
208                               merged_meta_dir):
209  """Merge META/dynamic_partitions_info.txt."""
210  framework_dynamic_partitions_dict = common.LoadDictionaryFromFile(
211      os.path.join(framework_meta_dir, 'dynamic_partitions_info.txt'))
212  vendor_dynamic_partitions_dict = common.LoadDictionaryFromFile(
213      os.path.join(vendor_meta_dir, 'dynamic_partitions_info.txt'))
214
215  merged_dynamic_partitions_dict = common.MergeDynamicPartitionInfoDicts(
216      framework_dict=framework_dynamic_partitions_dict,
217      vendor_dict=vendor_dynamic_partitions_dict)
218
219  merge_utils.WriteSortedData(
220      data=merged_dynamic_partitions_dict,
221      path=os.path.join(merged_meta_dir, 'dynamic_partitions_info.txt'))
222
223  # Merge misc info keys used for Dynamic Partitions.
224  OPTIONS.merged_misc_info.update(merged_dynamic_partitions_dict)
225  # Ensure that add_img_to_target_files rebuilds super split images for
226  # devices that retrofit dynamic partitions. This flag may have been set to
227  # false in the partial builds to prevent duplicate building of super.img.
228  OPTIONS.merged_misc_info['build_super_partition'] = 'true'
229
230
231def MergePackageKeys(framework_meta_dir, vendor_meta_dir, merged_meta_dir,
232                     file_name):
233  """Merges APK/APEX key list files."""
234
235  if file_name not in ('apkcerts.txt', 'apexkeys.txt'):
236    raise ExternalError(
237        'Unexpected file_name provided to merge_package_keys_txt: %s',
238        file_name)
239
240  def read_helper(d):
241    temp = {}
242    with open(os.path.join(d, file_name)) as f:
243      for line in f.read().splitlines():
244        line = line.strip()
245        if line:
246          name_search = MODULE_KEY_PATTERN.search(line.split()[0])
247          temp[name_search.group(1)] = line
248    return temp
249
250  framework_dict = read_helper(framework_meta_dir)
251  vendor_dict = read_helper(vendor_meta_dir)
252  merged_dict = {}
253
254  def filter_into_merged_dict(item_dict, partition_set):
255    for key, value in item_dict.items():
256      tag_search = PARTITION_TAG_PATTERN.search(value)
257
258      if tag_search is None:
259        raise ValueError('Entry missing partition tag: %s' % value)
260
261      partition_tag = tag_search.group(1)
262
263      if partition_tag in partition_set:
264        if key in merged_dict:
265          if OPTIONS.allow_duplicate_apkapex_keys:
266            # TODO(b/150582573) Always raise on duplicates.
267            logger.warning('Duplicate key %s' % key)
268            continue
269          else:
270            raise ValueError('Duplicate key %s' % key)
271
272        merged_dict[key] = value
273
274  # Prioritize framework keys first.
275  # Duplicate keys from vendor are an error, or ignored.
276  filter_into_merged_dict(framework_dict, OPTIONS.framework_partition_set)
277  filter_into_merged_dict(vendor_dict, OPTIONS.vendor_partition_set)
278
279  # The following code is similar to WriteSortedData, but different enough
280  # that we couldn't use that function. We need the output to be sorted by the
281  # basename of the apex/apk (without the ".apex" or ".apk" suffix). This
282  # allows the sort to be consistent with the framework/vendor input data and
283  # eases comparison of input data with merged data.
284  with open(os.path.join(merged_meta_dir, file_name), 'w') as output:
285    for key, value in sorted(merged_dict.items()):
286      output.write(value + '\n')
287
288
289def CopyNamedFileContexts(framework_meta_dir, vendor_meta_dir, merged_meta_dir):
290  """Creates named copies of each partial build's file_contexts.bin.
291
292  Used when regenerating images from the partial build.
293  """
294
295  def copy_fc_file(source_dir, file_name):
296    for name in (file_name, 'file_contexts.bin'):
297      fc_path = os.path.join(source_dir, name)
298      if os.path.exists(fc_path):
299        shutil.copyfile(fc_path, os.path.join(merged_meta_dir, file_name))
300        return
301    raise ValueError('Missing file_contexts file from %s: %s', source_dir,
302                     file_name)
303
304  copy_fc_file(framework_meta_dir, 'framework_file_contexts.bin')
305  copy_fc_file(vendor_meta_dir, 'vendor_file_contexts.bin')
306
307  # Replace <image>_selinux_fc values with framework or vendor file_contexts.bin
308  # depending on which dictionary the key came from.
309  # Only the file basename is required because all selinux_fc properties are
310  # replaced with the full path to the file under META/ when misc_info.txt is
311  # loaded from target files for repacking. See common.py LoadInfoDict().
312  for key in OPTIONS.vendor_misc_info:
313    if key.endswith('_selinux_fc'):
314      OPTIONS.merged_misc_info[key] = 'vendor_file_contexts.bin'
315  for key in OPTIONS.framework_misc_info:
316    if key.endswith('_selinux_fc'):
317      OPTIONS.merged_misc_info[key] = 'framework_file_contexts.bin'
318
319
320def UpdateCareMapImageSizeProps(images_dir):
321  """Sets <partition>_image_size props in misc_info.
322
323  add_images_to_target_files uses these props to generate META/care_map.pb.
324  Regenerated images will have this property set during regeneration.
325
326  However, images copied directly from input partial target files packages
327  need this value calculated here.
328  """
329  for partition in common.PARTITIONS_WITH_CARE_MAP:
330    image_path = os.path.join(images_dir, '{}.img'.format(partition))
331    if os.path.exists(image_path):
332      partition_size = sparse_img.GetImagePartitionSize(image_path)
333      image_props = build_image.ImagePropFromGlobalDict(
334          OPTIONS.merged_misc_info, partition)
335      verity_image_builder = verity_utils.CreateVerityImageBuilder(image_props)
336      image_size = verity_image_builder.CalculateMaxImageSize(partition_size)
337      OPTIONS.merged_misc_info['{}_image_size'.format(partition)] = image_size
338