1#!/usr/bin/env python3
2#
3# Copyright (C) 2011 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Builds output_image from the given input_directory, properties_file,
19and writes the image to target_output_directory.
20
21Usage:  build_image input_directory properties_file output_image \\
22            target_output_directory
23"""
24
25import datetime
26
27import argparse
28import glob
29import logging
30import os
31import os.path
32import re
33import shlex
34import shutil
35import sys
36import uuid
37import tempfile
38
39import common
40import verity_utils
41
42
43logger = logging.getLogger(__name__)
44
45OPTIONS = common.OPTIONS
46BLOCK_SIZE = common.BLOCK_SIZE
47BYTES_IN_MB = 1024 * 1024
48
49# Use a fixed timestamp (01/01/2009 00:00:00 UTC) for files when packaging
50# images. (b/24377993, b/80600931)
51FIXED_FILE_TIMESTAMP = int((
52    datetime.datetime(2009, 1, 1, 0, 0, 0, 0, None) -
53    datetime.datetime.utcfromtimestamp(0)).total_seconds())
54
55
56class BuildImageError(Exception):
57  """An Exception raised during image building."""
58
59  def __init__(self, message):
60    Exception.__init__(self, message)
61
62
63def GetDiskUsage(path):
64  """Returns the number of bytes that "path" occupies on host.
65
66  Args:
67    path: The directory or file to calculate size on.
68
69  Returns:
70    The number of bytes based on a 1K block_size.
71  """
72  cmd = ["du", "-b", "-k", "-s", path]
73  output = common.RunAndCheckOutput(cmd, verbose=False)
74  return int(output.split()[0]) * 1024
75
76
77def GetInodeUsage(path):
78  """Returns the number of inodes that "path" occupies on host.
79
80  Args:
81    path: The directory or file to calculate inode number on.
82
83  Returns:
84    The number of inodes used.
85  """
86  cmd = ["find", path, "-print"]
87  output = common.RunAndCheckOutput(cmd, verbose=False)
88  # increase by > 6% as number of files and directories is not whole picture.
89  inodes = output.count('\n')
90  spare_inodes = inodes * 6 // 100
91  min_spare_inodes = 12
92  if spare_inodes < min_spare_inodes:
93    spare_inodes = min_spare_inodes
94  return inodes + spare_inodes
95
96
97def GetFilesystemCharacteristics(fs_type, image_path, sparse_image=True):
98  """Returns various filesystem characteristics of "image_path".
99
100  Args:
101    image_path: The file to analyze.
102    sparse_image: Image is sparse
103
104  Returns:
105    The characteristics dictionary.
106  """
107  unsparse_image_path = image_path
108  if sparse_image:
109    unsparse_image_path = UnsparseImage(image_path, replace=False)
110
111  if fs_type.startswith("ext"):
112    cmd = ["tune2fs", "-l", unsparse_image_path]
113  elif fs_type.startswith("f2fs"):
114    cmd = ["fsck.f2fs", "-l", unsparse_image_path]
115
116  try:
117    output = common.RunAndCheckOutput(cmd, verbose=False)
118  finally:
119    if sparse_image:
120      os.remove(unsparse_image_path)
121  fs_dict = {}
122  for line in output.splitlines():
123    fields = line.split(":")
124    if len(fields) == 2:
125      fs_dict[fields[0].strip()] = fields[1].strip()
126  return fs_dict
127
128
129def UnsparseImage(sparse_image_path, replace=True):
130  img_dir = os.path.dirname(sparse_image_path)
131  unsparse_image_path = "unsparse_" + os.path.basename(sparse_image_path)
132  unsparse_image_path = os.path.join(img_dir, unsparse_image_path)
133  if os.path.exists(unsparse_image_path):
134    if replace:
135      os.unlink(unsparse_image_path)
136    else:
137      return unsparse_image_path
138  inflate_command = ["simg2img", sparse_image_path, unsparse_image_path]
139  try:
140    common.RunAndCheckOutput(inflate_command)
141  except:
142    os.remove(unsparse_image_path)
143    raise
144  return unsparse_image_path
145
146
147def ConvertBlockMapToBaseFs(block_map_file):
148  base_fs_file = common.MakeTempFile(prefix="script_gen_", suffix=".base_fs")
149  convert_command = ["blk_alloc_to_base_fs", block_map_file, base_fs_file]
150  common.RunAndCheckOutput(convert_command)
151  return base_fs_file
152
153
154def SetUpInDirAndFsConfig(origin_in, prop_dict):
155  """Returns the in_dir and fs_config that should be used for image building.
156
157  When building system.img for all targets, it creates and returns a staged dir
158  that combines the contents of /system (i.e. in the given in_dir) and root.
159
160  Args:
161    origin_in: Path to the input directory.
162    prop_dict: A property dict that contains info like partition size. Values
163        may be updated.
164
165  Returns:
166    A tuple of in_dir and fs_config that should be used to build the image.
167  """
168  fs_config = prop_dict.get("fs_config")
169
170  if prop_dict["mount_point"] == "system_other":
171    prop_dict["mount_point"] = "system"
172    return origin_in, fs_config
173
174  if prop_dict["mount_point"] != "system":
175    return origin_in, fs_config
176
177  if "first_pass" in prop_dict:
178    prop_dict["mount_point"] = "/"
179    return prop_dict["first_pass"]
180
181  # Construct a staging directory of the root file system.
182  in_dir = common.MakeTempDir()
183  root_dir = prop_dict.get("root_dir")
184  if root_dir:
185    shutil.rmtree(in_dir)
186    shutil.copytree(root_dir, in_dir, symlinks=True)
187  in_dir_system = os.path.join(in_dir, "system")
188  shutil.rmtree(in_dir_system, ignore_errors=True)
189  shutil.copytree(origin_in, in_dir_system, symlinks=True)
190
191  # Change the mount point to "/".
192  prop_dict["mount_point"] = "/"
193  if fs_config:
194    # We need to merge the fs_config files of system and root.
195    merged_fs_config = common.MakeTempFile(
196        prefix="merged_fs_config", suffix=".txt")
197    with open(merged_fs_config, "w") as fw:
198      if "root_fs_config" in prop_dict:
199        with open(prop_dict["root_fs_config"]) as fr:
200          fw.writelines(fr.readlines())
201      with open(fs_config) as fr:
202        fw.writelines(fr.readlines())
203    fs_config = merged_fs_config
204  prop_dict["first_pass"] = (in_dir, fs_config)
205  return in_dir, fs_config
206
207
208def CheckHeadroom(ext4fs_output, prop_dict):
209  """Checks if there's enough headroom space available.
210
211  Headroom is the reserved space on system image (via PRODUCT_SYSTEM_HEADROOM),
212  which is useful for devices with low disk space that have system image
213  variation between builds. The 'partition_headroom' in prop_dict is the size
214  in bytes, while the numbers in 'ext4fs_output' are for 4K-blocks.
215
216  Args:
217    ext4fs_output: The output string from mke2fs command.
218    prop_dict: The property dict.
219
220  Raises:
221    AssertionError: On invalid input.
222    BuildImageError: On check failure.
223  """
224  assert ext4fs_output is not None
225  assert prop_dict.get('fs_type', '').startswith('ext4')
226  assert 'partition_headroom' in prop_dict
227  assert 'mount_point' in prop_dict
228
229  ext4fs_stats = re.compile(
230      r'Created filesystem with .* (?P<used_blocks>[0-9]+)/'
231      r'(?P<total_blocks>[0-9]+) blocks')
232  last_line = ext4fs_output.strip().split('\n')[-1]
233  m = ext4fs_stats.match(last_line)
234  used_blocks = int(m.groupdict().get('used_blocks'))
235  total_blocks = int(m.groupdict().get('total_blocks'))
236  headroom_blocks = int(prop_dict['partition_headroom']) // BLOCK_SIZE
237  adjusted_blocks = total_blocks - headroom_blocks
238  if used_blocks > adjusted_blocks:
239    mount_point = prop_dict["mount_point"]
240    raise BuildImageError(
241        "Error: Not enough room on {} (total: {} blocks, used: {} blocks, "
242        "headroom: {} blocks, available: {} blocks)".format(
243            mount_point, total_blocks, used_blocks, headroom_blocks,
244            adjusted_blocks))
245
246
247def CalculateSizeAndReserved(prop_dict, size):
248  fs_type = prop_dict.get("fs_type", "")
249  partition_headroom = int(prop_dict.get("partition_headroom", 0))
250  # If not specified, give us 16MB margin for GetDiskUsage error ...
251  reserved_size = int(prop_dict.get(
252      "partition_reserved_size", BYTES_IN_MB * 16))
253
254  if fs_type == "erofs":
255    reserved_size = int(prop_dict.get("partition_reserved_size", 0))
256    if reserved_size == 0:
257      # give .3% margin or a minimum size for AVB footer
258      return max(size * 1003 // 1000, 256 * 1024)
259
260  if fs_type.startswith("ext4") and partition_headroom > reserved_size:
261    reserved_size = partition_headroom
262
263  return int(size * 1.1) + reserved_size
264
265
266def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config):
267  """Builds a pure image for the files under in_dir and writes it to out_file.
268
269  Args:
270    in_dir: Path to input directory.
271    prop_dict: A property dict that contains info like partition size. Values
272        will be updated with computed values.
273    out_file: The output image file.
274    target_out: Path to the TARGET_OUT directory as in Makefile. It actually
275        points to the /system directory under PRODUCT_OUT. fs_config (the one
276        under system/core/libcutils) reads device specific FS config files from
277        there.
278    fs_config: The fs_config file that drives the prototype
279
280  Raises:
281    BuildImageError: On build image failures.
282  """
283  build_command = []
284  fs_type = prop_dict.get("fs_type", "")
285  run_fsck = None
286  needs_projid = prop_dict.get("needs_projid", 0)
287  needs_casefold = prop_dict.get("needs_casefold", 0)
288  needs_compress = prop_dict.get("needs_compress", 0)
289
290  disable_sparse = "disable_sparse" in prop_dict
291  manual_sparse = False
292
293  if fs_type.startswith("ext"):
294    build_command = [prop_dict["ext_mkuserimg"]]
295    if "extfs_sparse_flag" in prop_dict and not disable_sparse:
296      build_command.append(prop_dict["extfs_sparse_flag"])
297      run_fsck = RunE2fsck
298    build_command.extend([in_dir, out_file, fs_type,
299                          prop_dict["mount_point"]])
300    build_command.append(prop_dict["image_size"])
301    if "journal_size" in prop_dict:
302      build_command.extend(["-j", prop_dict["journal_size"]])
303    if "timestamp" in prop_dict:
304      build_command.extend(["-T", str(prop_dict["timestamp"])])
305    if fs_config:
306      build_command.extend(["-C", fs_config])
307    if target_out:
308      build_command.extend(["-D", target_out])
309    if "block_list" in prop_dict:
310      build_command.extend(["-B", prop_dict["block_list"]])
311    if "base_fs_file" in prop_dict:
312      base_fs_file = ConvertBlockMapToBaseFs(prop_dict["base_fs_file"])
313      build_command.extend(["-d", base_fs_file])
314    build_command.extend(["-L", prop_dict["mount_point"]])
315    if "extfs_inode_count" in prop_dict:
316      build_command.extend(["-i", prop_dict["extfs_inode_count"]])
317    if "extfs_rsv_pct" in prop_dict:
318      build_command.extend(["-M", prop_dict["extfs_rsv_pct"]])
319    if "flash_erase_block_size" in prop_dict:
320      build_command.extend(["-e", prop_dict["flash_erase_block_size"]])
321    if "flash_logical_block_size" in prop_dict:
322      build_command.extend(["-o", prop_dict["flash_logical_block_size"]])
323    # Specify UUID and hash_seed if using mke2fs.
324    if os.path.basename(prop_dict["ext_mkuserimg"]) == "mkuserimg_mke2fs":
325      if "uuid" in prop_dict:
326        build_command.extend(["-U", prop_dict["uuid"]])
327      if "hash_seed" in prop_dict:
328        build_command.extend(["-S", prop_dict["hash_seed"]])
329    if prop_dict.get("ext4_share_dup_blocks") == "true":
330      build_command.append("-c")
331    if (needs_projid):
332      build_command.extend(["--inode_size", "512"])
333    else:
334      build_command.extend(["--inode_size", "256"])
335    if "selinux_fc" in prop_dict:
336      build_command.append(prop_dict["selinux_fc"])
337  elif fs_type.startswith("erofs"):
338    build_command = ["mkfs.erofs"]
339
340    compressor = None
341    if "erofs_default_compressor" in prop_dict:
342      compressor = prop_dict["erofs_default_compressor"]
343    if "erofs_compressor" in prop_dict:
344      compressor = prop_dict["erofs_compressor"]
345    if compressor and compressor != "none":
346      build_command.extend(["-z", compressor])
347
348    compress_hints = None
349    if "erofs_default_compress_hints" in prop_dict:
350      compress_hints = prop_dict["erofs_default_compress_hints"]
351    if "erofs_compress_hints" in prop_dict:
352      compress_hints = prop_dict["erofs_compress_hints"]
353    if compress_hints:
354      build_command.extend(["--compress-hints", compress_hints])
355
356    build_command.extend(["-b", prop_dict.get("erofs_blocksize", "4096")])
357
358    build_command.extend(["--mount-point", prop_dict["mount_point"]])
359    if target_out:
360      build_command.extend(["--product-out", target_out])
361    if fs_config:
362      build_command.extend(["--fs-config-file", fs_config])
363    if "selinux_fc" in prop_dict:
364      build_command.extend(["--file-contexts", prop_dict["selinux_fc"]])
365    if "timestamp" in prop_dict:
366      build_command.extend(["-T", str(prop_dict["timestamp"])])
367    if "uuid" in prop_dict:
368      build_command.extend(["-U", prop_dict["uuid"]])
369    if "block_list" in prop_dict:
370      build_command.extend(["--block-list-file", prop_dict["block_list"]])
371    if "erofs_pcluster_size" in prop_dict:
372      build_command.extend(["-C", prop_dict["erofs_pcluster_size"]])
373    if "erofs_share_dup_blocks" in prop_dict:
374      build_command.extend(["--chunksize", "4096"])
375    if "erofs_use_legacy_compression" in prop_dict:
376      build_command.extend(["-E", "legacy-compress"])
377
378    build_command.extend([out_file, in_dir])
379    if "erofs_sparse_flag" in prop_dict and not disable_sparse:
380      manual_sparse = True
381
382    run_fsck = RunErofsFsck
383  elif fs_type.startswith("squash"):
384    build_command = ["mksquashfsimage"]
385    build_command.extend([in_dir, out_file])
386    if "squashfs_sparse_flag" in prop_dict and not disable_sparse:
387      build_command.extend([prop_dict["squashfs_sparse_flag"]])
388    build_command.extend(["-m", prop_dict["mount_point"]])
389    if target_out:
390      build_command.extend(["-d", target_out])
391    if fs_config:
392      build_command.extend(["-C", fs_config])
393    if "selinux_fc" in prop_dict:
394      build_command.extend(["-c", prop_dict["selinux_fc"]])
395    if "block_list" in prop_dict:
396      build_command.extend(["-B", prop_dict["block_list"]])
397    if "squashfs_block_size" in prop_dict:
398      build_command.extend(["-b", prop_dict["squashfs_block_size"]])
399    if "squashfs_compressor" in prop_dict:
400      build_command.extend(["-z", prop_dict["squashfs_compressor"]])
401    if "squashfs_compressor_opt" in prop_dict:
402      build_command.extend(["-zo", prop_dict["squashfs_compressor_opt"]])
403    if prop_dict.get("squashfs_disable_4k_align") == "true":
404      build_command.extend(["-a"])
405  elif fs_type.startswith("f2fs"):
406    build_command = ["mkf2fsuserimg"]
407    build_command.extend([out_file, prop_dict["image_size"]])
408    if "f2fs_sparse_flag" in prop_dict and not disable_sparse:
409      build_command.extend([prop_dict["f2fs_sparse_flag"]])
410    if fs_config:
411      build_command.extend(["-C", fs_config])
412    build_command.extend(["-f", in_dir])
413    if target_out:
414      build_command.extend(["-D", target_out])
415    if "selinux_fc" in prop_dict:
416      build_command.extend(["-s", prop_dict["selinux_fc"]])
417    build_command.extend(["-t", prop_dict["mount_point"]])
418    if "timestamp" in prop_dict:
419      build_command.extend(["-T", str(prop_dict["timestamp"])])
420    if "block_list" in prop_dict:
421      build_command.extend(["-B", prop_dict["block_list"]])
422    build_command.extend(["-L", prop_dict["mount_point"]])
423    if (needs_projid):
424      build_command.append("--prjquota")
425    if (needs_casefold):
426      build_command.append("--casefold")
427    if (needs_compress or prop_dict.get("f2fs_compress") == "true"):
428      build_command.append("--compression")
429    if "ro_mount_point" in prop_dict:
430      build_command.append("--readonly")
431    if (prop_dict.get("f2fs_compress") == "true"):
432      build_command.append("--sldc")
433      if (prop_dict.get("f2fs_sldc_flags") == None):
434        build_command.append(str(0))
435      else:
436        sldc_flags_str = prop_dict.get("f2fs_sldc_flags")
437        sldc_flags = sldc_flags_str.split()
438        build_command.append(str(len(sldc_flags)))
439        build_command.extend(sldc_flags)
440    f2fs_blocksize = prop_dict.get("f2fs_blocksize", "4096")
441    build_command.extend(["-b", f2fs_blocksize])
442  else:
443    raise BuildImageError(
444        "Error: unknown filesystem type: {}".format(fs_type))
445
446  try:
447    mkfs_output = common.RunAndCheckOutput(build_command)
448  except:
449    try:
450      du = GetDiskUsage(in_dir)
451      du_str = "{} bytes ({} MB)".format(du, du // BYTES_IN_MB)
452    # Suppress any errors from GetDiskUsage() to avoid hiding the real errors
453    # from common.RunAndCheckOutput().
454    except Exception:  # pylint: disable=broad-except
455      logger.exception("Failed to compute disk usage with du")
456      du_str = "unknown"
457    print(
458        "Out of space? Out of inodes? The tree size of {} is {}, "
459        "with reserved space of {} bytes ({} MB).".format(
460            in_dir, du_str,
461            int(prop_dict.get("partition_reserved_size", 0)),
462            int(prop_dict.get("partition_reserved_size", 0)) // BYTES_IN_MB))
463    if ("image_size" in prop_dict and "partition_size" in prop_dict):
464      print(
465          "The max image size for filesystem files is {} bytes ({} MB), "
466          "out of a total partition size of {} bytes ({} MB).".format(
467              int(prop_dict["image_size"]),
468              int(prop_dict["image_size"]) // BYTES_IN_MB,
469              int(prop_dict["partition_size"]),
470              int(prop_dict["partition_size"]) // BYTES_IN_MB))
471    raise
472
473  if run_fsck and prop_dict.get("skip_fsck") != "true":
474    run_fsck(out_file)
475
476  if manual_sparse:
477    temp_file = out_file + ".sparse"
478    img2simg_argv = ["img2simg", out_file, temp_file]
479    common.RunAndCheckOutput(img2simg_argv)
480    os.rename(temp_file, out_file)
481
482  return mkfs_output
483
484
485def RunE2fsck(out_file):
486  unsparse_image = UnsparseImage(out_file, replace=False)
487
488  # Run e2fsck on the inflated image file
489  e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image]
490  try:
491    common.RunAndCheckOutput(e2fsck_command)
492  finally:
493    os.remove(unsparse_image)
494
495
496def RunErofsFsck(out_file):
497  fsck_command = ["fsck.erofs", "--extract", out_file]
498  try:
499    common.RunAndCheckOutput(fsck_command)
500  except:
501    print("Check failed for EROFS image {}".format(out_file))
502    raise
503
504
505def SetUUIDIfNotExist(image_props):
506
507  # Use repeatable ext4 FS UUID and hash_seed UUID (based on partition name and
508  # build fingerprint). Also use the legacy build id, because the vbmeta digest
509  # isn't available at this point.
510  what = image_props["mount_point"]
511  fingerprint = image_props.get("fingerprint", "")
512  uuid_seed = what + "-" + fingerprint
513  logger.info("Using fingerprint %s for partition %s", fingerprint, what)
514  image_props["uuid"] = str(uuid.uuid5(uuid.NAMESPACE_URL, uuid_seed))
515  hash_seed = "hash_seed-" + uuid_seed
516  image_props["hash_seed"] = str(uuid.uuid5(uuid.NAMESPACE_URL, hash_seed))
517
518
519def BuildImage(in_dir, prop_dict, out_file, target_out=None):
520  """Builds an image for the files under in_dir and writes it to out_file.
521
522  Args:
523    in_dir: Path to input directory.
524    prop_dict: A property dict that contains info like partition size. Values
525        will be updated with computed values.
526    out_file: The output image file.
527    target_out: Path to the TARGET_OUT directory as in Makefile. It actually
528        points to the /system directory under PRODUCT_OUT. fs_config (the one
529        under system/core/libcutils) reads device specific FS config files from
530        there.
531
532  Raises:
533    BuildImageError: On build image failures.
534  """
535  in_dir, fs_config = SetUpInDirAndFsConfig(in_dir, prop_dict)
536  SetUUIDIfNotExist(prop_dict)
537
538  build_command = []
539  fs_type = prop_dict.get("fs_type", "")
540
541  fs_spans_partition = True
542  if fs_type.startswith("squash") or fs_type.startswith("erofs"):
543    fs_spans_partition = False
544  elif fs_type.startswith("f2fs") and prop_dict.get("f2fs_compress") == "true":
545    fs_spans_partition = False
546
547  # Get a builder for creating an image that's to be verified by Verified Boot,
548  # or None if not applicable.
549  verity_image_builder = verity_utils.CreateVerityImageBuilder(prop_dict)
550
551  disable_sparse = "disable_sparse" in prop_dict
552  mkfs_output = None
553  if (prop_dict.get("use_dynamic_partition_size") == "true" and
554          "partition_size" not in prop_dict):
555    # If partition_size is not defined, use output of `du' + reserved_size.
556    # For compressed file system, it's better to use the compressed size to avoid wasting space.
557    if fs_type.startswith("erofs"):
558      mkfs_output = BuildImageMkfs(
559          in_dir, prop_dict, out_file, target_out, fs_config)
560      if "erofs_sparse_flag" in prop_dict and not disable_sparse:
561        image_path = UnsparseImage(out_file, replace=False)
562        size = GetDiskUsage(image_path)
563        os.remove(image_path)
564      else:
565        size = GetDiskUsage(out_file)
566    else:
567      size = GetDiskUsage(in_dir)
568    logger.info(
569        "The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB)
570    size = CalculateSizeAndReserved(prop_dict, size)
571    # Round this up to a multiple of 4K so that avbtool works
572    size = common.RoundUpTo4K(size)
573    if fs_type.startswith("ext"):
574      prop_dict["partition_size"] = str(size)
575      prop_dict["image_size"] = str(size)
576      if "extfs_inode_count" not in prop_dict:
577        prop_dict["extfs_inode_count"] = str(GetInodeUsage(in_dir))
578      logger.info(
579          "First Pass based on estimates of %d MB and %s inodes.",
580          size // BYTES_IN_MB, prop_dict["extfs_inode_count"])
581      BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config)
582      sparse_image = False
583      if "extfs_sparse_flag" in prop_dict and not disable_sparse:
584        sparse_image = True
585      fs_dict = GetFilesystemCharacteristics(fs_type, out_file, sparse_image)
586      os.remove(out_file)
587      block_size = int(fs_dict.get("Block size", "4096"))
588      free_size = int(fs_dict.get("Free blocks", "0")) * block_size
589      reserved_size = int(prop_dict.get("partition_reserved_size", 0))
590      partition_headroom = int(fs_dict.get("partition_headroom", 0))
591      if fs_type.startswith("ext4") and partition_headroom > reserved_size:
592        reserved_size = partition_headroom
593      if free_size <= reserved_size:
594        logger.info(
595            "Not worth reducing image %d <= %d.", free_size, reserved_size)
596      else:
597        size -= free_size
598        size += reserved_size
599        if reserved_size == 0:
600          # add .3% margin
601          size = size * 1003 // 1000
602        # Use a minimum size, otherwise we will fail to calculate an AVB footer
603        # or fail to construct an ext4 image.
604        size = max(size, 256 * 1024)
605        if block_size <= 4096:
606          size = common.RoundUpTo4K(size)
607        else:
608          size = ((size + block_size - 1) // block_size) * block_size
609      extfs_inode_count = prop_dict["extfs_inode_count"]
610      inodes = int(fs_dict.get("Inode count", extfs_inode_count))
611      inodes -= int(fs_dict.get("Free inodes", "0"))
612      # add .2% margin or 1 inode, whichever is greater
613      spare_inodes = inodes * 2 // 1000
614      min_spare_inodes = 1
615      if spare_inodes < min_spare_inodes:
616        spare_inodes = min_spare_inodes
617      inodes += spare_inodes
618      prop_dict["extfs_inode_count"] = str(inodes)
619      prop_dict["partition_size"] = str(size)
620      logger.info(
621          "Allocating %d Inodes for %s.", inodes, out_file)
622    elif fs_type.startswith("f2fs") and prop_dict.get("f2fs_compress") == "true":
623      prop_dict["partition_size"] = str(size)
624      prop_dict["image_size"] = str(size)
625      BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config)
626      sparse_image = False
627      if "f2fs_sparse_flag" in prop_dict and not disable_sparse:
628        sparse_image = True
629      fs_dict = GetFilesystemCharacteristics(fs_type, out_file, sparse_image)
630      os.remove(out_file)
631      block_count = int(fs_dict.get("block_count", "0"))
632      log_blocksize = int(fs_dict.get("log_blocksize", "12"))
633      size = block_count << log_blocksize
634      prop_dict["partition_size"] = str(size)
635    if verity_image_builder:
636      size = verity_image_builder.CalculateDynamicPartitionSize(size)
637    prop_dict["partition_size"] = str(size)
638    logger.info(
639        "Allocating %d MB for %s", size // BYTES_IN_MB, out_file)
640
641  prop_dict["image_size"] = prop_dict["partition_size"]
642
643  # Adjust the image size to make room for the hashes if this is to be verified.
644  if verity_image_builder:
645    max_image_size = verity_image_builder.CalculateMaxImageSize()
646    prop_dict["image_size"] = str(max_image_size)
647
648  if not mkfs_output:
649    mkfs_output = BuildImageMkfs(
650        in_dir, prop_dict, out_file, target_out, fs_config)
651
652  # Update the image (eg filesystem size). This can be different eg if mkfs
653  # rounds the requested size down due to alignment.
654  prop_dict["image_size"] = common.sparse_img.GetImagePartitionSize(out_file)
655
656  # Check if there's enough headroom space available for ext4 image.
657  if "partition_headroom" in prop_dict and fs_type.startswith("ext4"):
658    CheckHeadroom(mkfs_output, prop_dict)
659
660  if not fs_spans_partition and verity_image_builder:
661    verity_image_builder.PadSparseImage(out_file)
662
663  # Create the verified image if this is to be verified.
664  if verity_image_builder:
665    verity_image_builder.Build(out_file)
666
667
668def TryParseFingerprint(glob_dict: dict):
669  for (key, val) in glob_dict.items():
670    if not key.endswith("_add_hashtree_footer_args") and not key.endswith("_add_hash_footer_args"):
671      continue
672    for arg in shlex.split(val):
673      m = re.match(r"^com\.android\.build\.\w+\.fingerprint:", arg)
674      if m is None:
675        continue
676      fingerprint = arg[len(m.group()):]
677      glob_dict["fingerprint"] = fingerprint
678      return
679
680
681def ImagePropFromGlobalDict(glob_dict, mount_point):
682  """Build an image property dictionary from the global dictionary.
683
684  Args:
685    glob_dict: the global dictionary from the build system.
686    mount_point: such as "system", "data" etc.
687  """
688  d = {}
689  TryParseFingerprint(glob_dict)
690
691  # Set fixed timestamp for building the OTA package.
692  if "use_fixed_timestamp" in glob_dict:
693    d["timestamp"] = FIXED_FILE_TIMESTAMP
694  if "build.prop" in glob_dict:
695    timestamp = glob_dict["build.prop"].GetProp("ro.build.date.utc")
696    if timestamp:
697      d["timestamp"] = timestamp
698
699  def copy_prop(src_p, dest_p):
700    """Copy a property from the global dictionary.
701
702    Args:
703      src_p: The source property in the global dictionary.
704      dest_p: The destination property.
705    Returns:
706      True if property was found and copied, False otherwise.
707    """
708    if src_p in glob_dict:
709      d[dest_p] = str(glob_dict[src_p])
710      return True
711    return False
712
713  common_props = (
714      "extfs_sparse_flag",
715      "erofs_default_compressor",
716      "erofs_default_compress_hints",
717      "erofs_pcluster_size",
718      "erofs_blocksize",
719      "erofs_share_dup_blocks",
720      "erofs_sparse_flag",
721      "erofs_use_legacy_compression",
722      "squashfs_sparse_flag",
723      "system_f2fs_compress",
724      "system_f2fs_sldc_flags",
725      "f2fs_sparse_flag",
726      "f2fs_blocksize",
727      "skip_fsck",
728      "ext_mkuserimg",
729      "avb_enable",
730      "avb_avbtool",
731      "use_dynamic_partition_size",
732      "fingerprint",
733  )
734  for p in common_props:
735    copy_prop(p, p)
736
737  ro_mount_points = set([
738      "odm",
739      "odm_dlkm",
740      "oem",
741      "product",
742      "system",
743      "system_dlkm",
744      "system_ext",
745      "system_other",
746      "vendor",
747      "vendor_dlkm",
748  ])
749
750  # Tuple layout: (readonly, specific prop, general prop)
751  fmt_props = (
752      # Generic first, then specific file type.
753      (False, "fs_type", "fs_type"),
754      (False, "{}_fs_type", "fs_type"),
755
756      # Ordering for these doesn't matter.
757      (False, "{}_selinux_fc", "selinux_fc"),
758      (False, "{}_size", "partition_size"),
759      (True, "avb_{}_add_hashtree_footer_args", "avb_add_hashtree_footer_args"),
760      (True, "avb_{}_algorithm", "avb_algorithm"),
761      (True, "avb_{}_hashtree_enable", "avb_hashtree_enable"),
762      (True, "avb_{}_key_path", "avb_key_path"),
763      (True, "avb_{}_salt", "avb_salt"),
764      (True, "erofs_use_legacy_compression", "erofs_use_legacy_compression"),
765      (True, "ext4_share_dup_blocks", "ext4_share_dup_blocks"),
766      (True, "{}_base_fs_file", "base_fs_file"),
767      (True, "{}_disable_sparse", "disable_sparse"),
768      (True, "{}_erofs_compressor", "erofs_compressor"),
769      (True, "{}_erofs_compress_hints", "erofs_compress_hints"),
770      (True, "{}_erofs_pcluster_size", "erofs_pcluster_size"),
771      (True, "{}_erofs_blocksize", "erofs_blocksize"),
772      (True, "{}_erofs_share_dup_blocks", "erofs_share_dup_blocks"),
773      (True, "{}_extfs_inode_count", "extfs_inode_count"),
774      (True, "{}_f2fs_compress", "f2fs_compress"),
775      (True, "{}_f2fs_sldc_flags", "f2fs_sldc_flags"),
776      (True, "{}_f2fs_blocksize", "f2fs_block_size"),
777      (True, "{}_reserved_size", "partition_reserved_size"),
778      (True, "{}_squashfs_block_size", "squashfs_block_size"),
779      (True, "{}_squashfs_compressor", "squashfs_compressor"),
780      (True, "{}_squashfs_compressor_opt", "squashfs_compressor_opt"),
781      (True, "{}_squashfs_disable_4k_align", "squashfs_disable_4k_align"),
782      (True, "{}_verity_block_device", "verity_block_device"),
783  )
784
785  # Translate prefixed properties into generic ones.
786  if mount_point == "data":
787    prefix = "userdata"
788  else:
789    prefix = mount_point
790
791  for readonly, src_prop, dest_prop in fmt_props:
792    if readonly and mount_point not in ro_mount_points:
793      continue
794
795    if src_prop == "fs_type":
796      # This property is legacy and only used on a few partitions. b/202600377
797      allowed_partitions = set(["system", "system_other", "data", "oem"])
798      if mount_point not in allowed_partitions:
799        continue
800
801    if (mount_point == "system_other") and (dest_prop != "partition_size"):
802      # Propagate system properties to system_other. They'll get overridden
803      # after as needed.
804      copy_prop(src_prop.format("system"), dest_prop)
805
806    copy_prop(src_prop.format(prefix), dest_prop)
807
808  # Set prefixed properties that need a default value.
809  if mount_point in ro_mount_points:
810    prop = "{}_journal_size".format(prefix)
811    if not copy_prop(prop, "journal_size"):
812      d["journal_size"] = "0"
813
814    prop = "{}_extfs_rsv_pct".format(prefix)
815    if not copy_prop(prop, "extfs_rsv_pct"):
816      d["extfs_rsv_pct"] = "0"
817
818    d["ro_mount_point"] = "1"
819
820  # Copy partition-specific properties.
821  d["mount_point"] = mount_point
822  if mount_point == "system":
823    copy_prop("system_headroom", "partition_headroom")
824    copy_prop("root_dir", "root_dir")
825    copy_prop("root_fs_config", "root_fs_config")
826  elif mount_point == "data":
827    # Copy the generic fs type first, override with specific one if available.
828    copy_prop("flash_logical_block_size", "flash_logical_block_size")
829    copy_prop("flash_erase_block_size", "flash_erase_block_size")
830    copy_prop("needs_casefold", "needs_casefold")
831    copy_prop("needs_projid", "needs_projid")
832    copy_prop("needs_compress", "needs_compress")
833  d["partition_name"] = mount_point
834  return d
835
836
837def LoadGlobalDict(filename):
838  """Load "name=value" pairs from filename"""
839  d = {}
840  f = open(filename)
841  for line in f:
842    line = line.strip()
843    if not line or line.startswith("#"):
844      continue
845    k, v = line.split("=", 1)
846    d[k] = v
847  f.close()
848  return d
849
850
851def GlobalDictFromImageProp(image_prop, mount_point):
852  d = {}
853
854  def copy_prop(src_p, dest_p):
855    if src_p in image_prop:
856      d[dest_p] = image_prop[src_p]
857      return True
858    return False
859
860  if mount_point == "system":
861    copy_prop("partition_size", "system_size")
862  elif mount_point == "system_other":
863    copy_prop("partition_size", "system_other_size")
864  elif mount_point == "vendor":
865    copy_prop("partition_size", "vendor_size")
866  elif mount_point == "odm":
867    copy_prop("partition_size", "odm_size")
868  elif mount_point == "vendor_dlkm":
869    copy_prop("partition_size", "vendor_dlkm_size")
870  elif mount_point == "odm_dlkm":
871    copy_prop("partition_size", "odm_dlkm_size")
872  elif mount_point == "system_dlkm":
873    copy_prop("partition_size", "system_dlkm_size")
874  elif mount_point == "product":
875    copy_prop("partition_size", "product_size")
876  elif mount_point == "system_ext":
877    copy_prop("partition_size", "system_ext_size")
878  return d
879
880
881def BuildVBMeta(in_dir, glob_dict, output_path):
882  """Creates a VBMeta image.
883
884  It generates the requested VBMeta image. The requested image could be for
885  top-level or chained VBMeta image, which is determined based on the name.
886
887  Args:
888    output_path: Path to generated vbmeta.img
889    partitions: A dict that's keyed by partition names with image paths as
890        values. Only valid partition names are accepted, as partitions listed
891        in common.AVB_PARTITIONS and custom partitions listed in
892        OPTIONS.info_dict.get("avb_custom_images_partition_list")
893    name: Name of the VBMeta partition, e.g. 'vbmeta', 'vbmeta_system'.
894    needed_partitions: Partitions whose descriptors should be included into the
895        generated VBMeta image.
896
897  Returns:
898    Path to the created image.
899
900  Raises:
901    AssertionError: On invalid input args.
902  """
903  vbmeta_partitions = common.AVB_PARTITIONS[:]
904  name = os.path.basename(output_path).rstrip(".img")
905  vbmeta_system = glob_dict.get("avb_vbmeta_system", "").strip()
906  vbmeta_vendor = glob_dict.get("avb_vbmeta_vendor", "").strip()
907  if "vbmeta_system" in name:
908    vbmeta_partitions = vbmeta_system.split()
909  elif "vbmeta_vendor" in name:
910    vbmeta_partitions = vbmeta_vendor.split()
911  else:
912    if vbmeta_system:
913      vbmeta_partitions = [
914          item for item in vbmeta_partitions
915          if item not in vbmeta_system.split()]
916      vbmeta_partitions.append("vbmeta_system")
917
918    if vbmeta_vendor:
919      vbmeta_partitions = [
920          item for item in vbmeta_partitions
921          if item not in vbmeta_vendor.split()]
922      vbmeta_partitions.append("vbmeta_vendor")
923
924  partitions = {part: os.path.join(in_dir, part + ".img")
925                for part in vbmeta_partitions}
926  partitions = {part: path for (part, path) in partitions.items() if os.path.exists(path)}
927  common.BuildVBMeta(output_path, partitions, name, vbmeta_partitions)
928
929
930def BuildImageOrVBMeta(input_directory, target_out, glob_dict, image_properties, out_file):
931  try:
932    if "vbmeta" in os.path.basename(out_file):
933      OPTIONS.info_dict = glob_dict
934      BuildVBMeta(input_directory, glob_dict, out_file)
935    else:
936      BuildImage(input_directory, image_properties, out_file, target_out)
937  except:
938    logger.error("Failed to build %s from %s", out_file, input_directory)
939    raise
940
941
942def CopyInputDirectory(src, dst, filter_file):
943  with open(filter_file, 'r') as f:
944    for line in f:
945      line = line.strip()
946      if not line:
947        return
948      if line != os.path.normpath(line):
949        sys.exit(f"{line}: not normalized")
950      if line.startswith("../") or line.startswith('/'):
951        sys.exit(f"{line}: escapes staging directory by starting with ../ or /")
952      full_src = os.path.join(src, line)
953      full_dst = os.path.join(dst, line)
954      if os.path.isdir(full_src):
955        os.makedirs(full_dst, exist_ok=True)
956      else:
957        os.makedirs(os.path.dirname(full_dst), exist_ok=True)
958        os.link(full_src, full_dst, follow_symlinks=False)
959
960
961def main(argv):
962  parser = argparse.ArgumentParser(
963    description="Builds output_image from the given input_directory and properties_file, and "
964    "writes the image to target_output_directory.")
965  parser.add_argument("--input-directory-filter-file",
966    help="the path to a file that contains a list of all files in the input_directory. If this "
967    "option is provided, all files under the input_directory that are not listed in this file will "
968    "be deleted before building the image. This is to work around the fact that building a module "
969    "will install in by default, so there could be files in the input_directory that are not "
970    "actually supposed to be part of the partition. The paths in this file must be relative to "
971    "input_directory.")
972  parser.add_argument("input_directory",
973    help="the staging directory to be converted to an image file")
974  parser.add_argument("properties_file",
975    help="a file containing the 'global dictionary' of properties that affect how the image is "
976    "built")
977  parser.add_argument("out_file",
978    help="the output file to write")
979  parser.add_argument("target_out",
980    help="the path to $(TARGET_OUT). Certain tools will use this to look through multiple staging "
981    "directories for fs config files.")
982  parser.add_argument("-v", action="store_true",
983                      help="Enable verbose logging", dest="verbose")
984  args = parser.parse_args()
985  if args.verbose:
986    OPTIONS.verbose = True
987
988  common.InitLogging()
989
990  glob_dict = LoadGlobalDict(args.properties_file)
991  if "mount_point" in glob_dict:
992    # The caller knows the mount point and provides a dictionary needed by
993    # BuildImage().
994    image_properties = glob_dict
995  else:
996    image_filename = os.path.basename(args.out_file)
997    mount_point = ""
998    if image_filename == "system.img":
999      mount_point = "system"
1000    elif image_filename == "system_other.img":
1001      mount_point = "system_other"
1002    elif image_filename == "userdata.img":
1003      mount_point = "data"
1004    elif image_filename == "cache.img":
1005      mount_point = "cache"
1006    elif image_filename == "vendor.img":
1007      mount_point = "vendor"
1008    elif image_filename == "odm.img":
1009      mount_point = "odm"
1010    elif image_filename == "vendor_dlkm.img":
1011      mount_point = "vendor_dlkm"
1012    elif image_filename == "odm_dlkm.img":
1013      mount_point = "odm_dlkm"
1014    elif image_filename == "system_dlkm.img":
1015      mount_point = "system_dlkm"
1016    elif image_filename == "oem.img":
1017      mount_point = "oem"
1018    elif image_filename == "product.img":
1019      mount_point = "product"
1020    elif image_filename == "system_ext.img":
1021      mount_point = "system_ext"
1022    elif "vbmeta" in image_filename:
1023      mount_point = "vbmeta"
1024    else:
1025      logger.error("Unknown image file name %s", image_filename)
1026      sys.exit(1)
1027
1028    if "vbmeta" != mount_point:
1029      image_properties = ImagePropFromGlobalDict(glob_dict, mount_point)
1030
1031  if args.input_directory_filter_file and not os.environ.get("BUILD_BROKEN_INCORRECT_PARTITION_IMAGES"):
1032    with tempfile.TemporaryDirectory(dir=os.path.dirname(args.input_directory)) as new_input_directory:
1033      CopyInputDirectory(args.input_directory, new_input_directory, args.input_directory_filter_file)
1034      BuildImageOrVBMeta(new_input_directory, args.target_out, glob_dict, image_properties, args.out_file)
1035  else:
1036    BuildImageOrVBMeta(args.input_directory, args.target_out, glob_dict, image_properties, args.out_file)
1037
1038
1039if __name__ == '__main__':
1040  try:
1041    main(sys.argv[1:])
1042  finally:
1043    common.Cleanup()
1044