From 621fd65adc825b69a59367b97a7c2aa73e382909 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 6 Jan 2026 09:27:31 -0700 Subject: scripts/make_fit: Speed up operation The kernel is likely at least 16MB so we may as well use that as a step size when reallocating space for the FIT in memory. Pack the FIT at the end, so there is no wasted space. This reduces the time to pack by an order of magnitude, or so. Signed-off-by: Simon Glass Reviewed-by: Nicolas Schier Reviewed-by: Ahmad Fatoum Tested-by: Chen-Yu Tsai Link: https://patch.msgid.link/20260106162738.2605574-2-sjg@chromium.org Signed-off-by: Nathan Chancellor --- scripts/make_fit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'scripts/make_fit.py') diff --git a/scripts/make_fit.py b/scripts/make_fit.py index 1683e5ec6e67..0f5e7c4b8aed 100755 --- a/scripts/make_fit.py +++ b/scripts/make_fit.py @@ -98,7 +98,7 @@ def setup_fit(fsw, name): fsw (libfdt.FdtSw): Object to use for writing name (str): Name of kernel image """ - fsw.INC_SIZE = 65536 + fsw.INC_SIZE = 16 << 20 fsw.finish_reservemap() fsw.begin_node('') fsw.property_string('description', f'{name} with devicetree set') @@ -299,7 +299,9 @@ def build_fit(args): finish_fit(fsw, entries) # Include the kernel itself in the returned file count - return fsw.as_fdt().as_bytearray(), seq + 1, size + fdt = fsw.as_fdt() + fdt.pack() + return fdt.as_bytearray(), seq + 1, size def run_make_fit(): -- cgit v1.2.3 From 26428e7dd6a51e328776dd333f613445d1951658 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 6 Jan 2026 09:27:32 -0700 Subject: scripts/make_fit: Support an initial ramdisk FIT (Flat Image Tree) allows a ramdisk to be included in each configuration. Add support for this to the script. This feature is not available via 'make image.fit' since the ramdisk likely needs to be built separately anyway, e.g. using modules from the kernel build. Future work may provide support for doing that. Note that the uncompressed size is not correct when a ramdisk is used, since it is too expensive to decompress the ramdisk. Signed-off-by: Simon Glass Reviewed-by: Nicolas Schier Reviewed-by: Ahmad Fatoum Link: https://patch.msgid.link/20260106162738.2605574-3-sjg@chromium.org Signed-off-by: Nathan Chancellor --- scripts/make_fit.py | 52 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 8 deletions(-) (limited to 'scripts/make_fit.py') diff --git a/scripts/make_fit.py b/scripts/make_fit.py index 0f5e7c4b8aed..66cd73711506 100755 --- a/scripts/make_fit.py +++ b/scripts/make_fit.py @@ -10,10 +10,14 @@ Usage: make_fit.py -A arm64 -n 'Linux-6.6' -O linux -o arch/arm64/boot/image.fit -k /tmp/kern/arch/arm64/boot/image.itk - @arch/arm64/boot/dts/dtbs-list -E -c gzip + -r /boot/initrd.img-6.14.0-27-generic @arch/arm64/boot/dts/dtbs-list + -E -c gzip -Creates a FIT containing the supplied kernel and a set of devicetree files, -either specified individually or listed in a file (with an '@' prefix). +Creates a FIT containing the supplied kernel, an optional ramdisk, and a set of +devicetree files, either specified individually or listed in a file (with an +'@' prefix). + +Use -r to specify an existing ramdisk/initrd file. Use -E to generate an external FIT (where the data is placed after the FIT data structure). This allows parsing of the data without loading @@ -29,8 +33,6 @@ looks at the .cmd files produced by the kernel build. The resulting FIT can be booted by bootloaders which support FIT, such as U-Boot, Linuxboot, Tianocore, etc. - -Note that this tool does not yet support adding a ramdisk / initrd. """ import argparse @@ -81,6 +83,8 @@ def parse_args(): help='Specifies the operating system') parser.add_argument('-k', '--kernel', type=str, required=True, help='Specifies the (uncompressed) kernel input file (.itk)') + parser.add_argument('-r', '--ramdisk', type=str, + help='Specifies the ramdisk/initrd input file') parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output') parser.add_argument('dtbs', type=str, nargs='*', @@ -133,7 +137,28 @@ def write_kernel(fsw, data, args): fsw.property_u32('entry', 0) -def finish_fit(fsw, entries): +def write_ramdisk(fsw, data, args): + """Write out the ramdisk image + + Writes a ramdisk node along with the required properties + + Args: + fsw (libfdt.FdtSw): Object to use for writing + data (bytes): Data to write (possibly compressed) + args (Namespace): Contains necessary strings: + arch: FIT architecture, e.g. 'arm64' + fit_os: Operating Systems, e.g. 'linux' + """ + with fsw.add_node('ramdisk'): + fsw.property_string('description', 'Ramdisk') + fsw.property_string('type', 'ramdisk') + fsw.property_string('arch', args.arch) + fsw.property_string('compression', 'none') + fsw.property_string('os', args.os) + fsw.property('data', data) + + +def finish_fit(fsw, entries, has_ramdisk=False): """Finish the FIT ready for use Writes the /configurations node and subnodes @@ -143,6 +168,7 @@ def finish_fit(fsw, entries): entries (list of tuple): List of configurations: str: Description of model str: Compatible stringlist + has_ramdisk (bool): True if a ramdisk is included in the FIT """ fsw.end_node() seq = 0 @@ -154,6 +180,8 @@ def finish_fit(fsw, entries): fsw.property_string('description', model) fsw.property('fdt', bytes(''.join(f'fdt-{x}\x00' for x in files), "ascii")) fsw.property_string('kernel', 'kernel') + if has_ramdisk: + fsw.property_string('ramdisk', 'ramdisk') fsw.end_node() @@ -274,6 +302,14 @@ def build_fit(args): size += os.path.getsize(args.kernel) write_kernel(fsw, comp_data, args) + # Handle the ramdisk if provided. Compression is not supported as it is + # already compressed. + if args.ramdisk: + with open(args.ramdisk, 'rb') as inf: + data = inf.read() + size += len(data) + write_ramdisk(fsw, data, args) + for fname in args.dtbs: # Ignore non-DTB (*.dtb) files if os.path.splitext(fname)[1] != '.dtb': @@ -296,12 +332,12 @@ def build_fit(args): entries.append([model, compat, files_seq]) - finish_fit(fsw, entries) + finish_fit(fsw, entries, bool(args.ramdisk)) # Include the kernel itself in the returned file count fdt = fsw.as_fdt() fdt.pack() - return fdt.as_bytearray(), seq + 1, size + return fdt.as_bytearray(), seq + 1 + bool(args.ramdisk), size def run_make_fit(): -- cgit v1.2.3 From 873c2836982e1f7389d487afca4cc42ed373ba4b Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 6 Jan 2026 09:27:33 -0700 Subject: scripts/make_fit: Move dtb processing into a function Since build_fit() is getting quite long, move the dtb processing into a separate function. Change the double quotes in the write() call to single, to match the rest of the script. Signed-off-by: Simon Glass Reviewed-by: Nicolas Schier Reviewed-by: Ahmad Fatoum Reviewed-by: Chen-Yu Tsai Link: https://patch.msgid.link/20260106162738.2605574-4-sjg@chromium.org Signed-off-by: Nathan Chancellor --- scripts/make_fit.py | 67 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 23 deletions(-) (limited to 'scripts/make_fit.py') diff --git a/scripts/make_fit.py b/scripts/make_fit.py index 66cd73711506..7ddaf3c0d783 100755 --- a/scripts/make_fit.py +++ b/scripts/make_fit.py @@ -277,6 +277,47 @@ def process_dtb(fname, args): return (model, compat, files) + +def _process_dtbs(args, fsw, entries, fdts): + """Process all DTB files and add them to the FIT + + Args: + args: Program arguments + fsw: FIT writer object + entries: List to append entries to + fdts: Dictionary of processed DTBs + + Returns: + tuple: + Number of files processed + Total size of files processed + """ + seq = 0 + size = 0 + for fname in args.dtbs: + # Ignore non-DTB (*.dtb) files + if os.path.splitext(fname)[1] != '.dtb': + continue + + try: + (model, compat, files) = process_dtb(fname, args) + except Exception as e: + sys.stderr.write(f'Error processing {fname}:\n') + raise e + + for fn in files: + if fn not in fdts: + seq += 1 + size += os.path.getsize(fn) + output_dtb(fsw, seq, fn, args.arch, args.compress) + fdts[fn] = seq + + files_seq = [fdts[fn] for fn in files] + entries.append([model, compat, files_seq]) + + return seq, size + + def build_fit(args): """Build the FIT from the provided files and arguments @@ -289,7 +330,6 @@ def build_fit(args): int: Number of configurations generated size: Total uncompressed size of data """ - seq = 0 size = 0 fsw = libfdt.FdtSw() setup_fit(fsw, args.name) @@ -310,34 +350,15 @@ def build_fit(args): size += len(data) write_ramdisk(fsw, data, args) - for fname in args.dtbs: - # Ignore non-DTB (*.dtb) files - if os.path.splitext(fname)[1] != '.dtb': - continue - - try: - (model, compat, files) = process_dtb(fname, args) - except Exception as e: - sys.stderr.write(f"Error processing {fname}:\n") - raise e - - for fn in files: - if fn not in fdts: - seq += 1 - size += os.path.getsize(fn) - output_dtb(fsw, seq, fn, args.arch, args.compress) - fdts[fn] = seq - - files_seq = [fdts[fn] for fn in files] - - entries.append([model, compat, files_seq]) + count, fdt_size = _process_dtbs(args, fsw, entries, fdts) + size += fdt_size finish_fit(fsw, entries, bool(args.ramdisk)) # Include the kernel itself in the returned file count fdt = fsw.as_fdt() fdt.pack() - return fdt.as_bytearray(), seq + 1 + bool(args.ramdisk), size + return fdt.as_bytearray(), count + 1 + bool(args.ramdisk), size def run_make_fit(): -- cgit v1.2.3 From fcdcf22a34b0768471d6c834254ef041e3d9c3dc Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 6 Jan 2026 09:27:35 -0700 Subject: scripts/make_fit: Support a few more parallel compressors Add support for pbzip2, xz and plzip which can compress in parallel. This speeds up the ramdisk compression. Signed-off-by: Simon Glass Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/20260106162738.2605574-6-sjg@chromium.org Signed-off-by: Nathan Chancellor --- scripts/make_fit.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'scripts/make_fit.py') diff --git a/scripts/make_fit.py b/scripts/make_fit.py index 7ddaf3c0d783..defb87f223d7 100755 --- a/scripts/make_fit.py +++ b/scripts/make_fit.py @@ -50,11 +50,12 @@ import libfdt CompTool = collections.namedtuple('CompTool', 'ext,tools') COMP_TOOLS = { - 'bzip2': CompTool('.bz2', 'bzip2'), + 'bzip2': CompTool('.bz2', 'pbzip2,bzip2'), 'gzip': CompTool('.gz', 'pigz,gzip'), 'lz4': CompTool('.lz4', 'lz4'), - 'lzma': CompTool('.lzma', 'lzma'), + 'lzma': CompTool('.lzma', 'plzip,lzma'), 'lzo': CompTool('.lzo', 'lzop'), + 'xz': CompTool('.xz', 'xz'), 'zstd': CompTool('.zstd', 'zstd'), } @@ -207,7 +208,12 @@ def compress_data(inf, compress): done = False for tool in comp.tools.split(','): try: - subprocess.call([tool, '-c'], stdin=inf, stdout=outf) + # Add parallel flags for tools that support them + cmd = [tool] + if tool in ('zstd', 'xz'): + cmd.extend(['-T0']) # Use all available cores + cmd.append('-c') + subprocess.call(cmd, stdin=inf, stdout=outf) done = True break except FileNotFoundError: -- cgit v1.2.3 From c7c88b20cd4226af0c9fbdb365fb6f221501c7da Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 6 Jan 2026 09:27:36 -0700 Subject: scripts/make_fit: Compress dtbs in parallel When there are 1500 device tree files it takes quite a while to compress them. Do it in parallel. Signed-off-by: Simon Glass Link: https://patch.msgid.link/20260106162738.2605574-7-sjg@chromium.org Signed-off-by: Nathan Chancellor --- scripts/make_fit.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 6 deletions(-) (limited to 'scripts/make_fit.py') diff --git a/scripts/make_fit.py b/scripts/make_fit.py index defb87f223d7..e923cc8b05b7 100755 --- a/scripts/make_fit.py +++ b/scripts/make_fit.py @@ -37,6 +37,7 @@ as U-Boot, Linuxboot, Tianocore, etc. import argparse import collections +import multiprocessing import os import subprocess import sys @@ -225,15 +226,31 @@ def compress_data(inf, compress): return comp_data -def output_dtb(fsw, seq, fname, arch, compress): +def compress_dtb(fname, compress): + """Compress a single DTB file + + Args: + fname (str): Filename containing the DTB + compress (str): Compression algorithm, e.g. 'gzip' + + Returns: + tuple: (str: fname, bytes: compressed_data) + """ + with open(fname, 'rb') as inf: + compressed = compress_data(inf, compress) + return fname, compressed + + +def output_dtb(fsw, seq, fname, arch, compress, data=None): """Write out a single devicetree to the FIT Args: fsw (libfdt.FdtSw): Object to use for writing seq (int): Sequence number (1 for first) fname (str): Filename containing the DTB - arch: FIT architecture, e.g. 'arm64' + arch (str): FIT architecture, e.g. 'arm64' compress (str): Compressed algorithm, e.g. 'gzip' + data (bytes): Pre-compressed data (optional) """ with fsw.add_node(f'fdt-{seq}'): fsw.property_string('description', os.path.basename(fname)) @@ -241,9 +258,10 @@ def output_dtb(fsw, seq, fname, arch, compress): fsw.property_string('arch', arch) fsw.property_string('compression', compress) - with open(fname, 'rb') as inf: - compressed = compress_data(inf, compress) - fsw.property('data', compressed) + if data is None: + with open(fname, 'rb') as inf: + data = compress_data(inf, compress) + fsw.property('data', data) def process_dtb(fname, args): @@ -300,6 +318,11 @@ def _process_dtbs(args, fsw, entries, fdts): """ seq = 0 size = 0 + + # First figure out the unique DTB files that need compression + todo = [] + file_info = [] # List of (fname, model, compat, files) tuples + for fname in args.dtbs: # Ignore non-DTB (*.dtb) files if os.path.splitext(fname)[1] != '.dtb': @@ -311,11 +334,32 @@ def _process_dtbs(args, fsw, entries, fdts): sys.stderr.write(f'Error processing {fname}:\n') raise e + file_info.append((fname, model, compat, files)) + for fn in files: + if fn not in fdts and fn not in todo: + todo.append(fn) + + # Compress all DTBs in parallel + cache = {} + if todo and args.compress != 'none': + if args.verbose: + print(f'Compressing {len(todo)} DTBs...') + + with multiprocessing.Pool() as pool: + compress_args = [(fn, args.compress) for fn in todo] + # unpacks each tuple, calls compress_dtb(fn, compress) in parallel + results = pool.starmap(compress_dtb, compress_args) + + cache = dict(results) + + # Now write all DTBs to the FIT using pre-compressed data + for fname, model, compat, files in file_info: for fn in files: if fn not in fdts: seq += 1 size += os.path.getsize(fn) - output_dtb(fsw, seq, fn, args.arch, args.compress) + output_dtb(fsw, seq, fn, args.arch, args.compress, + cache.get(fn)) fdts[fn] = seq files_seq = [fdts[fn] for fn in files] -- cgit v1.2.3 From 59f18d88d96e87e8c1faa171904dae43b19729db Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Thu, 12 Feb 2026 15:43:07 +0800 Subject: scripts/make_fit.py: Drop explicit LZMA parallel compression Parallel compression for LZMA was added using the plzip tool. However plzip produces lzip format output, which is different from the raw LZMA format that the lzma tool produces. This causes depthcharge (the second stage bootloader on Chromebooks) to fail to load the payload. Drop the explicit LZMA parallel compression toolchain. If the lzma tool on the build machine is from xz-utils, then there's a chance parallel compression is already enabled. The xz-utils manpage says the following for the -T (threads) argument: Specify the number of worker threads to use. Setting threads to a special value 0 makes xz use up to as many threads as the processor(s) on the system support. The actual number of threads can be fewer than threads if the input file is not big enough for threading with the given settings or if using more threads would exceed the memory usage limit. [...] The default value for threads is 0. In xz 5.4.x and older the default is 1. Fixes: fcdcf22a34b0 ("scripts/make_fit: Support a few more parallel compressors") Signed-off-by: Chen-Yu Tsai Reviewed-by: Simon Glass Link: https://patch.msgid.link/20260212074308.2189032-1-wenst@chromium.org Signed-off-by: Nathan Chancellor --- scripts/make_fit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/make_fit.py') diff --git a/scripts/make_fit.py b/scripts/make_fit.py index e923cc8b05b7..15ba26974fd7 100755 --- a/scripts/make_fit.py +++ b/scripts/make_fit.py @@ -54,7 +54,7 @@ COMP_TOOLS = { 'bzip2': CompTool('.bz2', 'pbzip2,bzip2'), 'gzip': CompTool('.gz', 'pigz,gzip'), 'lz4': CompTool('.lz4', 'lz4'), - 'lzma': CompTool('.lzma', 'plzip,lzma'), + 'lzma': CompTool('.lzma', 'lzma'), 'lzo': CompTool('.lzo', 'lzop'), 'xz': CompTool('.xz', 'xz'), 'zstd': CompTool('.zstd', 'zstd'), -- cgit v1.2.3